blob: 7810ae57b357e7b4eb461b8c5e06ced6fae82e39 [file] [log] [blame]
Alexei Starovoitov99c55f72014-09-26 00:16:57 -07001/* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
2 *
3 * This program is free software; you can redistribute it and/or
4 * modify it under the terms of version 2 of the GNU General Public
5 * License as published by the Free Software Foundation.
6 */
7#ifndef _LINUX_BPF_H
8#define _LINUX_BPF_H 1
9
10#include <uapi/linux/bpf.h>
Daniel Borkmann74451e662017-02-16 22:24:50 +010011
Alexei Starovoitov99c55f72014-09-26 00:16:57 -070012#include <linux/workqueue.h>
Alexei Starovoitovdb20fd22014-09-26 00:16:59 -070013#include <linux/file.h>
Alexei Starovoitovb121d1e2016-03-07 21:57:13 -080014#include <linux/percpu.h>
Zi Shen Lim002245c2016-06-08 21:18:47 -070015#include <linux/err.h>
Daniel Borkmann74451e662017-02-16 22:24:50 +010016#include <linux/rbtree_latch.h>
David S. Millerd6e1e462017-08-19 23:34:03 -070017#include <linux/numa.h>
Jakub Kicinskiab3f0062017-11-03 13:56:17 -070018#include <linux/wait.h>
Alexei Starovoitov99c55f72014-09-26 00:16:57 -070019
Jakub Kicinskicae19272017-12-27 18:39:05 -080020struct bpf_verifier_env;
Daniel Borkmann3b1efb12016-06-15 22:47:14 +020021struct perf_event;
John Fastabend174a79f2017-08-15 22:32:47 -070022struct bpf_prog;
Alexei Starovoitov99c55f72014-09-26 00:16:57 -070023struct bpf_map;
24
25/* map is generic key/value storage optionally accesible by eBPF programs */
26struct bpf_map_ops {
27 /* funcs callable from userspace (via syscall) */
28 struct bpf_map *(*map_alloc)(union bpf_attr *attr);
Daniel Borkmann61d1b6a2016-06-15 22:47:12 +020029 void (*map_release)(struct bpf_map *map, struct file *map_file);
30 void (*map_free)(struct bpf_map *map);
Alexei Starovoitovdb20fd22014-09-26 00:16:59 -070031 int (*map_get_next_key)(struct bpf_map *map, void *key, void *next_key);
32
33 /* funcs callable from userspace and from eBPF programs */
34 void *(*map_lookup_elem)(struct bpf_map *map, void *key);
Alexei Starovoitov3274f522014-11-13 17:36:44 -080035 int (*map_update_elem)(struct bpf_map *map, void *key, void *value, u64 flags);
Alexei Starovoitovdb20fd22014-09-26 00:16:59 -070036 int (*map_delete_elem)(struct bpf_map *map, void *key);
Wang Nan2a36f0b2015-08-06 07:02:33 +000037
38 /* funcs called by prog_array and perf_event_array map */
Daniel Borkmannd056a782016-06-15 22:47:13 +020039 void *(*map_fd_get_ptr)(struct bpf_map *map, struct file *map_file,
40 int fd);
41 void (*map_fd_put_ptr)(void *ptr);
Alexei Starovoitov81ed18a2017-03-15 18:26:42 -070042 u32 (*map_gen_lookup)(struct bpf_map *map, struct bpf_insn *insn_buf);
Martin KaFai Lau14dc6f02017-06-27 23:08:34 -070043 u32 (*map_fd_sys_lookup_elem)(void *ptr);
Alexei Starovoitov99c55f72014-09-26 00:16:57 -070044};
45
46struct bpf_map {
47 atomic_t refcnt;
48 enum bpf_map_type map_type;
49 u32 key_size;
50 u32 value_size;
51 u32 max_entries;
Alexei Starovoitov6c905982016-03-07 21:57:15 -080052 u32 map_flags;
Alexei Starovoitovaaac3ba2015-10-07 22:23:22 -070053 u32 pages;
Martin KaFai Lauf3f1c052017-06-05 12:15:47 -070054 u32 id;
Martin KaFai Lau96eabe72017-08-18 11:28:00 -070055 int numa_node;
Alexei Starovoitovaaac3ba2015-10-07 22:23:22 -070056 struct user_struct *user;
Daniel Borkmanna2c83ff2015-03-01 12:31:42 +010057 const struct bpf_map_ops *ops;
Alexei Starovoitov99c55f72014-09-26 00:16:57 -070058 struct work_struct work;
Daniel Borkmannc9da1612015-11-24 21:28:15 +010059 atomic_t usercnt;
Martin KaFai Lau56f668d2017-03-22 10:00:33 -070060 struct bpf_map *inner_map_meta;
Martin KaFai Lau067cae42017-10-05 21:52:12 -070061 char name[BPF_OBJ_NAME_LEN];
Chenbo Fengafdb09c2017-10-18 13:00:24 -070062#ifdef CONFIG_SECURITY
63 void *security;
64#endif
Alexei Starovoitov99c55f72014-09-26 00:16:57 -070065};
66
Alexei Starovoitov17a52672014-09-26 00:17:06 -070067/* function argument constraints */
68enum bpf_arg_type {
Daniel Borkmann80f1d682015-03-12 17:21:42 +010069 ARG_DONTCARE = 0, /* unused argument in helper function */
Alexei Starovoitov17a52672014-09-26 00:17:06 -070070
71 /* the following constraints used to prototype
72 * bpf_map_lookup/update/delete_elem() functions
73 */
74 ARG_CONST_MAP_PTR, /* const argument used as pointer to bpf_map */
75 ARG_PTR_TO_MAP_KEY, /* pointer to stack used as map key */
76 ARG_PTR_TO_MAP_VALUE, /* pointer to stack used as map value */
77
78 /* the following constraints used to prototype bpf_memcmp() and other
79 * functions that access data on eBPF program stack
80 */
Alexei Starovoitov39f19ebb2017-01-09 10:19:50 -080081 ARG_PTR_TO_MEM, /* pointer to valid memory (stack, packet, map value) */
Gianluca Borellodb1ac492017-11-22 18:32:53 +000082 ARG_PTR_TO_MEM_OR_NULL, /* pointer to valid memory or NULL */
Alexei Starovoitov39f19ebb2017-01-09 10:19:50 -080083 ARG_PTR_TO_UNINIT_MEM, /* pointer to memory does not need to be initialized,
84 * helper function must fill all bytes or clear
85 * them in error case.
Daniel Borkmann435faee12016-04-13 00:10:51 +020086 */
87
Alexei Starovoitov39f19ebb2017-01-09 10:19:50 -080088 ARG_CONST_SIZE, /* number of bytes accessed from memory */
89 ARG_CONST_SIZE_OR_ZERO, /* number of bytes accessed from memory or 0 */
Daniel Borkmann80f1d682015-03-12 17:21:42 +010090
Alexei Starovoitov608cd712015-03-26 19:53:57 -070091 ARG_PTR_TO_CTX, /* pointer to context */
Daniel Borkmann80f1d682015-03-12 17:21:42 +010092 ARG_ANYTHING, /* any (initialized) argument is ok */
Alexei Starovoitov17a52672014-09-26 00:17:06 -070093};
94
95/* type of values returned from helper functions */
96enum bpf_return_type {
97 RET_INTEGER, /* function returns integer */
98 RET_VOID, /* function doesn't return anything */
99 RET_PTR_TO_MAP_VALUE_OR_NULL, /* returns a pointer to map elem value or NULL */
100};
101
Alexei Starovoitov09756af2014-09-26 00:17:00 -0700102/* eBPF function prototype used by verifier to allow BPF_CALLs from eBPF programs
103 * to in-kernel helper functions and for adjusting imm32 field in BPF_CALL
104 * instructions after verifying
105 */
106struct bpf_func_proto {
107 u64 (*func)(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
108 bool gpl_only;
Daniel Borkmann36bbef52016-09-20 00:26:13 +0200109 bool pkt_access;
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700110 enum bpf_return_type ret_type;
111 enum bpf_arg_type arg1_type;
112 enum bpf_arg_type arg2_type;
113 enum bpf_arg_type arg3_type;
114 enum bpf_arg_type arg4_type;
115 enum bpf_arg_type arg5_type;
116};
117
118/* bpf_context is intentionally undefined structure. Pointer to bpf_context is
119 * the first argument to eBPF programs.
120 * For socket filters: 'struct bpf_context *' == 'struct sk_buff *'
121 */
122struct bpf_context;
123
124enum bpf_access_type {
125 BPF_READ = 1,
126 BPF_WRITE = 2
Alexei Starovoitov09756af2014-09-26 00:17:00 -0700127};
128
Alexei Starovoitov19de99f2016-06-15 18:25:38 -0700129/* types of values stored in eBPF registers */
Edward Creef1174f72017-08-07 15:26:19 +0100130/* Pointer types represent:
131 * pointer
132 * pointer + imm
133 * pointer + (u16) var
134 * pointer + (u16) var + imm
135 * if (range > 0) then [ptr, ptr + range - off) is safe to access
136 * if (id > 0) means that some 'var' was added
137 * if (off > 0) means that 'imm' was added
138 */
Alexei Starovoitov19de99f2016-06-15 18:25:38 -0700139enum bpf_reg_type {
140 NOT_INIT = 0, /* nothing was written into register */
Edward Creef1174f72017-08-07 15:26:19 +0100141 SCALAR_VALUE, /* reg doesn't contain a valid pointer */
Alexei Starovoitov19de99f2016-06-15 18:25:38 -0700142 PTR_TO_CTX, /* reg points to bpf_context */
143 CONST_PTR_TO_MAP, /* reg points to struct bpf_map */
144 PTR_TO_MAP_VALUE, /* reg points to map element value */
145 PTR_TO_MAP_VALUE_OR_NULL,/* points to map elem value or NULL */
Edward Creef1174f72017-08-07 15:26:19 +0100146 PTR_TO_STACK, /* reg == frame_pointer + offset */
Daniel Borkmannde8f3a82017-09-25 02:25:51 +0200147 PTR_TO_PACKET_META, /* skb->data - meta_len */
Edward Creef1174f72017-08-07 15:26:19 +0100148 PTR_TO_PACKET, /* reg points to skb->data */
Alexei Starovoitov19de99f2016-06-15 18:25:38 -0700149 PTR_TO_PACKET_END, /* skb->data + headlen */
150};
151
Yonghong Song23994632017-06-22 15:07:39 -0700152/* The information passed from prog-specific *_is_valid_access
153 * back to the verifier.
154 */
155struct bpf_insn_access_aux {
156 enum bpf_reg_type reg_type;
157 int ctx_field_size;
Yonghong Song23994632017-06-22 15:07:39 -0700158};
159
Daniel Borkmannf96da092017-07-02 02:13:27 +0200160static inline void
161bpf_ctx_record_field_size(struct bpf_insn_access_aux *aux, u32 size)
162{
163 aux->ctx_field_size = size;
164}
165
Jakub Kicinski7de16e32017-10-16 16:40:53 -0700166struct bpf_prog_ops {
167 int (*test_run)(struct bpf_prog *prog, const union bpf_attr *kattr,
168 union bpf_attr __user *uattr);
169};
170
Alexei Starovoitov09756af2014-09-26 00:17:00 -0700171struct bpf_verifier_ops {
172 /* return eBPF function prototype for verification */
173 const struct bpf_func_proto *(*get_func_proto)(enum bpf_func_id func_id);
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700174
175 /* return true if 'size' wide access at offset 'off' within bpf_context
176 * with 'type' (read or write) is allowed
177 */
Alexei Starovoitov19de99f2016-06-15 18:25:38 -0700178 bool (*is_valid_access)(int off, int size, enum bpf_access_type type,
Yonghong Song23994632017-06-22 15:07:39 -0700179 struct bpf_insn_access_aux *info);
Daniel Borkmann36bbef52016-09-20 00:26:13 +0200180 int (*gen_prologue)(struct bpf_insn *insn, bool direct_write,
181 const struct bpf_prog *prog);
Daniel Borkmann6b8cc1d2017-01-12 11:51:32 +0100182 u32 (*convert_ctx_access)(enum bpf_access_type type,
183 const struct bpf_insn *src,
184 struct bpf_insn *dst,
Daniel Borkmannf96da092017-07-02 02:13:27 +0200185 struct bpf_prog *prog, u32 *target_size);
Alexei Starovoitov09756af2014-09-26 00:17:00 -0700186};
187
Jakub Kicinskicae19272017-12-27 18:39:05 -0800188struct bpf_prog_offload_ops {
189 int (*insn_hook)(struct bpf_verifier_env *env,
190 int insn_idx, int prev_insn_idx);
191};
192
Jakub Kicinskiab3f0062017-11-03 13:56:17 -0700193struct bpf_dev_offload {
194 struct bpf_prog *prog;
195 struct net_device *netdev;
196 void *dev_priv;
197 struct list_head offloads;
198 bool dev_state;
Jakub Kicinskicae19272017-12-27 18:39:05 -0800199 const struct bpf_prog_offload_ops *dev_ops;
Jakub Kicinskiab3f0062017-11-03 13:56:17 -0700200};
201
Alexei Starovoitov09756af2014-09-26 00:17:00 -0700202struct bpf_prog_aux {
203 atomic_t refcnt;
Daniel Borkmann24701ec2015-03-01 12:31:47 +0100204 u32 used_map_cnt;
Alexei Starovoitov32bbe002016-04-06 18:43:28 -0700205 u32 max_ctx_offset;
Alexei Starovoitov87266792017-05-30 13:31:29 -0700206 u32 stack_depth;
Martin KaFai Laudc4bb0e2017-06-05 12:15:46 -0700207 u32 id;
Alexei Starovoitov1c2a0882017-12-14 17:55:15 -0800208 u32 func_cnt;
Jakub Kicinski9a18eed2017-12-27 18:39:04 -0800209 bool offload_requested;
Alexei Starovoitov1c2a0882017-12-14 17:55:15 -0800210 struct bpf_prog **func;
211 void *jit_data; /* JIT specific data. arch dependent */
Daniel Borkmann74451e662017-02-16 22:24:50 +0100212 struct latch_tree_node ksym_tnode;
213 struct list_head ksym_lnode;
Jakub Kicinski7de16e32017-10-16 16:40:53 -0700214 const struct bpf_prog_ops *ops;
Alexei Starovoitov09756af2014-09-26 00:17:00 -0700215 struct bpf_map **used_maps;
Alexei Starovoitov09756af2014-09-26 00:17:00 -0700216 struct bpf_prog *prog;
Alexei Starovoitovaaac3ba2015-10-07 22:23:22 -0700217 struct user_struct *user;
Martin KaFai Laucb4d2b32017-09-27 14:37:52 -0700218 u64 load_time; /* ns since boottime */
Martin KaFai Lau067cae42017-10-05 21:52:12 -0700219 char name[BPF_OBJ_NAME_LEN];
Chenbo Fengafdb09c2017-10-18 13:00:24 -0700220#ifdef CONFIG_SECURITY
221 void *security;
222#endif
Jakub Kicinskiab3f0062017-11-03 13:56:17 -0700223 struct bpf_dev_offload *offload;
Alexei Starovoitovabf2e7d2015-05-28 19:26:02 -0700224 union {
225 struct work_struct work;
226 struct rcu_head rcu;
227 };
Alexei Starovoitov09756af2014-09-26 00:17:00 -0700228};
229
Alexei Starovoitov04fd61ab2015-05-19 16:59:03 -0700230struct bpf_array {
231 struct bpf_map map;
232 u32 elem_size;
233 /* 'ownership' of prog_array is claimed by the first program that
234 * is going to use this map or by the first program which FD is stored
235 * in the map to make sure that all callers and callees have the same
236 * prog_type and JITed flag
237 */
238 enum bpf_prog_type owner_prog_type;
239 bool owner_jited;
240 union {
241 char value[0] __aligned(8);
Wang Nan2a36f0b2015-08-06 07:02:33 +0000242 void *ptrs[0] __aligned(8);
Alexei Starovoitova10423b2016-02-01 22:39:54 -0800243 void __percpu *pptrs[0] __aligned(8);
Alexei Starovoitov04fd61ab2015-05-19 16:59:03 -0700244 };
245};
Daniel Borkmann3b1efb12016-06-15 22:47:14 +0200246
Alexei Starovoitov04fd61ab2015-05-19 16:59:03 -0700247#define MAX_TAIL_CALL_CNT 32
248
Daniel Borkmann3b1efb12016-06-15 22:47:14 +0200249struct bpf_event_entry {
250 struct perf_event *event;
251 struct file *perf_file;
252 struct file *map_file;
253 struct rcu_head rcu;
254};
255
Alexei Starovoitov04fd61ab2015-05-19 16:59:03 -0700256bool bpf_prog_array_compatible(struct bpf_array *array, const struct bpf_prog *fp);
Daniel Borkmannf1f77142017-01-13 23:38:15 +0100257int bpf_prog_calc_tag(struct bpf_prog *fp);
Daniel Borkmannbd570ff2016-04-18 21:01:24 +0200258
Alexei Starovoitov0756ea32015-06-12 19:39:13 -0700259const struct bpf_func_proto *bpf_get_trace_printk_proto(void);
Daniel Borkmann555c8a82016-07-14 18:08:05 +0200260
261typedef unsigned long (*bpf_ctx_copy_t)(void *dst, const void *src,
Daniel Borkmannaa7145c2016-07-22 01:19:42 +0200262 unsigned long off, unsigned long len);
Daniel Borkmann555c8a82016-07-14 18:08:05 +0200263
264u64 bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size,
265 void *ctx, u64 ctx_size, bpf_ctx_copy_t ctx_copy);
Alexei Starovoitov04fd61ab2015-05-19 16:59:03 -0700266
Alexei Starovoitov1cf1cae2017-03-30 21:45:38 -0700267int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr,
268 union bpf_attr __user *uattr);
269int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr,
270 union bpf_attr __user *uattr);
271
Alexei Starovoitov324bda9e62017-10-02 22:50:21 -0700272/* an array of programs to be executed under rcu_lock.
273 *
274 * Typical usage:
275 * ret = BPF_PROG_RUN_ARRAY(&bpf_prog_array, ctx, BPF_PROG_RUN);
276 *
277 * the structure returned by bpf_prog_array_alloc() should be populated
278 * with program pointers and the last pointer must be NULL.
279 * The user has to keep refcnt on the program and make sure the program
280 * is removed from the array before bpf_prog_put().
281 * The 'struct bpf_prog_array *' should only be replaced with xchg()
282 * since other cpus are walking the array of pointers in parallel.
283 */
284struct bpf_prog_array {
285 struct rcu_head rcu;
286 struct bpf_prog *progs[0];
287};
288
289struct bpf_prog_array __rcu *bpf_prog_array_alloc(u32 prog_cnt, gfp_t flags);
290void bpf_prog_array_free(struct bpf_prog_array __rcu *progs);
Alexei Starovoitov468e2f62017-10-02 22:50:22 -0700291int bpf_prog_array_length(struct bpf_prog_array __rcu *progs);
292int bpf_prog_array_copy_to_user(struct bpf_prog_array __rcu *progs,
293 __u32 __user *prog_ids, u32 cnt);
Alexei Starovoitov324bda9e62017-10-02 22:50:21 -0700294
Yonghong Songe87c6bc382017-10-23 23:53:08 -0700295void bpf_prog_array_delete_safe(struct bpf_prog_array __rcu *progs,
296 struct bpf_prog *old_prog);
Yonghong Songf371b302017-12-11 11:39:02 -0800297int bpf_prog_array_copy_info(struct bpf_prog_array __rcu *array,
298 __u32 __user *prog_ids, u32 request_cnt,
299 __u32 __user *prog_cnt);
Yonghong Songe87c6bc382017-10-23 23:53:08 -0700300int bpf_prog_array_copy(struct bpf_prog_array __rcu *old_array,
301 struct bpf_prog *exclude_prog,
302 struct bpf_prog *include_prog,
303 struct bpf_prog_array **new_array);
304
305#define __BPF_PROG_RUN_ARRAY(array, ctx, func, check_non_null) \
Alexei Starovoitov324bda9e62017-10-02 22:50:21 -0700306 ({ \
Yonghong Songe87c6bc382017-10-23 23:53:08 -0700307 struct bpf_prog **_prog, *__prog; \
308 struct bpf_prog_array *_array; \
Alexei Starovoitov324bda9e62017-10-02 22:50:21 -0700309 u32 _ret = 1; \
310 rcu_read_lock(); \
Yonghong Songe87c6bc382017-10-23 23:53:08 -0700311 _array = rcu_dereference(array); \
312 if (unlikely(check_non_null && !_array))\
313 goto _out; \
314 _prog = _array->progs; \
315 while ((__prog = READ_ONCE(*_prog))) { \
316 _ret &= func(__prog, ctx); \
317 _prog++; \
318 } \
319_out: \
Alexei Starovoitov324bda9e62017-10-02 22:50:21 -0700320 rcu_read_unlock(); \
321 _ret; \
322 })
323
Yonghong Songe87c6bc382017-10-23 23:53:08 -0700324#define BPF_PROG_RUN_ARRAY(array, ctx, func) \
325 __BPF_PROG_RUN_ARRAY(array, ctx, func, false)
326
327#define BPF_PROG_RUN_ARRAY_CHECK(array, ctx, func) \
328 __BPF_PROG_RUN_ARRAY(array, ctx, func, true)
329
Alexei Starovoitov89aa0752014-12-01 15:06:35 -0800330#ifdef CONFIG_BPF_SYSCALL
Alexei Starovoitovb121d1e2016-03-07 21:57:13 -0800331DECLARE_PER_CPU(int, bpf_prog_active);
332
Chenbo Fengf66e4482017-10-18 13:00:26 -0700333extern const struct file_operations bpf_map_fops;
334extern const struct file_operations bpf_prog_fops;
335
Jakub Kicinski7de16e32017-10-16 16:40:53 -0700336#define BPF_PROG_TYPE(_id, _name) \
337 extern const struct bpf_prog_ops _name ## _prog_ops; \
338 extern const struct bpf_verifier_ops _name ## _verifier_ops;
Johannes Berg40077e02017-04-11 15:34:58 +0200339#define BPF_MAP_TYPE(_id, _ops) \
340 extern const struct bpf_map_ops _ops;
Johannes Bergbe9370a2017-04-11 15:34:57 +0200341#include <linux/bpf_types.h>
342#undef BPF_PROG_TYPE
Johannes Berg40077e02017-04-11 15:34:58 +0200343#undef BPF_MAP_TYPE
Daniel Borkmann0fc174d2015-03-01 12:31:44 +0100344
Jakub Kicinskiab3f0062017-11-03 13:56:17 -0700345extern const struct bpf_prog_ops bpf_offload_prog_ops;
Jakub Kicinski4f9218a2017-10-16 16:40:55 -0700346extern const struct bpf_verifier_ops tc_cls_act_analyzer_ops;
347extern const struct bpf_verifier_ops xdp_analyzer_ops;
348
Alexei Starovoitov09756af2014-09-26 00:17:00 -0700349struct bpf_prog *bpf_prog_get(u32 ufd);
Jakub Kicinski248f3462017-11-03 13:56:20 -0700350struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, enum bpf_prog_type type,
Jakub Kicinski288b3de52017-11-20 15:21:54 -0800351 bool attach_drv);
Daniel Borkmann6d67942dd2016-11-19 01:45:03 +0100352struct bpf_prog * __must_check bpf_prog_add(struct bpf_prog *prog, int i);
Daniel Borkmannc5405942016-11-09 22:02:34 +0100353void bpf_prog_sub(struct bpf_prog *prog, int i);
Daniel Borkmann6d67942dd2016-11-19 01:45:03 +0100354struct bpf_prog * __must_check bpf_prog_inc(struct bpf_prog *prog);
John Fastabenda6f6df62017-08-15 22:32:22 -0700355struct bpf_prog * __must_check bpf_prog_inc_not_zero(struct bpf_prog *prog);
Daniel Borkmann61e021f32015-03-02 15:21:55 +0100356void bpf_prog_put(struct bpf_prog *prog);
Daniel Borkmann5ccb0712016-12-18 01:52:58 +0100357int __bpf_prog_charge(struct user_struct *user, u32 pages);
358void __bpf_prog_uncharge(struct user_struct *user, u32 pages);
Daniel Borkmann61e021f32015-03-02 15:21:55 +0100359
Jakub Kicinskiad8ad792017-12-27 18:39:07 -0800360void bpf_prog_free_id(struct bpf_prog *prog, bool do_idr_lock);
361
Daniel Borkmannc9da1612015-11-24 21:28:15 +0100362struct bpf_map *bpf_map_get_with_uref(u32 ufd);
Daniel Borkmannc2101292015-10-29 14:58:07 +0100363struct bpf_map *__bpf_map_get(struct fd f);
Daniel Borkmann6d67942dd2016-11-19 01:45:03 +0100364struct bpf_map * __must_check bpf_map_inc(struct bpf_map *map, bool uref);
Daniel Borkmannc9da1612015-11-24 21:28:15 +0100365void bpf_map_put_with_uref(struct bpf_map *map);
Daniel Borkmann61e021f32015-03-02 15:21:55 +0100366void bpf_map_put(struct bpf_map *map);
Alexei Starovoitov6c905982016-03-07 21:57:15 -0800367int bpf_map_precharge_memlock(u32 pages);
Martin KaFai Lau96eabe72017-08-18 11:28:00 -0700368void *bpf_map_area_alloc(size_t size, int numa_node);
Daniel Borkmannd407bd22017-01-18 15:14:17 +0100369void bpf_map_area_free(void *base);
Daniel Borkmann61e021f32015-03-02 15:21:55 +0100370
Alexei Starovoitov1be7f752015-10-07 22:23:21 -0700371extern int sysctl_unprivileged_bpf_disabled;
372
Chenbo Feng6e71b042017-10-18 13:00:22 -0700373int bpf_map_new_fd(struct bpf_map *map, int flags);
Daniel Borkmannb2197752015-10-29 14:58:09 +0100374int bpf_prog_new_fd(struct bpf_prog *prog);
375
376int bpf_obj_pin_user(u32 ufd, const char __user *pathname);
Chenbo Feng6e71b042017-10-18 13:00:22 -0700377int bpf_obj_get_user(const char __user *pathname, int flags);
Daniel Borkmannb2197752015-10-29 14:58:09 +0100378
Alexei Starovoitov15a07b32016-02-01 22:39:55 -0800379int bpf_percpu_hash_copy(struct bpf_map *map, void *key, void *value);
380int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value);
381int bpf_percpu_hash_update(struct bpf_map *map, void *key, void *value,
382 u64 flags);
383int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value,
384 u64 flags);
Daniel Borkmannd056a782016-06-15 22:47:13 +0200385
Alexei Starovoitov557c0c62016-03-07 21:57:17 -0800386int bpf_stackmap_copy(struct bpf_map *map, void *key, void *value);
Alexei Starovoitov15a07b32016-02-01 22:39:55 -0800387
Daniel Borkmannd056a782016-06-15 22:47:13 +0200388int bpf_fd_array_map_update_elem(struct bpf_map *map, struct file *map_file,
389 void *key, void *value, u64 map_flags);
Martin KaFai Lau14dc6f02017-06-27 23:08:34 -0700390int bpf_fd_array_map_lookup_elem(struct bpf_map *map, void *key, u32 *value);
Daniel Borkmannd056a782016-06-15 22:47:13 +0200391void bpf_fd_array_map_clear(struct bpf_map *map);
Martin KaFai Laubcc6b1b2017-03-22 10:00:34 -0700392int bpf_fd_htab_map_update_elem(struct bpf_map *map, struct file *map_file,
393 void *key, void *value, u64 map_flags);
Martin KaFai Lau14dc6f02017-06-27 23:08:34 -0700394int bpf_fd_htab_map_lookup_elem(struct bpf_map *map, void *key, u32 *value);
Daniel Borkmannd056a782016-06-15 22:47:13 +0200395
Chenbo Feng6e71b042017-10-18 13:00:22 -0700396int bpf_get_file_flag(int flags);
397
Alexei Starovoitov15a07b32016-02-01 22:39:55 -0800398/* memcpy that is used with 8-byte aligned pointers, power-of-8 size and
399 * forced to use 'long' read/writes to try to atomically copy long counters.
400 * Best-effort only. No barriers here, since it _will_ race with concurrent
401 * updates from BPF programs. Called from bpf syscall and mostly used with
402 * size 8 or 16 bytes, so ask compiler to inline it.
403 */
404static inline void bpf_long_memcpy(void *dst, const void *src, u32 size)
405{
406 const long *lsrc = src;
407 long *ldst = dst;
408
409 size /= sizeof(long);
410 while (size--)
411 *ldst++ = *lsrc++;
412}
413
Daniel Borkmann61e021f32015-03-02 15:21:55 +0100414/* verify correctness of eBPF program */
Alexei Starovoitov9bac3d62015-03-13 11:57:42 -0700415int bpf_check(struct bpf_prog **fp, union bpf_attr *attr);
Alexei Starovoitov1ea47e02017-12-14 17:55:13 -0800416void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth);
John Fastabend46f55cf2017-07-17 21:56:48 -0700417
418/* Map specifics */
419struct net_device *__dev_map_lookup_elem(struct bpf_map *map, u32 key);
420void __dev_map_insert_ctx(struct bpf_map *map, u32 index);
421void __dev_map_flush(struct bpf_map *map);
422
Jesper Dangaard Brouer9c270af2017-10-16 12:19:34 +0200423struct bpf_cpu_map_entry *__cpu_map_lookup_elem(struct bpf_map *map, u32 key);
424void __cpu_map_insert_ctx(struct bpf_map *map, u32 index);
425void __cpu_map_flush(struct bpf_map *map);
426struct xdp_buff;
427int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_buff *xdp,
428 struct net_device *dev_rx);
429
Martin KaFai Lau96eabe72017-08-18 11:28:00 -0700430/* Return map's numa specified by userspace */
431static inline int bpf_map_attr_numa_node(const union bpf_attr *attr)
432{
433 return (attr->map_flags & BPF_F_NUMA_NODE) ?
434 attr->numa_node : NUMA_NO_NODE;
435}
436
Jesper Dangaard Brouer9c270af2017-10-16 12:19:34 +0200437#else /* !CONFIG_BPF_SYSCALL */
Daniel Borkmann0fc174d2015-03-01 12:31:44 +0100438static inline struct bpf_prog *bpf_prog_get(u32 ufd)
439{
440 return ERR_PTR(-EOPNOTSUPP);
441}
442
Jakub Kicinski248f3462017-11-03 13:56:20 -0700443static inline struct bpf_prog *bpf_prog_get_type_dev(u32 ufd,
444 enum bpf_prog_type type,
Jakub Kicinski288b3de52017-11-20 15:21:54 -0800445 bool attach_drv)
Jakub Kicinski248f3462017-11-03 13:56:20 -0700446{
447 return ERR_PTR(-EOPNOTSUPP);
448}
449
Daniel Borkmann6d67942dd2016-11-19 01:45:03 +0100450static inline struct bpf_prog * __must_check bpf_prog_add(struct bpf_prog *prog,
451 int i)
Brenden Blancocc2e0b32016-07-20 07:55:52 -0700452{
453 return ERR_PTR(-EOPNOTSUPP);
454}
Daniel Borkmann113214b2016-06-30 17:24:44 +0200455
Daniel Borkmannc5405942016-11-09 22:02:34 +0100456static inline void bpf_prog_sub(struct bpf_prog *prog, int i)
457{
458}
459
Daniel Borkmann0fc174d2015-03-01 12:31:44 +0100460static inline void bpf_prog_put(struct bpf_prog *prog)
461{
462}
Daniel Borkmann6d67942dd2016-11-19 01:45:03 +0100463
464static inline struct bpf_prog * __must_check bpf_prog_inc(struct bpf_prog *prog)
Alexei Starovoitovaa6a5f32016-09-01 18:37:24 -0700465{
466 return ERR_PTR(-EOPNOTSUPP);
467}
Daniel Borkmann5ccb0712016-12-18 01:52:58 +0100468
John Fastabenda6f6df62017-08-15 22:32:22 -0700469static inline struct bpf_prog *__must_check
470bpf_prog_inc_not_zero(struct bpf_prog *prog)
471{
472 return ERR_PTR(-EOPNOTSUPP);
473}
474
Daniel Borkmann5ccb0712016-12-18 01:52:58 +0100475static inline int __bpf_prog_charge(struct user_struct *user, u32 pages)
476{
477 return 0;
478}
479
480static inline void __bpf_prog_uncharge(struct user_struct *user, u32 pages)
481{
482}
John Fastabend46f55cf2017-07-17 21:56:48 -0700483
Chenbo Feng6e71b042017-10-18 13:00:22 -0700484static inline int bpf_obj_get_user(const char __user *pathname, int flags)
Shmulik Ladkani98589a02017-10-09 15:27:15 +0300485{
486 return -EOPNOTSUPP;
487}
488
John Fastabend46f55cf2017-07-17 21:56:48 -0700489static inline struct net_device *__dev_map_lookup_elem(struct bpf_map *map,
490 u32 key)
491{
492 return NULL;
493}
494
495static inline void __dev_map_insert_ctx(struct bpf_map *map, u32 index)
496{
497}
498
499static inline void __dev_map_flush(struct bpf_map *map)
500{
501}
Jesper Dangaard Brouer9c270af2017-10-16 12:19:34 +0200502
503static inline
504struct bpf_cpu_map_entry *__cpu_map_lookup_elem(struct bpf_map *map, u32 key)
505{
506 return NULL;
507}
508
509static inline void __cpu_map_insert_ctx(struct bpf_map *map, u32 index)
510{
511}
512
513static inline void __cpu_map_flush(struct bpf_map *map)
514{
515}
516
517struct xdp_buff;
518static inline int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu,
519 struct xdp_buff *xdp,
520 struct net_device *dev_rx)
521{
522 return 0;
523}
Daniel Borkmann61e021f32015-03-02 15:21:55 +0100524#endif /* CONFIG_BPF_SYSCALL */
Alexei Starovoitov09756af2014-09-26 00:17:00 -0700525
Jakub Kicinski479321e2017-11-20 15:21:56 -0800526static inline struct bpf_prog *bpf_prog_get_type(u32 ufd,
527 enum bpf_prog_type type)
528{
529 return bpf_prog_get_type_dev(ufd, type, false);
530}
531
Jakub Kicinskiab3f0062017-11-03 13:56:17 -0700532int bpf_prog_offload_compile(struct bpf_prog *prog);
533void bpf_prog_offload_destroy(struct bpf_prog *prog);
Jakub Kicinski675fc272017-12-27 18:39:09 -0800534int bpf_prog_offload_info_fill(struct bpf_prog_info *info,
535 struct bpf_prog *prog);
Jakub Kicinskiab3f0062017-11-03 13:56:17 -0700536
537#if defined(CONFIG_NET) && defined(CONFIG_BPF_SYSCALL)
538int bpf_prog_offload_init(struct bpf_prog *prog, union bpf_attr *attr);
539
540static inline bool bpf_prog_is_dev_bound(struct bpf_prog_aux *aux)
541{
Jakub Kicinski9a18eed2017-12-27 18:39:04 -0800542 return aux->offload_requested;
Jakub Kicinskiab3f0062017-11-03 13:56:17 -0700543}
544#else
545static inline int bpf_prog_offload_init(struct bpf_prog *prog,
546 union bpf_attr *attr)
547{
548 return -EOPNOTSUPP;
549}
550
551static inline bool bpf_prog_is_dev_bound(struct bpf_prog_aux *aux)
552{
553 return false;
554}
555#endif /* CONFIG_NET && CONFIG_BPF_SYSCALL */
556
John Fastabend6bdc9c42017-08-16 15:02:32 -0700557#if defined(CONFIG_STREAM_PARSER) && defined(CONFIG_BPF_SYSCALL)
558struct sock *__sock_map_lookup_elem(struct bpf_map *map, u32 key);
John Fastabend5a67da22017-09-08 14:00:49 -0700559int sock_map_prog(struct bpf_map *map, struct bpf_prog *prog, u32 type);
John Fastabend6bdc9c42017-08-16 15:02:32 -0700560#else
561static inline struct sock *__sock_map_lookup_elem(struct bpf_map *map, u32 key)
562{
563 return NULL;
564}
John Fastabend464bc0f2017-08-28 07:10:04 -0700565
John Fastabend5a67da22017-09-08 14:00:49 -0700566static inline int sock_map_prog(struct bpf_map *map,
567 struct bpf_prog *prog,
568 u32 type)
John Fastabend464bc0f2017-08-28 07:10:04 -0700569{
570 return -EOPNOTSUPP;
571}
John Fastabend6bdc9c42017-08-16 15:02:32 -0700572#endif
573
Alexei Starovoitovd0003ec2014-11-13 17:36:49 -0800574/* verifier prototypes for helper functions called from eBPF programs */
Daniel Borkmanna2c83ff2015-03-01 12:31:42 +0100575extern const struct bpf_func_proto bpf_map_lookup_elem_proto;
576extern const struct bpf_func_proto bpf_map_update_elem_proto;
577extern const struct bpf_func_proto bpf_map_delete_elem_proto;
Alexei Starovoitovd0003ec2014-11-13 17:36:49 -0800578
Daniel Borkmann03e69b52015-03-14 02:27:16 +0100579extern const struct bpf_func_proto bpf_get_prandom_u32_proto;
Daniel Borkmannc04167c2015-03-14 02:27:17 +0100580extern const struct bpf_func_proto bpf_get_smp_processor_id_proto;
Daniel Borkmann2d0e30c2016-10-21 12:46:33 +0200581extern const struct bpf_func_proto bpf_get_numa_node_id_proto;
Alexei Starovoitov04fd61ab2015-05-19 16:59:03 -0700582extern const struct bpf_func_proto bpf_tail_call_proto;
Daniel Borkmann17ca8cb2015-05-29 23:23:06 +0200583extern const struct bpf_func_proto bpf_ktime_get_ns_proto;
Alexei Starovoitovffeedaf2015-06-12 19:39:12 -0700584extern const struct bpf_func_proto bpf_get_current_pid_tgid_proto;
585extern const struct bpf_func_proto bpf_get_current_uid_gid_proto;
586extern const struct bpf_func_proto bpf_get_current_comm_proto;
Alexei Starovoitov4e10df92015-07-20 20:34:18 -0700587extern const struct bpf_func_proto bpf_skb_vlan_push_proto;
588extern const struct bpf_func_proto bpf_skb_vlan_pop_proto;
Alexei Starovoitovd5a3b1f2016-02-17 19:58:58 -0800589extern const struct bpf_func_proto bpf_get_stackid_proto;
John Fastabend174a79f2017-08-15 22:32:47 -0700590extern const struct bpf_func_proto bpf_sock_map_update_proto;
Daniel Borkmann03e69b52015-03-14 02:27:16 +0100591
Daniel Borkmann3ad00402015-10-08 01:20:39 +0200592/* Shared helpers among cBPF and eBPF. */
593void bpf_user_rnd_init_once(void);
594u64 bpf_user_rnd_u32(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
595
Josef Bacik92ace992017-12-11 11:36:46 -0500596#if defined(__KERNEL__) && !defined(__ASSEMBLY__)
597#ifdef CONFIG_BPF_KPROBE_OVERRIDE
598#define BPF_ALLOW_ERROR_INJECTION(fname) \
599static unsigned long __used \
600 __attribute__((__section__("_kprobe_error_inject_list"))) \
601 _eil_addr_##fname = (unsigned long)fname;
602#else
603#define BPF_ALLOW_ERROR_INJECTION(fname)
604#endif
605#endif
606
Alexei Starovoitov99c55f72014-09-26 00:16:57 -0700607#endif /* _LINUX_BPF_H */