blob: 24678d6ecbcfe9b40b8165d35a87c8e20aa94041 [file] [log] [blame]
Thomas Gleixner25763b32019-05-28 10:10:09 -07001/* SPDX-License-Identifier: GPL-2.0-only */
Alexei Starovoitov99c55f72014-09-26 00:16:57 -07002/* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
Alexei Starovoitov99c55f72014-09-26 00:16:57 -07003 */
4#ifndef _LINUX_BPF_H
5#define _LINUX_BPF_H 1
6
7#include <uapi/linux/bpf.h>
Daniel Borkmann74451e662017-02-16 22:24:50 +01008
Alexei Starovoitov99c55f72014-09-26 00:16:57 -07009#include <linux/workqueue.h>
Alexei Starovoitovdb20fd22014-09-26 00:16:59 -070010#include <linux/file.h>
Alexei Starovoitovb121d1e2016-03-07 21:57:13 -080011#include <linux/percpu.h>
Zi Shen Lim002245c2016-06-08 21:18:47 -070012#include <linux/err.h>
Daniel Borkmann74451e662017-02-16 22:24:50 +010013#include <linux/rbtree_latch.h>
David S. Millerd6e1e462017-08-19 23:34:03 -070014#include <linux/numa.h>
Andrii Nakryikofc970222019-11-17 09:28:04 -080015#include <linux/mm_types.h>
Jakub Kicinskiab3f0062017-11-03 13:56:17 -070016#include <linux/wait.h>
Alexei Starovoitovfec56f52019-11-14 10:57:04 -080017#include <linux/refcount.h>
18#include <linux/mutex.h>
Martin KaFai Lau85d33df2020-01-08 16:35:05 -080019#include <linux/module.h>
Jiri Olsabfea9a82020-03-12 20:55:59 +010020#include <linux/kallsyms.h>
Alexei Starovoitov2c78ee82020-05-13 16:03:54 -070021#include <linux/capability.h>
Roman Gushchin48edc1f2020-12-01 13:58:32 -080022#include <linux/sched/mm.h>
23#include <linux/slab.h>
Alexei Starovoitove21aa342021-03-16 14:00:07 -070024#include <linux/percpu-refcount.h>
Alexei Starovoitov99c55f72014-09-26 00:16:57 -070025
Jakub Kicinskicae19272017-12-27 18:39:05 -080026struct bpf_verifier_env;
Alexei Starovoitov9e15db62019-10-15 20:25:00 -070027struct bpf_verifier_log;
Daniel Borkmann3b1efb12016-06-15 22:47:14 +020028struct perf_event;
John Fastabend174a79f2017-08-15 22:32:47 -070029struct bpf_prog;
Daniel Borkmannda765a22019-11-22 21:07:58 +010030struct bpf_prog_aux;
Alexei Starovoitov99c55f72014-09-26 00:16:57 -070031struct bpf_map;
John Fastabend4f738ad2018-03-18 12:57:10 -070032struct sock;
Martin KaFai Laua26ca7c2018-04-18 15:56:03 -070033struct seq_file;
Roman Gushchin1b2b2342018-12-10 15:43:00 -080034struct btf;
Daniel Borkmanne8d2bec2018-08-12 01:59:17 +020035struct btf_type;
Alexei Starovoitov3dec5412019-10-15 20:25:03 -070036struct exception_table_entry;
Yonghong Songae243452020-05-09 10:58:59 -070037struct seq_operations;
Yonghong Songf9c79272020-07-23 11:41:10 -070038struct bpf_iter_aux_info;
KP Singhf836a562020-08-25 20:29:15 +020039struct bpf_local_storage;
40struct bpf_local_storage_map;
Andrii Nakryiko36e68442020-11-09 17:19:31 -080041struct kobject;
Roman Gushchin48edc1f2020-12-01 13:58:32 -080042struct mem_cgroup;
Yonghong Song69c087b2021-02-26 12:49:25 -080043struct bpf_func_state;
Alexei Starovoitov99c55f72014-09-26 00:16:57 -070044
Quentin Monnet1b9ed842019-08-20 10:31:50 +010045extern struct idr btf_idr;
46extern spinlock_t btf_idr_lock;
Andrii Nakryiko36e68442020-11-09 17:19:31 -080047extern struct kobject *btf_kobj;
Quentin Monnet1b9ed842019-08-20 10:31:50 +010048
Yonghong Songf9c79272020-07-23 11:41:10 -070049typedef int (*bpf_iter_init_seq_priv_t)(void *private_data,
50 struct bpf_iter_aux_info *aux);
Yonghong Song14fc6bd62020-07-23 11:41:09 -070051typedef void (*bpf_iter_fini_seq_priv_t)(void *private_data);
52struct bpf_iter_seq_info {
53 const struct seq_operations *seq_ops;
54 bpf_iter_init_seq_priv_t init_seq_private;
55 bpf_iter_fini_seq_priv_t fini_seq_private;
56 u32 seq_priv_size;
57};
58
Ricardo Ribalda5d903492021-03-18 21:22:22 +010059/* map is generic key/value storage optionally accessible by eBPF programs */
Alexei Starovoitov99c55f72014-09-26 00:16:57 -070060struct bpf_map_ops {
61 /* funcs callable from userspace (via syscall) */
Jakub Kicinski1110f3a2018-01-11 20:29:03 -080062 int (*map_alloc_check)(union bpf_attr *attr);
Alexei Starovoitov99c55f72014-09-26 00:16:57 -070063 struct bpf_map *(*map_alloc)(union bpf_attr *attr);
Daniel Borkmann61d1b6a2016-06-15 22:47:12 +020064 void (*map_release)(struct bpf_map *map, struct file *map_file);
65 void (*map_free)(struct bpf_map *map);
Alexei Starovoitovdb20fd22014-09-26 00:16:59 -070066 int (*map_get_next_key)(struct bpf_map *map, void *key, void *next_key);
John Fastabendba6b8de2018-04-23 15:39:23 -070067 void (*map_release_uref)(struct bpf_map *map);
Daniel Borkmannc6110222019-05-14 01:18:55 +020068 void *(*map_lookup_elem_sys_only)(struct bpf_map *map, void *key);
Brian Vazquezcb4d03a2020-01-15 10:43:01 -080069 int (*map_lookup_batch)(struct bpf_map *map, const union bpf_attr *attr,
70 union bpf_attr __user *uattr);
Yonghong Song05799632020-01-15 10:43:04 -080071 int (*map_lookup_and_delete_batch)(struct bpf_map *map,
72 const union bpf_attr *attr,
73 union bpf_attr __user *uattr);
Brian Vazquezaa2e93b2020-01-15 10:43:02 -080074 int (*map_update_batch)(struct bpf_map *map, const union bpf_attr *attr,
75 union bpf_attr __user *uattr);
76 int (*map_delete_batch)(struct bpf_map *map, const union bpf_attr *attr,
77 union bpf_attr __user *uattr);
Alexei Starovoitovdb20fd22014-09-26 00:16:59 -070078
79 /* funcs callable from userspace and from eBPF programs */
80 void *(*map_lookup_elem)(struct bpf_map *map, void *key);
Alexei Starovoitov3274f522014-11-13 17:36:44 -080081 int (*map_update_elem)(struct bpf_map *map, void *key, void *value, u64 flags);
Alexei Starovoitovdb20fd22014-09-26 00:16:59 -070082 int (*map_delete_elem)(struct bpf_map *map, void *key);
Mauricio Vasquez Bf1a2e442018-10-18 15:16:25 +020083 int (*map_push_elem)(struct bpf_map *map, void *value, u64 flags);
84 int (*map_pop_elem)(struct bpf_map *map, void *value);
85 int (*map_peek_elem)(struct bpf_map *map, void *value);
Wang Nan2a36f0b2015-08-06 07:02:33 +000086
87 /* funcs called by prog_array and perf_event_array map */
Daniel Borkmannd056a782016-06-15 22:47:13 +020088 void *(*map_fd_get_ptr)(struct bpf_map *map, struct file *map_file,
89 int fd);
90 void (*map_fd_put_ptr)(void *ptr);
Daniel Borkmann4a8f87e2020-10-11 01:40:03 +020091 int (*map_gen_lookup)(struct bpf_map *map, struct bpf_insn *insn_buf);
Martin KaFai Lau14dc6f02017-06-27 23:08:34 -070092 u32 (*map_fd_sys_lookup_elem)(void *ptr);
Martin KaFai Laua26ca7c2018-04-18 15:56:03 -070093 void (*map_seq_show_elem)(struct bpf_map *map, void *key,
94 struct seq_file *m);
Daniel Borkmanne8d2bec2018-08-12 01:59:17 +020095 int (*map_check_btf)(const struct bpf_map *map,
Roman Gushchin1b2b2342018-12-10 15:43:00 -080096 const struct btf *btf,
Daniel Borkmanne8d2bec2018-08-12 01:59:17 +020097 const struct btf_type *key_type,
98 const struct btf_type *value_type);
Daniel Borkmannd8eca5b2019-04-09 23:20:03 +020099
Daniel Borkmannda765a22019-11-22 21:07:58 +0100100 /* Prog poke tracking helpers. */
101 int (*map_poke_track)(struct bpf_map *map, struct bpf_prog_aux *aux);
102 void (*map_poke_untrack)(struct bpf_map *map, struct bpf_prog_aux *aux);
103 void (*map_poke_run)(struct bpf_map *map, u32 key, struct bpf_prog *old,
104 struct bpf_prog *new);
105
Daniel Borkmannd8eca5b2019-04-09 23:20:03 +0200106 /* Direct value access helpers. */
107 int (*map_direct_value_addr)(const struct bpf_map *map,
108 u64 *imm, u32 off);
109 int (*map_direct_value_meta)(const struct bpf_map *map,
110 u64 imm, u32 *off);
Andrii Nakryikofc970222019-11-17 09:28:04 -0800111 int (*map_mmap)(struct bpf_map *map, struct vm_area_struct *vma);
Andrii Nakryiko457f44362020-05-29 00:54:20 -0700112 __poll_t (*map_poll)(struct bpf_map *map, struct file *filp,
113 struct poll_table_struct *pts);
Andrey Ignatov41c48f32020-06-19 14:11:43 -0700114
KP Singhf836a562020-08-25 20:29:15 +0200115 /* Functions called by bpf_local_storage maps */
116 int (*map_local_storage_charge)(struct bpf_local_storage_map *smap,
117 void *owner, u32 size);
118 void (*map_local_storage_uncharge)(struct bpf_local_storage_map *smap,
119 void *owner, u32 size);
120 struct bpf_local_storage __rcu ** (*map_owner_storage_ptr)(void *owner);
Martin KaFai Lauf4d05252020-08-27 18:18:06 -0700121
Björn Töpele6a47502021-03-08 12:29:06 +0100122 /* Misc helpers.*/
123 int (*map_redirect)(struct bpf_map *map, u32 ifindex, u64 flags);
124
Martin KaFai Lauf4d05252020-08-27 18:18:06 -0700125 /* map_meta_equal must be implemented for maps that can be
126 * used as an inner map. It is a runtime check to ensure
127 * an inner map can be inserted to an outer map.
128 *
129 * Some properties of the inner map has been used during the
130 * verification time. When inserting an inner map at the runtime,
131 * map_meta_equal has to ensure the inserting map has the same
132 * properties that the verifier has used earlier.
133 */
134 bool (*map_meta_equal)(const struct bpf_map *meta0,
135 const struct bpf_map *meta1);
136
Yonghong Song69c087b2021-02-26 12:49:25 -0800137
138 int (*map_set_for_each_callback_args)(struct bpf_verifier_env *env,
139 struct bpf_func_state *caller,
140 struct bpf_func_state *callee);
141 int (*map_for_each_callback)(struct bpf_map *map, void *callback_fn,
142 void *callback_ctx, u64 flags);
143
Andrey Ignatov41c48f32020-06-19 14:11:43 -0700144 /* BTF name and id of struct allocated by map_alloc */
145 const char * const map_btf_name;
146 int *map_btf_id;
Yonghong Songa5cbe052020-07-23 11:41:12 -0700147
148 /* bpf_iter info used to open a seq_file */
149 const struct bpf_iter_seq_info *iter_seq_info;
Alexei Starovoitov99c55f72014-09-26 00:16:57 -0700150};
151
152struct bpf_map {
Martin KaFai Laua26ca7c2018-04-18 15:56:03 -0700153 /* The first two cachelines with read-mostly members of which some
Daniel Borkmannbe95a842018-01-09 13:17:44 +0100154 * are also accessed in fast-path (e.g. ops, max_entries).
155 */
156 const struct bpf_map_ops *ops ____cacheline_aligned;
157 struct bpf_map *inner_map_meta;
158#ifdef CONFIG_SECURITY
159 void *security;
160#endif
Alexei Starovoitov99c55f72014-09-26 00:16:57 -0700161 enum bpf_map_type map_type;
162 u32 key_size;
163 u32 value_size;
164 u32 max_entries;
Alexei Starovoitov6c905982016-03-07 21:57:15 -0800165 u32 map_flags;
Alexei Starovoitovd83525c2019-01-31 15:40:04 -0800166 int spin_lock_off; /* >=0 valid offset, <0 error */
Martin KaFai Lauf3f1c052017-06-05 12:15:47 -0700167 u32 id;
Martin KaFai Lau96eabe72017-08-18 11:28:00 -0700168 int numa_node;
Martin KaFai Lau9b2cf322018-05-22 14:57:21 -0700169 u32 btf_key_type_id;
170 u32 btf_value_type_id;
Martin KaFai Laua26ca7c2018-04-18 15:56:03 -0700171 struct btf *btf;
Roman Gushchin48edc1f2020-12-01 13:58:32 -0800172#ifdef CONFIG_MEMCG_KMEM
173 struct mem_cgroup *memcg;
174#endif
Andrii Nakryikofc970222019-11-17 09:28:04 -0800175 char name[BPF_OBJ_NAME_LEN];
Martin KaFai Lau85d33df2020-01-08 16:35:05 -0800176 u32 btf_vmlinux_value_type_id;
Alexei Starovoitov2c78ee82020-05-13 16:03:54 -0700177 bool bypass_spec_v1;
Andrii Nakryikofc970222019-11-17 09:28:04 -0800178 bool frozen; /* write-once; write-protected by freeze_mutex */
179 /* 22 bytes hole */
Daniel Borkmannbe95a842018-01-09 13:17:44 +0100180
Martin KaFai Laua26ca7c2018-04-18 15:56:03 -0700181 /* The 3rd and 4th cacheline with misc members to avoid false sharing
Daniel Borkmannbe95a842018-01-09 13:17:44 +0100182 * particularly with refcounting.
183 */
Andrii Nakryiko1e0bd5a2019-11-17 09:28:02 -0800184 atomic64_t refcnt ____cacheline_aligned;
185 atomic64_t usercnt;
Daniel Borkmannbe95a842018-01-09 13:17:44 +0100186 struct work_struct work;
Andrii Nakryikofc970222019-11-17 09:28:04 -0800187 struct mutex freeze_mutex;
188 u64 writecnt; /* writable mmap cnt; protected by freeze_mutex */
Alexei Starovoitov99c55f72014-09-26 00:16:57 -0700189};
190
Alexei Starovoitovd83525c2019-01-31 15:40:04 -0800191static inline bool map_value_has_spin_lock(const struct bpf_map *map)
192{
193 return map->spin_lock_off >= 0;
194}
195
196static inline void check_and_init_map_lock(struct bpf_map *map, void *dst)
197{
198 if (likely(!map_value_has_spin_lock(map)))
199 return;
200 *(struct bpf_spin_lock *)(dst + map->spin_lock_off) =
201 (struct bpf_spin_lock){};
202}
203
204/* copy everything but bpf_spin_lock */
205static inline void copy_map_value(struct bpf_map *map, void *dst, void *src)
206{
207 if (unlikely(map_value_has_spin_lock(map))) {
208 u32 off = map->spin_lock_off;
209
210 memcpy(dst, src, off);
211 memcpy(dst + off + sizeof(struct bpf_spin_lock),
212 src + off + sizeof(struct bpf_spin_lock),
213 map->value_size - off - sizeof(struct bpf_spin_lock));
214 } else {
215 memcpy(dst, src, map->value_size);
216 }
217}
Alexei Starovoitov96049f32019-01-31 15:40:09 -0800218void copy_map_value_locked(struct bpf_map *map, void *dst, void *src,
219 bool lock_src);
Martin KaFai Lau8e7ae252020-03-13 18:02:09 -0700220int bpf_obj_name_cpy(char *dst, const char *src, unsigned int size);
Alexei Starovoitovd83525c2019-01-31 15:40:04 -0800221
Jakub Kicinski602144c2018-07-17 10:53:25 -0700222struct bpf_offload_dev;
Jakub Kicinskia3884572018-01-11 20:29:09 -0800223struct bpf_offloaded_map;
224
225struct bpf_map_dev_ops {
226 int (*map_get_next_key)(struct bpf_offloaded_map *map,
227 void *key, void *next_key);
228 int (*map_lookup_elem)(struct bpf_offloaded_map *map,
229 void *key, void *value);
230 int (*map_update_elem)(struct bpf_offloaded_map *map,
231 void *key, void *value, u64 flags);
232 int (*map_delete_elem)(struct bpf_offloaded_map *map, void *key);
233};
234
235struct bpf_offloaded_map {
236 struct bpf_map map;
237 struct net_device *netdev;
238 const struct bpf_map_dev_ops *dev_ops;
239 void *dev_priv;
240 struct list_head offloads;
241};
242
243static inline struct bpf_offloaded_map *map_to_offmap(struct bpf_map *map)
244{
245 return container_of(map, struct bpf_offloaded_map, map);
246}
247
Jakub Kicinski0cd3cbe2018-05-03 18:37:08 -0700248static inline bool bpf_map_offload_neutral(const struct bpf_map *map)
249{
250 return map->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY;
251}
252
Martin KaFai Laua26ca7c2018-04-18 15:56:03 -0700253static inline bool bpf_map_support_seq_show(const struct bpf_map *map)
254{
Martin KaFai Lau85d33df2020-01-08 16:35:05 -0800255 return (map->btf_value_type_id || map->btf_vmlinux_value_type_id) &&
256 map->ops->map_seq_show_elem;
Martin KaFai Laua26ca7c2018-04-18 15:56:03 -0700257}
258
Daniel Borkmanne8d2bec2018-08-12 01:59:17 +0200259int map_check_no_btf(const struct bpf_map *map,
Roman Gushchin1b2b2342018-12-10 15:43:00 -0800260 const struct btf *btf,
Daniel Borkmanne8d2bec2018-08-12 01:59:17 +0200261 const struct btf_type *key_type,
262 const struct btf_type *value_type);
263
Martin KaFai Lauf4d05252020-08-27 18:18:06 -0700264bool bpf_map_meta_equal(const struct bpf_map *meta0,
265 const struct bpf_map *meta1);
266
Jakub Kicinskia3884572018-01-11 20:29:09 -0800267extern const struct bpf_map_ops bpf_map_offload_ops;
268
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700269/* function argument constraints */
270enum bpf_arg_type {
Daniel Borkmann80f1d682015-03-12 17:21:42 +0100271 ARG_DONTCARE = 0, /* unused argument in helper function */
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700272
273 /* the following constraints used to prototype
274 * bpf_map_lookup/update/delete_elem() functions
275 */
276 ARG_CONST_MAP_PTR, /* const argument used as pointer to bpf_map */
277 ARG_PTR_TO_MAP_KEY, /* pointer to stack used as map key */
278 ARG_PTR_TO_MAP_VALUE, /* pointer to stack used as map value */
Mauricio Vasquez B2ea864c2018-10-18 15:16:20 +0200279 ARG_PTR_TO_UNINIT_MAP_VALUE, /* pointer to valid memory used to store a map value */
Martin KaFai Lau6ac99e82019-04-26 16:39:39 -0700280 ARG_PTR_TO_MAP_VALUE_OR_NULL, /* pointer to stack used as map value or NULL */
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700281
282 /* the following constraints used to prototype bpf_memcmp() and other
283 * functions that access data on eBPF program stack
284 */
Alexei Starovoitov39f19ebb2017-01-09 10:19:50 -0800285 ARG_PTR_TO_MEM, /* pointer to valid memory (stack, packet, map value) */
Gianluca Borellodb1ac492017-11-22 18:32:53 +0000286 ARG_PTR_TO_MEM_OR_NULL, /* pointer to valid memory or NULL */
Alexei Starovoitov39f19ebb2017-01-09 10:19:50 -0800287 ARG_PTR_TO_UNINIT_MEM, /* pointer to memory does not need to be initialized,
288 * helper function must fill all bytes or clear
289 * them in error case.
Daniel Borkmann435faee12016-04-13 00:10:51 +0200290 */
291
Alexei Starovoitov39f19ebb2017-01-09 10:19:50 -0800292 ARG_CONST_SIZE, /* number of bytes accessed from memory */
293 ARG_CONST_SIZE_OR_ZERO, /* number of bytes accessed from memory or 0 */
Daniel Borkmann80f1d682015-03-12 17:21:42 +0100294
Alexei Starovoitov608cd712015-03-26 19:53:57 -0700295 ARG_PTR_TO_CTX, /* pointer to context */
Daniel Borkmannf3189032020-03-27 16:58:52 +0100296 ARG_PTR_TO_CTX_OR_NULL, /* pointer to context or NULL */
Daniel Borkmann80f1d682015-03-12 17:21:42 +0100297 ARG_ANYTHING, /* any (initialized) argument is ok */
Alexei Starovoitovd83525c2019-01-31 15:40:04 -0800298 ARG_PTR_TO_SPIN_LOCK, /* pointer to bpf_spin_lock */
Martin KaFai Lau46f8bc92019-02-09 23:22:20 -0800299 ARG_PTR_TO_SOCK_COMMON, /* pointer to sock_common */
Andrey Ignatov57c3bb72019-03-18 16:57:10 -0700300 ARG_PTR_TO_INT, /* pointer to int */
301 ARG_PTR_TO_LONG, /* pointer to long */
Martin KaFai Lau6ac99e82019-04-26 16:39:39 -0700302 ARG_PTR_TO_SOCKET, /* pointer to bpf_sock (fullsock) */
Jakub Sitnickie9ddbb72020-07-17 12:35:23 +0200303 ARG_PTR_TO_SOCKET_OR_NULL, /* pointer to bpf_sock (fullsock) or NULL */
Alexei Starovoitova7658e12019-10-15 20:25:04 -0700304 ARG_PTR_TO_BTF_ID, /* pointer to in-kernel struct */
Andrii Nakryiko457f44362020-05-29 00:54:20 -0700305 ARG_PTR_TO_ALLOC_MEM, /* pointer to dynamically allocated memory */
306 ARG_PTR_TO_ALLOC_MEM_OR_NULL, /* pointer to dynamically allocated memory or NULL */
307 ARG_CONST_ALLOC_SIZE_OR_ZERO, /* number of allocated bytes requested */
Martin KaFai Lau1df8f552020-09-24 17:03:50 -0700308 ARG_PTR_TO_BTF_ID_SOCK_COMMON, /* pointer to in-kernel sock_common or bpf-mirrored bpf_sock */
Hao Luoeaa6bcb2020-09-29 16:50:47 -0700309 ARG_PTR_TO_PERCPU_BTF_ID, /* pointer to in-kernel percpu type */
Yonghong Song69c087b2021-02-26 12:49:25 -0800310 ARG_PTR_TO_FUNC, /* pointer to a bpf program function */
311 ARG_PTR_TO_STACK_OR_NULL, /* pointer to stack or NULL */
Lorenz Bauerf79e7ea2020-09-21 13:12:27 +0100312 __BPF_ARG_TYPE_MAX,
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700313};
314
315/* type of values returned from helper functions */
316enum bpf_return_type {
317 RET_INTEGER, /* function returns integer */
318 RET_VOID, /* function doesn't return anything */
Roman Gushchin3e6a4b32018-08-02 14:27:22 -0700319 RET_PTR_TO_MAP_VALUE, /* returns a pointer to map elem value */
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700320 RET_PTR_TO_MAP_VALUE_OR_NULL, /* returns a pointer to map elem value or NULL */
Joe Stringerc64b7982018-10-02 13:35:33 -0700321 RET_PTR_TO_SOCKET_OR_NULL, /* returns a pointer to a socket or NULL */
Martin KaFai Lau655a51e2019-02-09 23:22:24 -0800322 RET_PTR_TO_TCP_SOCK_OR_NULL, /* returns a pointer to a tcp_sock or NULL */
Lorenz Bauer85a51f82019-03-22 09:54:00 +0800323 RET_PTR_TO_SOCK_COMMON_OR_NULL, /* returns a pointer to a sock_common or NULL */
Andrii Nakryiko457f44362020-05-29 00:54:20 -0700324 RET_PTR_TO_ALLOC_MEM_OR_NULL, /* returns a pointer to dynamically allocated memory or NULL */
Yonghong Songaf7ec132020-06-23 16:08:09 -0700325 RET_PTR_TO_BTF_ID_OR_NULL, /* returns a pointer to a btf_id or NULL */
Hao Luoeaa6bcb2020-09-29 16:50:47 -0700326 RET_PTR_TO_MEM_OR_BTF_ID_OR_NULL, /* returns a pointer to a valid memory or a btf_id or NULL */
Hao Luo63d9b802020-09-29 16:50:48 -0700327 RET_PTR_TO_MEM_OR_BTF_ID, /* returns a pointer to a valid memory or a btf_id */
KP Singh3ca10322020-11-06 10:37:43 +0000328 RET_PTR_TO_BTF_ID, /* returns a pointer to a btf_id */
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700329};
330
Alexei Starovoitov09756af2014-09-26 00:17:00 -0700331/* eBPF function prototype used by verifier to allow BPF_CALLs from eBPF programs
332 * to in-kernel helper functions and for adjusting imm32 field in BPF_CALL
333 * instructions after verifying
334 */
335struct bpf_func_proto {
336 u64 (*func)(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
337 bool gpl_only;
Daniel Borkmann36bbef52016-09-20 00:26:13 +0200338 bool pkt_access;
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700339 enum bpf_return_type ret_type;
Alexei Starovoitova7658e12019-10-15 20:25:04 -0700340 union {
341 struct {
342 enum bpf_arg_type arg1_type;
343 enum bpf_arg_type arg2_type;
344 enum bpf_arg_type arg3_type;
345 enum bpf_arg_type arg4_type;
346 enum bpf_arg_type arg5_type;
347 };
348 enum bpf_arg_type arg_type[5];
349 };
Lorenz Bauer9436ef62020-09-21 13:12:20 +0100350 union {
351 struct {
352 u32 *arg1_btf_id;
353 u32 *arg2_btf_id;
354 u32 *arg3_btf_id;
355 u32 *arg4_btf_id;
356 u32 *arg5_btf_id;
357 };
358 u32 *arg_btf_id[5];
359 };
Yonghong Songaf7ec132020-06-23 16:08:09 -0700360 int *ret_btf_id; /* return value btf_id */
Jiri Olsaeae2e832020-08-25 21:21:19 +0200361 bool (*allowed)(const struct bpf_prog *prog);
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700362};
363
364/* bpf_context is intentionally undefined structure. Pointer to bpf_context is
365 * the first argument to eBPF programs.
366 * For socket filters: 'struct bpf_context *' == 'struct sk_buff *'
367 */
368struct bpf_context;
369
370enum bpf_access_type {
371 BPF_READ = 1,
372 BPF_WRITE = 2
Alexei Starovoitov09756af2014-09-26 00:17:00 -0700373};
374
Alexei Starovoitov19de99f2016-06-15 18:25:38 -0700375/* types of values stored in eBPF registers */
Edward Creef1174f72017-08-07 15:26:19 +0100376/* Pointer types represent:
377 * pointer
378 * pointer + imm
379 * pointer + (u16) var
380 * pointer + (u16) var + imm
381 * if (range > 0) then [ptr, ptr + range - off) is safe to access
382 * if (id > 0) means that some 'var' was added
383 * if (off > 0) means that 'imm' was added
384 */
Alexei Starovoitov19de99f2016-06-15 18:25:38 -0700385enum bpf_reg_type {
386 NOT_INIT = 0, /* nothing was written into register */
Edward Creef1174f72017-08-07 15:26:19 +0100387 SCALAR_VALUE, /* reg doesn't contain a valid pointer */
Alexei Starovoitov19de99f2016-06-15 18:25:38 -0700388 PTR_TO_CTX, /* reg points to bpf_context */
389 CONST_PTR_TO_MAP, /* reg points to struct bpf_map */
390 PTR_TO_MAP_VALUE, /* reg points to map element value */
391 PTR_TO_MAP_VALUE_OR_NULL,/* points to map elem value or NULL */
Edward Creef1174f72017-08-07 15:26:19 +0100392 PTR_TO_STACK, /* reg == frame_pointer + offset */
Daniel Borkmannde8f3a82017-09-25 02:25:51 +0200393 PTR_TO_PACKET_META, /* skb->data - meta_len */
Edward Creef1174f72017-08-07 15:26:19 +0100394 PTR_TO_PACKET, /* reg points to skb->data */
Alexei Starovoitov19de99f2016-06-15 18:25:38 -0700395 PTR_TO_PACKET_END, /* skb->data + headlen */
Petar Penkovd58e4682018-09-14 07:46:18 -0700396 PTR_TO_FLOW_KEYS, /* reg points to bpf_flow_keys */
Joe Stringerc64b7982018-10-02 13:35:33 -0700397 PTR_TO_SOCKET, /* reg points to struct bpf_sock */
398 PTR_TO_SOCKET_OR_NULL, /* reg points to struct bpf_sock or NULL */
Martin KaFai Lau46f8bc92019-02-09 23:22:20 -0800399 PTR_TO_SOCK_COMMON, /* reg points to sock_common */
400 PTR_TO_SOCK_COMMON_OR_NULL, /* reg points to sock_common or NULL */
Martin KaFai Lau655a51e2019-02-09 23:22:24 -0800401 PTR_TO_TCP_SOCK, /* reg points to struct tcp_sock */
402 PTR_TO_TCP_SOCK_OR_NULL, /* reg points to struct tcp_sock or NULL */
Matt Mullins9df1c282019-04-26 11:49:47 -0700403 PTR_TO_TP_BUFFER, /* reg points to a writable raw tp's buffer */
Jonathan Lemonfada7fd2019-06-06 13:59:40 -0700404 PTR_TO_XDP_SOCK, /* reg points to struct xdp_sock */
John Fastabendba5f4cf2020-09-24 12:58:40 -0700405 /* PTR_TO_BTF_ID points to a kernel struct that does not need
406 * to be null checked by the BPF program. This does not imply the
407 * pointer is _not_ null and in practice this can easily be a null
408 * pointer when reading pointer chains. The assumption is program
409 * context will handle null pointer dereference typically via fault
410 * handling. The verifier must keep this in mind and can make no
411 * assumptions about null or non-null when doing branch analysis.
412 * Further, when passed into helpers the helpers can not, without
413 * additional context, assume the value is non-null.
414 */
415 PTR_TO_BTF_ID,
416 /* PTR_TO_BTF_ID_OR_NULL points to a kernel struct that has not
417 * been checked for null. Used primarily to inform the verifier
418 * an explicit null check is required for this struct.
419 */
420 PTR_TO_BTF_ID_OR_NULL,
Andrii Nakryiko457f44362020-05-29 00:54:20 -0700421 PTR_TO_MEM, /* reg points to valid memory region */
422 PTR_TO_MEM_OR_NULL, /* reg points to valid memory region or NULL */
Yonghong Songafbf21d2020-07-23 11:41:11 -0700423 PTR_TO_RDONLY_BUF, /* reg points to a readonly buffer */
424 PTR_TO_RDONLY_BUF_OR_NULL, /* reg points to a readonly buffer or NULL */
425 PTR_TO_RDWR_BUF, /* reg points to a read/write buffer */
426 PTR_TO_RDWR_BUF_OR_NULL, /* reg points to a read/write buffer or NULL */
Hao Luoeaa6bcb2020-09-29 16:50:47 -0700427 PTR_TO_PERCPU_BTF_ID, /* reg points to a percpu kernel variable */
Yonghong Song69c087b2021-02-26 12:49:25 -0800428 PTR_TO_FUNC, /* reg points to a bpf program function */
429 PTR_TO_MAP_KEY, /* reg points to a map element key */
Alexei Starovoitov19de99f2016-06-15 18:25:38 -0700430};
431
Yonghong Song23994632017-06-22 15:07:39 -0700432/* The information passed from prog-specific *_is_valid_access
433 * back to the verifier.
434 */
435struct bpf_insn_access_aux {
436 enum bpf_reg_type reg_type;
Alexei Starovoitov9e15db62019-10-15 20:25:00 -0700437 union {
438 int ctx_field_size;
Andrii Nakryiko22dc4a02020-12-03 12:46:29 -0800439 struct {
440 struct btf *btf;
441 u32 btf_id;
442 };
Alexei Starovoitov9e15db62019-10-15 20:25:00 -0700443 };
444 struct bpf_verifier_log *log; /* for verbose logs */
Yonghong Song23994632017-06-22 15:07:39 -0700445};
446
Daniel Borkmannf96da092017-07-02 02:13:27 +0200447static inline void
448bpf_ctx_record_field_size(struct bpf_insn_access_aux *aux, u32 size)
449{
450 aux->ctx_field_size = size;
451}
452
Jakub Kicinski7de16e32017-10-16 16:40:53 -0700453struct bpf_prog_ops {
454 int (*test_run)(struct bpf_prog *prog, const union bpf_attr *kattr,
455 union bpf_attr __user *uattr);
456};
457
Alexei Starovoitov09756af2014-09-26 00:17:00 -0700458struct bpf_verifier_ops {
459 /* return eBPF function prototype for verification */
Andrey Ignatov5e43f892018-03-30 15:08:00 -0700460 const struct bpf_func_proto *
461 (*get_func_proto)(enum bpf_func_id func_id,
462 const struct bpf_prog *prog);
Alexei Starovoitov17a52672014-09-26 00:17:06 -0700463
464 /* return true if 'size' wide access at offset 'off' within bpf_context
465 * with 'type' (read or write) is allowed
466 */
Alexei Starovoitov19de99f2016-06-15 18:25:38 -0700467 bool (*is_valid_access)(int off, int size, enum bpf_access_type type,
Andrey Ignatov5e43f892018-03-30 15:08:00 -0700468 const struct bpf_prog *prog,
Yonghong Song23994632017-06-22 15:07:39 -0700469 struct bpf_insn_access_aux *info);
Daniel Borkmann36bbef52016-09-20 00:26:13 +0200470 int (*gen_prologue)(struct bpf_insn *insn, bool direct_write,
471 const struct bpf_prog *prog);
Daniel Borkmanne0cea7c2018-05-04 01:08:14 +0200472 int (*gen_ld_abs)(const struct bpf_insn *orig,
473 struct bpf_insn *insn_buf);
Daniel Borkmann6b8cc1d2017-01-12 11:51:32 +0100474 u32 (*convert_ctx_access)(enum bpf_access_type type,
475 const struct bpf_insn *src,
476 struct bpf_insn *dst,
Daniel Borkmannf96da092017-07-02 02:13:27 +0200477 struct bpf_prog *prog, u32 *target_size);
Martin KaFai Lau27ae79972020-01-08 16:35:03 -0800478 int (*btf_struct_access)(struct bpf_verifier_log *log,
Andrii Nakryiko22dc4a02020-12-03 12:46:29 -0800479 const struct btf *btf,
Martin KaFai Lau27ae79972020-01-08 16:35:03 -0800480 const struct btf_type *t, int off, int size,
481 enum bpf_access_type atype,
482 u32 *next_btf_id);
Alexei Starovoitov09756af2014-09-26 00:17:00 -0700483};
484
Jakub Kicinskicae19272017-12-27 18:39:05 -0800485struct bpf_prog_offload_ops {
Jakub Kicinski08ca90a2019-01-22 22:45:24 -0800486 /* verifier basic callbacks */
Jakub Kicinskicae19272017-12-27 18:39:05 -0800487 int (*insn_hook)(struct bpf_verifier_env *env,
488 int insn_idx, int prev_insn_idx);
Quentin Monnetc941ce92018-10-07 12:56:47 +0100489 int (*finalize)(struct bpf_verifier_env *env);
Jakub Kicinski08ca90a2019-01-22 22:45:24 -0800490 /* verifier optimization callbacks (called after .finalize) */
491 int (*replace_insn)(struct bpf_verifier_env *env, u32 off,
492 struct bpf_insn *insn);
493 int (*remove_insns)(struct bpf_verifier_env *env, u32 off, u32 cnt);
494 /* program management callbacks */
Quentin Monnet16a8cb5c2018-11-09 13:03:32 +0000495 int (*prepare)(struct bpf_prog *prog);
496 int (*translate)(struct bpf_prog *prog);
Quentin Monneteb911942018-11-09 13:03:30 +0000497 void (*destroy)(struct bpf_prog *prog);
Jakub Kicinskicae19272017-12-27 18:39:05 -0800498};
499
Jakub Kicinski0a9c1992018-01-11 20:29:07 -0800500struct bpf_prog_offload {
Jakub Kicinskiab3f0062017-11-03 13:56:17 -0700501 struct bpf_prog *prog;
502 struct net_device *netdev;
Quentin Monnet341b3e72018-11-09 13:03:26 +0000503 struct bpf_offload_dev *offdev;
Jakub Kicinskiab3f0062017-11-03 13:56:17 -0700504 void *dev_priv;
505 struct list_head offloads;
506 bool dev_state;
Jakub Kicinski08ca90a2019-01-22 22:45:24 -0800507 bool opt_failed;
Jiong Wangfcfb1262018-01-16 16:05:19 -0800508 void *jited_image;
509 u32 jited_len;
Jakub Kicinskiab3f0062017-11-03 13:56:17 -0700510};
511
Roman Gushchin8bad74f2018-09-28 14:45:36 +0000512enum bpf_cgroup_storage_type {
513 BPF_CGROUP_STORAGE_SHARED,
Roman Gushchinb741f162018-09-28 14:45:43 +0000514 BPF_CGROUP_STORAGE_PERCPU,
Roman Gushchin8bad74f2018-09-28 14:45:36 +0000515 __BPF_CGROUP_STORAGE_MAX
516};
517
518#define MAX_BPF_CGROUP_STORAGE_TYPE __BPF_CGROUP_STORAGE_MAX
519
Alexei Starovoitovf1b95092019-10-30 15:32:11 -0700520/* The longest tracepoint has 12 args.
521 * See include/trace/bpf_probe.h
522 */
523#define MAX_BPF_FUNC_ARGS 12
524
Dmitrii Banshchikov523a4cf2021-02-26 00:26:29 +0400525/* The maximum number of arguments passed through registers
526 * a single function may have.
527 */
528#define MAX_BPF_FUNC_REG_ARGS 5
529
Alexei Starovoitovfec56f52019-11-14 10:57:04 -0800530struct btf_func_model {
531 u8 ret_size;
532 u8 nr_args;
533 u8 arg_size[MAX_BPF_FUNC_ARGS];
534};
535
536/* Restore arguments before returning from trampoline to let original function
537 * continue executing. This flag is used for fentry progs when there are no
538 * fexit progs.
539 */
540#define BPF_TRAMP_F_RESTORE_REGS BIT(0)
541/* Call original function after fentry progs, but before fexit progs.
542 * Makes sense for fentry/fexit, normal calls and indirect calls.
543 */
544#define BPF_TRAMP_F_CALL_ORIG BIT(1)
545/* Skip current frame and return to parent. Makes sense for fentry/fexit
546 * programs only. Should not be used with normal calls and indirect calls.
547 */
548#define BPF_TRAMP_F_SKIP_FRAME BIT(2)
549
KP Singh88fd9e52020-03-04 20:18:47 +0100550/* Each call __bpf_prog_enter + call bpf_func + call __bpf_prog_exit is ~50
551 * bytes on x86. Pick a number to fit into BPF_IMAGE_SIZE / 2
552 */
Alexei Starovoitovca06f552021-02-09 19:36:29 -0800553#define BPF_MAX_TRAMP_PROGS 38
KP Singh88fd9e52020-03-04 20:18:47 +0100554
555struct bpf_tramp_progs {
556 struct bpf_prog *progs[BPF_MAX_TRAMP_PROGS];
557 int nr_progs;
558};
559
Alexei Starovoitovfec56f52019-11-14 10:57:04 -0800560/* Different use cases for BPF trampoline:
561 * 1. replace nop at the function entry (kprobe equivalent)
562 * flags = BPF_TRAMP_F_RESTORE_REGS
563 * fentry = a set of programs to run before returning from trampoline
564 *
565 * 2. replace nop at the function entry (kprobe + kretprobe equivalent)
566 * flags = BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_SKIP_FRAME
567 * orig_call = fentry_ip + MCOUNT_INSN_SIZE
568 * fentry = a set of program to run before calling original function
569 * fexit = a set of program to run after original function
570 *
571 * 3. replace direct call instruction anywhere in the function body
572 * or assign a function pointer for indirect call (like tcp_congestion_ops->cong_avoid)
573 * With flags = 0
574 * fentry = a set of programs to run before returning from trampoline
575 * With flags = BPF_TRAMP_F_CALL_ORIG
576 * orig_call = original callback addr or direct function addr
577 * fentry = a set of program to run before calling original function
578 * fexit = a set of program to run after original function
579 */
Alexei Starovoitove21aa342021-03-16 14:00:07 -0700580struct bpf_tramp_image;
581int arch_prepare_bpf_trampoline(struct bpf_tramp_image *tr, void *image, void *image_end,
Martin KaFai Lau85d33df2020-01-08 16:35:05 -0800582 const struct btf_func_model *m, u32 flags,
KP Singh88fd9e52020-03-04 20:18:47 +0100583 struct bpf_tramp_progs *tprogs,
Alexei Starovoitovfec56f52019-11-14 10:57:04 -0800584 void *orig_call);
585/* these two functions are called from generated trampoline */
Alexei Starovoitovca06f552021-02-09 19:36:29 -0800586u64 notrace __bpf_prog_enter(struct bpf_prog *prog);
Alexei Starovoitovfec56f52019-11-14 10:57:04 -0800587void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start);
Alexei Starovoitovca06f552021-02-09 19:36:29 -0800588u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog *prog);
Alexei Starovoitovf2dd3b32021-02-09 19:36:28 -0800589void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start);
Alexei Starovoitove21aa342021-03-16 14:00:07 -0700590void notrace __bpf_tramp_enter(struct bpf_tramp_image *tr);
591void notrace __bpf_tramp_exit(struct bpf_tramp_image *tr);
Alexei Starovoitovfec56f52019-11-14 10:57:04 -0800592
Jiri Olsa535911c2020-03-12 20:55:58 +0100593struct bpf_ksym {
594 unsigned long start;
595 unsigned long end;
Jiri Olsabfea9a82020-03-12 20:55:59 +0100596 char name[KSYM_NAME_LEN];
Jiri Olsaecb60d12020-03-12 20:56:00 +0100597 struct list_head lnode;
Jiri Olsaca4424c2020-03-12 20:56:01 +0100598 struct latch_tree_node tnode;
Jiri Olsacbd76f8d2020-03-12 20:56:03 +0100599 bool prog;
Jiri Olsa535911c2020-03-12 20:55:58 +0100600};
601
Alexei Starovoitovfec56f52019-11-14 10:57:04 -0800602enum bpf_tramp_prog_type {
603 BPF_TRAMP_FENTRY,
604 BPF_TRAMP_FEXIT,
KP Singhae240822020-03-04 20:18:49 +0100605 BPF_TRAMP_MODIFY_RETURN,
Alexei Starovoitovbe8704f2020-01-20 16:53:46 -0800606 BPF_TRAMP_MAX,
607 BPF_TRAMP_REPLACE, /* more than MAX */
Alexei Starovoitovfec56f52019-11-14 10:57:04 -0800608};
609
Alexei Starovoitove21aa342021-03-16 14:00:07 -0700610struct bpf_tramp_image {
611 void *image;
612 struct bpf_ksym ksym;
613 struct percpu_ref pcref;
614 void *ip_after_call;
615 void *ip_epilogue;
616 union {
617 struct rcu_head rcu;
618 struct work_struct work;
619 };
620};
621
Alexei Starovoitovfec56f52019-11-14 10:57:04 -0800622struct bpf_trampoline {
623 /* hlist for trampoline_table */
624 struct hlist_node hlist;
625 /* serializes access to fields of this trampoline */
626 struct mutex mutex;
627 refcount_t refcnt;
628 u64 key;
629 struct {
630 struct btf_func_model model;
631 void *addr;
Alexei Starovoitovb91e014f2019-12-08 16:01:13 -0800632 bool ftrace_managed;
Alexei Starovoitovfec56f52019-11-14 10:57:04 -0800633 } func;
Alexei Starovoitovbe8704f2020-01-20 16:53:46 -0800634 /* if !NULL this is BPF_PROG_TYPE_EXT program that extends another BPF
635 * program by replacing one of its functions. func.addr is the address
636 * of the function it replaced.
637 */
638 struct bpf_prog *extension_prog;
Alexei Starovoitovfec56f52019-11-14 10:57:04 -0800639 /* list of BPF programs using this trampoline */
640 struct hlist_head progs_hlist[BPF_TRAMP_MAX];
641 /* Number of attached programs. A counter per kind. */
642 int progs_cnt[BPF_TRAMP_MAX];
643 /* Executable image of trampoline */
Alexei Starovoitove21aa342021-03-16 14:00:07 -0700644 struct bpf_tramp_image *cur_image;
Alexei Starovoitovfec56f52019-11-14 10:57:04 -0800645 u64 selector;
Alexei Starovoitovfec56f52019-11-14 10:57:04 -0800646};
Björn Töpel75ccbef2019-12-13 18:51:08 +0100647
Toke Høiland-Jørgensenf7b12b62020-09-25 23:25:02 +0200648struct bpf_attach_target_info {
649 struct btf_func_model fmodel;
650 long tgt_addr;
651 const char *tgt_name;
652 const struct btf_type *tgt_type;
653};
654
Björn Töpel116eb782019-12-13 18:51:12 +0100655#define BPF_DISPATCHER_MAX 48 /* Fits in 2048B */
Björn Töpel75ccbef2019-12-13 18:51:08 +0100656
657struct bpf_dispatcher_prog {
658 struct bpf_prog *prog;
659 refcount_t users;
660};
661
662struct bpf_dispatcher {
663 /* dispatcher mutex */
664 struct mutex mutex;
665 void *func;
666 struct bpf_dispatcher_prog progs[BPF_DISPATCHER_MAX];
667 int num_progs;
668 void *image;
669 u32 image_off;
Jiri Olsa517b75e2020-03-12 20:56:06 +0100670 struct bpf_ksym ksym;
Björn Töpel75ccbef2019-12-13 18:51:08 +0100671};
672
Björn Töpel6a640372020-03-12 20:55:57 +0100673static __always_inline unsigned int bpf_dispatcher_nop_func(
Björn Töpel7e6897f2019-12-13 18:51:09 +0100674 const void *ctx,
675 const struct bpf_insn *insnsi,
676 unsigned int (*bpf_func)(const void *,
677 const struct bpf_insn *))
678{
679 return bpf_func(ctx, insnsi);
680}
Alexei Starovoitovfec56f52019-11-14 10:57:04 -0800681#ifdef CONFIG_BPF_JIT
Toke Høiland-Jørgensen3aac1ea2020-09-29 14:45:50 +0200682int bpf_trampoline_link_prog(struct bpf_prog *prog, struct bpf_trampoline *tr);
683int bpf_trampoline_unlink_prog(struct bpf_prog *prog, struct bpf_trampoline *tr);
Toke Høiland-Jørgensenf7b12b62020-09-25 23:25:02 +0200684struct bpf_trampoline *bpf_trampoline_get(u64 key,
685 struct bpf_attach_target_info *tgt_info);
Alexei Starovoitovfec56f52019-11-14 10:57:04 -0800686void bpf_trampoline_put(struct bpf_trampoline *tr);
Jiri Olsa517b75e2020-03-12 20:56:06 +0100687#define BPF_DISPATCHER_INIT(_name) { \
688 .mutex = __MUTEX_INITIALIZER(_name.mutex), \
689 .func = &_name##_func, \
690 .progs = {}, \
691 .num_progs = 0, \
692 .image = NULL, \
693 .image_off = 0, \
694 .ksym = { \
695 .name = #_name, \
696 .lnode = LIST_HEAD_INIT(_name.ksym.lnode), \
697 }, \
Björn Töpel75ccbef2019-12-13 18:51:08 +0100698}
699
700#define DEFINE_BPF_DISPATCHER(name) \
Björn Töpel6a640372020-03-12 20:55:57 +0100701 noinline unsigned int bpf_dispatcher_##name##_func( \
Björn Töpel75ccbef2019-12-13 18:51:08 +0100702 const void *ctx, \
703 const struct bpf_insn *insnsi, \
704 unsigned int (*bpf_func)(const void *, \
705 const struct bpf_insn *)) \
706 { \
707 return bpf_func(ctx, insnsi); \
708 } \
Björn Töpel6a640372020-03-12 20:55:57 +0100709 EXPORT_SYMBOL(bpf_dispatcher_##name##_func); \
710 struct bpf_dispatcher bpf_dispatcher_##name = \
711 BPF_DISPATCHER_INIT(bpf_dispatcher_##name);
Björn Töpel75ccbef2019-12-13 18:51:08 +0100712#define DECLARE_BPF_DISPATCHER(name) \
Björn Töpel6a640372020-03-12 20:55:57 +0100713 unsigned int bpf_dispatcher_##name##_func( \
Björn Töpel75ccbef2019-12-13 18:51:08 +0100714 const void *ctx, \
715 const struct bpf_insn *insnsi, \
716 unsigned int (*bpf_func)(const void *, \
717 const struct bpf_insn *)); \
Björn Töpel6a640372020-03-12 20:55:57 +0100718 extern struct bpf_dispatcher bpf_dispatcher_##name;
719#define BPF_DISPATCHER_FUNC(name) bpf_dispatcher_##name##_func
720#define BPF_DISPATCHER_PTR(name) (&bpf_dispatcher_##name)
Björn Töpel75ccbef2019-12-13 18:51:08 +0100721void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from,
722 struct bpf_prog *to);
Jiri Olsadba122fb2020-03-12 20:56:04 +0100723/* Called only from JIT-enabled code, so there's no need for stubs. */
Jiri Olsa7ac88eb2020-03-12 20:56:07 +0100724void *bpf_jit_alloc_exec_page(void);
Jiri Olsaa108f7d2020-03-12 20:56:05 +0100725void bpf_image_ksym_add(void *data, struct bpf_ksym *ksym);
726void bpf_image_ksym_del(struct bpf_ksym *ksym);
Jiri Olsadba122fb2020-03-12 20:56:04 +0100727void bpf_ksym_add(struct bpf_ksym *ksym);
728void bpf_ksym_del(struct bpf_ksym *ksym);
Alexei Starovoitove21aa342021-03-16 14:00:07 -0700729int bpf_jit_charge_modmem(u32 pages);
730void bpf_jit_uncharge_modmem(u32 pages);
Alexei Starovoitovfec56f52019-11-14 10:57:04 -0800731#else
Toke Høiland-Jørgensen3aac1ea2020-09-29 14:45:50 +0200732static inline int bpf_trampoline_link_prog(struct bpf_prog *prog,
733 struct bpf_trampoline *tr)
Alexei Starovoitovfec56f52019-11-14 10:57:04 -0800734{
735 return -ENOTSUPP;
736}
Toke Høiland-Jørgensen3aac1ea2020-09-29 14:45:50 +0200737static inline int bpf_trampoline_unlink_prog(struct bpf_prog *prog,
738 struct bpf_trampoline *tr)
Alexei Starovoitovfec56f52019-11-14 10:57:04 -0800739{
740 return -ENOTSUPP;
741}
Toke Høiland-Jørgensenf7b12b62020-09-25 23:25:02 +0200742static inline struct bpf_trampoline *bpf_trampoline_get(u64 key,
743 struct bpf_attach_target_info *tgt_info)
744{
745 return ERR_PTR(-EOPNOTSUPP);
746}
Alexei Starovoitovfec56f52019-11-14 10:57:04 -0800747static inline void bpf_trampoline_put(struct bpf_trampoline *tr) {}
Björn Töpel75ccbef2019-12-13 18:51:08 +0100748#define DEFINE_BPF_DISPATCHER(name)
749#define DECLARE_BPF_DISPATCHER(name)
Björn Töpel6a640372020-03-12 20:55:57 +0100750#define BPF_DISPATCHER_FUNC(name) bpf_dispatcher_nop_func
Björn Töpel75ccbef2019-12-13 18:51:08 +0100751#define BPF_DISPATCHER_PTR(name) NULL
752static inline void bpf_dispatcher_change_prog(struct bpf_dispatcher *d,
753 struct bpf_prog *from,
754 struct bpf_prog *to) {}
Jiri Olsae9b4e602020-01-23 17:15:07 +0100755static inline bool is_bpf_image_address(unsigned long address)
756{
757 return false;
758}
Alexei Starovoitovfec56f52019-11-14 10:57:04 -0800759#endif
760
Alexei Starovoitov8c1b6e62019-11-14 10:57:16 -0800761struct bpf_func_info_aux {
Alexei Starovoitov51c39bb2020-01-09 22:41:20 -0800762 u16 linkage;
Alexei Starovoitov8c1b6e62019-11-14 10:57:16 -0800763 bool unreliable;
764};
765
Daniel Borkmanna66886f2019-11-22 21:07:57 +0100766enum bpf_jit_poke_reason {
767 BPF_POKE_REASON_TAIL_CALL,
768};
769
770/* Descriptor of pokes pointing /into/ the JITed image. */
771struct bpf_jit_poke_descriptor {
Maciej Fijalkowskicf71b172020-09-16 23:10:06 +0200772 void *tailcall_target;
Maciej Fijalkowskiebf7d1f52020-09-16 23:10:08 +0200773 void *tailcall_bypass;
774 void *bypass_addr;
Daniel Borkmanna66886f2019-11-22 21:07:57 +0100775 union {
776 struct {
777 struct bpf_map *map;
778 u32 key;
779 } tail_call;
780 };
Maciej Fijalkowskicf71b172020-09-16 23:10:06 +0200781 bool tailcall_target_stable;
Daniel Borkmanna66886f2019-11-22 21:07:57 +0100782 u8 adj_off;
783 u16 reason;
Maciej Fijalkowskia748c692020-09-16 23:10:05 +0200784 u32 insn_idx;
Daniel Borkmanna66886f2019-11-22 21:07:57 +0100785};
786
Yonghong Song3c32cc12020-05-13 11:02:21 -0700787/* reg_type info for ctx arguments */
788struct bpf_ctx_arg_aux {
789 u32 offset;
790 enum bpf_reg_type reg_type;
Yonghong Song951cf362020-07-20 09:34:03 -0700791 u32 btf_id;
Yonghong Song3c32cc12020-05-13 11:02:21 -0700792};
793
Andrii Nakryiko541c3ba2021-01-11 23:55:18 -0800794struct btf_mod_pair {
795 struct btf *btf;
796 struct module *module;
797};
798
Alexei Starovoitov09756af2014-09-26 00:17:00 -0700799struct bpf_prog_aux {
Andrii Nakryiko85192dbf2019-11-17 09:28:03 -0800800 atomic64_t refcnt;
Daniel Borkmann24701ec2015-03-01 12:31:47 +0100801 u32 used_map_cnt;
Andrii Nakryiko541c3ba2021-01-11 23:55:18 -0800802 u32 used_btf_cnt;
Alexei Starovoitov32bbe002016-04-06 18:43:28 -0700803 u32 max_ctx_offset;
Jiong Wange6478152018-11-08 04:08:42 -0500804 u32 max_pkt_offset;
Matt Mullins9df1c282019-04-26 11:49:47 -0700805 u32 max_tp_access;
Alexei Starovoitov87266792017-05-30 13:31:29 -0700806 u32 stack_depth;
Martin KaFai Laudc4bb0e2017-06-05 12:15:46 -0700807 u32 id;
Yonghong Songba64e7d2018-11-24 23:20:44 -0800808 u32 func_cnt; /* used by non-func prog as the number of func progs */
809 u32 func_idx; /* 0 for non-func prog, the index in func array for func prog */
Alexei Starovoitovccfe29eb22019-10-15 20:24:58 -0700810 u32 attach_btf_id; /* in-kernel BTF type id to attach to */
Yonghong Song3c32cc12020-05-13 11:02:21 -0700811 u32 ctx_arg_info_size;
Yonghong Songafbf21d2020-07-23 11:41:11 -0700812 u32 max_rdonly_access;
813 u32 max_rdwr_access;
Andrii Nakryiko22dc4a02020-12-03 12:46:29 -0800814 struct btf *attach_btf;
Yonghong Song3c32cc12020-05-13 11:02:21 -0700815 const struct bpf_ctx_arg_aux *ctx_arg_info;
Toke Høiland-Jørgensen3aac1ea2020-09-29 14:45:50 +0200816 struct mutex dst_mutex; /* protects dst_* pointers below, *after* prog becomes visible */
817 struct bpf_prog *dst_prog;
818 struct bpf_trampoline *dst_trampoline;
Toke Høiland-Jørgensen4a1e7c02020-09-29 14:45:51 +0200819 enum bpf_prog_type saved_dst_prog_type;
820 enum bpf_attach_type saved_dst_attach_type;
Jiong Wanga4b1d3c2019-05-24 23:25:15 +0100821 bool verifier_zext; /* Zero extensions has been inserted by verifier. */
Jakub Kicinski9a18eed2017-12-27 18:39:04 -0800822 bool offload_requested;
Martin KaFai Lau38207292019-10-24 17:18:11 -0700823 bool attach_btf_trace; /* true if attaching to BTF-enabled raw tp */
Alexei Starovoitov8c1b6e62019-11-14 10:57:16 -0800824 bool func_proto_unreliable;
Alexei Starovoitov1e6c62a2020-08-27 15:01:11 -0700825 bool sleepable;
Maciej Fijalkowskiebf7d1f52020-09-16 23:10:08 +0200826 bool tail_call_reachable;
Alexei Starovoitovfec56f52019-11-14 10:57:04 -0800827 struct hlist_node tramp_hlist;
Martin KaFai Lau38207292019-10-24 17:18:11 -0700828 /* BTF_KIND_FUNC_PROTO for valid attach_btf_id */
829 const struct btf_type *attach_func_proto;
830 /* function name for valid attach_btf_id */
831 const char *attach_func_name;
Alexei Starovoitov1c2a0882017-12-14 17:55:15 -0800832 struct bpf_prog **func;
833 void *jit_data; /* JIT specific data. arch dependent */
Daniel Borkmanna66886f2019-11-22 21:07:57 +0100834 struct bpf_jit_poke_descriptor *poke_tab;
835 u32 size_poke_tab;
Jiri Olsa535911c2020-03-12 20:55:58 +0100836 struct bpf_ksym ksym;
Jakub Kicinski7de16e32017-10-16 16:40:53 -0700837 const struct bpf_prog_ops *ops;
Alexei Starovoitov09756af2014-09-26 00:17:00 -0700838 struct bpf_map **used_maps;
YiFei Zhu984fe942020-09-15 16:45:39 -0700839 struct mutex used_maps_mutex; /* mutex for used_maps and used_map_cnt */
Andrii Nakryiko541c3ba2021-01-11 23:55:18 -0800840 struct btf_mod_pair *used_btfs;
Alexei Starovoitov09756af2014-09-26 00:17:00 -0700841 struct bpf_prog *prog;
Alexei Starovoitovaaac3ba2015-10-07 22:23:22 -0700842 struct user_struct *user;
Martin KaFai Laucb4d2b32017-09-27 14:37:52 -0700843 u64 load_time; /* ns since boottime */
Roman Gushchin8bad74f2018-09-28 14:45:36 +0000844 struct bpf_map *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE];
Martin KaFai Lau067cae42017-10-05 21:52:12 -0700845 char name[BPF_OBJ_NAME_LEN];
Chenbo Fengafdb09c2017-10-18 13:00:24 -0700846#ifdef CONFIG_SECURITY
847 void *security;
848#endif
Jakub Kicinski0a9c1992018-01-11 20:29:07 -0800849 struct bpf_prog_offload *offload;
Yonghong Song838e9692018-11-19 15:29:11 -0800850 struct btf *btf;
Yonghong Songba64e7d2018-11-24 23:20:44 -0800851 struct bpf_func_info *func_info;
Alexei Starovoitov8c1b6e62019-11-14 10:57:16 -0800852 struct bpf_func_info_aux *func_info_aux;
Martin KaFai Lauc454a462018-12-07 16:42:25 -0800853 /* bpf_line_info loaded from userspace. linfo->insn_off
854 * has the xlated insn offset.
855 * Both the main and sub prog share the same linfo.
856 * The subprog can access its first linfo by
857 * using the linfo_idx.
858 */
859 struct bpf_line_info *linfo;
860 /* jited_linfo is the jited addr of the linfo. It has a
861 * one to one mapping to linfo:
862 * jited_linfo[i] is the jited addr for the linfo[i]->insn_off.
863 * Both the main and sub prog share the same jited_linfo.
864 * The subprog can access its first jited_linfo by
865 * using the linfo_idx.
866 */
867 void **jited_linfo;
Yonghong Songba64e7d2018-11-24 23:20:44 -0800868 u32 func_info_cnt;
Martin KaFai Lauc454a462018-12-07 16:42:25 -0800869 u32 nr_linfo;
870 /* subprog can use linfo_idx to access its first linfo and
871 * jited_linfo.
872 * main prog always has linfo_idx == 0
873 */
874 u32 linfo_idx;
Alexei Starovoitov3dec5412019-10-15 20:25:03 -0700875 u32 num_exentries;
876 struct exception_table_entry *extable;
Alexei Starovoitovabf2e7d2015-05-28 19:26:02 -0700877 union {
878 struct work_struct work;
879 struct rcu_head rcu;
880 };
Alexei Starovoitov09756af2014-09-26 00:17:00 -0700881};
882
Daniel Borkmann2beee5f2019-11-22 21:07:56 +0100883struct bpf_array_aux {
884 /* 'Ownership' of prog array is claimed by the first program that
885 * is going to use this map or by the first program which FD is
886 * stored in the map to make sure that all callers and callees have
887 * the same prog type and JITed flag.
888 */
889 enum bpf_prog_type type;
890 bool jited;
Daniel Borkmannda765a22019-11-22 21:07:58 +0100891 /* Programs with direct jumps into programs part of this array. */
892 struct list_head poke_progs;
893 struct bpf_map *map;
894 struct mutex poke_mutex;
895 struct work_struct work;
Daniel Borkmann2beee5f2019-11-22 21:07:56 +0100896};
897
Andrii Nakryiko6cc7d1e2020-07-21 23:45:54 -0700898struct bpf_link {
899 atomic64_t refcnt;
900 u32 id;
901 enum bpf_link_type type;
902 const struct bpf_link_ops *ops;
903 struct bpf_prog *prog;
904 struct work_struct work;
905};
906
907struct bpf_link_ops {
908 void (*release)(struct bpf_link *link);
909 void (*dealloc)(struct bpf_link *link);
Andrii Nakryiko73b11c2a2020-07-31 11:28:26 -0700910 int (*detach)(struct bpf_link *link);
Andrii Nakryiko6cc7d1e2020-07-21 23:45:54 -0700911 int (*update_prog)(struct bpf_link *link, struct bpf_prog *new_prog,
912 struct bpf_prog *old_prog);
913 void (*show_fdinfo)(const struct bpf_link *link, struct seq_file *seq);
914 int (*fill_link_info)(const struct bpf_link *link,
915 struct bpf_link_info *info);
916};
917
918struct bpf_link_primer {
919 struct bpf_link *link;
920 struct file *file;
921 int fd;
922 u32 id;
923};
924
Martin KaFai Lau85d33df2020-01-08 16:35:05 -0800925struct bpf_struct_ops_value;
Martin KaFai Lau27ae79972020-01-08 16:35:03 -0800926struct btf_type;
927struct btf_member;
928
929#define BPF_STRUCT_OPS_MAX_NR_MEMBERS 64
930struct bpf_struct_ops {
931 const struct bpf_verifier_ops *verifier_ops;
932 int (*init)(struct btf *btf);
933 int (*check_member)(const struct btf_type *t,
934 const struct btf_member *member);
Martin KaFai Lau85d33df2020-01-08 16:35:05 -0800935 int (*init_member)(const struct btf_type *t,
936 const struct btf_member *member,
937 void *kdata, const void *udata);
938 int (*reg)(void *kdata);
939 void (*unreg)(void *kdata);
Martin KaFai Lau27ae79972020-01-08 16:35:03 -0800940 const struct btf_type *type;
Martin KaFai Lau85d33df2020-01-08 16:35:05 -0800941 const struct btf_type *value_type;
Martin KaFai Lau27ae79972020-01-08 16:35:03 -0800942 const char *name;
943 struct btf_func_model func_models[BPF_STRUCT_OPS_MAX_NR_MEMBERS];
944 u32 type_id;
Martin KaFai Lau85d33df2020-01-08 16:35:05 -0800945 u32 value_id;
Martin KaFai Lau27ae79972020-01-08 16:35:03 -0800946};
947
948#if defined(CONFIG_BPF_JIT) && defined(CONFIG_BPF_SYSCALL)
Martin KaFai Lau85d33df2020-01-08 16:35:05 -0800949#define BPF_MODULE_OWNER ((void *)((0xeB9FUL << 2) + POISON_POINTER_DELTA))
Martin KaFai Lau27ae79972020-01-08 16:35:03 -0800950const struct bpf_struct_ops *bpf_struct_ops_find(u32 type_id);
Martin KaFai Laud3e42bb2020-01-27 09:51:45 -0800951void bpf_struct_ops_init(struct btf *btf, struct bpf_verifier_log *log);
Martin KaFai Lau85d33df2020-01-08 16:35:05 -0800952bool bpf_struct_ops_get(const void *kdata);
953void bpf_struct_ops_put(const void *kdata);
954int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map, void *key,
955 void *value);
956static inline bool bpf_try_module_get(const void *data, struct module *owner)
957{
958 if (owner == BPF_MODULE_OWNER)
959 return bpf_struct_ops_get(data);
960 else
961 return try_module_get(owner);
962}
963static inline void bpf_module_put(const void *data, struct module *owner)
964{
965 if (owner == BPF_MODULE_OWNER)
966 bpf_struct_ops_put(data);
967 else
968 module_put(owner);
969}
Martin KaFai Lau27ae79972020-01-08 16:35:03 -0800970#else
971static inline const struct bpf_struct_ops *bpf_struct_ops_find(u32 type_id)
972{
973 return NULL;
974}
Martin KaFai Laud3e42bb2020-01-27 09:51:45 -0800975static inline void bpf_struct_ops_init(struct btf *btf,
976 struct bpf_verifier_log *log)
977{
978}
Martin KaFai Lau85d33df2020-01-08 16:35:05 -0800979static inline bool bpf_try_module_get(const void *data, struct module *owner)
980{
981 return try_module_get(owner);
982}
983static inline void bpf_module_put(const void *data, struct module *owner)
984{
985 module_put(owner);
986}
987static inline int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map,
988 void *key,
989 void *value)
990{
991 return -EINVAL;
992}
Martin KaFai Lau27ae79972020-01-08 16:35:03 -0800993#endif
994
Alexei Starovoitov04fd61ab2015-05-19 16:59:03 -0700995struct bpf_array {
996 struct bpf_map map;
997 u32 elem_size;
Alexei Starovoitovb2157392018-01-07 17:33:02 -0800998 u32 index_mask;
Daniel Borkmann2beee5f2019-11-22 21:07:56 +0100999 struct bpf_array_aux *aux;
Alexei Starovoitov04fd61ab2015-05-19 16:59:03 -07001000 union {
1001 char value[0] __aligned(8);
Wang Nan2a36f0b2015-08-06 07:02:33 +00001002 void *ptrs[0] __aligned(8);
Alexei Starovoitova10423b2016-02-01 22:39:54 -08001003 void __percpu *pptrs[0] __aligned(8);
Alexei Starovoitov04fd61ab2015-05-19 16:59:03 -07001004 };
1005};
Daniel Borkmann3b1efb12016-06-15 22:47:14 +02001006
Alexei Starovoitovc04c0d22019-04-01 21:27:45 -07001007#define BPF_COMPLEXITY_LIMIT_INSNS 1000000 /* yes. 1M insns */
Alexei Starovoitov04fd61ab2015-05-19 16:59:03 -07001008#define MAX_TAIL_CALL_CNT 32
1009
Daniel Borkmann591fe982019-04-09 23:20:05 +02001010#define BPF_F_ACCESS_MASK (BPF_F_RDONLY | \
1011 BPF_F_RDONLY_PROG | \
1012 BPF_F_WRONLY | \
1013 BPF_F_WRONLY_PROG)
1014
1015#define BPF_MAP_CAN_READ BIT(0)
1016#define BPF_MAP_CAN_WRITE BIT(1)
1017
1018static inline u32 bpf_map_flags_to_cap(struct bpf_map *map)
1019{
1020 u32 access_flags = map->map_flags & (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG);
1021
1022 /* Combination of BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG is
1023 * not possible.
1024 */
1025 if (access_flags & BPF_F_RDONLY_PROG)
1026 return BPF_MAP_CAN_READ;
1027 else if (access_flags & BPF_F_WRONLY_PROG)
1028 return BPF_MAP_CAN_WRITE;
1029 else
1030 return BPF_MAP_CAN_READ | BPF_MAP_CAN_WRITE;
1031}
1032
1033static inline bool bpf_map_flags_access_ok(u32 access_flags)
1034{
1035 return (access_flags & (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG)) !=
1036 (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG);
1037}
1038
Daniel Borkmann3b1efb12016-06-15 22:47:14 +02001039struct bpf_event_entry {
1040 struct perf_event *event;
1041 struct file *perf_file;
1042 struct file *map_file;
1043 struct rcu_head rcu;
1044};
1045
Alexei Starovoitov04fd61ab2015-05-19 16:59:03 -07001046bool bpf_prog_array_compatible(struct bpf_array *array, const struct bpf_prog *fp);
Daniel Borkmannf1f77142017-01-13 23:38:15 +01001047int bpf_prog_calc_tag(struct bpf_prog *fp);
Daniel Borkmannbd570ff2016-04-18 21:01:24 +02001048
Alexei Starovoitov0756ea32015-06-12 19:39:13 -07001049const struct bpf_func_proto *bpf_get_trace_printk_proto(void);
Daniel Borkmann555c8a82016-07-14 18:08:05 +02001050
1051typedef unsigned long (*bpf_ctx_copy_t)(void *dst, const void *src,
Daniel Borkmannaa7145c2016-07-22 01:19:42 +02001052 unsigned long off, unsigned long len);
Joe Stringerc64b7982018-10-02 13:35:33 -07001053typedef u32 (*bpf_convert_ctx_access_t)(enum bpf_access_type type,
1054 const struct bpf_insn *src,
1055 struct bpf_insn *dst,
1056 struct bpf_prog *prog,
1057 u32 *target_size);
Daniel Borkmann555c8a82016-07-14 18:08:05 +02001058
1059u64 bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size,
1060 void *ctx, u64 ctx_size, bpf_ctx_copy_t ctx_copy);
Alexei Starovoitov04fd61ab2015-05-19 16:59:03 -07001061
Alexei Starovoitov324bda9e62017-10-02 22:50:21 -07001062/* an array of programs to be executed under rcu_lock.
1063 *
1064 * Typical usage:
1065 * ret = BPF_PROG_RUN_ARRAY(&bpf_prog_array, ctx, BPF_PROG_RUN);
1066 *
1067 * the structure returned by bpf_prog_array_alloc() should be populated
1068 * with program pointers and the last pointer must be NULL.
1069 * The user has to keep refcnt on the program and make sure the program
1070 * is removed from the array before bpf_prog_put().
1071 * The 'struct bpf_prog_array *' should only be replaced with xchg()
1072 * since other cpus are walking the array of pointers in parallel.
1073 */
Roman Gushchin394e40a2018-08-02 14:27:21 -07001074struct bpf_prog_array_item {
1075 struct bpf_prog *prog;
Roman Gushchin8bad74f2018-09-28 14:45:36 +00001076 struct bpf_cgroup_storage *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE];
Roman Gushchin394e40a2018-08-02 14:27:21 -07001077};
1078
Alexei Starovoitov324bda9e62017-10-02 22:50:21 -07001079struct bpf_prog_array {
1080 struct rcu_head rcu;
Gustavo A. R. Silvad7f10df2020-02-26 18:17:44 -06001081 struct bpf_prog_array_item items[];
Alexei Starovoitov324bda9e62017-10-02 22:50:21 -07001082};
1083
Roman Gushchind29ab6e2018-07-13 12:41:10 -07001084struct bpf_prog_array *bpf_prog_array_alloc(u32 prog_cnt, gfp_t flags);
Stanislav Fomichev54e9c9d2019-05-28 14:14:41 -07001085void bpf_prog_array_free(struct bpf_prog_array *progs);
1086int bpf_prog_array_length(struct bpf_prog_array *progs);
Stanislav Fomichev0d01da62019-06-27 13:38:47 -07001087bool bpf_prog_array_is_empty(struct bpf_prog_array *array);
Stanislav Fomichev54e9c9d2019-05-28 14:14:41 -07001088int bpf_prog_array_copy_to_user(struct bpf_prog_array *progs,
Alexei Starovoitov468e2f62017-10-02 22:50:22 -07001089 __u32 __user *prog_ids, u32 cnt);
Alexei Starovoitov324bda9e62017-10-02 22:50:21 -07001090
Stanislav Fomichev54e9c9d2019-05-28 14:14:41 -07001091void bpf_prog_array_delete_safe(struct bpf_prog_array *progs,
Yonghong Songe87c6bc382017-10-23 23:53:08 -07001092 struct bpf_prog *old_prog);
Jakub Sitnickice3aa9c2020-07-17 12:35:22 +02001093int bpf_prog_array_delete_safe_at(struct bpf_prog_array *array, int index);
1094int bpf_prog_array_update_at(struct bpf_prog_array *array, int index,
1095 struct bpf_prog *prog);
Stanislav Fomichev54e9c9d2019-05-28 14:14:41 -07001096int bpf_prog_array_copy_info(struct bpf_prog_array *array,
Yonghong Song3a38bb92018-04-10 09:37:32 -07001097 u32 *prog_ids, u32 request_cnt,
1098 u32 *prog_cnt);
Stanislav Fomichev54e9c9d2019-05-28 14:14:41 -07001099int bpf_prog_array_copy(struct bpf_prog_array *old_array,
Yonghong Songe87c6bc382017-10-23 23:53:08 -07001100 struct bpf_prog *exclude_prog,
1101 struct bpf_prog *include_prog,
1102 struct bpf_prog_array **new_array);
1103
Stanislav Fomichev77241212021-01-27 11:31:39 -08001104/* BPF program asks to bypass CAP_NET_BIND_SERVICE in bind. */
1105#define BPF_RET_BIND_NO_CAP_NET_BIND_SERVICE (1 << 0)
1106/* BPF program asks to set CN on the packet. */
1107#define BPF_RET_SET_CN (1 << 0)
1108
1109#define BPF_PROG_RUN_ARRAY_FLAGS(array, ctx, func, ret_flags) \
1110 ({ \
1111 struct bpf_prog_array_item *_item; \
1112 struct bpf_prog *_prog; \
1113 struct bpf_prog_array *_array; \
1114 u32 _ret = 1; \
1115 u32 func_ret; \
1116 migrate_disable(); \
1117 rcu_read_lock(); \
1118 _array = rcu_dereference(array); \
1119 _item = &_array->items[0]; \
1120 while ((_prog = READ_ONCE(_item->prog))) { \
1121 bpf_cgroup_storage_set(_item->cgroup_storage); \
1122 func_ret = func(_prog, ctx); \
1123 _ret &= (func_ret & 1); \
1124 *(ret_flags) |= (func_ret >> 1); \
1125 _item++; \
1126 } \
1127 rcu_read_unlock(); \
1128 migrate_enable(); \
1129 _ret; \
1130 })
1131
Yonghong Song05a68ce2021-03-09 10:50:28 -08001132#define __BPF_PROG_RUN_ARRAY(array, ctx, func, check_non_null, set_cg_storage) \
Alexei Starovoitov324bda9e62017-10-02 22:50:21 -07001133 ({ \
Roman Gushchin394e40a2018-08-02 14:27:21 -07001134 struct bpf_prog_array_item *_item; \
1135 struct bpf_prog *_prog; \
Yonghong Songe87c6bc382017-10-23 23:53:08 -07001136 struct bpf_prog_array *_array; \
Alexei Starovoitov324bda9e62017-10-02 22:50:21 -07001137 u32 _ret = 1; \
David Miller2a916f22020-02-24 15:01:46 +01001138 migrate_disable(); \
Alexei Starovoitov324bda9e62017-10-02 22:50:21 -07001139 rcu_read_lock(); \
Yonghong Songe87c6bc382017-10-23 23:53:08 -07001140 _array = rcu_dereference(array); \
1141 if (unlikely(check_non_null && !_array))\
1142 goto _out; \
Roman Gushchin394e40a2018-08-02 14:27:21 -07001143 _item = &_array->items[0]; \
1144 while ((_prog = READ_ONCE(_item->prog))) { \
Yonghong Song05a68ce2021-03-09 10:50:28 -08001145 if (set_cg_storage) \
1146 bpf_cgroup_storage_set(_item->cgroup_storage); \
Roman Gushchin394e40a2018-08-02 14:27:21 -07001147 _ret &= func(_prog, ctx); \
1148 _item++; \
Yonghong Songe87c6bc382017-10-23 23:53:08 -07001149 } \
1150_out: \
Alexei Starovoitov324bda9e62017-10-02 22:50:21 -07001151 rcu_read_unlock(); \
David Miller2a916f22020-02-24 15:01:46 +01001152 migrate_enable(); \
Alexei Starovoitov324bda9e62017-10-02 22:50:21 -07001153 _ret; \
1154 })
1155
brakmo1f52f6c2019-05-28 16:59:35 -07001156/* To be used by __cgroup_bpf_run_filter_skb for EGRESS BPF progs
1157 * so BPF programs can request cwr for TCP packets.
1158 *
1159 * Current cgroup skb programs can only return 0 or 1 (0 to drop the
1160 * packet. This macro changes the behavior so the low order bit
1161 * indicates whether the packet should be dropped (0) or not (1)
1162 * and the next bit is a congestion notification bit. This could be
1163 * used by TCP to call tcp_enter_cwr()
1164 *
1165 * Hence, new allowed return values of CGROUP EGRESS BPF programs are:
1166 * 0: drop packet
1167 * 1: keep packet
1168 * 2: drop packet and cn
1169 * 3: keep packet and cn
1170 *
1171 * This macro then converts it to one of the NET_XMIT or an error
1172 * code that is then interpreted as drop packet (and no cn):
1173 * 0: NET_XMIT_SUCCESS skb should be transmitted
1174 * 1: NET_XMIT_DROP skb should be dropped and cn
1175 * 2: NET_XMIT_CN skb should be transmitted and cn
1176 * 3: -EPERM skb should be dropped
1177 */
1178#define BPF_PROG_CGROUP_INET_EGRESS_RUN_ARRAY(array, ctx, func) \
1179 ({ \
Stanislav Fomichev77241212021-01-27 11:31:39 -08001180 u32 _flags = 0; \
1181 bool _cn; \
1182 u32 _ret; \
1183 _ret = BPF_PROG_RUN_ARRAY_FLAGS(array, ctx, func, &_flags); \
1184 _cn = _flags & BPF_RET_SET_CN; \
brakmo1f52f6c2019-05-28 16:59:35 -07001185 if (_ret) \
1186 _ret = (_cn ? NET_XMIT_CN : NET_XMIT_SUCCESS); \
1187 else \
1188 _ret = (_cn ? NET_XMIT_DROP : -EPERM); \
1189 _ret; \
1190 })
1191
Yonghong Songe87c6bc382017-10-23 23:53:08 -07001192#define BPF_PROG_RUN_ARRAY(array, ctx, func) \
Yonghong Song05a68ce2021-03-09 10:50:28 -08001193 __BPF_PROG_RUN_ARRAY(array, ctx, func, false, true)
Yonghong Songe87c6bc382017-10-23 23:53:08 -07001194
1195#define BPF_PROG_RUN_ARRAY_CHECK(array, ctx, func) \
Yonghong Song05a68ce2021-03-09 10:50:28 -08001196 __BPF_PROG_RUN_ARRAY(array, ctx, func, true, false)
Yonghong Songe87c6bc382017-10-23 23:53:08 -07001197
Alexei Starovoitov89aa0752014-12-01 15:06:35 -08001198#ifdef CONFIG_BPF_SYSCALL
Alexei Starovoitovb121d1e2016-03-07 21:57:13 -08001199DECLARE_PER_CPU(int, bpf_prog_active);
Song Liud46edd62020-04-30 00:15:04 -07001200extern struct mutex bpf_stats_enabled_mutex;
Alexei Starovoitovb121d1e2016-03-07 21:57:13 -08001201
Thomas Gleixnerc518cfa2020-02-24 15:01:47 +01001202/*
1203 * Block execution of BPF programs attached to instrumentation (perf,
1204 * kprobes, tracepoints) to prevent deadlocks on map operations as any of
1205 * these events can happen inside a region which holds a map bucket lock
1206 * and can deadlock on it.
1207 *
1208 * Use the preemption safe inc/dec variants on RT because migrate disable
1209 * is preemptible on RT and preemption in the middle of the RMW operation
1210 * might lead to inconsistent state. Use the raw variants for non RT
1211 * kernels as migrate_disable() maps to preempt_disable() so the slightly
1212 * more expensive save operation can be avoided.
1213 */
1214static inline void bpf_disable_instrumentation(void)
1215{
1216 migrate_disable();
1217 if (IS_ENABLED(CONFIG_PREEMPT_RT))
1218 this_cpu_inc(bpf_prog_active);
1219 else
1220 __this_cpu_inc(bpf_prog_active);
1221}
1222
1223static inline void bpf_enable_instrumentation(void)
1224{
1225 if (IS_ENABLED(CONFIG_PREEMPT_RT))
1226 this_cpu_dec(bpf_prog_active);
1227 else
1228 __this_cpu_dec(bpf_prog_active);
1229 migrate_enable();
1230}
1231
Chenbo Fengf66e4482017-10-18 13:00:26 -07001232extern const struct file_operations bpf_map_fops;
1233extern const struct file_operations bpf_prog_fops;
Yonghong Song367ec3e2020-05-09 10:59:06 -07001234extern const struct file_operations bpf_iter_fops;
Chenbo Fengf66e4482017-10-18 13:00:26 -07001235
Alexei Starovoitov91cc1a92019-11-14 10:57:15 -08001236#define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \
Jakub Kicinski7de16e32017-10-16 16:40:53 -07001237 extern const struct bpf_prog_ops _name ## _prog_ops; \
1238 extern const struct bpf_verifier_ops _name ## _verifier_ops;
Johannes Berg40077e02017-04-11 15:34:58 +02001239#define BPF_MAP_TYPE(_id, _ops) \
1240 extern const struct bpf_map_ops _ops;
Andrii Nakryikof2e10bf2020-04-28 17:16:08 -07001241#define BPF_LINK_TYPE(_id, _name)
Johannes Bergbe9370a2017-04-11 15:34:57 +02001242#include <linux/bpf_types.h>
1243#undef BPF_PROG_TYPE
Johannes Berg40077e02017-04-11 15:34:58 +02001244#undef BPF_MAP_TYPE
Andrii Nakryikof2e10bf2020-04-28 17:16:08 -07001245#undef BPF_LINK_TYPE
Daniel Borkmann0fc174d2015-03-01 12:31:44 +01001246
Jakub Kicinskiab3f0062017-11-03 13:56:17 -07001247extern const struct bpf_prog_ops bpf_offload_prog_ops;
Jakub Kicinski4f9218a2017-10-16 16:40:55 -07001248extern const struct bpf_verifier_ops tc_cls_act_analyzer_ops;
1249extern const struct bpf_verifier_ops xdp_analyzer_ops;
1250
Alexei Starovoitov09756af2014-09-26 00:17:00 -07001251struct bpf_prog *bpf_prog_get(u32 ufd);
Jakub Kicinski248f3462017-11-03 13:56:20 -07001252struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, enum bpf_prog_type type,
Jakub Kicinski288b3de52017-11-20 15:21:54 -08001253 bool attach_drv);
Andrii Nakryiko85192dbf2019-11-17 09:28:03 -08001254void bpf_prog_add(struct bpf_prog *prog, int i);
Daniel Borkmannc5405942016-11-09 22:02:34 +01001255void bpf_prog_sub(struct bpf_prog *prog, int i);
Andrii Nakryiko85192dbf2019-11-17 09:28:03 -08001256void bpf_prog_inc(struct bpf_prog *prog);
John Fastabenda6f6df62017-08-15 22:32:22 -07001257struct bpf_prog * __must_check bpf_prog_inc_not_zero(struct bpf_prog *prog);
Daniel Borkmann61e021f32015-03-02 15:21:55 +01001258void bpf_prog_put(struct bpf_prog *prog);
1259
Jakub Kicinskiad8ad792017-12-27 18:39:07 -08001260void bpf_prog_free_id(struct bpf_prog *prog, bool do_idr_lock);
Jakub Kicinskia3884572018-01-11 20:29:09 -08001261void bpf_map_free_id(struct bpf_map *map, bool do_idr_lock);
Jakub Kicinskiad8ad792017-12-27 18:39:07 -08001262
Martin KaFai Lau1ed4d922020-02-25 15:04:21 -08001263struct bpf_map *bpf_map_get(u32 ufd);
Daniel Borkmannc9da1612015-11-24 21:28:15 +01001264struct bpf_map *bpf_map_get_with_uref(u32 ufd);
Daniel Borkmannc2101292015-10-29 14:58:07 +01001265struct bpf_map *__bpf_map_get(struct fd f);
Andrii Nakryiko1e0bd5a2019-11-17 09:28:02 -08001266void bpf_map_inc(struct bpf_map *map);
1267void bpf_map_inc_with_uref(struct bpf_map *map);
1268struct bpf_map * __must_check bpf_map_inc_not_zero(struct bpf_map *map);
Daniel Borkmannc9da1612015-11-24 21:28:15 +01001269void bpf_map_put_with_uref(struct bpf_map *map);
Daniel Borkmann61e021f32015-03-02 15:21:55 +01001270void bpf_map_put(struct bpf_map *map);
Daniel Borkmann196e8ca2019-11-20 23:04:44 +01001271void *bpf_map_area_alloc(u64 size, int numa_node);
1272void *bpf_map_area_mmapable_alloc(u64 size, int numa_node);
Daniel Borkmannd407bd22017-01-18 15:14:17 +01001273void bpf_map_area_free(void *base);
Jakub Kicinskibd475642018-01-11 20:29:06 -08001274void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr);
Brian Vazquezcb4d03a2020-01-15 10:43:01 -08001275int generic_map_lookup_batch(struct bpf_map *map,
1276 const union bpf_attr *attr,
1277 union bpf_attr __user *uattr);
Brian Vazquezaa2e93b2020-01-15 10:43:02 -08001278int generic_map_update_batch(struct bpf_map *map,
1279 const union bpf_attr *attr,
1280 union bpf_attr __user *uattr);
1281int generic_map_delete_batch(struct bpf_map *map,
1282 const union bpf_attr *attr,
1283 union bpf_attr __user *uattr);
Yonghong Song6086d292020-05-09 10:59:09 -07001284struct bpf_map *bpf_map_get_curr_or_next(u32 *id);
Alexei Starovoitova228a642020-07-01 18:10:18 -07001285struct bpf_prog *bpf_prog_get_curr_or_next(u32 *id);
Daniel Borkmann61e021f32015-03-02 15:21:55 +01001286
Roman Gushchin48edc1f2020-12-01 13:58:32 -08001287#ifdef CONFIG_MEMCG_KMEM
1288void *bpf_map_kmalloc_node(const struct bpf_map *map, size_t size, gfp_t flags,
1289 int node);
1290void *bpf_map_kzalloc(const struct bpf_map *map, size_t size, gfp_t flags);
1291void __percpu *bpf_map_alloc_percpu(const struct bpf_map *map, size_t size,
1292 size_t align, gfp_t flags);
1293#else
1294static inline void *
1295bpf_map_kmalloc_node(const struct bpf_map *map, size_t size, gfp_t flags,
1296 int node)
1297{
1298 return kmalloc_node(size, flags, node);
1299}
1300
1301static inline void *
1302bpf_map_kzalloc(const struct bpf_map *map, size_t size, gfp_t flags)
1303{
1304 return kzalloc(size, flags);
1305}
1306
1307static inline void __percpu *
1308bpf_map_alloc_percpu(const struct bpf_map *map, size_t size, size_t align,
1309 gfp_t flags)
1310{
1311 return __alloc_percpu_gfp(size, align, flags);
1312}
1313#endif
1314
Alexei Starovoitov1be7f752015-10-07 22:23:21 -07001315extern int sysctl_unprivileged_bpf_disabled;
1316
Alexei Starovoitov2c78ee82020-05-13 16:03:54 -07001317static inline bool bpf_allow_ptr_leaks(void)
1318{
1319 return perfmon_capable();
1320}
1321
Andrei Matei01f810a2021-02-06 20:10:24 -05001322static inline bool bpf_allow_uninit_stack(void)
1323{
1324 return perfmon_capable();
1325}
1326
Andrey Ignatov41c48f32020-06-19 14:11:43 -07001327static inline bool bpf_allow_ptr_to_map_access(void)
1328{
1329 return perfmon_capable();
1330}
1331
Alexei Starovoitov2c78ee82020-05-13 16:03:54 -07001332static inline bool bpf_bypass_spec_v1(void)
1333{
1334 return perfmon_capable();
1335}
1336
1337static inline bool bpf_bypass_spec_v4(void)
1338{
1339 return perfmon_capable();
1340}
1341
Chenbo Feng6e71b042017-10-18 13:00:22 -07001342int bpf_map_new_fd(struct bpf_map *map, int flags);
Daniel Borkmannb2197752015-10-29 14:58:09 +01001343int bpf_prog_new_fd(struct bpf_prog *prog);
1344
Andrii Nakryikof2e10bf2020-04-28 17:16:08 -07001345void bpf_link_init(struct bpf_link *link, enum bpf_link_type type,
Andrii Nakryikoa3b80e12020-04-28 17:16:06 -07001346 const struct bpf_link_ops *ops, struct bpf_prog *prog);
1347int bpf_link_prime(struct bpf_link *link, struct bpf_link_primer *primer);
1348int bpf_link_settle(struct bpf_link_primer *primer);
1349void bpf_link_cleanup(struct bpf_link_primer *primer);
Andrii Nakryiko70ed5062020-03-02 20:31:57 -08001350void bpf_link_inc(struct bpf_link *link);
1351void bpf_link_put(struct bpf_link *link);
1352int bpf_link_new_fd(struct bpf_link *link);
Andrii Nakryikobabf3162020-03-09 16:10:51 -07001353struct file *bpf_link_new_file(struct bpf_link *link, int *reserved_fd);
Andrii Nakryiko70ed5062020-03-02 20:31:57 -08001354struct bpf_link *bpf_link_get_from_fd(u32 ufd);
1355
Daniel Borkmannb2197752015-10-29 14:58:09 +01001356int bpf_obj_pin_user(u32 ufd, const char __user *pathname);
Chenbo Feng6e71b042017-10-18 13:00:22 -07001357int bpf_obj_get_user(const char __user *pathname, int flags);
Daniel Borkmannb2197752015-10-29 14:58:09 +01001358
Yonghong Song21aef702020-05-13 11:02:16 -07001359#define BPF_ITER_FUNC_PREFIX "bpf_iter_"
Yonghong Songe5158d92020-05-09 10:59:07 -07001360#define DEFINE_BPF_ITER_FUNC(target, args...) \
Yonghong Song21aef702020-05-13 11:02:16 -07001361 extern int bpf_iter_ ## target(args); \
1362 int __init bpf_iter_ ## target(args) { return 0; }
Yonghong Song15d83c42020-05-09 10:59:00 -07001363
Yonghong Songf9c79272020-07-23 11:41:10 -07001364struct bpf_iter_aux_info {
Yonghong Songa5cbe052020-07-23 11:41:12 -07001365 struct bpf_map *map;
Yonghong Songf9c79272020-07-23 11:41:10 -07001366};
1367
Yonghong Song5e7b3022020-08-04 22:50:56 -07001368typedef int (*bpf_iter_attach_target_t)(struct bpf_prog *prog,
1369 union bpf_iter_link_info *linfo,
1370 struct bpf_iter_aux_info *aux);
1371typedef void (*bpf_iter_detach_target_t)(struct bpf_iter_aux_info *aux);
Yonghong Song6b0a2492020-08-21 11:44:18 -07001372typedef void (*bpf_iter_show_fdinfo_t) (const struct bpf_iter_aux_info *aux,
1373 struct seq_file *seq);
1374typedef int (*bpf_iter_fill_link_info_t)(const struct bpf_iter_aux_info *aux,
1375 struct bpf_link_info *info);
Yonghong Songa5cbe052020-07-23 11:41:12 -07001376
Yonghong Songcf83b2d2020-10-27 23:10:54 -07001377enum bpf_iter_feature {
1378 BPF_ITER_RESCHED = BIT(0),
1379};
1380
Yonghong Song3c32cc12020-05-13 11:02:21 -07001381#define BPF_ITER_CTX_ARG_MAX 2
Yonghong Songae243452020-05-09 10:58:59 -07001382struct bpf_iter_reg {
1383 const char *target;
Yonghong Song5e7b3022020-08-04 22:50:56 -07001384 bpf_iter_attach_target_t attach_target;
1385 bpf_iter_detach_target_t detach_target;
Yonghong Song6b0a2492020-08-21 11:44:18 -07001386 bpf_iter_show_fdinfo_t show_fdinfo;
1387 bpf_iter_fill_link_info_t fill_link_info;
Yonghong Song3c32cc12020-05-13 11:02:21 -07001388 u32 ctx_arg_info_size;
Yonghong Songcf83b2d2020-10-27 23:10:54 -07001389 u32 feature;
Yonghong Song3c32cc12020-05-13 11:02:21 -07001390 struct bpf_ctx_arg_aux ctx_arg_info[BPF_ITER_CTX_ARG_MAX];
Yonghong Song14fc6bd62020-07-23 11:41:09 -07001391 const struct bpf_iter_seq_info *seq_info;
Yonghong Songae243452020-05-09 10:58:59 -07001392};
1393
Yonghong Songe5158d92020-05-09 10:59:07 -07001394struct bpf_iter_meta {
1395 __bpf_md_ptr(struct seq_file *, seq);
1396 u64 session_id;
1397 u64 seq_num;
1398};
1399
Yonghong Songa5cbe052020-07-23 11:41:12 -07001400struct bpf_iter__bpf_map_elem {
1401 __bpf_md_ptr(struct bpf_iter_meta *, meta);
1402 __bpf_md_ptr(struct bpf_map *, map);
1403 __bpf_md_ptr(void *, key);
1404 __bpf_md_ptr(void *, value);
1405};
1406
Yonghong Song15172a42020-05-13 11:02:19 -07001407int bpf_iter_reg_target(const struct bpf_iter_reg *reg_info);
Yonghong Songab2ee4f2020-05-13 11:02:20 -07001408void bpf_iter_unreg_target(const struct bpf_iter_reg *reg_info);
Yonghong Song15d83c42020-05-09 10:59:00 -07001409bool bpf_iter_prog_supported(struct bpf_prog *prog);
Yonghong Songde4e05c2020-05-09 10:59:01 -07001410int bpf_iter_link_attach(const union bpf_attr *attr, struct bpf_prog *prog);
Yonghong Songac51d992020-05-09 10:59:05 -07001411int bpf_iter_new_fd(struct bpf_link *link);
Yonghong Song367ec3e2020-05-09 10:59:06 -07001412bool bpf_link_is_iter(struct bpf_link *link);
Yonghong Songe5158d92020-05-09 10:59:07 -07001413struct bpf_prog *bpf_iter_get_info(struct bpf_iter_meta *meta, bool in_stop);
1414int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx);
Yonghong Songb76f2222020-08-21 11:44:19 -07001415void bpf_iter_map_show_fdinfo(const struct bpf_iter_aux_info *aux,
1416 struct seq_file *seq);
1417int bpf_iter_map_fill_link_info(const struct bpf_iter_aux_info *aux,
1418 struct bpf_link_info *info);
Yonghong Songae243452020-05-09 10:58:59 -07001419
Yonghong Song314ee052021-02-26 12:49:27 -08001420int map_set_for_each_callback_args(struct bpf_verifier_env *env,
1421 struct bpf_func_state *caller,
1422 struct bpf_func_state *callee);
1423
Alexei Starovoitov15a07b32016-02-01 22:39:55 -08001424int bpf_percpu_hash_copy(struct bpf_map *map, void *key, void *value);
1425int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value);
1426int bpf_percpu_hash_update(struct bpf_map *map, void *key, void *value,
1427 u64 flags);
1428int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value,
1429 u64 flags);
Daniel Borkmannd056a782016-06-15 22:47:13 +02001430
Alexei Starovoitov557c0c62016-03-07 21:57:17 -08001431int bpf_stackmap_copy(struct bpf_map *map, void *key, void *value);
Alexei Starovoitov15a07b32016-02-01 22:39:55 -08001432
Daniel Borkmannd056a782016-06-15 22:47:13 +02001433int bpf_fd_array_map_update_elem(struct bpf_map *map, struct file *map_file,
1434 void *key, void *value, u64 map_flags);
Martin KaFai Lau14dc6f02017-06-27 23:08:34 -07001435int bpf_fd_array_map_lookup_elem(struct bpf_map *map, void *key, u32 *value);
Martin KaFai Laubcc6b1b2017-03-22 10:00:34 -07001436int bpf_fd_htab_map_update_elem(struct bpf_map *map, struct file *map_file,
1437 void *key, void *value, u64 map_flags);
Martin KaFai Lau14dc6f02017-06-27 23:08:34 -07001438int bpf_fd_htab_map_lookup_elem(struct bpf_map *map, void *key, u32 *value);
Daniel Borkmannd056a782016-06-15 22:47:13 +02001439
Chenbo Feng6e71b042017-10-18 13:00:22 -07001440int bpf_get_file_flag(int flags);
Martin KaFai Laudcab51f2018-05-22 15:03:31 -07001441int bpf_check_uarg_tail_zero(void __user *uaddr, size_t expected_size,
1442 size_t actual_size);
Chenbo Feng6e71b042017-10-18 13:00:22 -07001443
Alexei Starovoitov15a07b32016-02-01 22:39:55 -08001444/* memcpy that is used with 8-byte aligned pointers, power-of-8 size and
1445 * forced to use 'long' read/writes to try to atomically copy long counters.
1446 * Best-effort only. No barriers here, since it _will_ race with concurrent
1447 * updates from BPF programs. Called from bpf syscall and mostly used with
1448 * size 8 or 16 bytes, so ask compiler to inline it.
1449 */
1450static inline void bpf_long_memcpy(void *dst, const void *src, u32 size)
1451{
1452 const long *lsrc = src;
1453 long *ldst = dst;
1454
1455 size /= sizeof(long);
1456 while (size--)
1457 *ldst++ = *lsrc++;
1458}
1459
Daniel Borkmann61e021f32015-03-02 15:21:55 +01001460/* verify correctness of eBPF program */
Yonghong Song838e9692018-11-19 15:29:11 -08001461int bpf_check(struct bpf_prog **fp, union bpf_attr *attr,
1462 union bpf_attr __user *uattr);
Andrii Nakryikoa643bff2021-01-11 23:55:14 -08001463
1464#ifndef CONFIG_BPF_JIT_ALWAYS_ON
Alexei Starovoitov1ea47e02017-12-14 17:55:13 -08001465void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth);
Andrii Nakryikoa643bff2021-01-11 23:55:14 -08001466#endif
John Fastabend46f55cf2017-07-17 21:56:48 -07001467
Alan Maguire76654e62020-09-28 12:31:03 +01001468struct btf *bpf_get_btf_vmlinux(void);
1469
John Fastabend46f55cf2017-07-17 21:56:48 -07001470/* Map specifics */
Jesper Dangaard Brouer67f29e02018-05-24 16:45:46 +02001471struct xdp_buff;
Toshiaki Makita6d5fc192018-06-14 11:07:42 +09001472struct sk_buff;
Björn Töpele6a47502021-03-08 12:29:06 +01001473struct bpf_dtab_netdev;
1474struct bpf_cpu_map_entry;
Jesper Dangaard Brouer67f29e02018-05-24 16:45:46 +02001475
Toke Høiland-Jørgensen1d233882020-01-16 16:14:45 +01001476void __dev_flush(void);
1477int dev_xdp_enqueue(struct net_device *dev, struct xdp_buff *xdp,
1478 struct net_device *dev_rx);
Jesper Dangaard Brouer38edddb2018-05-24 16:45:57 +02001479int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_buff *xdp,
1480 struct net_device *dev_rx);
Toshiaki Makita6d5fc192018-06-14 11:07:42 +09001481int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, struct sk_buff *skb,
1482 struct bpf_prog *xdp_prog);
David Ahernfbee97f2020-05-29 16:07:13 -06001483bool dev_map_can_have_prog(struct bpf_map *map);
John Fastabend46f55cf2017-07-17 21:56:48 -07001484
Björn Töpelcdfafe92019-12-19 07:10:04 +01001485void __cpu_map_flush(void);
Jesper Dangaard Brouer9c270af2017-10-16 12:19:34 +02001486int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_buff *xdp,
1487 struct net_device *dev_rx);
Lorenzo Bianconi92164772020-07-14 15:56:38 +02001488bool cpu_map_prog_allowed(struct bpf_map *map);
Jesper Dangaard Brouer9c270af2017-10-16 12:19:34 +02001489
Martin KaFai Lau96eabe72017-08-18 11:28:00 -07001490/* Return map's numa specified by userspace */
1491static inline int bpf_map_attr_numa_node(const union bpf_attr *attr)
1492{
1493 return (attr->map_flags & BPF_F_NUMA_NODE) ?
1494 attr->numa_node : NUMA_NO_NODE;
1495}
1496
Al Viro040ee692017-12-02 20:20:38 -05001497struct bpf_prog *bpf_prog_get_type_path(const char *name, enum bpf_prog_type type);
Martin KaFai Lau5dc4c4b2018-08-08 01:01:24 -07001498int array_map_alloc_check(union bpf_attr *attr);
Al Viro040ee692017-12-02 20:20:38 -05001499
Stanislav Fomichevc6958652019-04-11 09:12:02 -07001500int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr,
1501 union bpf_attr __user *uattr);
1502int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr,
1503 union bpf_attr __user *uattr);
KP Singhda00d2f2020-03-04 20:18:52 +01001504int bpf_prog_test_run_tracing(struct bpf_prog *prog,
1505 const union bpf_attr *kattr,
1506 union bpf_attr __user *uattr);
Stanislav Fomichevc6958652019-04-11 09:12:02 -07001507int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog,
1508 const union bpf_attr *kattr,
1509 union bpf_attr __user *uattr);
Song Liu1b4d60e2020-09-25 13:54:29 -07001510int bpf_prog_test_run_raw_tp(struct bpf_prog *prog,
1511 const union bpf_attr *kattr,
1512 union bpf_attr __user *uattr);
Lorenz Bauer7c32e8f2021-03-03 10:18:13 +00001513int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog,
1514 const union bpf_attr *kattr,
1515 union bpf_attr __user *uattr);
Alexei Starovoitov9e15db62019-10-15 20:25:00 -07001516bool btf_ctx_access(int off, int size, enum bpf_access_type type,
1517 const struct bpf_prog *prog,
1518 struct bpf_insn_access_aux *info);
Andrii Nakryiko22dc4a02020-12-03 12:46:29 -08001519int btf_struct_access(struct bpf_verifier_log *log, const struct btf *btf,
Alexei Starovoitov9e15db62019-10-15 20:25:00 -07001520 const struct btf_type *t, int off, int size,
1521 enum bpf_access_type atype,
1522 u32 *next_btf_id);
Jiri Olsafaaf4a72020-08-25 21:21:18 +02001523bool btf_struct_ids_match(struct bpf_verifier_log *log,
Andrii Nakryiko22dc4a02020-12-03 12:46:29 -08001524 const struct btf *btf, u32 id, int off,
1525 const struct btf *need_btf, u32 need_type_id);
Alexei Starovoitov9e15db62019-10-15 20:25:00 -07001526
Alexei Starovoitovfec56f52019-11-14 10:57:04 -08001527int btf_distill_func_proto(struct bpf_verifier_log *log,
1528 struct btf *btf,
1529 const struct btf_type *func_proto,
1530 const char *func_name,
1531 struct btf_func_model *m);
1532
Alexei Starovoitov51c39bb2020-01-09 22:41:20 -08001533struct bpf_reg_state;
1534int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog,
1535 struct bpf_reg_state *regs);
1536int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog,
1537 struct bpf_reg_state *reg);
Toke Høiland-Jørgensenefc68152020-09-25 23:25:01 +02001538int btf_check_type_match(struct bpf_verifier_log *log, const struct bpf_prog *prog,
Alexei Starovoitovbe8704f2020-01-20 16:53:46 -08001539 struct btf *btf, const struct btf_type *t);
Alexei Starovoitov8c1b6e62019-11-14 10:57:16 -08001540
Björn Töpel7e6897f2019-12-13 18:51:09 +01001541struct bpf_prog *bpf_prog_by_id(u32 id);
Alexei Starovoitov005142b2020-08-18 21:27:56 -07001542struct bpf_link *bpf_link_by_id(u32 id);
Björn Töpel7e6897f2019-12-13 18:51:09 +01001543
Stanislav Fomichev68908962020-04-24 16:59:41 -07001544const struct bpf_func_proto *bpf_base_func_proto(enum bpf_func_id func_id);
Song Liua10787e2021-02-25 15:43:14 -08001545void bpf_task_storage_free(struct task_struct *task);
Jesper Dangaard Brouer9c270af2017-10-16 12:19:34 +02001546#else /* !CONFIG_BPF_SYSCALL */
Daniel Borkmann0fc174d2015-03-01 12:31:44 +01001547static inline struct bpf_prog *bpf_prog_get(u32 ufd)
1548{
1549 return ERR_PTR(-EOPNOTSUPP);
1550}
1551
Jakub Kicinski248f3462017-11-03 13:56:20 -07001552static inline struct bpf_prog *bpf_prog_get_type_dev(u32 ufd,
1553 enum bpf_prog_type type,
Jakub Kicinski288b3de52017-11-20 15:21:54 -08001554 bool attach_drv)
Jakub Kicinski248f3462017-11-03 13:56:20 -07001555{
1556 return ERR_PTR(-EOPNOTSUPP);
1557}
1558
Andrii Nakryiko85192dbf2019-11-17 09:28:03 -08001559static inline void bpf_prog_add(struct bpf_prog *prog, int i)
Brenden Blancocc2e0b32016-07-20 07:55:52 -07001560{
Brenden Blancocc2e0b32016-07-20 07:55:52 -07001561}
Daniel Borkmann113214b2016-06-30 17:24:44 +02001562
Daniel Borkmannc5405942016-11-09 22:02:34 +01001563static inline void bpf_prog_sub(struct bpf_prog *prog, int i)
1564{
1565}
1566
Daniel Borkmann0fc174d2015-03-01 12:31:44 +01001567static inline void bpf_prog_put(struct bpf_prog *prog)
1568{
1569}
Daniel Borkmann6d67942dd2016-11-19 01:45:03 +01001570
Andrii Nakryiko85192dbf2019-11-17 09:28:03 -08001571static inline void bpf_prog_inc(struct bpf_prog *prog)
Alexei Starovoitovaa6a5f32016-09-01 18:37:24 -07001572{
Alexei Starovoitovaa6a5f32016-09-01 18:37:24 -07001573}
Daniel Borkmann5ccb0712016-12-18 01:52:58 +01001574
John Fastabenda6f6df62017-08-15 22:32:22 -07001575static inline struct bpf_prog *__must_check
1576bpf_prog_inc_not_zero(struct bpf_prog *prog)
1577{
1578 return ERR_PTR(-EOPNOTSUPP);
1579}
1580
Andrii Nakryiko6cc7d1e2020-07-21 23:45:54 -07001581static inline void bpf_link_init(struct bpf_link *link, enum bpf_link_type type,
1582 const struct bpf_link_ops *ops,
1583 struct bpf_prog *prog)
1584{
1585}
1586
1587static inline int bpf_link_prime(struct bpf_link *link,
1588 struct bpf_link_primer *primer)
1589{
1590 return -EOPNOTSUPP;
1591}
1592
1593static inline int bpf_link_settle(struct bpf_link_primer *primer)
1594{
1595 return -EOPNOTSUPP;
1596}
1597
1598static inline void bpf_link_cleanup(struct bpf_link_primer *primer)
1599{
1600}
1601
1602static inline void bpf_link_inc(struct bpf_link *link)
1603{
1604}
1605
1606static inline void bpf_link_put(struct bpf_link *link)
1607{
1608}
1609
Chenbo Feng6e71b042017-10-18 13:00:22 -07001610static inline int bpf_obj_get_user(const char __user *pathname, int flags)
Shmulik Ladkani98589a02017-10-09 15:27:15 +03001611{
1612 return -EOPNOTSUPP;
1613}
1614
David Ahernfbee97f2020-05-29 16:07:13 -06001615static inline bool dev_map_can_have_prog(struct bpf_map *map)
1616{
1617 return false;
1618}
Toke Høiland-Jørgensen6f9d4512019-07-26 18:06:55 +02001619
Toke Høiland-Jørgensen1d233882020-01-16 16:14:45 +01001620static inline void __dev_flush(void)
John Fastabend46f55cf2017-07-17 21:56:48 -07001621{
1622}
Jesper Dangaard Brouer9c270af2017-10-16 12:19:34 +02001623
Jesper Dangaard Brouer67f29e02018-05-24 16:45:46 +02001624struct xdp_buff;
1625struct bpf_dtab_netdev;
Björn Töpele6a47502021-03-08 12:29:06 +01001626struct bpf_cpu_map_entry;
Jesper Dangaard Brouer67f29e02018-05-24 16:45:46 +02001627
1628static inline
Toke Høiland-Jørgensen1d233882020-01-16 16:14:45 +01001629int dev_xdp_enqueue(struct net_device *dev, struct xdp_buff *xdp,
1630 struct net_device *dev_rx)
1631{
1632 return 0;
1633}
1634
1635static inline
Jesper Dangaard Brouer38edddb2018-05-24 16:45:57 +02001636int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_buff *xdp,
1637 struct net_device *dev_rx)
Jesper Dangaard Brouer67f29e02018-05-24 16:45:46 +02001638{
1639 return 0;
1640}
1641
Toshiaki Makita6d5fc192018-06-14 11:07:42 +09001642struct sk_buff;
1643
1644static inline int dev_map_generic_redirect(struct bpf_dtab_netdev *dst,
1645 struct sk_buff *skb,
1646 struct bpf_prog *xdp_prog)
1647{
1648 return 0;
1649}
1650
Björn Töpelcdfafe92019-12-19 07:10:04 +01001651static inline void __cpu_map_flush(void)
Jesper Dangaard Brouer9c270af2017-10-16 12:19:34 +02001652{
1653}
1654
Jesper Dangaard Brouer9c270af2017-10-16 12:19:34 +02001655static inline int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu,
1656 struct xdp_buff *xdp,
1657 struct net_device *dev_rx)
1658{
1659 return 0;
1660}
Al Viro040ee692017-12-02 20:20:38 -05001661
Lorenzo Bianconi92164772020-07-14 15:56:38 +02001662static inline bool cpu_map_prog_allowed(struct bpf_map *map)
1663{
1664 return false;
1665}
1666
Al Viro040ee692017-12-02 20:20:38 -05001667static inline struct bpf_prog *bpf_prog_get_type_path(const char *name,
1668 enum bpf_prog_type type)
1669{
1670 return ERR_PTR(-EOPNOTSUPP);
1671}
Stanislav Fomichevc6958652019-04-11 09:12:02 -07001672
1673static inline int bpf_prog_test_run_xdp(struct bpf_prog *prog,
1674 const union bpf_attr *kattr,
1675 union bpf_attr __user *uattr)
1676{
1677 return -ENOTSUPP;
1678}
1679
1680static inline int bpf_prog_test_run_skb(struct bpf_prog *prog,
1681 const union bpf_attr *kattr,
1682 union bpf_attr __user *uattr)
1683{
1684 return -ENOTSUPP;
1685}
1686
KP Singhda00d2f2020-03-04 20:18:52 +01001687static inline int bpf_prog_test_run_tracing(struct bpf_prog *prog,
1688 const union bpf_attr *kattr,
1689 union bpf_attr __user *uattr)
1690{
1691 return -ENOTSUPP;
1692}
1693
Stanislav Fomichevc6958652019-04-11 09:12:02 -07001694static inline int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog,
1695 const union bpf_attr *kattr,
1696 union bpf_attr __user *uattr)
1697{
1698 return -ENOTSUPP;
1699}
Daniel Borkmann6332be02019-11-22 21:07:55 +01001700
Lorenz Bauer7c32e8f2021-03-03 10:18:13 +00001701static inline int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog,
1702 const union bpf_attr *kattr,
1703 union bpf_attr __user *uattr)
1704{
1705 return -ENOTSUPP;
1706}
1707
Daniel Borkmann6332be02019-11-22 21:07:55 +01001708static inline void bpf_map_put(struct bpf_map *map)
1709{
1710}
Björn Töpel7e6897f2019-12-13 18:51:09 +01001711
1712static inline struct bpf_prog *bpf_prog_by_id(u32 id)
1713{
1714 return ERR_PTR(-ENOTSUPP);
1715}
Stanislav Fomichev68908962020-04-24 16:59:41 -07001716
1717static inline const struct bpf_func_proto *
1718bpf_base_func_proto(enum bpf_func_id func_id)
1719{
1720 return NULL;
1721}
Song Liua10787e2021-02-25 15:43:14 -08001722
1723static inline void bpf_task_storage_free(struct task_struct *task)
1724{
1725}
Daniel Borkmann61e021f32015-03-02 15:21:55 +01001726#endif /* CONFIG_BPF_SYSCALL */
Alexei Starovoitov09756af2014-09-26 00:17:00 -07001727
Andrii Nakryiko541c3ba2021-01-11 23:55:18 -08001728void __bpf_free_used_btfs(struct bpf_prog_aux *aux,
1729 struct btf_mod_pair *used_btfs, u32 len);
1730
Jakub Kicinski479321e2017-11-20 15:21:56 -08001731static inline struct bpf_prog *bpf_prog_get_type(u32 ufd,
1732 enum bpf_prog_type type)
1733{
1734 return bpf_prog_get_type_dev(ufd, type, false);
1735}
1736
Andrii Nakryiko936f8942021-01-11 23:55:16 -08001737void __bpf_free_used_maps(struct bpf_prog_aux *aux,
1738 struct bpf_map **used_maps, u32 len);
1739
Al Viro040ee692017-12-02 20:20:38 -05001740bool bpf_prog_get_ok(struct bpf_prog *, enum bpf_prog_type *, bool);
1741
Jakub Kicinskiab3f0062017-11-03 13:56:17 -07001742int bpf_prog_offload_compile(struct bpf_prog *prog);
1743void bpf_prog_offload_destroy(struct bpf_prog *prog);
Jakub Kicinski675fc272017-12-27 18:39:09 -08001744int bpf_prog_offload_info_fill(struct bpf_prog_info *info,
1745 struct bpf_prog *prog);
Jakub Kicinskiab3f0062017-11-03 13:56:17 -07001746
Jakub Kicinski52775b32018-01-17 19:13:28 -08001747int bpf_map_offload_info_fill(struct bpf_map_info *info, struct bpf_map *map);
1748
Jakub Kicinskia3884572018-01-11 20:29:09 -08001749int bpf_map_offload_lookup_elem(struct bpf_map *map, void *key, void *value);
1750int bpf_map_offload_update_elem(struct bpf_map *map,
1751 void *key, void *value, u64 flags);
1752int bpf_map_offload_delete_elem(struct bpf_map *map, void *key);
1753int bpf_map_offload_get_next_key(struct bpf_map *map,
1754 void *key, void *next_key);
1755
Jakub Kicinski09728262018-07-17 10:53:23 -07001756bool bpf_offload_prog_map_match(struct bpf_prog *prog, struct bpf_map *map);
Jakub Kicinskia3884572018-01-11 20:29:09 -08001757
Quentin Monnet1385d752018-11-09 13:03:25 +00001758struct bpf_offload_dev *
Jakub Kicinskidd27c2e2019-02-12 00:20:39 -08001759bpf_offload_dev_create(const struct bpf_prog_offload_ops *ops, void *priv);
Jakub Kicinski602144c2018-07-17 10:53:25 -07001760void bpf_offload_dev_destroy(struct bpf_offload_dev *offdev);
Jakub Kicinskidd27c2e2019-02-12 00:20:39 -08001761void *bpf_offload_dev_priv(struct bpf_offload_dev *offdev);
Jakub Kicinski602144c2018-07-17 10:53:25 -07001762int bpf_offload_dev_netdev_register(struct bpf_offload_dev *offdev,
1763 struct net_device *netdev);
1764void bpf_offload_dev_netdev_unregister(struct bpf_offload_dev *offdev,
1765 struct net_device *netdev);
Jakub Kicinskifd4f2272018-07-17 10:53:26 -07001766bool bpf_offload_dev_match(struct bpf_prog *prog, struct net_device *netdev);
Jakub Kicinski9fd7c552018-07-17 10:53:24 -07001767
Jakub Kicinskiab3f0062017-11-03 13:56:17 -07001768#if defined(CONFIG_NET) && defined(CONFIG_BPF_SYSCALL)
1769int bpf_prog_offload_init(struct bpf_prog *prog, union bpf_attr *attr);
1770
Jakub Kicinski0d830032018-05-08 19:37:06 -07001771static inline bool bpf_prog_is_dev_bound(const struct bpf_prog_aux *aux)
Jakub Kicinskiab3f0062017-11-03 13:56:17 -07001772{
Jakub Kicinski9a18eed2017-12-27 18:39:04 -08001773 return aux->offload_requested;
Jakub Kicinskiab3f0062017-11-03 13:56:17 -07001774}
Jakub Kicinskia3884572018-01-11 20:29:09 -08001775
1776static inline bool bpf_map_is_dev_bound(struct bpf_map *map)
1777{
1778 return unlikely(map->ops == &bpf_map_offload_ops);
1779}
1780
1781struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr);
1782void bpf_map_offload_map_free(struct bpf_map *map);
Jakub Kicinskiab3f0062017-11-03 13:56:17 -07001783#else
1784static inline int bpf_prog_offload_init(struct bpf_prog *prog,
1785 union bpf_attr *attr)
1786{
1787 return -EOPNOTSUPP;
1788}
1789
1790static inline bool bpf_prog_is_dev_bound(struct bpf_prog_aux *aux)
1791{
1792 return false;
1793}
Jakub Kicinskia3884572018-01-11 20:29:09 -08001794
1795static inline bool bpf_map_is_dev_bound(struct bpf_map *map)
1796{
1797 return false;
1798}
1799
1800static inline struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr)
1801{
1802 return ERR_PTR(-EOPNOTSUPP);
1803}
1804
1805static inline void bpf_map_offload_map_free(struct bpf_map *map)
1806{
1807}
Jakub Kicinskiab3f0062017-11-03 13:56:17 -07001808#endif /* CONFIG_NET && CONFIG_BPF_SYSCALL */
1809
Cong Wang88759602021-02-23 10:49:26 -08001810#if defined(CONFIG_INET) && defined(CONFIG_BPF_SYSCALL)
Daniel Borkmann604326b2018-10-13 02:45:58 +02001811int sock_map_get_from_fd(const union bpf_attr *attr, struct bpf_prog *prog);
Lorenz Bauerbb0de312020-06-29 10:56:28 +01001812int sock_map_prog_detach(const union bpf_attr *attr, enum bpf_prog_type ptype);
Lorenz Bauer13b79d32020-08-21 11:29:45 +01001813int sock_map_update_elem_sys(struct bpf_map *map, void *key, void *value, u64 flags);
Lorenz Bauerf7476322020-03-09 11:12:36 +00001814void sock_map_unhash(struct sock *sk);
1815void sock_map_close(struct sock *sk, long timeout);
Cong Wang88759602021-02-23 10:49:26 -08001816
1817void bpf_sk_reuseport_detach(struct sock *sk);
1818int bpf_fd_reuseport_array_lookup_elem(struct bpf_map *map, void *key,
1819 void *value);
1820int bpf_fd_reuseport_array_update_elem(struct bpf_map *map, void *key,
1821 void *value, u64 map_flags);
John Fastabend6bdc9c42017-08-16 15:02:32 -07001822#else
Cong Wang88759602021-02-23 10:49:26 -08001823static inline void bpf_sk_reuseport_detach(struct sock *sk)
1824{
1825}
1826
1827#ifdef CONFIG_BPF_SYSCALL
Daniel Borkmann604326b2018-10-13 02:45:58 +02001828static inline int sock_map_get_from_fd(const union bpf_attr *attr,
1829 struct bpf_prog *prog)
Sean Youngfdb5c452018-06-19 00:04:24 +01001830{
1831 return -EINVAL;
1832}
Lorenz Bauerbb0de312020-06-29 10:56:28 +01001833
1834static inline int sock_map_prog_detach(const union bpf_attr *attr,
1835 enum bpf_prog_type ptype)
1836{
1837 return -EOPNOTSUPP;
1838}
Lorenz Bauer13b79d32020-08-21 11:29:45 +01001839
1840static inline int sock_map_update_elem_sys(struct bpf_map *map, void *key, void *value,
1841 u64 flags)
1842{
1843 return -EOPNOTSUPP;
1844}
John Fastabend6bdc9c42017-08-16 15:02:32 -07001845
Martin KaFai Lau5dc4c4b2018-08-08 01:01:24 -07001846static inline int bpf_fd_reuseport_array_lookup_elem(struct bpf_map *map,
1847 void *key, void *value)
1848{
1849 return -EOPNOTSUPP;
1850}
1851
1852static inline int bpf_fd_reuseport_array_update_elem(struct bpf_map *map,
1853 void *key, void *value,
1854 u64 map_flags)
1855{
1856 return -EOPNOTSUPP;
1857}
1858#endif /* CONFIG_BPF_SYSCALL */
1859#endif /* defined(CONFIG_INET) && defined(CONFIG_BPF_SYSCALL) */
1860
Alexei Starovoitovd0003ec2014-11-13 17:36:49 -08001861/* verifier prototypes for helper functions called from eBPF programs */
Daniel Borkmanna2c83ff2015-03-01 12:31:42 +01001862extern const struct bpf_func_proto bpf_map_lookup_elem_proto;
1863extern const struct bpf_func_proto bpf_map_update_elem_proto;
1864extern const struct bpf_func_proto bpf_map_delete_elem_proto;
Mauricio Vasquez Bf1a2e442018-10-18 15:16:25 +02001865extern const struct bpf_func_proto bpf_map_push_elem_proto;
1866extern const struct bpf_func_proto bpf_map_pop_elem_proto;
1867extern const struct bpf_func_proto bpf_map_peek_elem_proto;
Alexei Starovoitovd0003ec2014-11-13 17:36:49 -08001868
Daniel Borkmann03e69b52015-03-14 02:27:16 +01001869extern const struct bpf_func_proto bpf_get_prandom_u32_proto;
Daniel Borkmannc04167c2015-03-14 02:27:17 +01001870extern const struct bpf_func_proto bpf_get_smp_processor_id_proto;
Daniel Borkmann2d0e30c2016-10-21 12:46:33 +02001871extern const struct bpf_func_proto bpf_get_numa_node_id_proto;
Alexei Starovoitov04fd61ab2015-05-19 16:59:03 -07001872extern const struct bpf_func_proto bpf_tail_call_proto;
Daniel Borkmann17ca8cb2015-05-29 23:23:06 +02001873extern const struct bpf_func_proto bpf_ktime_get_ns_proto;
Maciej Żenczykowski71d19212020-04-26 09:15:25 -07001874extern const struct bpf_func_proto bpf_ktime_get_boot_ns_proto;
Alexei Starovoitovffeedaf2015-06-12 19:39:12 -07001875extern const struct bpf_func_proto bpf_get_current_pid_tgid_proto;
1876extern const struct bpf_func_proto bpf_get_current_uid_gid_proto;
1877extern const struct bpf_func_proto bpf_get_current_comm_proto;
Alexei Starovoitovd5a3b1f2016-02-17 19:58:58 -08001878extern const struct bpf_func_proto bpf_get_stackid_proto;
Yonghong Songc195651e2018-04-28 22:28:08 -07001879extern const struct bpf_func_proto bpf_get_stack_proto;
Song Liufa28dcb2020-06-29 23:28:44 -07001880extern const struct bpf_func_proto bpf_get_task_stack_proto;
Song Liu7b04d6d2020-07-23 11:06:44 -07001881extern const struct bpf_func_proto bpf_get_stackid_proto_pe;
1882extern const struct bpf_func_proto bpf_get_stack_proto_pe;
John Fastabend174a79f2017-08-15 22:32:47 -07001883extern const struct bpf_func_proto bpf_sock_map_update_proto;
John Fastabend81110382018-05-14 10:00:17 -07001884extern const struct bpf_func_proto bpf_sock_hash_update_proto;
Yonghong Songbf6fa2c82018-06-03 15:59:41 -07001885extern const struct bpf_func_proto bpf_get_current_cgroup_id_proto;
Daniel Borkmann0f09abd2020-03-27 16:58:54 +01001886extern const struct bpf_func_proto bpf_get_current_ancestor_cgroup_id_proto;
Daniel Borkmann604326b2018-10-13 02:45:58 +02001887extern const struct bpf_func_proto bpf_msg_redirect_hash_proto;
1888extern const struct bpf_func_proto bpf_msg_redirect_map_proto;
1889extern const struct bpf_func_proto bpf_sk_redirect_hash_proto;
1890extern const struct bpf_func_proto bpf_sk_redirect_map_proto;
Alexei Starovoitovd83525c2019-01-31 15:40:04 -08001891extern const struct bpf_func_proto bpf_spin_lock_proto;
1892extern const struct bpf_func_proto bpf_spin_unlock_proto;
Roman Gushchincd339432018-08-02 14:27:24 -07001893extern const struct bpf_func_proto bpf_get_local_storage_proto;
Andrey Ignatovd7a4cb92019-03-18 17:55:26 -07001894extern const struct bpf_func_proto bpf_strtol_proto;
1895extern const struct bpf_func_proto bpf_strtoul_proto;
Stanislav Fomichev0d01da62019-06-27 13:38:47 -07001896extern const struct bpf_func_proto bpf_tcp_sock_proto;
Martin KaFai Lau5576b992020-01-22 15:36:46 -08001897extern const struct bpf_func_proto bpf_jiffies64_proto;
Carlos Neirab4490c52020-03-04 17:41:56 -03001898extern const struct bpf_func_proto bpf_get_ns_current_pid_tgid_proto;
Stanislav Fomichev0456ea12020-04-20 10:46:10 -07001899extern const struct bpf_func_proto bpf_event_output_data_proto;
Andrii Nakryiko457f44362020-05-29 00:54:20 -07001900extern const struct bpf_func_proto bpf_ringbuf_output_proto;
1901extern const struct bpf_func_proto bpf_ringbuf_reserve_proto;
1902extern const struct bpf_func_proto bpf_ringbuf_submit_proto;
1903extern const struct bpf_func_proto bpf_ringbuf_discard_proto;
1904extern const struct bpf_func_proto bpf_ringbuf_query_proto;
Yonghong Songaf7ec132020-06-23 16:08:09 -07001905extern const struct bpf_func_proto bpf_skc_to_tcp6_sock_proto;
Yonghong Song478cfbd2020-06-23 16:08:11 -07001906extern const struct bpf_func_proto bpf_skc_to_tcp_sock_proto;
1907extern const struct bpf_func_proto bpf_skc_to_tcp_timewait_sock_proto;
1908extern const struct bpf_func_proto bpf_skc_to_tcp_request_sock_proto;
Yonghong Song0d4fad3e52020-06-23 16:08:15 -07001909extern const struct bpf_func_proto bpf_skc_to_udp6_sock_proto;
Alexei Starovoitov07be4c42020-08-27 15:01:12 -07001910extern const struct bpf_func_proto bpf_copy_from_user_proto;
Alan Maguirec4d0bfb2020-09-28 12:31:05 +01001911extern const struct bpf_func_proto bpf_snprintf_btf_proto;
Hao Luoeaa6bcb2020-09-29 16:50:47 -07001912extern const struct bpf_func_proto bpf_per_cpu_ptr_proto;
Hao Luo63d9b802020-09-29 16:50:48 -07001913extern const struct bpf_func_proto bpf_this_cpu_ptr_proto;
Dmitrii Banshchikovd0551262020-11-17 18:45:49 +00001914extern const struct bpf_func_proto bpf_ktime_get_coarse_ns_proto;
Florent Revestb60da492020-12-08 18:36:23 +01001915extern const struct bpf_func_proto bpf_sock_from_file_proto;
Florent Revestc5dbb892021-02-10 12:14:03 +01001916extern const struct bpf_func_proto bpf_get_socket_ptr_cookie_proto;
Song Liua10787e2021-02-25 15:43:14 -08001917extern const struct bpf_func_proto bpf_task_storage_get_proto;
1918extern const struct bpf_func_proto bpf_task_storage_delete_proto;
Yonghong Song69c087b2021-02-26 12:49:25 -08001919extern const struct bpf_func_proto bpf_for_each_map_elem_proto;
Roman Gushchincd339432018-08-02 14:27:24 -07001920
KP Singhfc611f42020-03-29 01:43:49 +01001921const struct bpf_func_proto *bpf_tracing_func_proto(
1922 enum bpf_func_id func_id, const struct bpf_prog *prog);
1923
Jiri Olsa958a3f22020-05-31 17:42:55 +02001924const struct bpf_func_proto *tracing_prog_func_proto(
1925 enum bpf_func_id func_id, const struct bpf_prog *prog);
1926
Daniel Borkmann3ad00402015-10-08 01:20:39 +02001927/* Shared helpers among cBPF and eBPF. */
1928void bpf_user_rnd_init_once(void);
1929u64 bpf_user_rnd_u32(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
Stanislav Fomichev68908962020-04-24 16:59:41 -07001930u64 bpf_get_raw_cpu_id(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
Daniel Borkmann3ad00402015-10-08 01:20:39 +02001931
Joe Stringerc64b7982018-10-02 13:35:33 -07001932#if defined(CONFIG_NET)
Martin KaFai Lau46f8bc92019-02-09 23:22:20 -08001933bool bpf_sock_common_is_valid_access(int off, int size,
1934 enum bpf_access_type type,
1935 struct bpf_insn_access_aux *info);
Joe Stringerc64b7982018-10-02 13:35:33 -07001936bool bpf_sock_is_valid_access(int off, int size, enum bpf_access_type type,
1937 struct bpf_insn_access_aux *info);
1938u32 bpf_sock_convert_ctx_access(enum bpf_access_type type,
1939 const struct bpf_insn *si,
1940 struct bpf_insn *insn_buf,
1941 struct bpf_prog *prog,
1942 u32 *target_size);
1943#else
Martin KaFai Lau46f8bc92019-02-09 23:22:20 -08001944static inline bool bpf_sock_common_is_valid_access(int off, int size,
1945 enum bpf_access_type type,
1946 struct bpf_insn_access_aux *info)
1947{
1948 return false;
1949}
Joe Stringerc64b7982018-10-02 13:35:33 -07001950static inline bool bpf_sock_is_valid_access(int off, int size,
1951 enum bpf_access_type type,
1952 struct bpf_insn_access_aux *info)
1953{
1954 return false;
1955}
1956static inline u32 bpf_sock_convert_ctx_access(enum bpf_access_type type,
1957 const struct bpf_insn *si,
1958 struct bpf_insn *insn_buf,
1959 struct bpf_prog *prog,
1960 u32 *target_size)
1961{
1962 return 0;
1963}
1964#endif
1965
Martin KaFai Lau655a51e2019-02-09 23:22:24 -08001966#ifdef CONFIG_INET
Alexei Starovoitov91cc1a92019-11-14 10:57:15 -08001967struct sk_reuseport_kern {
1968 struct sk_buff *skb;
1969 struct sock *sk;
1970 struct sock *selected_sk;
1971 void *data_end;
1972 u32 hash;
1973 u32 reuseport_id;
1974 bool bind_inany;
1975};
Martin KaFai Lau655a51e2019-02-09 23:22:24 -08001976bool bpf_tcp_sock_is_valid_access(int off, int size, enum bpf_access_type type,
1977 struct bpf_insn_access_aux *info);
1978
1979u32 bpf_tcp_sock_convert_ctx_access(enum bpf_access_type type,
1980 const struct bpf_insn *si,
1981 struct bpf_insn *insn_buf,
1982 struct bpf_prog *prog,
1983 u32 *target_size);
YueHaibing7f942082019-06-12 17:18:47 +08001984
1985bool bpf_xdp_sock_is_valid_access(int off, int size, enum bpf_access_type type,
1986 struct bpf_insn_access_aux *info);
1987
1988u32 bpf_xdp_sock_convert_ctx_access(enum bpf_access_type type,
1989 const struct bpf_insn *si,
1990 struct bpf_insn *insn_buf,
1991 struct bpf_prog *prog,
1992 u32 *target_size);
Martin KaFai Lau655a51e2019-02-09 23:22:24 -08001993#else
1994static inline bool bpf_tcp_sock_is_valid_access(int off, int size,
1995 enum bpf_access_type type,
1996 struct bpf_insn_access_aux *info)
1997{
1998 return false;
1999}
2000
2001static inline u32 bpf_tcp_sock_convert_ctx_access(enum bpf_access_type type,
2002 const struct bpf_insn *si,
2003 struct bpf_insn *insn_buf,
2004 struct bpf_prog *prog,
2005 u32 *target_size)
2006{
2007 return 0;
2008}
YueHaibing7f942082019-06-12 17:18:47 +08002009static inline bool bpf_xdp_sock_is_valid_access(int off, int size,
2010 enum bpf_access_type type,
2011 struct bpf_insn_access_aux *info)
2012{
2013 return false;
2014}
2015
2016static inline u32 bpf_xdp_sock_convert_ctx_access(enum bpf_access_type type,
2017 const struct bpf_insn *si,
2018 struct bpf_insn *insn_buf,
2019 struct bpf_prog *prog,
2020 u32 *target_size)
2021{
2022 return 0;
2023}
Martin KaFai Lau655a51e2019-02-09 23:22:24 -08002024#endif /* CONFIG_INET */
2025
Alexei Starovoitov5964b202019-11-14 10:57:03 -08002026enum bpf_text_poke_type {
Daniel Borkmannb553a6e2019-11-24 01:39:42 +01002027 BPF_MOD_CALL,
2028 BPF_MOD_JUMP,
Alexei Starovoitov5964b202019-11-14 10:57:03 -08002029};
Daniel Borkmann4b3da772019-11-22 21:07:54 +01002030
Alexei Starovoitov5964b202019-11-14 10:57:03 -08002031int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t,
2032 void *addr1, void *addr2);
2033
Jiri Olsaeae2e832020-08-25 21:21:19 +02002034struct btf_id_set;
Lorenz Bauer2af30f12020-09-21 13:12:17 +01002035bool btf_id_set_contains(const struct btf_id_set *set, u32 id);
Jiri Olsaeae2e832020-08-25 21:21:19 +02002036
Alexei Starovoitov99c55f72014-09-26 00:16:57 -07002037#endif /* _LINUX_BPF_H */