Thomas Gleixner | 25763b3 | 2019-05-28 10:10:09 -0700 | [diff] [blame] | 1 | /* SPDX-License-Identifier: GPL-2.0-only */ |
Alexei Starovoitov | 99c55f7 | 2014-09-26 00:16:57 -0700 | [diff] [blame] | 2 | /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com |
Alexei Starovoitov | 99c55f7 | 2014-09-26 00:16:57 -0700 | [diff] [blame] | 3 | */ |
| 4 | #ifndef _LINUX_BPF_H |
| 5 | #define _LINUX_BPF_H 1 |
| 6 | |
| 7 | #include <uapi/linux/bpf.h> |
Daniel Borkmann | 74451e66 | 2017-02-16 22:24:50 +0100 | [diff] [blame] | 8 | |
Alexei Starovoitov | 99c55f7 | 2014-09-26 00:16:57 -0700 | [diff] [blame] | 9 | #include <linux/workqueue.h> |
Alexei Starovoitov | db20fd2 | 2014-09-26 00:16:59 -0700 | [diff] [blame] | 10 | #include <linux/file.h> |
Alexei Starovoitov | b121d1e | 2016-03-07 21:57:13 -0800 | [diff] [blame] | 11 | #include <linux/percpu.h> |
Zi Shen Lim | 002245c | 2016-06-08 21:18:47 -0700 | [diff] [blame] | 12 | #include <linux/err.h> |
Daniel Borkmann | 74451e66 | 2017-02-16 22:24:50 +0100 | [diff] [blame] | 13 | #include <linux/rbtree_latch.h> |
David S. Miller | d6e1e46 | 2017-08-19 23:34:03 -0700 | [diff] [blame] | 14 | #include <linux/numa.h> |
Andrii Nakryiko | fc97022 | 2019-11-17 09:28:04 -0800 | [diff] [blame] | 15 | #include <linux/mm_types.h> |
Jakub Kicinski | ab3f006 | 2017-11-03 13:56:17 -0700 | [diff] [blame] | 16 | #include <linux/wait.h> |
Alexei Starovoitov | fec56f5 | 2019-11-14 10:57:04 -0800 | [diff] [blame] | 17 | #include <linux/refcount.h> |
| 18 | #include <linux/mutex.h> |
Martin KaFai Lau | 85d33df | 2020-01-08 16:35:05 -0800 | [diff] [blame] | 19 | #include <linux/module.h> |
Jiri Olsa | bfea9a8 | 2020-03-12 20:55:59 +0100 | [diff] [blame] | 20 | #include <linux/kallsyms.h> |
Alexei Starovoitov | 2c78ee8 | 2020-05-13 16:03:54 -0700 | [diff] [blame] | 21 | #include <linux/capability.h> |
Roman Gushchin | 48edc1f | 2020-12-01 13:58:32 -0800 | [diff] [blame] | 22 | #include <linux/sched/mm.h> |
| 23 | #include <linux/slab.h> |
Alexei Starovoitov | e21aa34 | 2021-03-16 14:00:07 -0700 | [diff] [blame] | 24 | #include <linux/percpu-refcount.h> |
Alexei Starovoitov | 99c55f7 | 2014-09-26 00:16:57 -0700 | [diff] [blame] | 25 | |
Jakub Kicinski | cae1927 | 2017-12-27 18:39:05 -0800 | [diff] [blame] | 26 | struct bpf_verifier_env; |
Alexei Starovoitov | 9e15db6 | 2019-10-15 20:25:00 -0700 | [diff] [blame] | 27 | struct bpf_verifier_log; |
Daniel Borkmann | 3b1efb1 | 2016-06-15 22:47:14 +0200 | [diff] [blame] | 28 | struct perf_event; |
John Fastabend | 174a79f | 2017-08-15 22:32:47 -0700 | [diff] [blame] | 29 | struct bpf_prog; |
Daniel Borkmann | da765a2 | 2019-11-22 21:07:58 +0100 | [diff] [blame] | 30 | struct bpf_prog_aux; |
Alexei Starovoitov | 99c55f7 | 2014-09-26 00:16:57 -0700 | [diff] [blame] | 31 | struct bpf_map; |
John Fastabend | 4f738ad | 2018-03-18 12:57:10 -0700 | [diff] [blame] | 32 | struct sock; |
Martin KaFai Lau | a26ca7c | 2018-04-18 15:56:03 -0700 | [diff] [blame] | 33 | struct seq_file; |
Roman Gushchin | 1b2b234 | 2018-12-10 15:43:00 -0800 | [diff] [blame] | 34 | struct btf; |
Daniel Borkmann | e8d2bec | 2018-08-12 01:59:17 +0200 | [diff] [blame] | 35 | struct btf_type; |
Alexei Starovoitov | 3dec541 | 2019-10-15 20:25:03 -0700 | [diff] [blame] | 36 | struct exception_table_entry; |
Yonghong Song | ae24345 | 2020-05-09 10:58:59 -0700 | [diff] [blame] | 37 | struct seq_operations; |
Yonghong Song | f9c7927 | 2020-07-23 11:41:10 -0700 | [diff] [blame] | 38 | struct bpf_iter_aux_info; |
KP Singh | f836a56 | 2020-08-25 20:29:15 +0200 | [diff] [blame] | 39 | struct bpf_local_storage; |
| 40 | struct bpf_local_storage_map; |
Andrii Nakryiko | 36e6844 | 2020-11-09 17:19:31 -0800 | [diff] [blame] | 41 | struct kobject; |
Roman Gushchin | 48edc1f | 2020-12-01 13:58:32 -0800 | [diff] [blame] | 42 | struct mem_cgroup; |
Yonghong Song | 69c087b | 2021-02-26 12:49:25 -0800 | [diff] [blame] | 43 | struct bpf_func_state; |
Alexei Starovoitov | 99c55f7 | 2014-09-26 00:16:57 -0700 | [diff] [blame] | 44 | |
Quentin Monnet | 1b9ed84 | 2019-08-20 10:31:50 +0100 | [diff] [blame] | 45 | extern struct idr btf_idr; |
| 46 | extern spinlock_t btf_idr_lock; |
Andrii Nakryiko | 36e6844 | 2020-11-09 17:19:31 -0800 | [diff] [blame] | 47 | extern struct kobject *btf_kobj; |
Quentin Monnet | 1b9ed84 | 2019-08-20 10:31:50 +0100 | [diff] [blame] | 48 | |
Yonghong Song | f9c7927 | 2020-07-23 11:41:10 -0700 | [diff] [blame] | 49 | typedef int (*bpf_iter_init_seq_priv_t)(void *private_data, |
| 50 | struct bpf_iter_aux_info *aux); |
Yonghong Song | 14fc6bd6 | 2020-07-23 11:41:09 -0700 | [diff] [blame] | 51 | typedef void (*bpf_iter_fini_seq_priv_t)(void *private_data); |
| 52 | struct bpf_iter_seq_info { |
| 53 | const struct seq_operations *seq_ops; |
| 54 | bpf_iter_init_seq_priv_t init_seq_private; |
| 55 | bpf_iter_fini_seq_priv_t fini_seq_private; |
| 56 | u32 seq_priv_size; |
| 57 | }; |
| 58 | |
Ricardo Ribalda | 5d90349 | 2021-03-18 21:22:22 +0100 | [diff] [blame^] | 59 | /* map is generic key/value storage optionally accessible by eBPF programs */ |
Alexei Starovoitov | 99c55f7 | 2014-09-26 00:16:57 -0700 | [diff] [blame] | 60 | struct bpf_map_ops { |
| 61 | /* funcs callable from userspace (via syscall) */ |
Jakub Kicinski | 1110f3a | 2018-01-11 20:29:03 -0800 | [diff] [blame] | 62 | int (*map_alloc_check)(union bpf_attr *attr); |
Alexei Starovoitov | 99c55f7 | 2014-09-26 00:16:57 -0700 | [diff] [blame] | 63 | struct bpf_map *(*map_alloc)(union bpf_attr *attr); |
Daniel Borkmann | 61d1b6a | 2016-06-15 22:47:12 +0200 | [diff] [blame] | 64 | void (*map_release)(struct bpf_map *map, struct file *map_file); |
| 65 | void (*map_free)(struct bpf_map *map); |
Alexei Starovoitov | db20fd2 | 2014-09-26 00:16:59 -0700 | [diff] [blame] | 66 | int (*map_get_next_key)(struct bpf_map *map, void *key, void *next_key); |
John Fastabend | ba6b8de | 2018-04-23 15:39:23 -0700 | [diff] [blame] | 67 | void (*map_release_uref)(struct bpf_map *map); |
Daniel Borkmann | c611022 | 2019-05-14 01:18:55 +0200 | [diff] [blame] | 68 | void *(*map_lookup_elem_sys_only)(struct bpf_map *map, void *key); |
Brian Vazquez | cb4d03a | 2020-01-15 10:43:01 -0800 | [diff] [blame] | 69 | int (*map_lookup_batch)(struct bpf_map *map, const union bpf_attr *attr, |
| 70 | union bpf_attr __user *uattr); |
Yonghong Song | 0579963 | 2020-01-15 10:43:04 -0800 | [diff] [blame] | 71 | int (*map_lookup_and_delete_batch)(struct bpf_map *map, |
| 72 | const union bpf_attr *attr, |
| 73 | union bpf_attr __user *uattr); |
Brian Vazquez | aa2e93b | 2020-01-15 10:43:02 -0800 | [diff] [blame] | 74 | int (*map_update_batch)(struct bpf_map *map, const union bpf_attr *attr, |
| 75 | union bpf_attr __user *uattr); |
| 76 | int (*map_delete_batch)(struct bpf_map *map, const union bpf_attr *attr, |
| 77 | union bpf_attr __user *uattr); |
Alexei Starovoitov | db20fd2 | 2014-09-26 00:16:59 -0700 | [diff] [blame] | 78 | |
| 79 | /* funcs callable from userspace and from eBPF programs */ |
| 80 | void *(*map_lookup_elem)(struct bpf_map *map, void *key); |
Alexei Starovoitov | 3274f52 | 2014-11-13 17:36:44 -0800 | [diff] [blame] | 81 | int (*map_update_elem)(struct bpf_map *map, void *key, void *value, u64 flags); |
Alexei Starovoitov | db20fd2 | 2014-09-26 00:16:59 -0700 | [diff] [blame] | 82 | int (*map_delete_elem)(struct bpf_map *map, void *key); |
Mauricio Vasquez B | f1a2e44 | 2018-10-18 15:16:25 +0200 | [diff] [blame] | 83 | int (*map_push_elem)(struct bpf_map *map, void *value, u64 flags); |
| 84 | int (*map_pop_elem)(struct bpf_map *map, void *value); |
| 85 | int (*map_peek_elem)(struct bpf_map *map, void *value); |
Wang Nan | 2a36f0b | 2015-08-06 07:02:33 +0000 | [diff] [blame] | 86 | |
| 87 | /* funcs called by prog_array and perf_event_array map */ |
Daniel Borkmann | d056a78 | 2016-06-15 22:47:13 +0200 | [diff] [blame] | 88 | void *(*map_fd_get_ptr)(struct bpf_map *map, struct file *map_file, |
| 89 | int fd); |
| 90 | void (*map_fd_put_ptr)(void *ptr); |
Daniel Borkmann | 4a8f87e | 2020-10-11 01:40:03 +0200 | [diff] [blame] | 91 | int (*map_gen_lookup)(struct bpf_map *map, struct bpf_insn *insn_buf); |
Martin KaFai Lau | 14dc6f0 | 2017-06-27 23:08:34 -0700 | [diff] [blame] | 92 | u32 (*map_fd_sys_lookup_elem)(void *ptr); |
Martin KaFai Lau | a26ca7c | 2018-04-18 15:56:03 -0700 | [diff] [blame] | 93 | void (*map_seq_show_elem)(struct bpf_map *map, void *key, |
| 94 | struct seq_file *m); |
Daniel Borkmann | e8d2bec | 2018-08-12 01:59:17 +0200 | [diff] [blame] | 95 | int (*map_check_btf)(const struct bpf_map *map, |
Roman Gushchin | 1b2b234 | 2018-12-10 15:43:00 -0800 | [diff] [blame] | 96 | const struct btf *btf, |
Daniel Borkmann | e8d2bec | 2018-08-12 01:59:17 +0200 | [diff] [blame] | 97 | const struct btf_type *key_type, |
| 98 | const struct btf_type *value_type); |
Daniel Borkmann | d8eca5b | 2019-04-09 23:20:03 +0200 | [diff] [blame] | 99 | |
Daniel Borkmann | da765a2 | 2019-11-22 21:07:58 +0100 | [diff] [blame] | 100 | /* Prog poke tracking helpers. */ |
| 101 | int (*map_poke_track)(struct bpf_map *map, struct bpf_prog_aux *aux); |
| 102 | void (*map_poke_untrack)(struct bpf_map *map, struct bpf_prog_aux *aux); |
| 103 | void (*map_poke_run)(struct bpf_map *map, u32 key, struct bpf_prog *old, |
| 104 | struct bpf_prog *new); |
| 105 | |
Daniel Borkmann | d8eca5b | 2019-04-09 23:20:03 +0200 | [diff] [blame] | 106 | /* Direct value access helpers. */ |
| 107 | int (*map_direct_value_addr)(const struct bpf_map *map, |
| 108 | u64 *imm, u32 off); |
| 109 | int (*map_direct_value_meta)(const struct bpf_map *map, |
| 110 | u64 imm, u32 *off); |
Andrii Nakryiko | fc97022 | 2019-11-17 09:28:04 -0800 | [diff] [blame] | 111 | int (*map_mmap)(struct bpf_map *map, struct vm_area_struct *vma); |
Andrii Nakryiko | 457f4436 | 2020-05-29 00:54:20 -0700 | [diff] [blame] | 112 | __poll_t (*map_poll)(struct bpf_map *map, struct file *filp, |
| 113 | struct poll_table_struct *pts); |
Andrey Ignatov | 41c48f3 | 2020-06-19 14:11:43 -0700 | [diff] [blame] | 114 | |
KP Singh | f836a56 | 2020-08-25 20:29:15 +0200 | [diff] [blame] | 115 | /* Functions called by bpf_local_storage maps */ |
| 116 | int (*map_local_storage_charge)(struct bpf_local_storage_map *smap, |
| 117 | void *owner, u32 size); |
| 118 | void (*map_local_storage_uncharge)(struct bpf_local_storage_map *smap, |
| 119 | void *owner, u32 size); |
| 120 | struct bpf_local_storage __rcu ** (*map_owner_storage_ptr)(void *owner); |
Martin KaFai Lau | f4d0525 | 2020-08-27 18:18:06 -0700 | [diff] [blame] | 121 | |
Björn Töpel | e6a4750 | 2021-03-08 12:29:06 +0100 | [diff] [blame] | 122 | /* Misc helpers.*/ |
| 123 | int (*map_redirect)(struct bpf_map *map, u32 ifindex, u64 flags); |
| 124 | |
Martin KaFai Lau | f4d0525 | 2020-08-27 18:18:06 -0700 | [diff] [blame] | 125 | /* map_meta_equal must be implemented for maps that can be |
| 126 | * used as an inner map. It is a runtime check to ensure |
| 127 | * an inner map can be inserted to an outer map. |
| 128 | * |
| 129 | * Some properties of the inner map has been used during the |
| 130 | * verification time. When inserting an inner map at the runtime, |
| 131 | * map_meta_equal has to ensure the inserting map has the same |
| 132 | * properties that the verifier has used earlier. |
| 133 | */ |
| 134 | bool (*map_meta_equal)(const struct bpf_map *meta0, |
| 135 | const struct bpf_map *meta1); |
| 136 | |
Yonghong Song | 69c087b | 2021-02-26 12:49:25 -0800 | [diff] [blame] | 137 | |
| 138 | int (*map_set_for_each_callback_args)(struct bpf_verifier_env *env, |
| 139 | struct bpf_func_state *caller, |
| 140 | struct bpf_func_state *callee); |
| 141 | int (*map_for_each_callback)(struct bpf_map *map, void *callback_fn, |
| 142 | void *callback_ctx, u64 flags); |
| 143 | |
Andrey Ignatov | 41c48f3 | 2020-06-19 14:11:43 -0700 | [diff] [blame] | 144 | /* BTF name and id of struct allocated by map_alloc */ |
| 145 | const char * const map_btf_name; |
| 146 | int *map_btf_id; |
Yonghong Song | a5cbe05 | 2020-07-23 11:41:12 -0700 | [diff] [blame] | 147 | |
| 148 | /* bpf_iter info used to open a seq_file */ |
| 149 | const struct bpf_iter_seq_info *iter_seq_info; |
Alexei Starovoitov | 99c55f7 | 2014-09-26 00:16:57 -0700 | [diff] [blame] | 150 | }; |
| 151 | |
| 152 | struct bpf_map { |
Martin KaFai Lau | a26ca7c | 2018-04-18 15:56:03 -0700 | [diff] [blame] | 153 | /* The first two cachelines with read-mostly members of which some |
Daniel Borkmann | be95a84 | 2018-01-09 13:17:44 +0100 | [diff] [blame] | 154 | * are also accessed in fast-path (e.g. ops, max_entries). |
| 155 | */ |
| 156 | const struct bpf_map_ops *ops ____cacheline_aligned; |
| 157 | struct bpf_map *inner_map_meta; |
| 158 | #ifdef CONFIG_SECURITY |
| 159 | void *security; |
| 160 | #endif |
Alexei Starovoitov | 99c55f7 | 2014-09-26 00:16:57 -0700 | [diff] [blame] | 161 | enum bpf_map_type map_type; |
| 162 | u32 key_size; |
| 163 | u32 value_size; |
| 164 | u32 max_entries; |
Alexei Starovoitov | 6c90598 | 2016-03-07 21:57:15 -0800 | [diff] [blame] | 165 | u32 map_flags; |
Alexei Starovoitov | d83525c | 2019-01-31 15:40:04 -0800 | [diff] [blame] | 166 | int spin_lock_off; /* >=0 valid offset, <0 error */ |
Martin KaFai Lau | f3f1c05 | 2017-06-05 12:15:47 -0700 | [diff] [blame] | 167 | u32 id; |
Martin KaFai Lau | 96eabe7 | 2017-08-18 11:28:00 -0700 | [diff] [blame] | 168 | int numa_node; |
Martin KaFai Lau | 9b2cf32 | 2018-05-22 14:57:21 -0700 | [diff] [blame] | 169 | u32 btf_key_type_id; |
| 170 | u32 btf_value_type_id; |
Martin KaFai Lau | a26ca7c | 2018-04-18 15:56:03 -0700 | [diff] [blame] | 171 | struct btf *btf; |
Roman Gushchin | 48edc1f | 2020-12-01 13:58:32 -0800 | [diff] [blame] | 172 | #ifdef CONFIG_MEMCG_KMEM |
| 173 | struct mem_cgroup *memcg; |
| 174 | #endif |
Andrii Nakryiko | fc97022 | 2019-11-17 09:28:04 -0800 | [diff] [blame] | 175 | char name[BPF_OBJ_NAME_LEN]; |
Martin KaFai Lau | 85d33df | 2020-01-08 16:35:05 -0800 | [diff] [blame] | 176 | u32 btf_vmlinux_value_type_id; |
Alexei Starovoitov | 2c78ee8 | 2020-05-13 16:03:54 -0700 | [diff] [blame] | 177 | bool bypass_spec_v1; |
Andrii Nakryiko | fc97022 | 2019-11-17 09:28:04 -0800 | [diff] [blame] | 178 | bool frozen; /* write-once; write-protected by freeze_mutex */ |
| 179 | /* 22 bytes hole */ |
Daniel Borkmann | be95a84 | 2018-01-09 13:17:44 +0100 | [diff] [blame] | 180 | |
Martin KaFai Lau | a26ca7c | 2018-04-18 15:56:03 -0700 | [diff] [blame] | 181 | /* The 3rd and 4th cacheline with misc members to avoid false sharing |
Daniel Borkmann | be95a84 | 2018-01-09 13:17:44 +0100 | [diff] [blame] | 182 | * particularly with refcounting. |
| 183 | */ |
Andrii Nakryiko | 1e0bd5a | 2019-11-17 09:28:02 -0800 | [diff] [blame] | 184 | atomic64_t refcnt ____cacheline_aligned; |
| 185 | atomic64_t usercnt; |
Daniel Borkmann | be95a84 | 2018-01-09 13:17:44 +0100 | [diff] [blame] | 186 | struct work_struct work; |
Andrii Nakryiko | fc97022 | 2019-11-17 09:28:04 -0800 | [diff] [blame] | 187 | struct mutex freeze_mutex; |
| 188 | u64 writecnt; /* writable mmap cnt; protected by freeze_mutex */ |
Alexei Starovoitov | 99c55f7 | 2014-09-26 00:16:57 -0700 | [diff] [blame] | 189 | }; |
| 190 | |
Alexei Starovoitov | d83525c | 2019-01-31 15:40:04 -0800 | [diff] [blame] | 191 | static inline bool map_value_has_spin_lock(const struct bpf_map *map) |
| 192 | { |
| 193 | return map->spin_lock_off >= 0; |
| 194 | } |
| 195 | |
| 196 | static inline void check_and_init_map_lock(struct bpf_map *map, void *dst) |
| 197 | { |
| 198 | if (likely(!map_value_has_spin_lock(map))) |
| 199 | return; |
| 200 | *(struct bpf_spin_lock *)(dst + map->spin_lock_off) = |
| 201 | (struct bpf_spin_lock){}; |
| 202 | } |
| 203 | |
| 204 | /* copy everything but bpf_spin_lock */ |
| 205 | static inline void copy_map_value(struct bpf_map *map, void *dst, void *src) |
| 206 | { |
| 207 | if (unlikely(map_value_has_spin_lock(map))) { |
| 208 | u32 off = map->spin_lock_off; |
| 209 | |
| 210 | memcpy(dst, src, off); |
| 211 | memcpy(dst + off + sizeof(struct bpf_spin_lock), |
| 212 | src + off + sizeof(struct bpf_spin_lock), |
| 213 | map->value_size - off - sizeof(struct bpf_spin_lock)); |
| 214 | } else { |
| 215 | memcpy(dst, src, map->value_size); |
| 216 | } |
| 217 | } |
Alexei Starovoitov | 96049f3 | 2019-01-31 15:40:09 -0800 | [diff] [blame] | 218 | void copy_map_value_locked(struct bpf_map *map, void *dst, void *src, |
| 219 | bool lock_src); |
Martin KaFai Lau | 8e7ae25 | 2020-03-13 18:02:09 -0700 | [diff] [blame] | 220 | int bpf_obj_name_cpy(char *dst, const char *src, unsigned int size); |
Alexei Starovoitov | d83525c | 2019-01-31 15:40:04 -0800 | [diff] [blame] | 221 | |
Jakub Kicinski | 602144c | 2018-07-17 10:53:25 -0700 | [diff] [blame] | 222 | struct bpf_offload_dev; |
Jakub Kicinski | a388457 | 2018-01-11 20:29:09 -0800 | [diff] [blame] | 223 | struct bpf_offloaded_map; |
| 224 | |
| 225 | struct bpf_map_dev_ops { |
| 226 | int (*map_get_next_key)(struct bpf_offloaded_map *map, |
| 227 | void *key, void *next_key); |
| 228 | int (*map_lookup_elem)(struct bpf_offloaded_map *map, |
| 229 | void *key, void *value); |
| 230 | int (*map_update_elem)(struct bpf_offloaded_map *map, |
| 231 | void *key, void *value, u64 flags); |
| 232 | int (*map_delete_elem)(struct bpf_offloaded_map *map, void *key); |
| 233 | }; |
| 234 | |
| 235 | struct bpf_offloaded_map { |
| 236 | struct bpf_map map; |
| 237 | struct net_device *netdev; |
| 238 | const struct bpf_map_dev_ops *dev_ops; |
| 239 | void *dev_priv; |
| 240 | struct list_head offloads; |
| 241 | }; |
| 242 | |
| 243 | static inline struct bpf_offloaded_map *map_to_offmap(struct bpf_map *map) |
| 244 | { |
| 245 | return container_of(map, struct bpf_offloaded_map, map); |
| 246 | } |
| 247 | |
Jakub Kicinski | 0cd3cbe | 2018-05-03 18:37:08 -0700 | [diff] [blame] | 248 | static inline bool bpf_map_offload_neutral(const struct bpf_map *map) |
| 249 | { |
| 250 | return map->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY; |
| 251 | } |
| 252 | |
Martin KaFai Lau | a26ca7c | 2018-04-18 15:56:03 -0700 | [diff] [blame] | 253 | static inline bool bpf_map_support_seq_show(const struct bpf_map *map) |
| 254 | { |
Martin KaFai Lau | 85d33df | 2020-01-08 16:35:05 -0800 | [diff] [blame] | 255 | return (map->btf_value_type_id || map->btf_vmlinux_value_type_id) && |
| 256 | map->ops->map_seq_show_elem; |
Martin KaFai Lau | a26ca7c | 2018-04-18 15:56:03 -0700 | [diff] [blame] | 257 | } |
| 258 | |
Daniel Borkmann | e8d2bec | 2018-08-12 01:59:17 +0200 | [diff] [blame] | 259 | int map_check_no_btf(const struct bpf_map *map, |
Roman Gushchin | 1b2b234 | 2018-12-10 15:43:00 -0800 | [diff] [blame] | 260 | const struct btf *btf, |
Daniel Borkmann | e8d2bec | 2018-08-12 01:59:17 +0200 | [diff] [blame] | 261 | const struct btf_type *key_type, |
| 262 | const struct btf_type *value_type); |
| 263 | |
Martin KaFai Lau | f4d0525 | 2020-08-27 18:18:06 -0700 | [diff] [blame] | 264 | bool bpf_map_meta_equal(const struct bpf_map *meta0, |
| 265 | const struct bpf_map *meta1); |
| 266 | |
Jakub Kicinski | a388457 | 2018-01-11 20:29:09 -0800 | [diff] [blame] | 267 | extern const struct bpf_map_ops bpf_map_offload_ops; |
| 268 | |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 269 | /* function argument constraints */ |
| 270 | enum bpf_arg_type { |
Daniel Borkmann | 80f1d68 | 2015-03-12 17:21:42 +0100 | [diff] [blame] | 271 | ARG_DONTCARE = 0, /* unused argument in helper function */ |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 272 | |
| 273 | /* the following constraints used to prototype |
| 274 | * bpf_map_lookup/update/delete_elem() functions |
| 275 | */ |
| 276 | ARG_CONST_MAP_PTR, /* const argument used as pointer to bpf_map */ |
| 277 | ARG_PTR_TO_MAP_KEY, /* pointer to stack used as map key */ |
| 278 | ARG_PTR_TO_MAP_VALUE, /* pointer to stack used as map value */ |
Mauricio Vasquez B | 2ea864c | 2018-10-18 15:16:20 +0200 | [diff] [blame] | 279 | ARG_PTR_TO_UNINIT_MAP_VALUE, /* pointer to valid memory used to store a map value */ |
Martin KaFai Lau | 6ac99e8 | 2019-04-26 16:39:39 -0700 | [diff] [blame] | 280 | ARG_PTR_TO_MAP_VALUE_OR_NULL, /* pointer to stack used as map value or NULL */ |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 281 | |
| 282 | /* the following constraints used to prototype bpf_memcmp() and other |
| 283 | * functions that access data on eBPF program stack |
| 284 | */ |
Alexei Starovoitov | 39f19ebb | 2017-01-09 10:19:50 -0800 | [diff] [blame] | 285 | ARG_PTR_TO_MEM, /* pointer to valid memory (stack, packet, map value) */ |
Gianluca Borello | db1ac49 | 2017-11-22 18:32:53 +0000 | [diff] [blame] | 286 | ARG_PTR_TO_MEM_OR_NULL, /* pointer to valid memory or NULL */ |
Alexei Starovoitov | 39f19ebb | 2017-01-09 10:19:50 -0800 | [diff] [blame] | 287 | ARG_PTR_TO_UNINIT_MEM, /* pointer to memory does not need to be initialized, |
| 288 | * helper function must fill all bytes or clear |
| 289 | * them in error case. |
Daniel Borkmann | 435faee1 | 2016-04-13 00:10:51 +0200 | [diff] [blame] | 290 | */ |
| 291 | |
Alexei Starovoitov | 39f19ebb | 2017-01-09 10:19:50 -0800 | [diff] [blame] | 292 | ARG_CONST_SIZE, /* number of bytes accessed from memory */ |
| 293 | ARG_CONST_SIZE_OR_ZERO, /* number of bytes accessed from memory or 0 */ |
Daniel Borkmann | 80f1d68 | 2015-03-12 17:21:42 +0100 | [diff] [blame] | 294 | |
Alexei Starovoitov | 608cd71 | 2015-03-26 19:53:57 -0700 | [diff] [blame] | 295 | ARG_PTR_TO_CTX, /* pointer to context */ |
Daniel Borkmann | f318903 | 2020-03-27 16:58:52 +0100 | [diff] [blame] | 296 | ARG_PTR_TO_CTX_OR_NULL, /* pointer to context or NULL */ |
Daniel Borkmann | 80f1d68 | 2015-03-12 17:21:42 +0100 | [diff] [blame] | 297 | ARG_ANYTHING, /* any (initialized) argument is ok */ |
Alexei Starovoitov | d83525c | 2019-01-31 15:40:04 -0800 | [diff] [blame] | 298 | ARG_PTR_TO_SPIN_LOCK, /* pointer to bpf_spin_lock */ |
Martin KaFai Lau | 46f8bc9 | 2019-02-09 23:22:20 -0800 | [diff] [blame] | 299 | ARG_PTR_TO_SOCK_COMMON, /* pointer to sock_common */ |
Andrey Ignatov | 57c3bb7 | 2019-03-18 16:57:10 -0700 | [diff] [blame] | 300 | ARG_PTR_TO_INT, /* pointer to int */ |
| 301 | ARG_PTR_TO_LONG, /* pointer to long */ |
Martin KaFai Lau | 6ac99e8 | 2019-04-26 16:39:39 -0700 | [diff] [blame] | 302 | ARG_PTR_TO_SOCKET, /* pointer to bpf_sock (fullsock) */ |
Jakub Sitnicki | e9ddbb7 | 2020-07-17 12:35:23 +0200 | [diff] [blame] | 303 | ARG_PTR_TO_SOCKET_OR_NULL, /* pointer to bpf_sock (fullsock) or NULL */ |
Alexei Starovoitov | a7658e1 | 2019-10-15 20:25:04 -0700 | [diff] [blame] | 304 | ARG_PTR_TO_BTF_ID, /* pointer to in-kernel struct */ |
Andrii Nakryiko | 457f4436 | 2020-05-29 00:54:20 -0700 | [diff] [blame] | 305 | ARG_PTR_TO_ALLOC_MEM, /* pointer to dynamically allocated memory */ |
| 306 | ARG_PTR_TO_ALLOC_MEM_OR_NULL, /* pointer to dynamically allocated memory or NULL */ |
| 307 | ARG_CONST_ALLOC_SIZE_OR_ZERO, /* number of allocated bytes requested */ |
Martin KaFai Lau | 1df8f55 | 2020-09-24 17:03:50 -0700 | [diff] [blame] | 308 | ARG_PTR_TO_BTF_ID_SOCK_COMMON, /* pointer to in-kernel sock_common or bpf-mirrored bpf_sock */ |
Hao Luo | eaa6bcb | 2020-09-29 16:50:47 -0700 | [diff] [blame] | 309 | ARG_PTR_TO_PERCPU_BTF_ID, /* pointer to in-kernel percpu type */ |
Yonghong Song | 69c087b | 2021-02-26 12:49:25 -0800 | [diff] [blame] | 310 | ARG_PTR_TO_FUNC, /* pointer to a bpf program function */ |
| 311 | ARG_PTR_TO_STACK_OR_NULL, /* pointer to stack or NULL */ |
Lorenz Bauer | f79e7ea | 2020-09-21 13:12:27 +0100 | [diff] [blame] | 312 | __BPF_ARG_TYPE_MAX, |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 313 | }; |
| 314 | |
| 315 | /* type of values returned from helper functions */ |
| 316 | enum bpf_return_type { |
| 317 | RET_INTEGER, /* function returns integer */ |
| 318 | RET_VOID, /* function doesn't return anything */ |
Roman Gushchin | 3e6a4b3 | 2018-08-02 14:27:22 -0700 | [diff] [blame] | 319 | RET_PTR_TO_MAP_VALUE, /* returns a pointer to map elem value */ |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 320 | RET_PTR_TO_MAP_VALUE_OR_NULL, /* returns a pointer to map elem value or NULL */ |
Joe Stringer | c64b798 | 2018-10-02 13:35:33 -0700 | [diff] [blame] | 321 | RET_PTR_TO_SOCKET_OR_NULL, /* returns a pointer to a socket or NULL */ |
Martin KaFai Lau | 655a51e | 2019-02-09 23:22:24 -0800 | [diff] [blame] | 322 | RET_PTR_TO_TCP_SOCK_OR_NULL, /* returns a pointer to a tcp_sock or NULL */ |
Lorenz Bauer | 85a51f8 | 2019-03-22 09:54:00 +0800 | [diff] [blame] | 323 | RET_PTR_TO_SOCK_COMMON_OR_NULL, /* returns a pointer to a sock_common or NULL */ |
Andrii Nakryiko | 457f4436 | 2020-05-29 00:54:20 -0700 | [diff] [blame] | 324 | RET_PTR_TO_ALLOC_MEM_OR_NULL, /* returns a pointer to dynamically allocated memory or NULL */ |
Yonghong Song | af7ec13 | 2020-06-23 16:08:09 -0700 | [diff] [blame] | 325 | RET_PTR_TO_BTF_ID_OR_NULL, /* returns a pointer to a btf_id or NULL */ |
Hao Luo | eaa6bcb | 2020-09-29 16:50:47 -0700 | [diff] [blame] | 326 | RET_PTR_TO_MEM_OR_BTF_ID_OR_NULL, /* returns a pointer to a valid memory or a btf_id or NULL */ |
Hao Luo | 63d9b80 | 2020-09-29 16:50:48 -0700 | [diff] [blame] | 327 | RET_PTR_TO_MEM_OR_BTF_ID, /* returns a pointer to a valid memory or a btf_id */ |
KP Singh | 3ca1032 | 2020-11-06 10:37:43 +0000 | [diff] [blame] | 328 | RET_PTR_TO_BTF_ID, /* returns a pointer to a btf_id */ |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 329 | }; |
| 330 | |
Alexei Starovoitov | 09756af | 2014-09-26 00:17:00 -0700 | [diff] [blame] | 331 | /* eBPF function prototype used by verifier to allow BPF_CALLs from eBPF programs |
| 332 | * to in-kernel helper functions and for adjusting imm32 field in BPF_CALL |
| 333 | * instructions after verifying |
| 334 | */ |
| 335 | struct bpf_func_proto { |
| 336 | u64 (*func)(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); |
| 337 | bool gpl_only; |
Daniel Borkmann | 36bbef5 | 2016-09-20 00:26:13 +0200 | [diff] [blame] | 338 | bool pkt_access; |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 339 | enum bpf_return_type ret_type; |
Alexei Starovoitov | a7658e1 | 2019-10-15 20:25:04 -0700 | [diff] [blame] | 340 | union { |
| 341 | struct { |
| 342 | enum bpf_arg_type arg1_type; |
| 343 | enum bpf_arg_type arg2_type; |
| 344 | enum bpf_arg_type arg3_type; |
| 345 | enum bpf_arg_type arg4_type; |
| 346 | enum bpf_arg_type arg5_type; |
| 347 | }; |
| 348 | enum bpf_arg_type arg_type[5]; |
| 349 | }; |
Lorenz Bauer | 9436ef6 | 2020-09-21 13:12:20 +0100 | [diff] [blame] | 350 | union { |
| 351 | struct { |
| 352 | u32 *arg1_btf_id; |
| 353 | u32 *arg2_btf_id; |
| 354 | u32 *arg3_btf_id; |
| 355 | u32 *arg4_btf_id; |
| 356 | u32 *arg5_btf_id; |
| 357 | }; |
| 358 | u32 *arg_btf_id[5]; |
| 359 | }; |
Yonghong Song | af7ec13 | 2020-06-23 16:08:09 -0700 | [diff] [blame] | 360 | int *ret_btf_id; /* return value btf_id */ |
Jiri Olsa | eae2e83 | 2020-08-25 21:21:19 +0200 | [diff] [blame] | 361 | bool (*allowed)(const struct bpf_prog *prog); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 362 | }; |
| 363 | |
| 364 | /* bpf_context is intentionally undefined structure. Pointer to bpf_context is |
| 365 | * the first argument to eBPF programs. |
| 366 | * For socket filters: 'struct bpf_context *' == 'struct sk_buff *' |
| 367 | */ |
| 368 | struct bpf_context; |
| 369 | |
| 370 | enum bpf_access_type { |
| 371 | BPF_READ = 1, |
| 372 | BPF_WRITE = 2 |
Alexei Starovoitov | 09756af | 2014-09-26 00:17:00 -0700 | [diff] [blame] | 373 | }; |
| 374 | |
Alexei Starovoitov | 19de99f | 2016-06-15 18:25:38 -0700 | [diff] [blame] | 375 | /* types of values stored in eBPF registers */ |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 376 | /* Pointer types represent: |
| 377 | * pointer |
| 378 | * pointer + imm |
| 379 | * pointer + (u16) var |
| 380 | * pointer + (u16) var + imm |
| 381 | * if (range > 0) then [ptr, ptr + range - off) is safe to access |
| 382 | * if (id > 0) means that some 'var' was added |
| 383 | * if (off > 0) means that 'imm' was added |
| 384 | */ |
Alexei Starovoitov | 19de99f | 2016-06-15 18:25:38 -0700 | [diff] [blame] | 385 | enum bpf_reg_type { |
| 386 | NOT_INIT = 0, /* nothing was written into register */ |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 387 | SCALAR_VALUE, /* reg doesn't contain a valid pointer */ |
Alexei Starovoitov | 19de99f | 2016-06-15 18:25:38 -0700 | [diff] [blame] | 388 | PTR_TO_CTX, /* reg points to bpf_context */ |
| 389 | CONST_PTR_TO_MAP, /* reg points to struct bpf_map */ |
| 390 | PTR_TO_MAP_VALUE, /* reg points to map element value */ |
| 391 | PTR_TO_MAP_VALUE_OR_NULL,/* points to map elem value or NULL */ |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 392 | PTR_TO_STACK, /* reg == frame_pointer + offset */ |
Daniel Borkmann | de8f3a8 | 2017-09-25 02:25:51 +0200 | [diff] [blame] | 393 | PTR_TO_PACKET_META, /* skb->data - meta_len */ |
Edward Cree | f1174f7 | 2017-08-07 15:26:19 +0100 | [diff] [blame] | 394 | PTR_TO_PACKET, /* reg points to skb->data */ |
Alexei Starovoitov | 19de99f | 2016-06-15 18:25:38 -0700 | [diff] [blame] | 395 | PTR_TO_PACKET_END, /* skb->data + headlen */ |
Petar Penkov | d58e468 | 2018-09-14 07:46:18 -0700 | [diff] [blame] | 396 | PTR_TO_FLOW_KEYS, /* reg points to bpf_flow_keys */ |
Joe Stringer | c64b798 | 2018-10-02 13:35:33 -0700 | [diff] [blame] | 397 | PTR_TO_SOCKET, /* reg points to struct bpf_sock */ |
| 398 | PTR_TO_SOCKET_OR_NULL, /* reg points to struct bpf_sock or NULL */ |
Martin KaFai Lau | 46f8bc9 | 2019-02-09 23:22:20 -0800 | [diff] [blame] | 399 | PTR_TO_SOCK_COMMON, /* reg points to sock_common */ |
| 400 | PTR_TO_SOCK_COMMON_OR_NULL, /* reg points to sock_common or NULL */ |
Martin KaFai Lau | 655a51e | 2019-02-09 23:22:24 -0800 | [diff] [blame] | 401 | PTR_TO_TCP_SOCK, /* reg points to struct tcp_sock */ |
| 402 | PTR_TO_TCP_SOCK_OR_NULL, /* reg points to struct tcp_sock or NULL */ |
Matt Mullins | 9df1c28 | 2019-04-26 11:49:47 -0700 | [diff] [blame] | 403 | PTR_TO_TP_BUFFER, /* reg points to a writable raw tp's buffer */ |
Jonathan Lemon | fada7fd | 2019-06-06 13:59:40 -0700 | [diff] [blame] | 404 | PTR_TO_XDP_SOCK, /* reg points to struct xdp_sock */ |
John Fastabend | ba5f4cf | 2020-09-24 12:58:40 -0700 | [diff] [blame] | 405 | /* PTR_TO_BTF_ID points to a kernel struct that does not need |
| 406 | * to be null checked by the BPF program. This does not imply the |
| 407 | * pointer is _not_ null and in practice this can easily be a null |
| 408 | * pointer when reading pointer chains. The assumption is program |
| 409 | * context will handle null pointer dereference typically via fault |
| 410 | * handling. The verifier must keep this in mind and can make no |
| 411 | * assumptions about null or non-null when doing branch analysis. |
| 412 | * Further, when passed into helpers the helpers can not, without |
| 413 | * additional context, assume the value is non-null. |
| 414 | */ |
| 415 | PTR_TO_BTF_ID, |
| 416 | /* PTR_TO_BTF_ID_OR_NULL points to a kernel struct that has not |
| 417 | * been checked for null. Used primarily to inform the verifier |
| 418 | * an explicit null check is required for this struct. |
| 419 | */ |
| 420 | PTR_TO_BTF_ID_OR_NULL, |
Andrii Nakryiko | 457f4436 | 2020-05-29 00:54:20 -0700 | [diff] [blame] | 421 | PTR_TO_MEM, /* reg points to valid memory region */ |
| 422 | PTR_TO_MEM_OR_NULL, /* reg points to valid memory region or NULL */ |
Yonghong Song | afbf21d | 2020-07-23 11:41:11 -0700 | [diff] [blame] | 423 | PTR_TO_RDONLY_BUF, /* reg points to a readonly buffer */ |
| 424 | PTR_TO_RDONLY_BUF_OR_NULL, /* reg points to a readonly buffer or NULL */ |
| 425 | PTR_TO_RDWR_BUF, /* reg points to a read/write buffer */ |
| 426 | PTR_TO_RDWR_BUF_OR_NULL, /* reg points to a read/write buffer or NULL */ |
Hao Luo | eaa6bcb | 2020-09-29 16:50:47 -0700 | [diff] [blame] | 427 | PTR_TO_PERCPU_BTF_ID, /* reg points to a percpu kernel variable */ |
Yonghong Song | 69c087b | 2021-02-26 12:49:25 -0800 | [diff] [blame] | 428 | PTR_TO_FUNC, /* reg points to a bpf program function */ |
| 429 | PTR_TO_MAP_KEY, /* reg points to a map element key */ |
Alexei Starovoitov | 19de99f | 2016-06-15 18:25:38 -0700 | [diff] [blame] | 430 | }; |
| 431 | |
Yonghong Song | 2399463 | 2017-06-22 15:07:39 -0700 | [diff] [blame] | 432 | /* The information passed from prog-specific *_is_valid_access |
| 433 | * back to the verifier. |
| 434 | */ |
| 435 | struct bpf_insn_access_aux { |
| 436 | enum bpf_reg_type reg_type; |
Alexei Starovoitov | 9e15db6 | 2019-10-15 20:25:00 -0700 | [diff] [blame] | 437 | union { |
| 438 | int ctx_field_size; |
Andrii Nakryiko | 22dc4a0 | 2020-12-03 12:46:29 -0800 | [diff] [blame] | 439 | struct { |
| 440 | struct btf *btf; |
| 441 | u32 btf_id; |
| 442 | }; |
Alexei Starovoitov | 9e15db6 | 2019-10-15 20:25:00 -0700 | [diff] [blame] | 443 | }; |
| 444 | struct bpf_verifier_log *log; /* for verbose logs */ |
Yonghong Song | 2399463 | 2017-06-22 15:07:39 -0700 | [diff] [blame] | 445 | }; |
| 446 | |
Daniel Borkmann | f96da09 | 2017-07-02 02:13:27 +0200 | [diff] [blame] | 447 | static inline void |
| 448 | bpf_ctx_record_field_size(struct bpf_insn_access_aux *aux, u32 size) |
| 449 | { |
| 450 | aux->ctx_field_size = size; |
| 451 | } |
| 452 | |
Jakub Kicinski | 7de16e3 | 2017-10-16 16:40:53 -0700 | [diff] [blame] | 453 | struct bpf_prog_ops { |
| 454 | int (*test_run)(struct bpf_prog *prog, const union bpf_attr *kattr, |
| 455 | union bpf_attr __user *uattr); |
| 456 | }; |
| 457 | |
Alexei Starovoitov | 09756af | 2014-09-26 00:17:00 -0700 | [diff] [blame] | 458 | struct bpf_verifier_ops { |
| 459 | /* return eBPF function prototype for verification */ |
Andrey Ignatov | 5e43f89 | 2018-03-30 15:08:00 -0700 | [diff] [blame] | 460 | const struct bpf_func_proto * |
| 461 | (*get_func_proto)(enum bpf_func_id func_id, |
| 462 | const struct bpf_prog *prog); |
Alexei Starovoitov | 17a5267 | 2014-09-26 00:17:06 -0700 | [diff] [blame] | 463 | |
| 464 | /* return true if 'size' wide access at offset 'off' within bpf_context |
| 465 | * with 'type' (read or write) is allowed |
| 466 | */ |
Alexei Starovoitov | 19de99f | 2016-06-15 18:25:38 -0700 | [diff] [blame] | 467 | bool (*is_valid_access)(int off, int size, enum bpf_access_type type, |
Andrey Ignatov | 5e43f89 | 2018-03-30 15:08:00 -0700 | [diff] [blame] | 468 | const struct bpf_prog *prog, |
Yonghong Song | 2399463 | 2017-06-22 15:07:39 -0700 | [diff] [blame] | 469 | struct bpf_insn_access_aux *info); |
Daniel Borkmann | 36bbef5 | 2016-09-20 00:26:13 +0200 | [diff] [blame] | 470 | int (*gen_prologue)(struct bpf_insn *insn, bool direct_write, |
| 471 | const struct bpf_prog *prog); |
Daniel Borkmann | e0cea7c | 2018-05-04 01:08:14 +0200 | [diff] [blame] | 472 | int (*gen_ld_abs)(const struct bpf_insn *orig, |
| 473 | struct bpf_insn *insn_buf); |
Daniel Borkmann | 6b8cc1d | 2017-01-12 11:51:32 +0100 | [diff] [blame] | 474 | u32 (*convert_ctx_access)(enum bpf_access_type type, |
| 475 | const struct bpf_insn *src, |
| 476 | struct bpf_insn *dst, |
Daniel Borkmann | f96da09 | 2017-07-02 02:13:27 +0200 | [diff] [blame] | 477 | struct bpf_prog *prog, u32 *target_size); |
Martin KaFai Lau | 27ae7997 | 2020-01-08 16:35:03 -0800 | [diff] [blame] | 478 | int (*btf_struct_access)(struct bpf_verifier_log *log, |
Andrii Nakryiko | 22dc4a0 | 2020-12-03 12:46:29 -0800 | [diff] [blame] | 479 | const struct btf *btf, |
Martin KaFai Lau | 27ae7997 | 2020-01-08 16:35:03 -0800 | [diff] [blame] | 480 | const struct btf_type *t, int off, int size, |
| 481 | enum bpf_access_type atype, |
| 482 | u32 *next_btf_id); |
Alexei Starovoitov | 09756af | 2014-09-26 00:17:00 -0700 | [diff] [blame] | 483 | }; |
| 484 | |
Jakub Kicinski | cae1927 | 2017-12-27 18:39:05 -0800 | [diff] [blame] | 485 | struct bpf_prog_offload_ops { |
Jakub Kicinski | 08ca90a | 2019-01-22 22:45:24 -0800 | [diff] [blame] | 486 | /* verifier basic callbacks */ |
Jakub Kicinski | cae1927 | 2017-12-27 18:39:05 -0800 | [diff] [blame] | 487 | int (*insn_hook)(struct bpf_verifier_env *env, |
| 488 | int insn_idx, int prev_insn_idx); |
Quentin Monnet | c941ce9 | 2018-10-07 12:56:47 +0100 | [diff] [blame] | 489 | int (*finalize)(struct bpf_verifier_env *env); |
Jakub Kicinski | 08ca90a | 2019-01-22 22:45:24 -0800 | [diff] [blame] | 490 | /* verifier optimization callbacks (called after .finalize) */ |
| 491 | int (*replace_insn)(struct bpf_verifier_env *env, u32 off, |
| 492 | struct bpf_insn *insn); |
| 493 | int (*remove_insns)(struct bpf_verifier_env *env, u32 off, u32 cnt); |
| 494 | /* program management callbacks */ |
Quentin Monnet | 16a8cb5c | 2018-11-09 13:03:32 +0000 | [diff] [blame] | 495 | int (*prepare)(struct bpf_prog *prog); |
| 496 | int (*translate)(struct bpf_prog *prog); |
Quentin Monnet | eb91194 | 2018-11-09 13:03:30 +0000 | [diff] [blame] | 497 | void (*destroy)(struct bpf_prog *prog); |
Jakub Kicinski | cae1927 | 2017-12-27 18:39:05 -0800 | [diff] [blame] | 498 | }; |
| 499 | |
Jakub Kicinski | 0a9c199 | 2018-01-11 20:29:07 -0800 | [diff] [blame] | 500 | struct bpf_prog_offload { |
Jakub Kicinski | ab3f006 | 2017-11-03 13:56:17 -0700 | [diff] [blame] | 501 | struct bpf_prog *prog; |
| 502 | struct net_device *netdev; |
Quentin Monnet | 341b3e7 | 2018-11-09 13:03:26 +0000 | [diff] [blame] | 503 | struct bpf_offload_dev *offdev; |
Jakub Kicinski | ab3f006 | 2017-11-03 13:56:17 -0700 | [diff] [blame] | 504 | void *dev_priv; |
| 505 | struct list_head offloads; |
| 506 | bool dev_state; |
Jakub Kicinski | 08ca90a | 2019-01-22 22:45:24 -0800 | [diff] [blame] | 507 | bool opt_failed; |
Jiong Wang | fcfb126 | 2018-01-16 16:05:19 -0800 | [diff] [blame] | 508 | void *jited_image; |
| 509 | u32 jited_len; |
Jakub Kicinski | ab3f006 | 2017-11-03 13:56:17 -0700 | [diff] [blame] | 510 | }; |
| 511 | |
Roman Gushchin | 8bad74f | 2018-09-28 14:45:36 +0000 | [diff] [blame] | 512 | enum bpf_cgroup_storage_type { |
| 513 | BPF_CGROUP_STORAGE_SHARED, |
Roman Gushchin | b741f16 | 2018-09-28 14:45:43 +0000 | [diff] [blame] | 514 | BPF_CGROUP_STORAGE_PERCPU, |
Roman Gushchin | 8bad74f | 2018-09-28 14:45:36 +0000 | [diff] [blame] | 515 | __BPF_CGROUP_STORAGE_MAX |
| 516 | }; |
| 517 | |
| 518 | #define MAX_BPF_CGROUP_STORAGE_TYPE __BPF_CGROUP_STORAGE_MAX |
| 519 | |
Alexei Starovoitov | f1b9509 | 2019-10-30 15:32:11 -0700 | [diff] [blame] | 520 | /* The longest tracepoint has 12 args. |
| 521 | * See include/trace/bpf_probe.h |
| 522 | */ |
| 523 | #define MAX_BPF_FUNC_ARGS 12 |
| 524 | |
Dmitrii Banshchikov | 523a4cf | 2021-02-26 00:26:29 +0400 | [diff] [blame] | 525 | /* The maximum number of arguments passed through registers |
| 526 | * a single function may have. |
| 527 | */ |
| 528 | #define MAX_BPF_FUNC_REG_ARGS 5 |
| 529 | |
Alexei Starovoitov | fec56f5 | 2019-11-14 10:57:04 -0800 | [diff] [blame] | 530 | struct btf_func_model { |
| 531 | u8 ret_size; |
| 532 | u8 nr_args; |
| 533 | u8 arg_size[MAX_BPF_FUNC_ARGS]; |
| 534 | }; |
| 535 | |
| 536 | /* Restore arguments before returning from trampoline to let original function |
| 537 | * continue executing. This flag is used for fentry progs when there are no |
| 538 | * fexit progs. |
| 539 | */ |
| 540 | #define BPF_TRAMP_F_RESTORE_REGS BIT(0) |
| 541 | /* Call original function after fentry progs, but before fexit progs. |
| 542 | * Makes sense for fentry/fexit, normal calls and indirect calls. |
| 543 | */ |
| 544 | #define BPF_TRAMP_F_CALL_ORIG BIT(1) |
| 545 | /* Skip current frame and return to parent. Makes sense for fentry/fexit |
| 546 | * programs only. Should not be used with normal calls and indirect calls. |
| 547 | */ |
| 548 | #define BPF_TRAMP_F_SKIP_FRAME BIT(2) |
| 549 | |
KP Singh | 88fd9e5 | 2020-03-04 20:18:47 +0100 | [diff] [blame] | 550 | /* Each call __bpf_prog_enter + call bpf_func + call __bpf_prog_exit is ~50 |
| 551 | * bytes on x86. Pick a number to fit into BPF_IMAGE_SIZE / 2 |
| 552 | */ |
Alexei Starovoitov | ca06f55 | 2021-02-09 19:36:29 -0800 | [diff] [blame] | 553 | #define BPF_MAX_TRAMP_PROGS 38 |
KP Singh | 88fd9e5 | 2020-03-04 20:18:47 +0100 | [diff] [blame] | 554 | |
| 555 | struct bpf_tramp_progs { |
| 556 | struct bpf_prog *progs[BPF_MAX_TRAMP_PROGS]; |
| 557 | int nr_progs; |
| 558 | }; |
| 559 | |
Alexei Starovoitov | fec56f5 | 2019-11-14 10:57:04 -0800 | [diff] [blame] | 560 | /* Different use cases for BPF trampoline: |
| 561 | * 1. replace nop at the function entry (kprobe equivalent) |
| 562 | * flags = BPF_TRAMP_F_RESTORE_REGS |
| 563 | * fentry = a set of programs to run before returning from trampoline |
| 564 | * |
| 565 | * 2. replace nop at the function entry (kprobe + kretprobe equivalent) |
| 566 | * flags = BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_SKIP_FRAME |
| 567 | * orig_call = fentry_ip + MCOUNT_INSN_SIZE |
| 568 | * fentry = a set of program to run before calling original function |
| 569 | * fexit = a set of program to run after original function |
| 570 | * |
| 571 | * 3. replace direct call instruction anywhere in the function body |
| 572 | * or assign a function pointer for indirect call (like tcp_congestion_ops->cong_avoid) |
| 573 | * With flags = 0 |
| 574 | * fentry = a set of programs to run before returning from trampoline |
| 575 | * With flags = BPF_TRAMP_F_CALL_ORIG |
| 576 | * orig_call = original callback addr or direct function addr |
| 577 | * fentry = a set of program to run before calling original function |
| 578 | * fexit = a set of program to run after original function |
| 579 | */ |
Alexei Starovoitov | e21aa34 | 2021-03-16 14:00:07 -0700 | [diff] [blame] | 580 | struct bpf_tramp_image; |
| 581 | int arch_prepare_bpf_trampoline(struct bpf_tramp_image *tr, void *image, void *image_end, |
Martin KaFai Lau | 85d33df | 2020-01-08 16:35:05 -0800 | [diff] [blame] | 582 | const struct btf_func_model *m, u32 flags, |
KP Singh | 88fd9e5 | 2020-03-04 20:18:47 +0100 | [diff] [blame] | 583 | struct bpf_tramp_progs *tprogs, |
Alexei Starovoitov | fec56f5 | 2019-11-14 10:57:04 -0800 | [diff] [blame] | 584 | void *orig_call); |
| 585 | /* these two functions are called from generated trampoline */ |
Alexei Starovoitov | ca06f55 | 2021-02-09 19:36:29 -0800 | [diff] [blame] | 586 | u64 notrace __bpf_prog_enter(struct bpf_prog *prog); |
Alexei Starovoitov | fec56f5 | 2019-11-14 10:57:04 -0800 | [diff] [blame] | 587 | void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start); |
Alexei Starovoitov | ca06f55 | 2021-02-09 19:36:29 -0800 | [diff] [blame] | 588 | u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog *prog); |
Alexei Starovoitov | f2dd3b3 | 2021-02-09 19:36:28 -0800 | [diff] [blame] | 589 | void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start); |
Alexei Starovoitov | e21aa34 | 2021-03-16 14:00:07 -0700 | [diff] [blame] | 590 | void notrace __bpf_tramp_enter(struct bpf_tramp_image *tr); |
| 591 | void notrace __bpf_tramp_exit(struct bpf_tramp_image *tr); |
Alexei Starovoitov | fec56f5 | 2019-11-14 10:57:04 -0800 | [diff] [blame] | 592 | |
Jiri Olsa | 535911c | 2020-03-12 20:55:58 +0100 | [diff] [blame] | 593 | struct bpf_ksym { |
| 594 | unsigned long start; |
| 595 | unsigned long end; |
Jiri Olsa | bfea9a8 | 2020-03-12 20:55:59 +0100 | [diff] [blame] | 596 | char name[KSYM_NAME_LEN]; |
Jiri Olsa | ecb60d1 | 2020-03-12 20:56:00 +0100 | [diff] [blame] | 597 | struct list_head lnode; |
Jiri Olsa | ca4424c | 2020-03-12 20:56:01 +0100 | [diff] [blame] | 598 | struct latch_tree_node tnode; |
Jiri Olsa | cbd76f8d | 2020-03-12 20:56:03 +0100 | [diff] [blame] | 599 | bool prog; |
Jiri Olsa | 535911c | 2020-03-12 20:55:58 +0100 | [diff] [blame] | 600 | }; |
| 601 | |
Alexei Starovoitov | fec56f5 | 2019-11-14 10:57:04 -0800 | [diff] [blame] | 602 | enum bpf_tramp_prog_type { |
| 603 | BPF_TRAMP_FENTRY, |
| 604 | BPF_TRAMP_FEXIT, |
KP Singh | ae24082 | 2020-03-04 20:18:49 +0100 | [diff] [blame] | 605 | BPF_TRAMP_MODIFY_RETURN, |
Alexei Starovoitov | be8704f | 2020-01-20 16:53:46 -0800 | [diff] [blame] | 606 | BPF_TRAMP_MAX, |
| 607 | BPF_TRAMP_REPLACE, /* more than MAX */ |
Alexei Starovoitov | fec56f5 | 2019-11-14 10:57:04 -0800 | [diff] [blame] | 608 | }; |
| 609 | |
Alexei Starovoitov | e21aa34 | 2021-03-16 14:00:07 -0700 | [diff] [blame] | 610 | struct bpf_tramp_image { |
| 611 | void *image; |
| 612 | struct bpf_ksym ksym; |
| 613 | struct percpu_ref pcref; |
| 614 | void *ip_after_call; |
| 615 | void *ip_epilogue; |
| 616 | union { |
| 617 | struct rcu_head rcu; |
| 618 | struct work_struct work; |
| 619 | }; |
| 620 | }; |
| 621 | |
Alexei Starovoitov | fec56f5 | 2019-11-14 10:57:04 -0800 | [diff] [blame] | 622 | struct bpf_trampoline { |
| 623 | /* hlist for trampoline_table */ |
| 624 | struct hlist_node hlist; |
| 625 | /* serializes access to fields of this trampoline */ |
| 626 | struct mutex mutex; |
| 627 | refcount_t refcnt; |
| 628 | u64 key; |
| 629 | struct { |
| 630 | struct btf_func_model model; |
| 631 | void *addr; |
Alexei Starovoitov | b91e014f | 2019-12-08 16:01:13 -0800 | [diff] [blame] | 632 | bool ftrace_managed; |
Alexei Starovoitov | fec56f5 | 2019-11-14 10:57:04 -0800 | [diff] [blame] | 633 | } func; |
Alexei Starovoitov | be8704f | 2020-01-20 16:53:46 -0800 | [diff] [blame] | 634 | /* if !NULL this is BPF_PROG_TYPE_EXT program that extends another BPF |
| 635 | * program by replacing one of its functions. func.addr is the address |
| 636 | * of the function it replaced. |
| 637 | */ |
| 638 | struct bpf_prog *extension_prog; |
Alexei Starovoitov | fec56f5 | 2019-11-14 10:57:04 -0800 | [diff] [blame] | 639 | /* list of BPF programs using this trampoline */ |
| 640 | struct hlist_head progs_hlist[BPF_TRAMP_MAX]; |
| 641 | /* Number of attached programs. A counter per kind. */ |
| 642 | int progs_cnt[BPF_TRAMP_MAX]; |
| 643 | /* Executable image of trampoline */ |
Alexei Starovoitov | e21aa34 | 2021-03-16 14:00:07 -0700 | [diff] [blame] | 644 | struct bpf_tramp_image *cur_image; |
Alexei Starovoitov | fec56f5 | 2019-11-14 10:57:04 -0800 | [diff] [blame] | 645 | u64 selector; |
Alexei Starovoitov | fec56f5 | 2019-11-14 10:57:04 -0800 | [diff] [blame] | 646 | }; |
Björn Töpel | 75ccbef | 2019-12-13 18:51:08 +0100 | [diff] [blame] | 647 | |
Toke Høiland-Jørgensen | f7b12b6 | 2020-09-25 23:25:02 +0200 | [diff] [blame] | 648 | struct bpf_attach_target_info { |
| 649 | struct btf_func_model fmodel; |
| 650 | long tgt_addr; |
| 651 | const char *tgt_name; |
| 652 | const struct btf_type *tgt_type; |
| 653 | }; |
| 654 | |
Björn Töpel | 116eb78 | 2019-12-13 18:51:12 +0100 | [diff] [blame] | 655 | #define BPF_DISPATCHER_MAX 48 /* Fits in 2048B */ |
Björn Töpel | 75ccbef | 2019-12-13 18:51:08 +0100 | [diff] [blame] | 656 | |
| 657 | struct bpf_dispatcher_prog { |
| 658 | struct bpf_prog *prog; |
| 659 | refcount_t users; |
| 660 | }; |
| 661 | |
| 662 | struct bpf_dispatcher { |
| 663 | /* dispatcher mutex */ |
| 664 | struct mutex mutex; |
| 665 | void *func; |
| 666 | struct bpf_dispatcher_prog progs[BPF_DISPATCHER_MAX]; |
| 667 | int num_progs; |
| 668 | void *image; |
| 669 | u32 image_off; |
Jiri Olsa | 517b75e | 2020-03-12 20:56:06 +0100 | [diff] [blame] | 670 | struct bpf_ksym ksym; |
Björn Töpel | 75ccbef | 2019-12-13 18:51:08 +0100 | [diff] [blame] | 671 | }; |
| 672 | |
Björn Töpel | 6a64037 | 2020-03-12 20:55:57 +0100 | [diff] [blame] | 673 | static __always_inline unsigned int bpf_dispatcher_nop_func( |
Björn Töpel | 7e6897f | 2019-12-13 18:51:09 +0100 | [diff] [blame] | 674 | const void *ctx, |
| 675 | const struct bpf_insn *insnsi, |
| 676 | unsigned int (*bpf_func)(const void *, |
| 677 | const struct bpf_insn *)) |
| 678 | { |
| 679 | return bpf_func(ctx, insnsi); |
| 680 | } |
Alexei Starovoitov | fec56f5 | 2019-11-14 10:57:04 -0800 | [diff] [blame] | 681 | #ifdef CONFIG_BPF_JIT |
Toke Høiland-Jørgensen | 3aac1ea | 2020-09-29 14:45:50 +0200 | [diff] [blame] | 682 | int bpf_trampoline_link_prog(struct bpf_prog *prog, struct bpf_trampoline *tr); |
| 683 | int bpf_trampoline_unlink_prog(struct bpf_prog *prog, struct bpf_trampoline *tr); |
Toke Høiland-Jørgensen | f7b12b6 | 2020-09-25 23:25:02 +0200 | [diff] [blame] | 684 | struct bpf_trampoline *bpf_trampoline_get(u64 key, |
| 685 | struct bpf_attach_target_info *tgt_info); |
Alexei Starovoitov | fec56f5 | 2019-11-14 10:57:04 -0800 | [diff] [blame] | 686 | void bpf_trampoline_put(struct bpf_trampoline *tr); |
Jiri Olsa | 517b75e | 2020-03-12 20:56:06 +0100 | [diff] [blame] | 687 | #define BPF_DISPATCHER_INIT(_name) { \ |
| 688 | .mutex = __MUTEX_INITIALIZER(_name.mutex), \ |
| 689 | .func = &_name##_func, \ |
| 690 | .progs = {}, \ |
| 691 | .num_progs = 0, \ |
| 692 | .image = NULL, \ |
| 693 | .image_off = 0, \ |
| 694 | .ksym = { \ |
| 695 | .name = #_name, \ |
| 696 | .lnode = LIST_HEAD_INIT(_name.ksym.lnode), \ |
| 697 | }, \ |
Björn Töpel | 75ccbef | 2019-12-13 18:51:08 +0100 | [diff] [blame] | 698 | } |
| 699 | |
| 700 | #define DEFINE_BPF_DISPATCHER(name) \ |
Björn Töpel | 6a64037 | 2020-03-12 20:55:57 +0100 | [diff] [blame] | 701 | noinline unsigned int bpf_dispatcher_##name##_func( \ |
Björn Töpel | 75ccbef | 2019-12-13 18:51:08 +0100 | [diff] [blame] | 702 | const void *ctx, \ |
| 703 | const struct bpf_insn *insnsi, \ |
| 704 | unsigned int (*bpf_func)(const void *, \ |
| 705 | const struct bpf_insn *)) \ |
| 706 | { \ |
| 707 | return bpf_func(ctx, insnsi); \ |
| 708 | } \ |
Björn Töpel | 6a64037 | 2020-03-12 20:55:57 +0100 | [diff] [blame] | 709 | EXPORT_SYMBOL(bpf_dispatcher_##name##_func); \ |
| 710 | struct bpf_dispatcher bpf_dispatcher_##name = \ |
| 711 | BPF_DISPATCHER_INIT(bpf_dispatcher_##name); |
Björn Töpel | 75ccbef | 2019-12-13 18:51:08 +0100 | [diff] [blame] | 712 | #define DECLARE_BPF_DISPATCHER(name) \ |
Björn Töpel | 6a64037 | 2020-03-12 20:55:57 +0100 | [diff] [blame] | 713 | unsigned int bpf_dispatcher_##name##_func( \ |
Björn Töpel | 75ccbef | 2019-12-13 18:51:08 +0100 | [diff] [blame] | 714 | const void *ctx, \ |
| 715 | const struct bpf_insn *insnsi, \ |
| 716 | unsigned int (*bpf_func)(const void *, \ |
| 717 | const struct bpf_insn *)); \ |
Björn Töpel | 6a64037 | 2020-03-12 20:55:57 +0100 | [diff] [blame] | 718 | extern struct bpf_dispatcher bpf_dispatcher_##name; |
| 719 | #define BPF_DISPATCHER_FUNC(name) bpf_dispatcher_##name##_func |
| 720 | #define BPF_DISPATCHER_PTR(name) (&bpf_dispatcher_##name) |
Björn Töpel | 75ccbef | 2019-12-13 18:51:08 +0100 | [diff] [blame] | 721 | void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from, |
| 722 | struct bpf_prog *to); |
Jiri Olsa | dba122fb | 2020-03-12 20:56:04 +0100 | [diff] [blame] | 723 | /* Called only from JIT-enabled code, so there's no need for stubs. */ |
Jiri Olsa | 7ac88eb | 2020-03-12 20:56:07 +0100 | [diff] [blame] | 724 | void *bpf_jit_alloc_exec_page(void); |
Jiri Olsa | a108f7d | 2020-03-12 20:56:05 +0100 | [diff] [blame] | 725 | void bpf_image_ksym_add(void *data, struct bpf_ksym *ksym); |
| 726 | void bpf_image_ksym_del(struct bpf_ksym *ksym); |
Jiri Olsa | dba122fb | 2020-03-12 20:56:04 +0100 | [diff] [blame] | 727 | void bpf_ksym_add(struct bpf_ksym *ksym); |
| 728 | void bpf_ksym_del(struct bpf_ksym *ksym); |
Alexei Starovoitov | e21aa34 | 2021-03-16 14:00:07 -0700 | [diff] [blame] | 729 | int bpf_jit_charge_modmem(u32 pages); |
| 730 | void bpf_jit_uncharge_modmem(u32 pages); |
Alexei Starovoitov | fec56f5 | 2019-11-14 10:57:04 -0800 | [diff] [blame] | 731 | #else |
Toke Høiland-Jørgensen | 3aac1ea | 2020-09-29 14:45:50 +0200 | [diff] [blame] | 732 | static inline int bpf_trampoline_link_prog(struct bpf_prog *prog, |
| 733 | struct bpf_trampoline *tr) |
Alexei Starovoitov | fec56f5 | 2019-11-14 10:57:04 -0800 | [diff] [blame] | 734 | { |
| 735 | return -ENOTSUPP; |
| 736 | } |
Toke Høiland-Jørgensen | 3aac1ea | 2020-09-29 14:45:50 +0200 | [diff] [blame] | 737 | static inline int bpf_trampoline_unlink_prog(struct bpf_prog *prog, |
| 738 | struct bpf_trampoline *tr) |
Alexei Starovoitov | fec56f5 | 2019-11-14 10:57:04 -0800 | [diff] [blame] | 739 | { |
| 740 | return -ENOTSUPP; |
| 741 | } |
Toke Høiland-Jørgensen | f7b12b6 | 2020-09-25 23:25:02 +0200 | [diff] [blame] | 742 | static inline struct bpf_trampoline *bpf_trampoline_get(u64 key, |
| 743 | struct bpf_attach_target_info *tgt_info) |
| 744 | { |
| 745 | return ERR_PTR(-EOPNOTSUPP); |
| 746 | } |
Alexei Starovoitov | fec56f5 | 2019-11-14 10:57:04 -0800 | [diff] [blame] | 747 | static inline void bpf_trampoline_put(struct bpf_trampoline *tr) {} |
Björn Töpel | 75ccbef | 2019-12-13 18:51:08 +0100 | [diff] [blame] | 748 | #define DEFINE_BPF_DISPATCHER(name) |
| 749 | #define DECLARE_BPF_DISPATCHER(name) |
Björn Töpel | 6a64037 | 2020-03-12 20:55:57 +0100 | [diff] [blame] | 750 | #define BPF_DISPATCHER_FUNC(name) bpf_dispatcher_nop_func |
Björn Töpel | 75ccbef | 2019-12-13 18:51:08 +0100 | [diff] [blame] | 751 | #define BPF_DISPATCHER_PTR(name) NULL |
| 752 | static inline void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, |
| 753 | struct bpf_prog *from, |
| 754 | struct bpf_prog *to) {} |
Jiri Olsa | e9b4e60 | 2020-01-23 17:15:07 +0100 | [diff] [blame] | 755 | static inline bool is_bpf_image_address(unsigned long address) |
| 756 | { |
| 757 | return false; |
| 758 | } |
Alexei Starovoitov | fec56f5 | 2019-11-14 10:57:04 -0800 | [diff] [blame] | 759 | #endif |
| 760 | |
Alexei Starovoitov | 8c1b6e6 | 2019-11-14 10:57:16 -0800 | [diff] [blame] | 761 | struct bpf_func_info_aux { |
Alexei Starovoitov | 51c39bb | 2020-01-09 22:41:20 -0800 | [diff] [blame] | 762 | u16 linkage; |
Alexei Starovoitov | 8c1b6e6 | 2019-11-14 10:57:16 -0800 | [diff] [blame] | 763 | bool unreliable; |
| 764 | }; |
| 765 | |
Daniel Borkmann | a66886f | 2019-11-22 21:07:57 +0100 | [diff] [blame] | 766 | enum bpf_jit_poke_reason { |
| 767 | BPF_POKE_REASON_TAIL_CALL, |
| 768 | }; |
| 769 | |
| 770 | /* Descriptor of pokes pointing /into/ the JITed image. */ |
| 771 | struct bpf_jit_poke_descriptor { |
Maciej Fijalkowski | cf71b17 | 2020-09-16 23:10:06 +0200 | [diff] [blame] | 772 | void *tailcall_target; |
Maciej Fijalkowski | ebf7d1f5 | 2020-09-16 23:10:08 +0200 | [diff] [blame] | 773 | void *tailcall_bypass; |
| 774 | void *bypass_addr; |
Daniel Borkmann | a66886f | 2019-11-22 21:07:57 +0100 | [diff] [blame] | 775 | union { |
| 776 | struct { |
| 777 | struct bpf_map *map; |
| 778 | u32 key; |
| 779 | } tail_call; |
| 780 | }; |
Maciej Fijalkowski | cf71b17 | 2020-09-16 23:10:06 +0200 | [diff] [blame] | 781 | bool tailcall_target_stable; |
Daniel Borkmann | a66886f | 2019-11-22 21:07:57 +0100 | [diff] [blame] | 782 | u8 adj_off; |
| 783 | u16 reason; |
Maciej Fijalkowski | a748c69 | 2020-09-16 23:10:05 +0200 | [diff] [blame] | 784 | u32 insn_idx; |
Daniel Borkmann | a66886f | 2019-11-22 21:07:57 +0100 | [diff] [blame] | 785 | }; |
| 786 | |
Yonghong Song | 3c32cc1 | 2020-05-13 11:02:21 -0700 | [diff] [blame] | 787 | /* reg_type info for ctx arguments */ |
| 788 | struct bpf_ctx_arg_aux { |
| 789 | u32 offset; |
| 790 | enum bpf_reg_type reg_type; |
Yonghong Song | 951cf36 | 2020-07-20 09:34:03 -0700 | [diff] [blame] | 791 | u32 btf_id; |
Yonghong Song | 3c32cc1 | 2020-05-13 11:02:21 -0700 | [diff] [blame] | 792 | }; |
| 793 | |
Andrii Nakryiko | 541c3ba | 2021-01-11 23:55:18 -0800 | [diff] [blame] | 794 | struct btf_mod_pair { |
| 795 | struct btf *btf; |
| 796 | struct module *module; |
| 797 | }; |
| 798 | |
Alexei Starovoitov | 09756af | 2014-09-26 00:17:00 -0700 | [diff] [blame] | 799 | struct bpf_prog_aux { |
Andrii Nakryiko | 85192dbf | 2019-11-17 09:28:03 -0800 | [diff] [blame] | 800 | atomic64_t refcnt; |
Daniel Borkmann | 24701ec | 2015-03-01 12:31:47 +0100 | [diff] [blame] | 801 | u32 used_map_cnt; |
Andrii Nakryiko | 541c3ba | 2021-01-11 23:55:18 -0800 | [diff] [blame] | 802 | u32 used_btf_cnt; |
Alexei Starovoitov | 32bbe00 | 2016-04-06 18:43:28 -0700 | [diff] [blame] | 803 | u32 max_ctx_offset; |
Jiong Wang | e647815 | 2018-11-08 04:08:42 -0500 | [diff] [blame] | 804 | u32 max_pkt_offset; |
Matt Mullins | 9df1c28 | 2019-04-26 11:49:47 -0700 | [diff] [blame] | 805 | u32 max_tp_access; |
Alexei Starovoitov | 8726679 | 2017-05-30 13:31:29 -0700 | [diff] [blame] | 806 | u32 stack_depth; |
Martin KaFai Lau | dc4bb0e | 2017-06-05 12:15:46 -0700 | [diff] [blame] | 807 | u32 id; |
Yonghong Song | ba64e7d | 2018-11-24 23:20:44 -0800 | [diff] [blame] | 808 | u32 func_cnt; /* used by non-func prog as the number of func progs */ |
| 809 | u32 func_idx; /* 0 for non-func prog, the index in func array for func prog */ |
Alexei Starovoitov | ccfe29eb2 | 2019-10-15 20:24:58 -0700 | [diff] [blame] | 810 | u32 attach_btf_id; /* in-kernel BTF type id to attach to */ |
Yonghong Song | 3c32cc1 | 2020-05-13 11:02:21 -0700 | [diff] [blame] | 811 | u32 ctx_arg_info_size; |
Yonghong Song | afbf21d | 2020-07-23 11:41:11 -0700 | [diff] [blame] | 812 | u32 max_rdonly_access; |
| 813 | u32 max_rdwr_access; |
Andrii Nakryiko | 22dc4a0 | 2020-12-03 12:46:29 -0800 | [diff] [blame] | 814 | struct btf *attach_btf; |
Yonghong Song | 3c32cc1 | 2020-05-13 11:02:21 -0700 | [diff] [blame] | 815 | const struct bpf_ctx_arg_aux *ctx_arg_info; |
Toke Høiland-Jørgensen | 3aac1ea | 2020-09-29 14:45:50 +0200 | [diff] [blame] | 816 | struct mutex dst_mutex; /* protects dst_* pointers below, *after* prog becomes visible */ |
| 817 | struct bpf_prog *dst_prog; |
| 818 | struct bpf_trampoline *dst_trampoline; |
Toke Høiland-Jørgensen | 4a1e7c0 | 2020-09-29 14:45:51 +0200 | [diff] [blame] | 819 | enum bpf_prog_type saved_dst_prog_type; |
| 820 | enum bpf_attach_type saved_dst_attach_type; |
Jiong Wang | a4b1d3c | 2019-05-24 23:25:15 +0100 | [diff] [blame] | 821 | bool verifier_zext; /* Zero extensions has been inserted by verifier. */ |
Jakub Kicinski | 9a18eed | 2017-12-27 18:39:04 -0800 | [diff] [blame] | 822 | bool offload_requested; |
Martin KaFai Lau | 3820729 | 2019-10-24 17:18:11 -0700 | [diff] [blame] | 823 | bool attach_btf_trace; /* true if attaching to BTF-enabled raw tp */ |
Alexei Starovoitov | 8c1b6e6 | 2019-11-14 10:57:16 -0800 | [diff] [blame] | 824 | bool func_proto_unreliable; |
Alexei Starovoitov | 1e6c62a | 2020-08-27 15:01:11 -0700 | [diff] [blame] | 825 | bool sleepable; |
Maciej Fijalkowski | ebf7d1f5 | 2020-09-16 23:10:08 +0200 | [diff] [blame] | 826 | bool tail_call_reachable; |
Alexei Starovoitov | fec56f5 | 2019-11-14 10:57:04 -0800 | [diff] [blame] | 827 | struct hlist_node tramp_hlist; |
Martin KaFai Lau | 3820729 | 2019-10-24 17:18:11 -0700 | [diff] [blame] | 828 | /* BTF_KIND_FUNC_PROTO for valid attach_btf_id */ |
| 829 | const struct btf_type *attach_func_proto; |
| 830 | /* function name for valid attach_btf_id */ |
| 831 | const char *attach_func_name; |
Alexei Starovoitov | 1c2a088 | 2017-12-14 17:55:15 -0800 | [diff] [blame] | 832 | struct bpf_prog **func; |
| 833 | void *jit_data; /* JIT specific data. arch dependent */ |
Daniel Borkmann | a66886f | 2019-11-22 21:07:57 +0100 | [diff] [blame] | 834 | struct bpf_jit_poke_descriptor *poke_tab; |
| 835 | u32 size_poke_tab; |
Jiri Olsa | 535911c | 2020-03-12 20:55:58 +0100 | [diff] [blame] | 836 | struct bpf_ksym ksym; |
Jakub Kicinski | 7de16e3 | 2017-10-16 16:40:53 -0700 | [diff] [blame] | 837 | const struct bpf_prog_ops *ops; |
Alexei Starovoitov | 09756af | 2014-09-26 00:17:00 -0700 | [diff] [blame] | 838 | struct bpf_map **used_maps; |
YiFei Zhu | 984fe94 | 2020-09-15 16:45:39 -0700 | [diff] [blame] | 839 | struct mutex used_maps_mutex; /* mutex for used_maps and used_map_cnt */ |
Andrii Nakryiko | 541c3ba | 2021-01-11 23:55:18 -0800 | [diff] [blame] | 840 | struct btf_mod_pair *used_btfs; |
Alexei Starovoitov | 09756af | 2014-09-26 00:17:00 -0700 | [diff] [blame] | 841 | struct bpf_prog *prog; |
Alexei Starovoitov | aaac3ba | 2015-10-07 22:23:22 -0700 | [diff] [blame] | 842 | struct user_struct *user; |
Martin KaFai Lau | cb4d2b3 | 2017-09-27 14:37:52 -0700 | [diff] [blame] | 843 | u64 load_time; /* ns since boottime */ |
Roman Gushchin | 8bad74f | 2018-09-28 14:45:36 +0000 | [diff] [blame] | 844 | struct bpf_map *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]; |
Martin KaFai Lau | 067cae4 | 2017-10-05 21:52:12 -0700 | [diff] [blame] | 845 | char name[BPF_OBJ_NAME_LEN]; |
Chenbo Feng | afdb09c | 2017-10-18 13:00:24 -0700 | [diff] [blame] | 846 | #ifdef CONFIG_SECURITY |
| 847 | void *security; |
| 848 | #endif |
Jakub Kicinski | 0a9c199 | 2018-01-11 20:29:07 -0800 | [diff] [blame] | 849 | struct bpf_prog_offload *offload; |
Yonghong Song | 838e969 | 2018-11-19 15:29:11 -0800 | [diff] [blame] | 850 | struct btf *btf; |
Yonghong Song | ba64e7d | 2018-11-24 23:20:44 -0800 | [diff] [blame] | 851 | struct bpf_func_info *func_info; |
Alexei Starovoitov | 8c1b6e6 | 2019-11-14 10:57:16 -0800 | [diff] [blame] | 852 | struct bpf_func_info_aux *func_info_aux; |
Martin KaFai Lau | c454a46 | 2018-12-07 16:42:25 -0800 | [diff] [blame] | 853 | /* bpf_line_info loaded from userspace. linfo->insn_off |
| 854 | * has the xlated insn offset. |
| 855 | * Both the main and sub prog share the same linfo. |
| 856 | * The subprog can access its first linfo by |
| 857 | * using the linfo_idx. |
| 858 | */ |
| 859 | struct bpf_line_info *linfo; |
| 860 | /* jited_linfo is the jited addr of the linfo. It has a |
| 861 | * one to one mapping to linfo: |
| 862 | * jited_linfo[i] is the jited addr for the linfo[i]->insn_off. |
| 863 | * Both the main and sub prog share the same jited_linfo. |
| 864 | * The subprog can access its first jited_linfo by |
| 865 | * using the linfo_idx. |
| 866 | */ |
| 867 | void **jited_linfo; |
Yonghong Song | ba64e7d | 2018-11-24 23:20:44 -0800 | [diff] [blame] | 868 | u32 func_info_cnt; |
Martin KaFai Lau | c454a46 | 2018-12-07 16:42:25 -0800 | [diff] [blame] | 869 | u32 nr_linfo; |
| 870 | /* subprog can use linfo_idx to access its first linfo and |
| 871 | * jited_linfo. |
| 872 | * main prog always has linfo_idx == 0 |
| 873 | */ |
| 874 | u32 linfo_idx; |
Alexei Starovoitov | 3dec541 | 2019-10-15 20:25:03 -0700 | [diff] [blame] | 875 | u32 num_exentries; |
| 876 | struct exception_table_entry *extable; |
Alexei Starovoitov | abf2e7d | 2015-05-28 19:26:02 -0700 | [diff] [blame] | 877 | union { |
| 878 | struct work_struct work; |
| 879 | struct rcu_head rcu; |
| 880 | }; |
Alexei Starovoitov | 09756af | 2014-09-26 00:17:00 -0700 | [diff] [blame] | 881 | }; |
| 882 | |
Daniel Borkmann | 2beee5f | 2019-11-22 21:07:56 +0100 | [diff] [blame] | 883 | struct bpf_array_aux { |
| 884 | /* 'Ownership' of prog array is claimed by the first program that |
| 885 | * is going to use this map or by the first program which FD is |
| 886 | * stored in the map to make sure that all callers and callees have |
| 887 | * the same prog type and JITed flag. |
| 888 | */ |
| 889 | enum bpf_prog_type type; |
| 890 | bool jited; |
Daniel Borkmann | da765a2 | 2019-11-22 21:07:58 +0100 | [diff] [blame] | 891 | /* Programs with direct jumps into programs part of this array. */ |
| 892 | struct list_head poke_progs; |
| 893 | struct bpf_map *map; |
| 894 | struct mutex poke_mutex; |
| 895 | struct work_struct work; |
Daniel Borkmann | 2beee5f | 2019-11-22 21:07:56 +0100 | [diff] [blame] | 896 | }; |
| 897 | |
Andrii Nakryiko | 6cc7d1e | 2020-07-21 23:45:54 -0700 | [diff] [blame] | 898 | struct bpf_link { |
| 899 | atomic64_t refcnt; |
| 900 | u32 id; |
| 901 | enum bpf_link_type type; |
| 902 | const struct bpf_link_ops *ops; |
| 903 | struct bpf_prog *prog; |
| 904 | struct work_struct work; |
| 905 | }; |
| 906 | |
| 907 | struct bpf_link_ops { |
| 908 | void (*release)(struct bpf_link *link); |
| 909 | void (*dealloc)(struct bpf_link *link); |
Andrii Nakryiko | 73b11c2a | 2020-07-31 11:28:26 -0700 | [diff] [blame] | 910 | int (*detach)(struct bpf_link *link); |
Andrii Nakryiko | 6cc7d1e | 2020-07-21 23:45:54 -0700 | [diff] [blame] | 911 | int (*update_prog)(struct bpf_link *link, struct bpf_prog *new_prog, |
| 912 | struct bpf_prog *old_prog); |
| 913 | void (*show_fdinfo)(const struct bpf_link *link, struct seq_file *seq); |
| 914 | int (*fill_link_info)(const struct bpf_link *link, |
| 915 | struct bpf_link_info *info); |
| 916 | }; |
| 917 | |
| 918 | struct bpf_link_primer { |
| 919 | struct bpf_link *link; |
| 920 | struct file *file; |
| 921 | int fd; |
| 922 | u32 id; |
| 923 | }; |
| 924 | |
Martin KaFai Lau | 85d33df | 2020-01-08 16:35:05 -0800 | [diff] [blame] | 925 | struct bpf_struct_ops_value; |
Martin KaFai Lau | 27ae7997 | 2020-01-08 16:35:03 -0800 | [diff] [blame] | 926 | struct btf_type; |
| 927 | struct btf_member; |
| 928 | |
| 929 | #define BPF_STRUCT_OPS_MAX_NR_MEMBERS 64 |
| 930 | struct bpf_struct_ops { |
| 931 | const struct bpf_verifier_ops *verifier_ops; |
| 932 | int (*init)(struct btf *btf); |
| 933 | int (*check_member)(const struct btf_type *t, |
| 934 | const struct btf_member *member); |
Martin KaFai Lau | 85d33df | 2020-01-08 16:35:05 -0800 | [diff] [blame] | 935 | int (*init_member)(const struct btf_type *t, |
| 936 | const struct btf_member *member, |
| 937 | void *kdata, const void *udata); |
| 938 | int (*reg)(void *kdata); |
| 939 | void (*unreg)(void *kdata); |
Martin KaFai Lau | 27ae7997 | 2020-01-08 16:35:03 -0800 | [diff] [blame] | 940 | const struct btf_type *type; |
Martin KaFai Lau | 85d33df | 2020-01-08 16:35:05 -0800 | [diff] [blame] | 941 | const struct btf_type *value_type; |
Martin KaFai Lau | 27ae7997 | 2020-01-08 16:35:03 -0800 | [diff] [blame] | 942 | const char *name; |
| 943 | struct btf_func_model func_models[BPF_STRUCT_OPS_MAX_NR_MEMBERS]; |
| 944 | u32 type_id; |
Martin KaFai Lau | 85d33df | 2020-01-08 16:35:05 -0800 | [diff] [blame] | 945 | u32 value_id; |
Martin KaFai Lau | 27ae7997 | 2020-01-08 16:35:03 -0800 | [diff] [blame] | 946 | }; |
| 947 | |
| 948 | #if defined(CONFIG_BPF_JIT) && defined(CONFIG_BPF_SYSCALL) |
Martin KaFai Lau | 85d33df | 2020-01-08 16:35:05 -0800 | [diff] [blame] | 949 | #define BPF_MODULE_OWNER ((void *)((0xeB9FUL << 2) + POISON_POINTER_DELTA)) |
Martin KaFai Lau | 27ae7997 | 2020-01-08 16:35:03 -0800 | [diff] [blame] | 950 | const struct bpf_struct_ops *bpf_struct_ops_find(u32 type_id); |
Martin KaFai Lau | d3e42bb | 2020-01-27 09:51:45 -0800 | [diff] [blame] | 951 | void bpf_struct_ops_init(struct btf *btf, struct bpf_verifier_log *log); |
Martin KaFai Lau | 85d33df | 2020-01-08 16:35:05 -0800 | [diff] [blame] | 952 | bool bpf_struct_ops_get(const void *kdata); |
| 953 | void bpf_struct_ops_put(const void *kdata); |
| 954 | int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map, void *key, |
| 955 | void *value); |
| 956 | static inline bool bpf_try_module_get(const void *data, struct module *owner) |
| 957 | { |
| 958 | if (owner == BPF_MODULE_OWNER) |
| 959 | return bpf_struct_ops_get(data); |
| 960 | else |
| 961 | return try_module_get(owner); |
| 962 | } |
| 963 | static inline void bpf_module_put(const void *data, struct module *owner) |
| 964 | { |
| 965 | if (owner == BPF_MODULE_OWNER) |
| 966 | bpf_struct_ops_put(data); |
| 967 | else |
| 968 | module_put(owner); |
| 969 | } |
Martin KaFai Lau | 27ae7997 | 2020-01-08 16:35:03 -0800 | [diff] [blame] | 970 | #else |
| 971 | static inline const struct bpf_struct_ops *bpf_struct_ops_find(u32 type_id) |
| 972 | { |
| 973 | return NULL; |
| 974 | } |
Martin KaFai Lau | d3e42bb | 2020-01-27 09:51:45 -0800 | [diff] [blame] | 975 | static inline void bpf_struct_ops_init(struct btf *btf, |
| 976 | struct bpf_verifier_log *log) |
| 977 | { |
| 978 | } |
Martin KaFai Lau | 85d33df | 2020-01-08 16:35:05 -0800 | [diff] [blame] | 979 | static inline bool bpf_try_module_get(const void *data, struct module *owner) |
| 980 | { |
| 981 | return try_module_get(owner); |
| 982 | } |
| 983 | static inline void bpf_module_put(const void *data, struct module *owner) |
| 984 | { |
| 985 | module_put(owner); |
| 986 | } |
| 987 | static inline int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map, |
| 988 | void *key, |
| 989 | void *value) |
| 990 | { |
| 991 | return -EINVAL; |
| 992 | } |
Martin KaFai Lau | 27ae7997 | 2020-01-08 16:35:03 -0800 | [diff] [blame] | 993 | #endif |
| 994 | |
Alexei Starovoitov | 04fd61ab | 2015-05-19 16:59:03 -0700 | [diff] [blame] | 995 | struct bpf_array { |
| 996 | struct bpf_map map; |
| 997 | u32 elem_size; |
Alexei Starovoitov | b215739 | 2018-01-07 17:33:02 -0800 | [diff] [blame] | 998 | u32 index_mask; |
Daniel Borkmann | 2beee5f | 2019-11-22 21:07:56 +0100 | [diff] [blame] | 999 | struct bpf_array_aux *aux; |
Alexei Starovoitov | 04fd61ab | 2015-05-19 16:59:03 -0700 | [diff] [blame] | 1000 | union { |
| 1001 | char value[0] __aligned(8); |
Wang Nan | 2a36f0b | 2015-08-06 07:02:33 +0000 | [diff] [blame] | 1002 | void *ptrs[0] __aligned(8); |
Alexei Starovoitov | a10423b | 2016-02-01 22:39:54 -0800 | [diff] [blame] | 1003 | void __percpu *pptrs[0] __aligned(8); |
Alexei Starovoitov | 04fd61ab | 2015-05-19 16:59:03 -0700 | [diff] [blame] | 1004 | }; |
| 1005 | }; |
Daniel Borkmann | 3b1efb1 | 2016-06-15 22:47:14 +0200 | [diff] [blame] | 1006 | |
Alexei Starovoitov | c04c0d2 | 2019-04-01 21:27:45 -0700 | [diff] [blame] | 1007 | #define BPF_COMPLEXITY_LIMIT_INSNS 1000000 /* yes. 1M insns */ |
Alexei Starovoitov | 04fd61ab | 2015-05-19 16:59:03 -0700 | [diff] [blame] | 1008 | #define MAX_TAIL_CALL_CNT 32 |
| 1009 | |
Daniel Borkmann | 591fe98 | 2019-04-09 23:20:05 +0200 | [diff] [blame] | 1010 | #define BPF_F_ACCESS_MASK (BPF_F_RDONLY | \ |
| 1011 | BPF_F_RDONLY_PROG | \ |
| 1012 | BPF_F_WRONLY | \ |
| 1013 | BPF_F_WRONLY_PROG) |
| 1014 | |
| 1015 | #define BPF_MAP_CAN_READ BIT(0) |
| 1016 | #define BPF_MAP_CAN_WRITE BIT(1) |
| 1017 | |
| 1018 | static inline u32 bpf_map_flags_to_cap(struct bpf_map *map) |
| 1019 | { |
| 1020 | u32 access_flags = map->map_flags & (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG); |
| 1021 | |
| 1022 | /* Combination of BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG is |
| 1023 | * not possible. |
| 1024 | */ |
| 1025 | if (access_flags & BPF_F_RDONLY_PROG) |
| 1026 | return BPF_MAP_CAN_READ; |
| 1027 | else if (access_flags & BPF_F_WRONLY_PROG) |
| 1028 | return BPF_MAP_CAN_WRITE; |
| 1029 | else |
| 1030 | return BPF_MAP_CAN_READ | BPF_MAP_CAN_WRITE; |
| 1031 | } |
| 1032 | |
| 1033 | static inline bool bpf_map_flags_access_ok(u32 access_flags) |
| 1034 | { |
| 1035 | return (access_flags & (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG)) != |
| 1036 | (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG); |
| 1037 | } |
| 1038 | |
Daniel Borkmann | 3b1efb1 | 2016-06-15 22:47:14 +0200 | [diff] [blame] | 1039 | struct bpf_event_entry { |
| 1040 | struct perf_event *event; |
| 1041 | struct file *perf_file; |
| 1042 | struct file *map_file; |
| 1043 | struct rcu_head rcu; |
| 1044 | }; |
| 1045 | |
Alexei Starovoitov | 04fd61ab | 2015-05-19 16:59:03 -0700 | [diff] [blame] | 1046 | bool bpf_prog_array_compatible(struct bpf_array *array, const struct bpf_prog *fp); |
Daniel Borkmann | f1f7714 | 2017-01-13 23:38:15 +0100 | [diff] [blame] | 1047 | int bpf_prog_calc_tag(struct bpf_prog *fp); |
Daniel Borkmann | bd570ff | 2016-04-18 21:01:24 +0200 | [diff] [blame] | 1048 | |
Alexei Starovoitov | 0756ea3 | 2015-06-12 19:39:13 -0700 | [diff] [blame] | 1049 | const struct bpf_func_proto *bpf_get_trace_printk_proto(void); |
Daniel Borkmann | 555c8a8 | 2016-07-14 18:08:05 +0200 | [diff] [blame] | 1050 | |
| 1051 | typedef unsigned long (*bpf_ctx_copy_t)(void *dst, const void *src, |
Daniel Borkmann | aa7145c | 2016-07-22 01:19:42 +0200 | [diff] [blame] | 1052 | unsigned long off, unsigned long len); |
Joe Stringer | c64b798 | 2018-10-02 13:35:33 -0700 | [diff] [blame] | 1053 | typedef u32 (*bpf_convert_ctx_access_t)(enum bpf_access_type type, |
| 1054 | const struct bpf_insn *src, |
| 1055 | struct bpf_insn *dst, |
| 1056 | struct bpf_prog *prog, |
| 1057 | u32 *target_size); |
Daniel Borkmann | 555c8a8 | 2016-07-14 18:08:05 +0200 | [diff] [blame] | 1058 | |
| 1059 | u64 bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size, |
| 1060 | void *ctx, u64 ctx_size, bpf_ctx_copy_t ctx_copy); |
Alexei Starovoitov | 04fd61ab | 2015-05-19 16:59:03 -0700 | [diff] [blame] | 1061 | |
Alexei Starovoitov | 324bda9e6 | 2017-10-02 22:50:21 -0700 | [diff] [blame] | 1062 | /* an array of programs to be executed under rcu_lock. |
| 1063 | * |
| 1064 | * Typical usage: |
| 1065 | * ret = BPF_PROG_RUN_ARRAY(&bpf_prog_array, ctx, BPF_PROG_RUN); |
| 1066 | * |
| 1067 | * the structure returned by bpf_prog_array_alloc() should be populated |
| 1068 | * with program pointers and the last pointer must be NULL. |
| 1069 | * The user has to keep refcnt on the program and make sure the program |
| 1070 | * is removed from the array before bpf_prog_put(). |
| 1071 | * The 'struct bpf_prog_array *' should only be replaced with xchg() |
| 1072 | * since other cpus are walking the array of pointers in parallel. |
| 1073 | */ |
Roman Gushchin | 394e40a | 2018-08-02 14:27:21 -0700 | [diff] [blame] | 1074 | struct bpf_prog_array_item { |
| 1075 | struct bpf_prog *prog; |
Roman Gushchin | 8bad74f | 2018-09-28 14:45:36 +0000 | [diff] [blame] | 1076 | struct bpf_cgroup_storage *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]; |
Roman Gushchin | 394e40a | 2018-08-02 14:27:21 -0700 | [diff] [blame] | 1077 | }; |
| 1078 | |
Alexei Starovoitov | 324bda9e6 | 2017-10-02 22:50:21 -0700 | [diff] [blame] | 1079 | struct bpf_prog_array { |
| 1080 | struct rcu_head rcu; |
Gustavo A. R. Silva | d7f10df | 2020-02-26 18:17:44 -0600 | [diff] [blame] | 1081 | struct bpf_prog_array_item items[]; |
Alexei Starovoitov | 324bda9e6 | 2017-10-02 22:50:21 -0700 | [diff] [blame] | 1082 | }; |
| 1083 | |
Roman Gushchin | d29ab6e | 2018-07-13 12:41:10 -0700 | [diff] [blame] | 1084 | struct bpf_prog_array *bpf_prog_array_alloc(u32 prog_cnt, gfp_t flags); |
Stanislav Fomichev | 54e9c9d | 2019-05-28 14:14:41 -0700 | [diff] [blame] | 1085 | void bpf_prog_array_free(struct bpf_prog_array *progs); |
| 1086 | int bpf_prog_array_length(struct bpf_prog_array *progs); |
Stanislav Fomichev | 0d01da6 | 2019-06-27 13:38:47 -0700 | [diff] [blame] | 1087 | bool bpf_prog_array_is_empty(struct bpf_prog_array *array); |
Stanislav Fomichev | 54e9c9d | 2019-05-28 14:14:41 -0700 | [diff] [blame] | 1088 | int bpf_prog_array_copy_to_user(struct bpf_prog_array *progs, |
Alexei Starovoitov | 468e2f6 | 2017-10-02 22:50:22 -0700 | [diff] [blame] | 1089 | __u32 __user *prog_ids, u32 cnt); |
Alexei Starovoitov | 324bda9e6 | 2017-10-02 22:50:21 -0700 | [diff] [blame] | 1090 | |
Stanislav Fomichev | 54e9c9d | 2019-05-28 14:14:41 -0700 | [diff] [blame] | 1091 | void bpf_prog_array_delete_safe(struct bpf_prog_array *progs, |
Yonghong Song | e87c6bc38 | 2017-10-23 23:53:08 -0700 | [diff] [blame] | 1092 | struct bpf_prog *old_prog); |
Jakub Sitnicki | ce3aa9c | 2020-07-17 12:35:22 +0200 | [diff] [blame] | 1093 | int bpf_prog_array_delete_safe_at(struct bpf_prog_array *array, int index); |
| 1094 | int bpf_prog_array_update_at(struct bpf_prog_array *array, int index, |
| 1095 | struct bpf_prog *prog); |
Stanislav Fomichev | 54e9c9d | 2019-05-28 14:14:41 -0700 | [diff] [blame] | 1096 | int bpf_prog_array_copy_info(struct bpf_prog_array *array, |
Yonghong Song | 3a38bb9 | 2018-04-10 09:37:32 -0700 | [diff] [blame] | 1097 | u32 *prog_ids, u32 request_cnt, |
| 1098 | u32 *prog_cnt); |
Stanislav Fomichev | 54e9c9d | 2019-05-28 14:14:41 -0700 | [diff] [blame] | 1099 | int bpf_prog_array_copy(struct bpf_prog_array *old_array, |
Yonghong Song | e87c6bc38 | 2017-10-23 23:53:08 -0700 | [diff] [blame] | 1100 | struct bpf_prog *exclude_prog, |
| 1101 | struct bpf_prog *include_prog, |
| 1102 | struct bpf_prog_array **new_array); |
| 1103 | |
Stanislav Fomichev | 7724121 | 2021-01-27 11:31:39 -0800 | [diff] [blame] | 1104 | /* BPF program asks to bypass CAP_NET_BIND_SERVICE in bind. */ |
| 1105 | #define BPF_RET_BIND_NO_CAP_NET_BIND_SERVICE (1 << 0) |
| 1106 | /* BPF program asks to set CN on the packet. */ |
| 1107 | #define BPF_RET_SET_CN (1 << 0) |
| 1108 | |
| 1109 | #define BPF_PROG_RUN_ARRAY_FLAGS(array, ctx, func, ret_flags) \ |
| 1110 | ({ \ |
| 1111 | struct bpf_prog_array_item *_item; \ |
| 1112 | struct bpf_prog *_prog; \ |
| 1113 | struct bpf_prog_array *_array; \ |
| 1114 | u32 _ret = 1; \ |
| 1115 | u32 func_ret; \ |
| 1116 | migrate_disable(); \ |
| 1117 | rcu_read_lock(); \ |
| 1118 | _array = rcu_dereference(array); \ |
| 1119 | _item = &_array->items[0]; \ |
| 1120 | while ((_prog = READ_ONCE(_item->prog))) { \ |
| 1121 | bpf_cgroup_storage_set(_item->cgroup_storage); \ |
| 1122 | func_ret = func(_prog, ctx); \ |
| 1123 | _ret &= (func_ret & 1); \ |
| 1124 | *(ret_flags) |= (func_ret >> 1); \ |
| 1125 | _item++; \ |
| 1126 | } \ |
| 1127 | rcu_read_unlock(); \ |
| 1128 | migrate_enable(); \ |
| 1129 | _ret; \ |
| 1130 | }) |
| 1131 | |
Yonghong Song | 05a68ce | 2021-03-09 10:50:28 -0800 | [diff] [blame] | 1132 | #define __BPF_PROG_RUN_ARRAY(array, ctx, func, check_non_null, set_cg_storage) \ |
Alexei Starovoitov | 324bda9e6 | 2017-10-02 22:50:21 -0700 | [diff] [blame] | 1133 | ({ \ |
Roman Gushchin | 394e40a | 2018-08-02 14:27:21 -0700 | [diff] [blame] | 1134 | struct bpf_prog_array_item *_item; \ |
| 1135 | struct bpf_prog *_prog; \ |
Yonghong Song | e87c6bc38 | 2017-10-23 23:53:08 -0700 | [diff] [blame] | 1136 | struct bpf_prog_array *_array; \ |
Alexei Starovoitov | 324bda9e6 | 2017-10-02 22:50:21 -0700 | [diff] [blame] | 1137 | u32 _ret = 1; \ |
David Miller | 2a916f2 | 2020-02-24 15:01:46 +0100 | [diff] [blame] | 1138 | migrate_disable(); \ |
Alexei Starovoitov | 324bda9e6 | 2017-10-02 22:50:21 -0700 | [diff] [blame] | 1139 | rcu_read_lock(); \ |
Yonghong Song | e87c6bc38 | 2017-10-23 23:53:08 -0700 | [diff] [blame] | 1140 | _array = rcu_dereference(array); \ |
| 1141 | if (unlikely(check_non_null && !_array))\ |
| 1142 | goto _out; \ |
Roman Gushchin | 394e40a | 2018-08-02 14:27:21 -0700 | [diff] [blame] | 1143 | _item = &_array->items[0]; \ |
| 1144 | while ((_prog = READ_ONCE(_item->prog))) { \ |
Yonghong Song | 05a68ce | 2021-03-09 10:50:28 -0800 | [diff] [blame] | 1145 | if (set_cg_storage) \ |
| 1146 | bpf_cgroup_storage_set(_item->cgroup_storage); \ |
Roman Gushchin | 394e40a | 2018-08-02 14:27:21 -0700 | [diff] [blame] | 1147 | _ret &= func(_prog, ctx); \ |
| 1148 | _item++; \ |
Yonghong Song | e87c6bc38 | 2017-10-23 23:53:08 -0700 | [diff] [blame] | 1149 | } \ |
| 1150 | _out: \ |
Alexei Starovoitov | 324bda9e6 | 2017-10-02 22:50:21 -0700 | [diff] [blame] | 1151 | rcu_read_unlock(); \ |
David Miller | 2a916f2 | 2020-02-24 15:01:46 +0100 | [diff] [blame] | 1152 | migrate_enable(); \ |
Alexei Starovoitov | 324bda9e6 | 2017-10-02 22:50:21 -0700 | [diff] [blame] | 1153 | _ret; \ |
| 1154 | }) |
| 1155 | |
brakmo | 1f52f6c | 2019-05-28 16:59:35 -0700 | [diff] [blame] | 1156 | /* To be used by __cgroup_bpf_run_filter_skb for EGRESS BPF progs |
| 1157 | * so BPF programs can request cwr for TCP packets. |
| 1158 | * |
| 1159 | * Current cgroup skb programs can only return 0 or 1 (0 to drop the |
| 1160 | * packet. This macro changes the behavior so the low order bit |
| 1161 | * indicates whether the packet should be dropped (0) or not (1) |
| 1162 | * and the next bit is a congestion notification bit. This could be |
| 1163 | * used by TCP to call tcp_enter_cwr() |
| 1164 | * |
| 1165 | * Hence, new allowed return values of CGROUP EGRESS BPF programs are: |
| 1166 | * 0: drop packet |
| 1167 | * 1: keep packet |
| 1168 | * 2: drop packet and cn |
| 1169 | * 3: keep packet and cn |
| 1170 | * |
| 1171 | * This macro then converts it to one of the NET_XMIT or an error |
| 1172 | * code that is then interpreted as drop packet (and no cn): |
| 1173 | * 0: NET_XMIT_SUCCESS skb should be transmitted |
| 1174 | * 1: NET_XMIT_DROP skb should be dropped and cn |
| 1175 | * 2: NET_XMIT_CN skb should be transmitted and cn |
| 1176 | * 3: -EPERM skb should be dropped |
| 1177 | */ |
| 1178 | #define BPF_PROG_CGROUP_INET_EGRESS_RUN_ARRAY(array, ctx, func) \ |
| 1179 | ({ \ |
Stanislav Fomichev | 7724121 | 2021-01-27 11:31:39 -0800 | [diff] [blame] | 1180 | u32 _flags = 0; \ |
| 1181 | bool _cn; \ |
| 1182 | u32 _ret; \ |
| 1183 | _ret = BPF_PROG_RUN_ARRAY_FLAGS(array, ctx, func, &_flags); \ |
| 1184 | _cn = _flags & BPF_RET_SET_CN; \ |
brakmo | 1f52f6c | 2019-05-28 16:59:35 -0700 | [diff] [blame] | 1185 | if (_ret) \ |
| 1186 | _ret = (_cn ? NET_XMIT_CN : NET_XMIT_SUCCESS); \ |
| 1187 | else \ |
| 1188 | _ret = (_cn ? NET_XMIT_DROP : -EPERM); \ |
| 1189 | _ret; \ |
| 1190 | }) |
| 1191 | |
Yonghong Song | e87c6bc38 | 2017-10-23 23:53:08 -0700 | [diff] [blame] | 1192 | #define BPF_PROG_RUN_ARRAY(array, ctx, func) \ |
Yonghong Song | 05a68ce | 2021-03-09 10:50:28 -0800 | [diff] [blame] | 1193 | __BPF_PROG_RUN_ARRAY(array, ctx, func, false, true) |
Yonghong Song | e87c6bc38 | 2017-10-23 23:53:08 -0700 | [diff] [blame] | 1194 | |
| 1195 | #define BPF_PROG_RUN_ARRAY_CHECK(array, ctx, func) \ |
Yonghong Song | 05a68ce | 2021-03-09 10:50:28 -0800 | [diff] [blame] | 1196 | __BPF_PROG_RUN_ARRAY(array, ctx, func, true, false) |
Yonghong Song | e87c6bc38 | 2017-10-23 23:53:08 -0700 | [diff] [blame] | 1197 | |
Alexei Starovoitov | 89aa075 | 2014-12-01 15:06:35 -0800 | [diff] [blame] | 1198 | #ifdef CONFIG_BPF_SYSCALL |
Alexei Starovoitov | b121d1e | 2016-03-07 21:57:13 -0800 | [diff] [blame] | 1199 | DECLARE_PER_CPU(int, bpf_prog_active); |
Song Liu | d46edd6 | 2020-04-30 00:15:04 -0700 | [diff] [blame] | 1200 | extern struct mutex bpf_stats_enabled_mutex; |
Alexei Starovoitov | b121d1e | 2016-03-07 21:57:13 -0800 | [diff] [blame] | 1201 | |
Thomas Gleixner | c518cfa | 2020-02-24 15:01:47 +0100 | [diff] [blame] | 1202 | /* |
| 1203 | * Block execution of BPF programs attached to instrumentation (perf, |
| 1204 | * kprobes, tracepoints) to prevent deadlocks on map operations as any of |
| 1205 | * these events can happen inside a region which holds a map bucket lock |
| 1206 | * and can deadlock on it. |
| 1207 | * |
| 1208 | * Use the preemption safe inc/dec variants on RT because migrate disable |
| 1209 | * is preemptible on RT and preemption in the middle of the RMW operation |
| 1210 | * might lead to inconsistent state. Use the raw variants for non RT |
| 1211 | * kernels as migrate_disable() maps to preempt_disable() so the slightly |
| 1212 | * more expensive save operation can be avoided. |
| 1213 | */ |
| 1214 | static inline void bpf_disable_instrumentation(void) |
| 1215 | { |
| 1216 | migrate_disable(); |
| 1217 | if (IS_ENABLED(CONFIG_PREEMPT_RT)) |
| 1218 | this_cpu_inc(bpf_prog_active); |
| 1219 | else |
| 1220 | __this_cpu_inc(bpf_prog_active); |
| 1221 | } |
| 1222 | |
| 1223 | static inline void bpf_enable_instrumentation(void) |
| 1224 | { |
| 1225 | if (IS_ENABLED(CONFIG_PREEMPT_RT)) |
| 1226 | this_cpu_dec(bpf_prog_active); |
| 1227 | else |
| 1228 | __this_cpu_dec(bpf_prog_active); |
| 1229 | migrate_enable(); |
| 1230 | } |
| 1231 | |
Chenbo Feng | f66e448 | 2017-10-18 13:00:26 -0700 | [diff] [blame] | 1232 | extern const struct file_operations bpf_map_fops; |
| 1233 | extern const struct file_operations bpf_prog_fops; |
Yonghong Song | 367ec3e | 2020-05-09 10:59:06 -0700 | [diff] [blame] | 1234 | extern const struct file_operations bpf_iter_fops; |
Chenbo Feng | f66e448 | 2017-10-18 13:00:26 -0700 | [diff] [blame] | 1235 | |
Alexei Starovoitov | 91cc1a9 | 2019-11-14 10:57:15 -0800 | [diff] [blame] | 1236 | #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \ |
Jakub Kicinski | 7de16e3 | 2017-10-16 16:40:53 -0700 | [diff] [blame] | 1237 | extern const struct bpf_prog_ops _name ## _prog_ops; \ |
| 1238 | extern const struct bpf_verifier_ops _name ## _verifier_ops; |
Johannes Berg | 40077e0 | 2017-04-11 15:34:58 +0200 | [diff] [blame] | 1239 | #define BPF_MAP_TYPE(_id, _ops) \ |
| 1240 | extern const struct bpf_map_ops _ops; |
Andrii Nakryiko | f2e10bf | 2020-04-28 17:16:08 -0700 | [diff] [blame] | 1241 | #define BPF_LINK_TYPE(_id, _name) |
Johannes Berg | be9370a | 2017-04-11 15:34:57 +0200 | [diff] [blame] | 1242 | #include <linux/bpf_types.h> |
| 1243 | #undef BPF_PROG_TYPE |
Johannes Berg | 40077e0 | 2017-04-11 15:34:58 +0200 | [diff] [blame] | 1244 | #undef BPF_MAP_TYPE |
Andrii Nakryiko | f2e10bf | 2020-04-28 17:16:08 -0700 | [diff] [blame] | 1245 | #undef BPF_LINK_TYPE |
Daniel Borkmann | 0fc174d | 2015-03-01 12:31:44 +0100 | [diff] [blame] | 1246 | |
Jakub Kicinski | ab3f006 | 2017-11-03 13:56:17 -0700 | [diff] [blame] | 1247 | extern const struct bpf_prog_ops bpf_offload_prog_ops; |
Jakub Kicinski | 4f9218a | 2017-10-16 16:40:55 -0700 | [diff] [blame] | 1248 | extern const struct bpf_verifier_ops tc_cls_act_analyzer_ops; |
| 1249 | extern const struct bpf_verifier_ops xdp_analyzer_ops; |
| 1250 | |
Alexei Starovoitov | 09756af | 2014-09-26 00:17:00 -0700 | [diff] [blame] | 1251 | struct bpf_prog *bpf_prog_get(u32 ufd); |
Jakub Kicinski | 248f346 | 2017-11-03 13:56:20 -0700 | [diff] [blame] | 1252 | struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, enum bpf_prog_type type, |
Jakub Kicinski | 288b3de5 | 2017-11-20 15:21:54 -0800 | [diff] [blame] | 1253 | bool attach_drv); |
Andrii Nakryiko | 85192dbf | 2019-11-17 09:28:03 -0800 | [diff] [blame] | 1254 | void bpf_prog_add(struct bpf_prog *prog, int i); |
Daniel Borkmann | c540594 | 2016-11-09 22:02:34 +0100 | [diff] [blame] | 1255 | void bpf_prog_sub(struct bpf_prog *prog, int i); |
Andrii Nakryiko | 85192dbf | 2019-11-17 09:28:03 -0800 | [diff] [blame] | 1256 | void bpf_prog_inc(struct bpf_prog *prog); |
John Fastabend | a6f6df6 | 2017-08-15 22:32:22 -0700 | [diff] [blame] | 1257 | struct bpf_prog * __must_check bpf_prog_inc_not_zero(struct bpf_prog *prog); |
Daniel Borkmann | 61e021f3 | 2015-03-02 15:21:55 +0100 | [diff] [blame] | 1258 | void bpf_prog_put(struct bpf_prog *prog); |
| 1259 | |
Jakub Kicinski | ad8ad79 | 2017-12-27 18:39:07 -0800 | [diff] [blame] | 1260 | void bpf_prog_free_id(struct bpf_prog *prog, bool do_idr_lock); |
Jakub Kicinski | a388457 | 2018-01-11 20:29:09 -0800 | [diff] [blame] | 1261 | void bpf_map_free_id(struct bpf_map *map, bool do_idr_lock); |
Jakub Kicinski | ad8ad79 | 2017-12-27 18:39:07 -0800 | [diff] [blame] | 1262 | |
Martin KaFai Lau | 1ed4d92 | 2020-02-25 15:04:21 -0800 | [diff] [blame] | 1263 | struct bpf_map *bpf_map_get(u32 ufd); |
Daniel Borkmann | c9da161 | 2015-11-24 21:28:15 +0100 | [diff] [blame] | 1264 | struct bpf_map *bpf_map_get_with_uref(u32 ufd); |
Daniel Borkmann | c210129 | 2015-10-29 14:58:07 +0100 | [diff] [blame] | 1265 | struct bpf_map *__bpf_map_get(struct fd f); |
Andrii Nakryiko | 1e0bd5a | 2019-11-17 09:28:02 -0800 | [diff] [blame] | 1266 | void bpf_map_inc(struct bpf_map *map); |
| 1267 | void bpf_map_inc_with_uref(struct bpf_map *map); |
| 1268 | struct bpf_map * __must_check bpf_map_inc_not_zero(struct bpf_map *map); |
Daniel Borkmann | c9da161 | 2015-11-24 21:28:15 +0100 | [diff] [blame] | 1269 | void bpf_map_put_with_uref(struct bpf_map *map); |
Daniel Borkmann | 61e021f3 | 2015-03-02 15:21:55 +0100 | [diff] [blame] | 1270 | void bpf_map_put(struct bpf_map *map); |
Daniel Borkmann | 196e8ca | 2019-11-20 23:04:44 +0100 | [diff] [blame] | 1271 | void *bpf_map_area_alloc(u64 size, int numa_node); |
| 1272 | void *bpf_map_area_mmapable_alloc(u64 size, int numa_node); |
Daniel Borkmann | d407bd2 | 2017-01-18 15:14:17 +0100 | [diff] [blame] | 1273 | void bpf_map_area_free(void *base); |
Jakub Kicinski | bd47564 | 2018-01-11 20:29:06 -0800 | [diff] [blame] | 1274 | void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr); |
Brian Vazquez | cb4d03a | 2020-01-15 10:43:01 -0800 | [diff] [blame] | 1275 | int generic_map_lookup_batch(struct bpf_map *map, |
| 1276 | const union bpf_attr *attr, |
| 1277 | union bpf_attr __user *uattr); |
Brian Vazquez | aa2e93b | 2020-01-15 10:43:02 -0800 | [diff] [blame] | 1278 | int generic_map_update_batch(struct bpf_map *map, |
| 1279 | const union bpf_attr *attr, |
| 1280 | union bpf_attr __user *uattr); |
| 1281 | int generic_map_delete_batch(struct bpf_map *map, |
| 1282 | const union bpf_attr *attr, |
| 1283 | union bpf_attr __user *uattr); |
Yonghong Song | 6086d29 | 2020-05-09 10:59:09 -0700 | [diff] [blame] | 1284 | struct bpf_map *bpf_map_get_curr_or_next(u32 *id); |
Alexei Starovoitov | a228a64 | 2020-07-01 18:10:18 -0700 | [diff] [blame] | 1285 | struct bpf_prog *bpf_prog_get_curr_or_next(u32 *id); |
Daniel Borkmann | 61e021f3 | 2015-03-02 15:21:55 +0100 | [diff] [blame] | 1286 | |
Roman Gushchin | 48edc1f | 2020-12-01 13:58:32 -0800 | [diff] [blame] | 1287 | #ifdef CONFIG_MEMCG_KMEM |
| 1288 | void *bpf_map_kmalloc_node(const struct bpf_map *map, size_t size, gfp_t flags, |
| 1289 | int node); |
| 1290 | void *bpf_map_kzalloc(const struct bpf_map *map, size_t size, gfp_t flags); |
| 1291 | void __percpu *bpf_map_alloc_percpu(const struct bpf_map *map, size_t size, |
| 1292 | size_t align, gfp_t flags); |
| 1293 | #else |
| 1294 | static inline void * |
| 1295 | bpf_map_kmalloc_node(const struct bpf_map *map, size_t size, gfp_t flags, |
| 1296 | int node) |
| 1297 | { |
| 1298 | return kmalloc_node(size, flags, node); |
| 1299 | } |
| 1300 | |
| 1301 | static inline void * |
| 1302 | bpf_map_kzalloc(const struct bpf_map *map, size_t size, gfp_t flags) |
| 1303 | { |
| 1304 | return kzalloc(size, flags); |
| 1305 | } |
| 1306 | |
| 1307 | static inline void __percpu * |
| 1308 | bpf_map_alloc_percpu(const struct bpf_map *map, size_t size, size_t align, |
| 1309 | gfp_t flags) |
| 1310 | { |
| 1311 | return __alloc_percpu_gfp(size, align, flags); |
| 1312 | } |
| 1313 | #endif |
| 1314 | |
Alexei Starovoitov | 1be7f75 | 2015-10-07 22:23:21 -0700 | [diff] [blame] | 1315 | extern int sysctl_unprivileged_bpf_disabled; |
| 1316 | |
Alexei Starovoitov | 2c78ee8 | 2020-05-13 16:03:54 -0700 | [diff] [blame] | 1317 | static inline bool bpf_allow_ptr_leaks(void) |
| 1318 | { |
| 1319 | return perfmon_capable(); |
| 1320 | } |
| 1321 | |
Andrei Matei | 01f810a | 2021-02-06 20:10:24 -0500 | [diff] [blame] | 1322 | static inline bool bpf_allow_uninit_stack(void) |
| 1323 | { |
| 1324 | return perfmon_capable(); |
| 1325 | } |
| 1326 | |
Andrey Ignatov | 41c48f3 | 2020-06-19 14:11:43 -0700 | [diff] [blame] | 1327 | static inline bool bpf_allow_ptr_to_map_access(void) |
| 1328 | { |
| 1329 | return perfmon_capable(); |
| 1330 | } |
| 1331 | |
Alexei Starovoitov | 2c78ee8 | 2020-05-13 16:03:54 -0700 | [diff] [blame] | 1332 | static inline bool bpf_bypass_spec_v1(void) |
| 1333 | { |
| 1334 | return perfmon_capable(); |
| 1335 | } |
| 1336 | |
| 1337 | static inline bool bpf_bypass_spec_v4(void) |
| 1338 | { |
| 1339 | return perfmon_capable(); |
| 1340 | } |
| 1341 | |
Chenbo Feng | 6e71b04 | 2017-10-18 13:00:22 -0700 | [diff] [blame] | 1342 | int bpf_map_new_fd(struct bpf_map *map, int flags); |
Daniel Borkmann | b219775 | 2015-10-29 14:58:09 +0100 | [diff] [blame] | 1343 | int bpf_prog_new_fd(struct bpf_prog *prog); |
| 1344 | |
Andrii Nakryiko | f2e10bf | 2020-04-28 17:16:08 -0700 | [diff] [blame] | 1345 | void bpf_link_init(struct bpf_link *link, enum bpf_link_type type, |
Andrii Nakryiko | a3b80e1 | 2020-04-28 17:16:06 -0700 | [diff] [blame] | 1346 | const struct bpf_link_ops *ops, struct bpf_prog *prog); |
| 1347 | int bpf_link_prime(struct bpf_link *link, struct bpf_link_primer *primer); |
| 1348 | int bpf_link_settle(struct bpf_link_primer *primer); |
| 1349 | void bpf_link_cleanup(struct bpf_link_primer *primer); |
Andrii Nakryiko | 70ed506 | 2020-03-02 20:31:57 -0800 | [diff] [blame] | 1350 | void bpf_link_inc(struct bpf_link *link); |
| 1351 | void bpf_link_put(struct bpf_link *link); |
| 1352 | int bpf_link_new_fd(struct bpf_link *link); |
Andrii Nakryiko | babf316 | 2020-03-09 16:10:51 -0700 | [diff] [blame] | 1353 | struct file *bpf_link_new_file(struct bpf_link *link, int *reserved_fd); |
Andrii Nakryiko | 70ed506 | 2020-03-02 20:31:57 -0800 | [diff] [blame] | 1354 | struct bpf_link *bpf_link_get_from_fd(u32 ufd); |
| 1355 | |
Daniel Borkmann | b219775 | 2015-10-29 14:58:09 +0100 | [diff] [blame] | 1356 | int bpf_obj_pin_user(u32 ufd, const char __user *pathname); |
Chenbo Feng | 6e71b04 | 2017-10-18 13:00:22 -0700 | [diff] [blame] | 1357 | int bpf_obj_get_user(const char __user *pathname, int flags); |
Daniel Borkmann | b219775 | 2015-10-29 14:58:09 +0100 | [diff] [blame] | 1358 | |
Yonghong Song | 21aef70 | 2020-05-13 11:02:16 -0700 | [diff] [blame] | 1359 | #define BPF_ITER_FUNC_PREFIX "bpf_iter_" |
Yonghong Song | e5158d9 | 2020-05-09 10:59:07 -0700 | [diff] [blame] | 1360 | #define DEFINE_BPF_ITER_FUNC(target, args...) \ |
Yonghong Song | 21aef70 | 2020-05-13 11:02:16 -0700 | [diff] [blame] | 1361 | extern int bpf_iter_ ## target(args); \ |
| 1362 | int __init bpf_iter_ ## target(args) { return 0; } |
Yonghong Song | 15d83c4 | 2020-05-09 10:59:00 -0700 | [diff] [blame] | 1363 | |
Yonghong Song | f9c7927 | 2020-07-23 11:41:10 -0700 | [diff] [blame] | 1364 | struct bpf_iter_aux_info { |
Yonghong Song | a5cbe05 | 2020-07-23 11:41:12 -0700 | [diff] [blame] | 1365 | struct bpf_map *map; |
Yonghong Song | f9c7927 | 2020-07-23 11:41:10 -0700 | [diff] [blame] | 1366 | }; |
| 1367 | |
Yonghong Song | 5e7b302 | 2020-08-04 22:50:56 -0700 | [diff] [blame] | 1368 | typedef int (*bpf_iter_attach_target_t)(struct bpf_prog *prog, |
| 1369 | union bpf_iter_link_info *linfo, |
| 1370 | struct bpf_iter_aux_info *aux); |
| 1371 | typedef void (*bpf_iter_detach_target_t)(struct bpf_iter_aux_info *aux); |
Yonghong Song | 6b0a249 | 2020-08-21 11:44:18 -0700 | [diff] [blame] | 1372 | typedef void (*bpf_iter_show_fdinfo_t) (const struct bpf_iter_aux_info *aux, |
| 1373 | struct seq_file *seq); |
| 1374 | typedef int (*bpf_iter_fill_link_info_t)(const struct bpf_iter_aux_info *aux, |
| 1375 | struct bpf_link_info *info); |
Yonghong Song | a5cbe05 | 2020-07-23 11:41:12 -0700 | [diff] [blame] | 1376 | |
Yonghong Song | cf83b2d | 2020-10-27 23:10:54 -0700 | [diff] [blame] | 1377 | enum bpf_iter_feature { |
| 1378 | BPF_ITER_RESCHED = BIT(0), |
| 1379 | }; |
| 1380 | |
Yonghong Song | 3c32cc1 | 2020-05-13 11:02:21 -0700 | [diff] [blame] | 1381 | #define BPF_ITER_CTX_ARG_MAX 2 |
Yonghong Song | ae24345 | 2020-05-09 10:58:59 -0700 | [diff] [blame] | 1382 | struct bpf_iter_reg { |
| 1383 | const char *target; |
Yonghong Song | 5e7b302 | 2020-08-04 22:50:56 -0700 | [diff] [blame] | 1384 | bpf_iter_attach_target_t attach_target; |
| 1385 | bpf_iter_detach_target_t detach_target; |
Yonghong Song | 6b0a249 | 2020-08-21 11:44:18 -0700 | [diff] [blame] | 1386 | bpf_iter_show_fdinfo_t show_fdinfo; |
| 1387 | bpf_iter_fill_link_info_t fill_link_info; |
Yonghong Song | 3c32cc1 | 2020-05-13 11:02:21 -0700 | [diff] [blame] | 1388 | u32 ctx_arg_info_size; |
Yonghong Song | cf83b2d | 2020-10-27 23:10:54 -0700 | [diff] [blame] | 1389 | u32 feature; |
Yonghong Song | 3c32cc1 | 2020-05-13 11:02:21 -0700 | [diff] [blame] | 1390 | struct bpf_ctx_arg_aux ctx_arg_info[BPF_ITER_CTX_ARG_MAX]; |
Yonghong Song | 14fc6bd6 | 2020-07-23 11:41:09 -0700 | [diff] [blame] | 1391 | const struct bpf_iter_seq_info *seq_info; |
Yonghong Song | ae24345 | 2020-05-09 10:58:59 -0700 | [diff] [blame] | 1392 | }; |
| 1393 | |
Yonghong Song | e5158d9 | 2020-05-09 10:59:07 -0700 | [diff] [blame] | 1394 | struct bpf_iter_meta { |
| 1395 | __bpf_md_ptr(struct seq_file *, seq); |
| 1396 | u64 session_id; |
| 1397 | u64 seq_num; |
| 1398 | }; |
| 1399 | |
Yonghong Song | a5cbe05 | 2020-07-23 11:41:12 -0700 | [diff] [blame] | 1400 | struct bpf_iter__bpf_map_elem { |
| 1401 | __bpf_md_ptr(struct bpf_iter_meta *, meta); |
| 1402 | __bpf_md_ptr(struct bpf_map *, map); |
| 1403 | __bpf_md_ptr(void *, key); |
| 1404 | __bpf_md_ptr(void *, value); |
| 1405 | }; |
| 1406 | |
Yonghong Song | 15172a4 | 2020-05-13 11:02:19 -0700 | [diff] [blame] | 1407 | int bpf_iter_reg_target(const struct bpf_iter_reg *reg_info); |
Yonghong Song | ab2ee4f | 2020-05-13 11:02:20 -0700 | [diff] [blame] | 1408 | void bpf_iter_unreg_target(const struct bpf_iter_reg *reg_info); |
Yonghong Song | 15d83c4 | 2020-05-09 10:59:00 -0700 | [diff] [blame] | 1409 | bool bpf_iter_prog_supported(struct bpf_prog *prog); |
Yonghong Song | de4e05c | 2020-05-09 10:59:01 -0700 | [diff] [blame] | 1410 | int bpf_iter_link_attach(const union bpf_attr *attr, struct bpf_prog *prog); |
Yonghong Song | ac51d99 | 2020-05-09 10:59:05 -0700 | [diff] [blame] | 1411 | int bpf_iter_new_fd(struct bpf_link *link); |
Yonghong Song | 367ec3e | 2020-05-09 10:59:06 -0700 | [diff] [blame] | 1412 | bool bpf_link_is_iter(struct bpf_link *link); |
Yonghong Song | e5158d9 | 2020-05-09 10:59:07 -0700 | [diff] [blame] | 1413 | struct bpf_prog *bpf_iter_get_info(struct bpf_iter_meta *meta, bool in_stop); |
| 1414 | int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx); |
Yonghong Song | b76f222 | 2020-08-21 11:44:19 -0700 | [diff] [blame] | 1415 | void bpf_iter_map_show_fdinfo(const struct bpf_iter_aux_info *aux, |
| 1416 | struct seq_file *seq); |
| 1417 | int bpf_iter_map_fill_link_info(const struct bpf_iter_aux_info *aux, |
| 1418 | struct bpf_link_info *info); |
Yonghong Song | ae24345 | 2020-05-09 10:58:59 -0700 | [diff] [blame] | 1419 | |
Yonghong Song | 314ee05 | 2021-02-26 12:49:27 -0800 | [diff] [blame] | 1420 | int map_set_for_each_callback_args(struct bpf_verifier_env *env, |
| 1421 | struct bpf_func_state *caller, |
| 1422 | struct bpf_func_state *callee); |
| 1423 | |
Alexei Starovoitov | 15a07b3 | 2016-02-01 22:39:55 -0800 | [diff] [blame] | 1424 | int bpf_percpu_hash_copy(struct bpf_map *map, void *key, void *value); |
| 1425 | int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value); |
| 1426 | int bpf_percpu_hash_update(struct bpf_map *map, void *key, void *value, |
| 1427 | u64 flags); |
| 1428 | int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value, |
| 1429 | u64 flags); |
Daniel Borkmann | d056a78 | 2016-06-15 22:47:13 +0200 | [diff] [blame] | 1430 | |
Alexei Starovoitov | 557c0c6 | 2016-03-07 21:57:17 -0800 | [diff] [blame] | 1431 | int bpf_stackmap_copy(struct bpf_map *map, void *key, void *value); |
Alexei Starovoitov | 15a07b3 | 2016-02-01 22:39:55 -0800 | [diff] [blame] | 1432 | |
Daniel Borkmann | d056a78 | 2016-06-15 22:47:13 +0200 | [diff] [blame] | 1433 | int bpf_fd_array_map_update_elem(struct bpf_map *map, struct file *map_file, |
| 1434 | void *key, void *value, u64 map_flags); |
Martin KaFai Lau | 14dc6f0 | 2017-06-27 23:08:34 -0700 | [diff] [blame] | 1435 | int bpf_fd_array_map_lookup_elem(struct bpf_map *map, void *key, u32 *value); |
Martin KaFai Lau | bcc6b1b | 2017-03-22 10:00:34 -0700 | [diff] [blame] | 1436 | int bpf_fd_htab_map_update_elem(struct bpf_map *map, struct file *map_file, |
| 1437 | void *key, void *value, u64 map_flags); |
Martin KaFai Lau | 14dc6f0 | 2017-06-27 23:08:34 -0700 | [diff] [blame] | 1438 | int bpf_fd_htab_map_lookup_elem(struct bpf_map *map, void *key, u32 *value); |
Daniel Borkmann | d056a78 | 2016-06-15 22:47:13 +0200 | [diff] [blame] | 1439 | |
Chenbo Feng | 6e71b04 | 2017-10-18 13:00:22 -0700 | [diff] [blame] | 1440 | int bpf_get_file_flag(int flags); |
Martin KaFai Lau | dcab51f | 2018-05-22 15:03:31 -0700 | [diff] [blame] | 1441 | int bpf_check_uarg_tail_zero(void __user *uaddr, size_t expected_size, |
| 1442 | size_t actual_size); |
Chenbo Feng | 6e71b04 | 2017-10-18 13:00:22 -0700 | [diff] [blame] | 1443 | |
Alexei Starovoitov | 15a07b3 | 2016-02-01 22:39:55 -0800 | [diff] [blame] | 1444 | /* memcpy that is used with 8-byte aligned pointers, power-of-8 size and |
| 1445 | * forced to use 'long' read/writes to try to atomically copy long counters. |
| 1446 | * Best-effort only. No barriers here, since it _will_ race with concurrent |
| 1447 | * updates from BPF programs. Called from bpf syscall and mostly used with |
| 1448 | * size 8 or 16 bytes, so ask compiler to inline it. |
| 1449 | */ |
| 1450 | static inline void bpf_long_memcpy(void *dst, const void *src, u32 size) |
| 1451 | { |
| 1452 | const long *lsrc = src; |
| 1453 | long *ldst = dst; |
| 1454 | |
| 1455 | size /= sizeof(long); |
| 1456 | while (size--) |
| 1457 | *ldst++ = *lsrc++; |
| 1458 | } |
| 1459 | |
Daniel Borkmann | 61e021f3 | 2015-03-02 15:21:55 +0100 | [diff] [blame] | 1460 | /* verify correctness of eBPF program */ |
Yonghong Song | 838e969 | 2018-11-19 15:29:11 -0800 | [diff] [blame] | 1461 | int bpf_check(struct bpf_prog **fp, union bpf_attr *attr, |
| 1462 | union bpf_attr __user *uattr); |
Andrii Nakryiko | a643bff | 2021-01-11 23:55:14 -0800 | [diff] [blame] | 1463 | |
| 1464 | #ifndef CONFIG_BPF_JIT_ALWAYS_ON |
Alexei Starovoitov | 1ea47e0 | 2017-12-14 17:55:13 -0800 | [diff] [blame] | 1465 | void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth); |
Andrii Nakryiko | a643bff | 2021-01-11 23:55:14 -0800 | [diff] [blame] | 1466 | #endif |
John Fastabend | 46f55cf | 2017-07-17 21:56:48 -0700 | [diff] [blame] | 1467 | |
Alan Maguire | 76654e6 | 2020-09-28 12:31:03 +0100 | [diff] [blame] | 1468 | struct btf *bpf_get_btf_vmlinux(void); |
| 1469 | |
John Fastabend | 46f55cf | 2017-07-17 21:56:48 -0700 | [diff] [blame] | 1470 | /* Map specifics */ |
Jesper Dangaard Brouer | 67f29e0 | 2018-05-24 16:45:46 +0200 | [diff] [blame] | 1471 | struct xdp_buff; |
Toshiaki Makita | 6d5fc19 | 2018-06-14 11:07:42 +0900 | [diff] [blame] | 1472 | struct sk_buff; |
Björn Töpel | e6a4750 | 2021-03-08 12:29:06 +0100 | [diff] [blame] | 1473 | struct bpf_dtab_netdev; |
| 1474 | struct bpf_cpu_map_entry; |
Jesper Dangaard Brouer | 67f29e0 | 2018-05-24 16:45:46 +0200 | [diff] [blame] | 1475 | |
Toke Høiland-Jørgensen | 1d23388 | 2020-01-16 16:14:45 +0100 | [diff] [blame] | 1476 | void __dev_flush(void); |
| 1477 | int dev_xdp_enqueue(struct net_device *dev, struct xdp_buff *xdp, |
| 1478 | struct net_device *dev_rx); |
Jesper Dangaard Brouer | 38edddb | 2018-05-24 16:45:57 +0200 | [diff] [blame] | 1479 | int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_buff *xdp, |
| 1480 | struct net_device *dev_rx); |
Toshiaki Makita | 6d5fc19 | 2018-06-14 11:07:42 +0900 | [diff] [blame] | 1481 | int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, struct sk_buff *skb, |
| 1482 | struct bpf_prog *xdp_prog); |
David Ahern | fbee97f | 2020-05-29 16:07:13 -0600 | [diff] [blame] | 1483 | bool dev_map_can_have_prog(struct bpf_map *map); |
John Fastabend | 46f55cf | 2017-07-17 21:56:48 -0700 | [diff] [blame] | 1484 | |
Björn Töpel | cdfafe9 | 2019-12-19 07:10:04 +0100 | [diff] [blame] | 1485 | void __cpu_map_flush(void); |
Jesper Dangaard Brouer | 9c270af | 2017-10-16 12:19:34 +0200 | [diff] [blame] | 1486 | int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_buff *xdp, |
| 1487 | struct net_device *dev_rx); |
Lorenzo Bianconi | 9216477 | 2020-07-14 15:56:38 +0200 | [diff] [blame] | 1488 | bool cpu_map_prog_allowed(struct bpf_map *map); |
Jesper Dangaard Brouer | 9c270af | 2017-10-16 12:19:34 +0200 | [diff] [blame] | 1489 | |
Martin KaFai Lau | 96eabe7 | 2017-08-18 11:28:00 -0700 | [diff] [blame] | 1490 | /* Return map's numa specified by userspace */ |
| 1491 | static inline int bpf_map_attr_numa_node(const union bpf_attr *attr) |
| 1492 | { |
| 1493 | return (attr->map_flags & BPF_F_NUMA_NODE) ? |
| 1494 | attr->numa_node : NUMA_NO_NODE; |
| 1495 | } |
| 1496 | |
Al Viro | 040ee69 | 2017-12-02 20:20:38 -0500 | [diff] [blame] | 1497 | struct bpf_prog *bpf_prog_get_type_path(const char *name, enum bpf_prog_type type); |
Martin KaFai Lau | 5dc4c4b | 2018-08-08 01:01:24 -0700 | [diff] [blame] | 1498 | int array_map_alloc_check(union bpf_attr *attr); |
Al Viro | 040ee69 | 2017-12-02 20:20:38 -0500 | [diff] [blame] | 1499 | |
Stanislav Fomichev | c695865 | 2019-04-11 09:12:02 -0700 | [diff] [blame] | 1500 | int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr, |
| 1501 | union bpf_attr __user *uattr); |
| 1502 | int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr, |
| 1503 | union bpf_attr __user *uattr); |
KP Singh | da00d2f | 2020-03-04 20:18:52 +0100 | [diff] [blame] | 1504 | int bpf_prog_test_run_tracing(struct bpf_prog *prog, |
| 1505 | const union bpf_attr *kattr, |
| 1506 | union bpf_attr __user *uattr); |
Stanislav Fomichev | c695865 | 2019-04-11 09:12:02 -0700 | [diff] [blame] | 1507 | int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog, |
| 1508 | const union bpf_attr *kattr, |
| 1509 | union bpf_attr __user *uattr); |
Song Liu | 1b4d60e | 2020-09-25 13:54:29 -0700 | [diff] [blame] | 1510 | int bpf_prog_test_run_raw_tp(struct bpf_prog *prog, |
| 1511 | const union bpf_attr *kattr, |
| 1512 | union bpf_attr __user *uattr); |
Lorenz Bauer | 7c32e8f | 2021-03-03 10:18:13 +0000 | [diff] [blame] | 1513 | int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog, |
| 1514 | const union bpf_attr *kattr, |
| 1515 | union bpf_attr __user *uattr); |
Alexei Starovoitov | 9e15db6 | 2019-10-15 20:25:00 -0700 | [diff] [blame] | 1516 | bool btf_ctx_access(int off, int size, enum bpf_access_type type, |
| 1517 | const struct bpf_prog *prog, |
| 1518 | struct bpf_insn_access_aux *info); |
Andrii Nakryiko | 22dc4a0 | 2020-12-03 12:46:29 -0800 | [diff] [blame] | 1519 | int btf_struct_access(struct bpf_verifier_log *log, const struct btf *btf, |
Alexei Starovoitov | 9e15db6 | 2019-10-15 20:25:00 -0700 | [diff] [blame] | 1520 | const struct btf_type *t, int off, int size, |
| 1521 | enum bpf_access_type atype, |
| 1522 | u32 *next_btf_id); |
Jiri Olsa | faaf4a7 | 2020-08-25 21:21:18 +0200 | [diff] [blame] | 1523 | bool btf_struct_ids_match(struct bpf_verifier_log *log, |
Andrii Nakryiko | 22dc4a0 | 2020-12-03 12:46:29 -0800 | [diff] [blame] | 1524 | const struct btf *btf, u32 id, int off, |
| 1525 | const struct btf *need_btf, u32 need_type_id); |
Alexei Starovoitov | 9e15db6 | 2019-10-15 20:25:00 -0700 | [diff] [blame] | 1526 | |
Alexei Starovoitov | fec56f5 | 2019-11-14 10:57:04 -0800 | [diff] [blame] | 1527 | int btf_distill_func_proto(struct bpf_verifier_log *log, |
| 1528 | struct btf *btf, |
| 1529 | const struct btf_type *func_proto, |
| 1530 | const char *func_name, |
| 1531 | struct btf_func_model *m); |
| 1532 | |
Alexei Starovoitov | 51c39bb | 2020-01-09 22:41:20 -0800 | [diff] [blame] | 1533 | struct bpf_reg_state; |
| 1534 | int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, |
| 1535 | struct bpf_reg_state *regs); |
| 1536 | int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog, |
| 1537 | struct bpf_reg_state *reg); |
Toke Høiland-Jørgensen | efc6815 | 2020-09-25 23:25:01 +0200 | [diff] [blame] | 1538 | int btf_check_type_match(struct bpf_verifier_log *log, const struct bpf_prog *prog, |
Alexei Starovoitov | be8704f | 2020-01-20 16:53:46 -0800 | [diff] [blame] | 1539 | struct btf *btf, const struct btf_type *t); |
Alexei Starovoitov | 8c1b6e6 | 2019-11-14 10:57:16 -0800 | [diff] [blame] | 1540 | |
Björn Töpel | 7e6897f | 2019-12-13 18:51:09 +0100 | [diff] [blame] | 1541 | struct bpf_prog *bpf_prog_by_id(u32 id); |
Alexei Starovoitov | 005142b | 2020-08-18 21:27:56 -0700 | [diff] [blame] | 1542 | struct bpf_link *bpf_link_by_id(u32 id); |
Björn Töpel | 7e6897f | 2019-12-13 18:51:09 +0100 | [diff] [blame] | 1543 | |
Stanislav Fomichev | 6890896 | 2020-04-24 16:59:41 -0700 | [diff] [blame] | 1544 | const struct bpf_func_proto *bpf_base_func_proto(enum bpf_func_id func_id); |
Song Liu | a10787e | 2021-02-25 15:43:14 -0800 | [diff] [blame] | 1545 | void bpf_task_storage_free(struct task_struct *task); |
Jesper Dangaard Brouer | 9c270af | 2017-10-16 12:19:34 +0200 | [diff] [blame] | 1546 | #else /* !CONFIG_BPF_SYSCALL */ |
Daniel Borkmann | 0fc174d | 2015-03-01 12:31:44 +0100 | [diff] [blame] | 1547 | static inline struct bpf_prog *bpf_prog_get(u32 ufd) |
| 1548 | { |
| 1549 | return ERR_PTR(-EOPNOTSUPP); |
| 1550 | } |
| 1551 | |
Jakub Kicinski | 248f346 | 2017-11-03 13:56:20 -0700 | [diff] [blame] | 1552 | static inline struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, |
| 1553 | enum bpf_prog_type type, |
Jakub Kicinski | 288b3de5 | 2017-11-20 15:21:54 -0800 | [diff] [blame] | 1554 | bool attach_drv) |
Jakub Kicinski | 248f346 | 2017-11-03 13:56:20 -0700 | [diff] [blame] | 1555 | { |
| 1556 | return ERR_PTR(-EOPNOTSUPP); |
| 1557 | } |
| 1558 | |
Andrii Nakryiko | 85192dbf | 2019-11-17 09:28:03 -0800 | [diff] [blame] | 1559 | static inline void bpf_prog_add(struct bpf_prog *prog, int i) |
Brenden Blanco | cc2e0b3 | 2016-07-20 07:55:52 -0700 | [diff] [blame] | 1560 | { |
Brenden Blanco | cc2e0b3 | 2016-07-20 07:55:52 -0700 | [diff] [blame] | 1561 | } |
Daniel Borkmann | 113214b | 2016-06-30 17:24:44 +0200 | [diff] [blame] | 1562 | |
Daniel Borkmann | c540594 | 2016-11-09 22:02:34 +0100 | [diff] [blame] | 1563 | static inline void bpf_prog_sub(struct bpf_prog *prog, int i) |
| 1564 | { |
| 1565 | } |
| 1566 | |
Daniel Borkmann | 0fc174d | 2015-03-01 12:31:44 +0100 | [diff] [blame] | 1567 | static inline void bpf_prog_put(struct bpf_prog *prog) |
| 1568 | { |
| 1569 | } |
Daniel Borkmann | 6d67942dd | 2016-11-19 01:45:03 +0100 | [diff] [blame] | 1570 | |
Andrii Nakryiko | 85192dbf | 2019-11-17 09:28:03 -0800 | [diff] [blame] | 1571 | static inline void bpf_prog_inc(struct bpf_prog *prog) |
Alexei Starovoitov | aa6a5f3 | 2016-09-01 18:37:24 -0700 | [diff] [blame] | 1572 | { |
Alexei Starovoitov | aa6a5f3 | 2016-09-01 18:37:24 -0700 | [diff] [blame] | 1573 | } |
Daniel Borkmann | 5ccb071 | 2016-12-18 01:52:58 +0100 | [diff] [blame] | 1574 | |
John Fastabend | a6f6df6 | 2017-08-15 22:32:22 -0700 | [diff] [blame] | 1575 | static inline struct bpf_prog *__must_check |
| 1576 | bpf_prog_inc_not_zero(struct bpf_prog *prog) |
| 1577 | { |
| 1578 | return ERR_PTR(-EOPNOTSUPP); |
| 1579 | } |
| 1580 | |
Andrii Nakryiko | 6cc7d1e | 2020-07-21 23:45:54 -0700 | [diff] [blame] | 1581 | static inline void bpf_link_init(struct bpf_link *link, enum bpf_link_type type, |
| 1582 | const struct bpf_link_ops *ops, |
| 1583 | struct bpf_prog *prog) |
| 1584 | { |
| 1585 | } |
| 1586 | |
| 1587 | static inline int bpf_link_prime(struct bpf_link *link, |
| 1588 | struct bpf_link_primer *primer) |
| 1589 | { |
| 1590 | return -EOPNOTSUPP; |
| 1591 | } |
| 1592 | |
| 1593 | static inline int bpf_link_settle(struct bpf_link_primer *primer) |
| 1594 | { |
| 1595 | return -EOPNOTSUPP; |
| 1596 | } |
| 1597 | |
| 1598 | static inline void bpf_link_cleanup(struct bpf_link_primer *primer) |
| 1599 | { |
| 1600 | } |
| 1601 | |
| 1602 | static inline void bpf_link_inc(struct bpf_link *link) |
| 1603 | { |
| 1604 | } |
| 1605 | |
| 1606 | static inline void bpf_link_put(struct bpf_link *link) |
| 1607 | { |
| 1608 | } |
| 1609 | |
Chenbo Feng | 6e71b04 | 2017-10-18 13:00:22 -0700 | [diff] [blame] | 1610 | static inline int bpf_obj_get_user(const char __user *pathname, int flags) |
Shmulik Ladkani | 98589a0 | 2017-10-09 15:27:15 +0300 | [diff] [blame] | 1611 | { |
| 1612 | return -EOPNOTSUPP; |
| 1613 | } |
| 1614 | |
David Ahern | fbee97f | 2020-05-29 16:07:13 -0600 | [diff] [blame] | 1615 | static inline bool dev_map_can_have_prog(struct bpf_map *map) |
| 1616 | { |
| 1617 | return false; |
| 1618 | } |
Toke Høiland-Jørgensen | 6f9d451 | 2019-07-26 18:06:55 +0200 | [diff] [blame] | 1619 | |
Toke Høiland-Jørgensen | 1d23388 | 2020-01-16 16:14:45 +0100 | [diff] [blame] | 1620 | static inline void __dev_flush(void) |
John Fastabend | 46f55cf | 2017-07-17 21:56:48 -0700 | [diff] [blame] | 1621 | { |
| 1622 | } |
Jesper Dangaard Brouer | 9c270af | 2017-10-16 12:19:34 +0200 | [diff] [blame] | 1623 | |
Jesper Dangaard Brouer | 67f29e0 | 2018-05-24 16:45:46 +0200 | [diff] [blame] | 1624 | struct xdp_buff; |
| 1625 | struct bpf_dtab_netdev; |
Björn Töpel | e6a4750 | 2021-03-08 12:29:06 +0100 | [diff] [blame] | 1626 | struct bpf_cpu_map_entry; |
Jesper Dangaard Brouer | 67f29e0 | 2018-05-24 16:45:46 +0200 | [diff] [blame] | 1627 | |
| 1628 | static inline |
Toke Høiland-Jørgensen | 1d23388 | 2020-01-16 16:14:45 +0100 | [diff] [blame] | 1629 | int dev_xdp_enqueue(struct net_device *dev, struct xdp_buff *xdp, |
| 1630 | struct net_device *dev_rx) |
| 1631 | { |
| 1632 | return 0; |
| 1633 | } |
| 1634 | |
| 1635 | static inline |
Jesper Dangaard Brouer | 38edddb | 2018-05-24 16:45:57 +0200 | [diff] [blame] | 1636 | int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_buff *xdp, |
| 1637 | struct net_device *dev_rx) |
Jesper Dangaard Brouer | 67f29e0 | 2018-05-24 16:45:46 +0200 | [diff] [blame] | 1638 | { |
| 1639 | return 0; |
| 1640 | } |
| 1641 | |
Toshiaki Makita | 6d5fc19 | 2018-06-14 11:07:42 +0900 | [diff] [blame] | 1642 | struct sk_buff; |
| 1643 | |
| 1644 | static inline int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, |
| 1645 | struct sk_buff *skb, |
| 1646 | struct bpf_prog *xdp_prog) |
| 1647 | { |
| 1648 | return 0; |
| 1649 | } |
| 1650 | |
Björn Töpel | cdfafe9 | 2019-12-19 07:10:04 +0100 | [diff] [blame] | 1651 | static inline void __cpu_map_flush(void) |
Jesper Dangaard Brouer | 9c270af | 2017-10-16 12:19:34 +0200 | [diff] [blame] | 1652 | { |
| 1653 | } |
| 1654 | |
Jesper Dangaard Brouer | 9c270af | 2017-10-16 12:19:34 +0200 | [diff] [blame] | 1655 | static inline int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, |
| 1656 | struct xdp_buff *xdp, |
| 1657 | struct net_device *dev_rx) |
| 1658 | { |
| 1659 | return 0; |
| 1660 | } |
Al Viro | 040ee69 | 2017-12-02 20:20:38 -0500 | [diff] [blame] | 1661 | |
Lorenzo Bianconi | 9216477 | 2020-07-14 15:56:38 +0200 | [diff] [blame] | 1662 | static inline bool cpu_map_prog_allowed(struct bpf_map *map) |
| 1663 | { |
| 1664 | return false; |
| 1665 | } |
| 1666 | |
Al Viro | 040ee69 | 2017-12-02 20:20:38 -0500 | [diff] [blame] | 1667 | static inline struct bpf_prog *bpf_prog_get_type_path(const char *name, |
| 1668 | enum bpf_prog_type type) |
| 1669 | { |
| 1670 | return ERR_PTR(-EOPNOTSUPP); |
| 1671 | } |
Stanislav Fomichev | c695865 | 2019-04-11 09:12:02 -0700 | [diff] [blame] | 1672 | |
| 1673 | static inline int bpf_prog_test_run_xdp(struct bpf_prog *prog, |
| 1674 | const union bpf_attr *kattr, |
| 1675 | union bpf_attr __user *uattr) |
| 1676 | { |
| 1677 | return -ENOTSUPP; |
| 1678 | } |
| 1679 | |
| 1680 | static inline int bpf_prog_test_run_skb(struct bpf_prog *prog, |
| 1681 | const union bpf_attr *kattr, |
| 1682 | union bpf_attr __user *uattr) |
| 1683 | { |
| 1684 | return -ENOTSUPP; |
| 1685 | } |
| 1686 | |
KP Singh | da00d2f | 2020-03-04 20:18:52 +0100 | [diff] [blame] | 1687 | static inline int bpf_prog_test_run_tracing(struct bpf_prog *prog, |
| 1688 | const union bpf_attr *kattr, |
| 1689 | union bpf_attr __user *uattr) |
| 1690 | { |
| 1691 | return -ENOTSUPP; |
| 1692 | } |
| 1693 | |
Stanislav Fomichev | c695865 | 2019-04-11 09:12:02 -0700 | [diff] [blame] | 1694 | static inline int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog, |
| 1695 | const union bpf_attr *kattr, |
| 1696 | union bpf_attr __user *uattr) |
| 1697 | { |
| 1698 | return -ENOTSUPP; |
| 1699 | } |
Daniel Borkmann | 6332be0 | 2019-11-22 21:07:55 +0100 | [diff] [blame] | 1700 | |
Lorenz Bauer | 7c32e8f | 2021-03-03 10:18:13 +0000 | [diff] [blame] | 1701 | static inline int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog, |
| 1702 | const union bpf_attr *kattr, |
| 1703 | union bpf_attr __user *uattr) |
| 1704 | { |
| 1705 | return -ENOTSUPP; |
| 1706 | } |
| 1707 | |
Daniel Borkmann | 6332be0 | 2019-11-22 21:07:55 +0100 | [diff] [blame] | 1708 | static inline void bpf_map_put(struct bpf_map *map) |
| 1709 | { |
| 1710 | } |
Björn Töpel | 7e6897f | 2019-12-13 18:51:09 +0100 | [diff] [blame] | 1711 | |
| 1712 | static inline struct bpf_prog *bpf_prog_by_id(u32 id) |
| 1713 | { |
| 1714 | return ERR_PTR(-ENOTSUPP); |
| 1715 | } |
Stanislav Fomichev | 6890896 | 2020-04-24 16:59:41 -0700 | [diff] [blame] | 1716 | |
| 1717 | static inline const struct bpf_func_proto * |
| 1718 | bpf_base_func_proto(enum bpf_func_id func_id) |
| 1719 | { |
| 1720 | return NULL; |
| 1721 | } |
Song Liu | a10787e | 2021-02-25 15:43:14 -0800 | [diff] [blame] | 1722 | |
| 1723 | static inline void bpf_task_storage_free(struct task_struct *task) |
| 1724 | { |
| 1725 | } |
Daniel Borkmann | 61e021f3 | 2015-03-02 15:21:55 +0100 | [diff] [blame] | 1726 | #endif /* CONFIG_BPF_SYSCALL */ |
Alexei Starovoitov | 09756af | 2014-09-26 00:17:00 -0700 | [diff] [blame] | 1727 | |
Andrii Nakryiko | 541c3ba | 2021-01-11 23:55:18 -0800 | [diff] [blame] | 1728 | void __bpf_free_used_btfs(struct bpf_prog_aux *aux, |
| 1729 | struct btf_mod_pair *used_btfs, u32 len); |
| 1730 | |
Jakub Kicinski | 479321e | 2017-11-20 15:21:56 -0800 | [diff] [blame] | 1731 | static inline struct bpf_prog *bpf_prog_get_type(u32 ufd, |
| 1732 | enum bpf_prog_type type) |
| 1733 | { |
| 1734 | return bpf_prog_get_type_dev(ufd, type, false); |
| 1735 | } |
| 1736 | |
Andrii Nakryiko | 936f894 | 2021-01-11 23:55:16 -0800 | [diff] [blame] | 1737 | void __bpf_free_used_maps(struct bpf_prog_aux *aux, |
| 1738 | struct bpf_map **used_maps, u32 len); |
| 1739 | |
Al Viro | 040ee69 | 2017-12-02 20:20:38 -0500 | [diff] [blame] | 1740 | bool bpf_prog_get_ok(struct bpf_prog *, enum bpf_prog_type *, bool); |
| 1741 | |
Jakub Kicinski | ab3f006 | 2017-11-03 13:56:17 -0700 | [diff] [blame] | 1742 | int bpf_prog_offload_compile(struct bpf_prog *prog); |
| 1743 | void bpf_prog_offload_destroy(struct bpf_prog *prog); |
Jakub Kicinski | 675fc27 | 2017-12-27 18:39:09 -0800 | [diff] [blame] | 1744 | int bpf_prog_offload_info_fill(struct bpf_prog_info *info, |
| 1745 | struct bpf_prog *prog); |
Jakub Kicinski | ab3f006 | 2017-11-03 13:56:17 -0700 | [diff] [blame] | 1746 | |
Jakub Kicinski | 52775b3 | 2018-01-17 19:13:28 -0800 | [diff] [blame] | 1747 | int bpf_map_offload_info_fill(struct bpf_map_info *info, struct bpf_map *map); |
| 1748 | |
Jakub Kicinski | a388457 | 2018-01-11 20:29:09 -0800 | [diff] [blame] | 1749 | int bpf_map_offload_lookup_elem(struct bpf_map *map, void *key, void *value); |
| 1750 | int bpf_map_offload_update_elem(struct bpf_map *map, |
| 1751 | void *key, void *value, u64 flags); |
| 1752 | int bpf_map_offload_delete_elem(struct bpf_map *map, void *key); |
| 1753 | int bpf_map_offload_get_next_key(struct bpf_map *map, |
| 1754 | void *key, void *next_key); |
| 1755 | |
Jakub Kicinski | 0972826 | 2018-07-17 10:53:23 -0700 | [diff] [blame] | 1756 | bool bpf_offload_prog_map_match(struct bpf_prog *prog, struct bpf_map *map); |
Jakub Kicinski | a388457 | 2018-01-11 20:29:09 -0800 | [diff] [blame] | 1757 | |
Quentin Monnet | 1385d75 | 2018-11-09 13:03:25 +0000 | [diff] [blame] | 1758 | struct bpf_offload_dev * |
Jakub Kicinski | dd27c2e | 2019-02-12 00:20:39 -0800 | [diff] [blame] | 1759 | bpf_offload_dev_create(const struct bpf_prog_offload_ops *ops, void *priv); |
Jakub Kicinski | 602144c | 2018-07-17 10:53:25 -0700 | [diff] [blame] | 1760 | void bpf_offload_dev_destroy(struct bpf_offload_dev *offdev); |
Jakub Kicinski | dd27c2e | 2019-02-12 00:20:39 -0800 | [diff] [blame] | 1761 | void *bpf_offload_dev_priv(struct bpf_offload_dev *offdev); |
Jakub Kicinski | 602144c | 2018-07-17 10:53:25 -0700 | [diff] [blame] | 1762 | int bpf_offload_dev_netdev_register(struct bpf_offload_dev *offdev, |
| 1763 | struct net_device *netdev); |
| 1764 | void bpf_offload_dev_netdev_unregister(struct bpf_offload_dev *offdev, |
| 1765 | struct net_device *netdev); |
Jakub Kicinski | fd4f227 | 2018-07-17 10:53:26 -0700 | [diff] [blame] | 1766 | bool bpf_offload_dev_match(struct bpf_prog *prog, struct net_device *netdev); |
Jakub Kicinski | 9fd7c55 | 2018-07-17 10:53:24 -0700 | [diff] [blame] | 1767 | |
Jakub Kicinski | ab3f006 | 2017-11-03 13:56:17 -0700 | [diff] [blame] | 1768 | #if defined(CONFIG_NET) && defined(CONFIG_BPF_SYSCALL) |
| 1769 | int bpf_prog_offload_init(struct bpf_prog *prog, union bpf_attr *attr); |
| 1770 | |
Jakub Kicinski | 0d83003 | 2018-05-08 19:37:06 -0700 | [diff] [blame] | 1771 | static inline bool bpf_prog_is_dev_bound(const struct bpf_prog_aux *aux) |
Jakub Kicinski | ab3f006 | 2017-11-03 13:56:17 -0700 | [diff] [blame] | 1772 | { |
Jakub Kicinski | 9a18eed | 2017-12-27 18:39:04 -0800 | [diff] [blame] | 1773 | return aux->offload_requested; |
Jakub Kicinski | ab3f006 | 2017-11-03 13:56:17 -0700 | [diff] [blame] | 1774 | } |
Jakub Kicinski | a388457 | 2018-01-11 20:29:09 -0800 | [diff] [blame] | 1775 | |
| 1776 | static inline bool bpf_map_is_dev_bound(struct bpf_map *map) |
| 1777 | { |
| 1778 | return unlikely(map->ops == &bpf_map_offload_ops); |
| 1779 | } |
| 1780 | |
| 1781 | struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr); |
| 1782 | void bpf_map_offload_map_free(struct bpf_map *map); |
Jakub Kicinski | ab3f006 | 2017-11-03 13:56:17 -0700 | [diff] [blame] | 1783 | #else |
| 1784 | static inline int bpf_prog_offload_init(struct bpf_prog *prog, |
| 1785 | union bpf_attr *attr) |
| 1786 | { |
| 1787 | return -EOPNOTSUPP; |
| 1788 | } |
| 1789 | |
| 1790 | static inline bool bpf_prog_is_dev_bound(struct bpf_prog_aux *aux) |
| 1791 | { |
| 1792 | return false; |
| 1793 | } |
Jakub Kicinski | a388457 | 2018-01-11 20:29:09 -0800 | [diff] [blame] | 1794 | |
| 1795 | static inline bool bpf_map_is_dev_bound(struct bpf_map *map) |
| 1796 | { |
| 1797 | return false; |
| 1798 | } |
| 1799 | |
| 1800 | static inline struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr) |
| 1801 | { |
| 1802 | return ERR_PTR(-EOPNOTSUPP); |
| 1803 | } |
| 1804 | |
| 1805 | static inline void bpf_map_offload_map_free(struct bpf_map *map) |
| 1806 | { |
| 1807 | } |
Jakub Kicinski | ab3f006 | 2017-11-03 13:56:17 -0700 | [diff] [blame] | 1808 | #endif /* CONFIG_NET && CONFIG_BPF_SYSCALL */ |
| 1809 | |
Cong Wang | 8875960 | 2021-02-23 10:49:26 -0800 | [diff] [blame] | 1810 | #if defined(CONFIG_INET) && defined(CONFIG_BPF_SYSCALL) |
Daniel Borkmann | 604326b | 2018-10-13 02:45:58 +0200 | [diff] [blame] | 1811 | int sock_map_get_from_fd(const union bpf_attr *attr, struct bpf_prog *prog); |
Lorenz Bauer | bb0de31 | 2020-06-29 10:56:28 +0100 | [diff] [blame] | 1812 | int sock_map_prog_detach(const union bpf_attr *attr, enum bpf_prog_type ptype); |
Lorenz Bauer | 13b79d3 | 2020-08-21 11:29:45 +0100 | [diff] [blame] | 1813 | int sock_map_update_elem_sys(struct bpf_map *map, void *key, void *value, u64 flags); |
Lorenz Bauer | f747632 | 2020-03-09 11:12:36 +0000 | [diff] [blame] | 1814 | void sock_map_unhash(struct sock *sk); |
| 1815 | void sock_map_close(struct sock *sk, long timeout); |
Cong Wang | 8875960 | 2021-02-23 10:49:26 -0800 | [diff] [blame] | 1816 | |
| 1817 | void bpf_sk_reuseport_detach(struct sock *sk); |
| 1818 | int bpf_fd_reuseport_array_lookup_elem(struct bpf_map *map, void *key, |
| 1819 | void *value); |
| 1820 | int bpf_fd_reuseport_array_update_elem(struct bpf_map *map, void *key, |
| 1821 | void *value, u64 map_flags); |
John Fastabend | 6bdc9c4 | 2017-08-16 15:02:32 -0700 | [diff] [blame] | 1822 | #else |
Cong Wang | 8875960 | 2021-02-23 10:49:26 -0800 | [diff] [blame] | 1823 | static inline void bpf_sk_reuseport_detach(struct sock *sk) |
| 1824 | { |
| 1825 | } |
| 1826 | |
| 1827 | #ifdef CONFIG_BPF_SYSCALL |
Daniel Borkmann | 604326b | 2018-10-13 02:45:58 +0200 | [diff] [blame] | 1828 | static inline int sock_map_get_from_fd(const union bpf_attr *attr, |
| 1829 | struct bpf_prog *prog) |
Sean Young | fdb5c45 | 2018-06-19 00:04:24 +0100 | [diff] [blame] | 1830 | { |
| 1831 | return -EINVAL; |
| 1832 | } |
Lorenz Bauer | bb0de31 | 2020-06-29 10:56:28 +0100 | [diff] [blame] | 1833 | |
| 1834 | static inline int sock_map_prog_detach(const union bpf_attr *attr, |
| 1835 | enum bpf_prog_type ptype) |
| 1836 | { |
| 1837 | return -EOPNOTSUPP; |
| 1838 | } |
Lorenz Bauer | 13b79d3 | 2020-08-21 11:29:45 +0100 | [diff] [blame] | 1839 | |
| 1840 | static inline int sock_map_update_elem_sys(struct bpf_map *map, void *key, void *value, |
| 1841 | u64 flags) |
| 1842 | { |
| 1843 | return -EOPNOTSUPP; |
| 1844 | } |
John Fastabend | 6bdc9c4 | 2017-08-16 15:02:32 -0700 | [diff] [blame] | 1845 | |
Martin KaFai Lau | 5dc4c4b | 2018-08-08 01:01:24 -0700 | [diff] [blame] | 1846 | static inline int bpf_fd_reuseport_array_lookup_elem(struct bpf_map *map, |
| 1847 | void *key, void *value) |
| 1848 | { |
| 1849 | return -EOPNOTSUPP; |
| 1850 | } |
| 1851 | |
| 1852 | static inline int bpf_fd_reuseport_array_update_elem(struct bpf_map *map, |
| 1853 | void *key, void *value, |
| 1854 | u64 map_flags) |
| 1855 | { |
| 1856 | return -EOPNOTSUPP; |
| 1857 | } |
| 1858 | #endif /* CONFIG_BPF_SYSCALL */ |
| 1859 | #endif /* defined(CONFIG_INET) && defined(CONFIG_BPF_SYSCALL) */ |
| 1860 | |
Alexei Starovoitov | d0003ec | 2014-11-13 17:36:49 -0800 | [diff] [blame] | 1861 | /* verifier prototypes for helper functions called from eBPF programs */ |
Daniel Borkmann | a2c83ff | 2015-03-01 12:31:42 +0100 | [diff] [blame] | 1862 | extern const struct bpf_func_proto bpf_map_lookup_elem_proto; |
| 1863 | extern const struct bpf_func_proto bpf_map_update_elem_proto; |
| 1864 | extern const struct bpf_func_proto bpf_map_delete_elem_proto; |
Mauricio Vasquez B | f1a2e44 | 2018-10-18 15:16:25 +0200 | [diff] [blame] | 1865 | extern const struct bpf_func_proto bpf_map_push_elem_proto; |
| 1866 | extern const struct bpf_func_proto bpf_map_pop_elem_proto; |
| 1867 | extern const struct bpf_func_proto bpf_map_peek_elem_proto; |
Alexei Starovoitov | d0003ec | 2014-11-13 17:36:49 -0800 | [diff] [blame] | 1868 | |
Daniel Borkmann | 03e69b5 | 2015-03-14 02:27:16 +0100 | [diff] [blame] | 1869 | extern const struct bpf_func_proto bpf_get_prandom_u32_proto; |
Daniel Borkmann | c04167c | 2015-03-14 02:27:17 +0100 | [diff] [blame] | 1870 | extern const struct bpf_func_proto bpf_get_smp_processor_id_proto; |
Daniel Borkmann | 2d0e30c | 2016-10-21 12:46:33 +0200 | [diff] [blame] | 1871 | extern const struct bpf_func_proto bpf_get_numa_node_id_proto; |
Alexei Starovoitov | 04fd61ab | 2015-05-19 16:59:03 -0700 | [diff] [blame] | 1872 | extern const struct bpf_func_proto bpf_tail_call_proto; |
Daniel Borkmann | 17ca8cb | 2015-05-29 23:23:06 +0200 | [diff] [blame] | 1873 | extern const struct bpf_func_proto bpf_ktime_get_ns_proto; |
Maciej Żenczykowski | 71d1921 | 2020-04-26 09:15:25 -0700 | [diff] [blame] | 1874 | extern const struct bpf_func_proto bpf_ktime_get_boot_ns_proto; |
Alexei Starovoitov | ffeedaf | 2015-06-12 19:39:12 -0700 | [diff] [blame] | 1875 | extern const struct bpf_func_proto bpf_get_current_pid_tgid_proto; |
| 1876 | extern const struct bpf_func_proto bpf_get_current_uid_gid_proto; |
| 1877 | extern const struct bpf_func_proto bpf_get_current_comm_proto; |
Alexei Starovoitov | d5a3b1f | 2016-02-17 19:58:58 -0800 | [diff] [blame] | 1878 | extern const struct bpf_func_proto bpf_get_stackid_proto; |
Yonghong Song | c195651e | 2018-04-28 22:28:08 -0700 | [diff] [blame] | 1879 | extern const struct bpf_func_proto bpf_get_stack_proto; |
Song Liu | fa28dcb | 2020-06-29 23:28:44 -0700 | [diff] [blame] | 1880 | extern const struct bpf_func_proto bpf_get_task_stack_proto; |
Song Liu | 7b04d6d | 2020-07-23 11:06:44 -0700 | [diff] [blame] | 1881 | extern const struct bpf_func_proto bpf_get_stackid_proto_pe; |
| 1882 | extern const struct bpf_func_proto bpf_get_stack_proto_pe; |
John Fastabend | 174a79f | 2017-08-15 22:32:47 -0700 | [diff] [blame] | 1883 | extern const struct bpf_func_proto bpf_sock_map_update_proto; |
John Fastabend | 8111038 | 2018-05-14 10:00:17 -0700 | [diff] [blame] | 1884 | extern const struct bpf_func_proto bpf_sock_hash_update_proto; |
Yonghong Song | bf6fa2c8 | 2018-06-03 15:59:41 -0700 | [diff] [blame] | 1885 | extern const struct bpf_func_proto bpf_get_current_cgroup_id_proto; |
Daniel Borkmann | 0f09abd | 2020-03-27 16:58:54 +0100 | [diff] [blame] | 1886 | extern const struct bpf_func_proto bpf_get_current_ancestor_cgroup_id_proto; |
Daniel Borkmann | 604326b | 2018-10-13 02:45:58 +0200 | [diff] [blame] | 1887 | extern const struct bpf_func_proto bpf_msg_redirect_hash_proto; |
| 1888 | extern const struct bpf_func_proto bpf_msg_redirect_map_proto; |
| 1889 | extern const struct bpf_func_proto bpf_sk_redirect_hash_proto; |
| 1890 | extern const struct bpf_func_proto bpf_sk_redirect_map_proto; |
Alexei Starovoitov | d83525c | 2019-01-31 15:40:04 -0800 | [diff] [blame] | 1891 | extern const struct bpf_func_proto bpf_spin_lock_proto; |
| 1892 | extern const struct bpf_func_proto bpf_spin_unlock_proto; |
Roman Gushchin | cd33943 | 2018-08-02 14:27:24 -0700 | [diff] [blame] | 1893 | extern const struct bpf_func_proto bpf_get_local_storage_proto; |
Andrey Ignatov | d7a4cb9 | 2019-03-18 17:55:26 -0700 | [diff] [blame] | 1894 | extern const struct bpf_func_proto bpf_strtol_proto; |
| 1895 | extern const struct bpf_func_proto bpf_strtoul_proto; |
Stanislav Fomichev | 0d01da6 | 2019-06-27 13:38:47 -0700 | [diff] [blame] | 1896 | extern const struct bpf_func_proto bpf_tcp_sock_proto; |
Martin KaFai Lau | 5576b99 | 2020-01-22 15:36:46 -0800 | [diff] [blame] | 1897 | extern const struct bpf_func_proto bpf_jiffies64_proto; |
Carlos Neira | b4490c5 | 2020-03-04 17:41:56 -0300 | [diff] [blame] | 1898 | extern const struct bpf_func_proto bpf_get_ns_current_pid_tgid_proto; |
Stanislav Fomichev | 0456ea1 | 2020-04-20 10:46:10 -0700 | [diff] [blame] | 1899 | extern const struct bpf_func_proto bpf_event_output_data_proto; |
Andrii Nakryiko | 457f4436 | 2020-05-29 00:54:20 -0700 | [diff] [blame] | 1900 | extern const struct bpf_func_proto bpf_ringbuf_output_proto; |
| 1901 | extern const struct bpf_func_proto bpf_ringbuf_reserve_proto; |
| 1902 | extern const struct bpf_func_proto bpf_ringbuf_submit_proto; |
| 1903 | extern const struct bpf_func_proto bpf_ringbuf_discard_proto; |
| 1904 | extern const struct bpf_func_proto bpf_ringbuf_query_proto; |
Yonghong Song | af7ec13 | 2020-06-23 16:08:09 -0700 | [diff] [blame] | 1905 | extern const struct bpf_func_proto bpf_skc_to_tcp6_sock_proto; |
Yonghong Song | 478cfbd | 2020-06-23 16:08:11 -0700 | [diff] [blame] | 1906 | extern const struct bpf_func_proto bpf_skc_to_tcp_sock_proto; |
| 1907 | extern const struct bpf_func_proto bpf_skc_to_tcp_timewait_sock_proto; |
| 1908 | extern const struct bpf_func_proto bpf_skc_to_tcp_request_sock_proto; |
Yonghong Song | 0d4fad3e5 | 2020-06-23 16:08:15 -0700 | [diff] [blame] | 1909 | extern const struct bpf_func_proto bpf_skc_to_udp6_sock_proto; |
Alexei Starovoitov | 07be4c4 | 2020-08-27 15:01:12 -0700 | [diff] [blame] | 1910 | extern const struct bpf_func_proto bpf_copy_from_user_proto; |
Alan Maguire | c4d0bfb | 2020-09-28 12:31:05 +0100 | [diff] [blame] | 1911 | extern const struct bpf_func_proto bpf_snprintf_btf_proto; |
Hao Luo | eaa6bcb | 2020-09-29 16:50:47 -0700 | [diff] [blame] | 1912 | extern const struct bpf_func_proto bpf_per_cpu_ptr_proto; |
Hao Luo | 63d9b80 | 2020-09-29 16:50:48 -0700 | [diff] [blame] | 1913 | extern const struct bpf_func_proto bpf_this_cpu_ptr_proto; |
Dmitrii Banshchikov | d055126 | 2020-11-17 18:45:49 +0000 | [diff] [blame] | 1914 | extern const struct bpf_func_proto bpf_ktime_get_coarse_ns_proto; |
Florent Revest | b60da49 | 2020-12-08 18:36:23 +0100 | [diff] [blame] | 1915 | extern const struct bpf_func_proto bpf_sock_from_file_proto; |
Florent Revest | c5dbb89 | 2021-02-10 12:14:03 +0100 | [diff] [blame] | 1916 | extern const struct bpf_func_proto bpf_get_socket_ptr_cookie_proto; |
Song Liu | a10787e | 2021-02-25 15:43:14 -0800 | [diff] [blame] | 1917 | extern const struct bpf_func_proto bpf_task_storage_get_proto; |
| 1918 | extern const struct bpf_func_proto bpf_task_storage_delete_proto; |
Yonghong Song | 69c087b | 2021-02-26 12:49:25 -0800 | [diff] [blame] | 1919 | extern const struct bpf_func_proto bpf_for_each_map_elem_proto; |
Roman Gushchin | cd33943 | 2018-08-02 14:27:24 -0700 | [diff] [blame] | 1920 | |
KP Singh | fc611f4 | 2020-03-29 01:43:49 +0100 | [diff] [blame] | 1921 | const struct bpf_func_proto *bpf_tracing_func_proto( |
| 1922 | enum bpf_func_id func_id, const struct bpf_prog *prog); |
| 1923 | |
Jiri Olsa | 958a3f2 | 2020-05-31 17:42:55 +0200 | [diff] [blame] | 1924 | const struct bpf_func_proto *tracing_prog_func_proto( |
| 1925 | enum bpf_func_id func_id, const struct bpf_prog *prog); |
| 1926 | |
Daniel Borkmann | 3ad0040 | 2015-10-08 01:20:39 +0200 | [diff] [blame] | 1927 | /* Shared helpers among cBPF and eBPF. */ |
| 1928 | void bpf_user_rnd_init_once(void); |
| 1929 | u64 bpf_user_rnd_u32(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); |
Stanislav Fomichev | 6890896 | 2020-04-24 16:59:41 -0700 | [diff] [blame] | 1930 | u64 bpf_get_raw_cpu_id(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); |
Daniel Borkmann | 3ad0040 | 2015-10-08 01:20:39 +0200 | [diff] [blame] | 1931 | |
Joe Stringer | c64b798 | 2018-10-02 13:35:33 -0700 | [diff] [blame] | 1932 | #if defined(CONFIG_NET) |
Martin KaFai Lau | 46f8bc9 | 2019-02-09 23:22:20 -0800 | [diff] [blame] | 1933 | bool bpf_sock_common_is_valid_access(int off, int size, |
| 1934 | enum bpf_access_type type, |
| 1935 | struct bpf_insn_access_aux *info); |
Joe Stringer | c64b798 | 2018-10-02 13:35:33 -0700 | [diff] [blame] | 1936 | bool bpf_sock_is_valid_access(int off, int size, enum bpf_access_type type, |
| 1937 | struct bpf_insn_access_aux *info); |
| 1938 | u32 bpf_sock_convert_ctx_access(enum bpf_access_type type, |
| 1939 | const struct bpf_insn *si, |
| 1940 | struct bpf_insn *insn_buf, |
| 1941 | struct bpf_prog *prog, |
| 1942 | u32 *target_size); |
| 1943 | #else |
Martin KaFai Lau | 46f8bc9 | 2019-02-09 23:22:20 -0800 | [diff] [blame] | 1944 | static inline bool bpf_sock_common_is_valid_access(int off, int size, |
| 1945 | enum bpf_access_type type, |
| 1946 | struct bpf_insn_access_aux *info) |
| 1947 | { |
| 1948 | return false; |
| 1949 | } |
Joe Stringer | c64b798 | 2018-10-02 13:35:33 -0700 | [diff] [blame] | 1950 | static inline bool bpf_sock_is_valid_access(int off, int size, |
| 1951 | enum bpf_access_type type, |
| 1952 | struct bpf_insn_access_aux *info) |
| 1953 | { |
| 1954 | return false; |
| 1955 | } |
| 1956 | static inline u32 bpf_sock_convert_ctx_access(enum bpf_access_type type, |
| 1957 | const struct bpf_insn *si, |
| 1958 | struct bpf_insn *insn_buf, |
| 1959 | struct bpf_prog *prog, |
| 1960 | u32 *target_size) |
| 1961 | { |
| 1962 | return 0; |
| 1963 | } |
| 1964 | #endif |
| 1965 | |
Martin KaFai Lau | 655a51e | 2019-02-09 23:22:24 -0800 | [diff] [blame] | 1966 | #ifdef CONFIG_INET |
Alexei Starovoitov | 91cc1a9 | 2019-11-14 10:57:15 -0800 | [diff] [blame] | 1967 | struct sk_reuseport_kern { |
| 1968 | struct sk_buff *skb; |
| 1969 | struct sock *sk; |
| 1970 | struct sock *selected_sk; |
| 1971 | void *data_end; |
| 1972 | u32 hash; |
| 1973 | u32 reuseport_id; |
| 1974 | bool bind_inany; |
| 1975 | }; |
Martin KaFai Lau | 655a51e | 2019-02-09 23:22:24 -0800 | [diff] [blame] | 1976 | bool bpf_tcp_sock_is_valid_access(int off, int size, enum bpf_access_type type, |
| 1977 | struct bpf_insn_access_aux *info); |
| 1978 | |
| 1979 | u32 bpf_tcp_sock_convert_ctx_access(enum bpf_access_type type, |
| 1980 | const struct bpf_insn *si, |
| 1981 | struct bpf_insn *insn_buf, |
| 1982 | struct bpf_prog *prog, |
| 1983 | u32 *target_size); |
YueHaibing | 7f94208 | 2019-06-12 17:18:47 +0800 | [diff] [blame] | 1984 | |
| 1985 | bool bpf_xdp_sock_is_valid_access(int off, int size, enum bpf_access_type type, |
| 1986 | struct bpf_insn_access_aux *info); |
| 1987 | |
| 1988 | u32 bpf_xdp_sock_convert_ctx_access(enum bpf_access_type type, |
| 1989 | const struct bpf_insn *si, |
| 1990 | struct bpf_insn *insn_buf, |
| 1991 | struct bpf_prog *prog, |
| 1992 | u32 *target_size); |
Martin KaFai Lau | 655a51e | 2019-02-09 23:22:24 -0800 | [diff] [blame] | 1993 | #else |
| 1994 | static inline bool bpf_tcp_sock_is_valid_access(int off, int size, |
| 1995 | enum bpf_access_type type, |
| 1996 | struct bpf_insn_access_aux *info) |
| 1997 | { |
| 1998 | return false; |
| 1999 | } |
| 2000 | |
| 2001 | static inline u32 bpf_tcp_sock_convert_ctx_access(enum bpf_access_type type, |
| 2002 | const struct bpf_insn *si, |
| 2003 | struct bpf_insn *insn_buf, |
| 2004 | struct bpf_prog *prog, |
| 2005 | u32 *target_size) |
| 2006 | { |
| 2007 | return 0; |
| 2008 | } |
YueHaibing | 7f94208 | 2019-06-12 17:18:47 +0800 | [diff] [blame] | 2009 | static inline bool bpf_xdp_sock_is_valid_access(int off, int size, |
| 2010 | enum bpf_access_type type, |
| 2011 | struct bpf_insn_access_aux *info) |
| 2012 | { |
| 2013 | return false; |
| 2014 | } |
| 2015 | |
| 2016 | static inline u32 bpf_xdp_sock_convert_ctx_access(enum bpf_access_type type, |
| 2017 | const struct bpf_insn *si, |
| 2018 | struct bpf_insn *insn_buf, |
| 2019 | struct bpf_prog *prog, |
| 2020 | u32 *target_size) |
| 2021 | { |
| 2022 | return 0; |
| 2023 | } |
Martin KaFai Lau | 655a51e | 2019-02-09 23:22:24 -0800 | [diff] [blame] | 2024 | #endif /* CONFIG_INET */ |
| 2025 | |
Alexei Starovoitov | 5964b20 | 2019-11-14 10:57:03 -0800 | [diff] [blame] | 2026 | enum bpf_text_poke_type { |
Daniel Borkmann | b553a6e | 2019-11-24 01:39:42 +0100 | [diff] [blame] | 2027 | BPF_MOD_CALL, |
| 2028 | BPF_MOD_JUMP, |
Alexei Starovoitov | 5964b20 | 2019-11-14 10:57:03 -0800 | [diff] [blame] | 2029 | }; |
Daniel Borkmann | 4b3da77 | 2019-11-22 21:07:54 +0100 | [diff] [blame] | 2030 | |
Alexei Starovoitov | 5964b20 | 2019-11-14 10:57:03 -0800 | [diff] [blame] | 2031 | int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t, |
| 2032 | void *addr1, void *addr2); |
| 2033 | |
Jiri Olsa | eae2e83 | 2020-08-25 21:21:19 +0200 | [diff] [blame] | 2034 | struct btf_id_set; |
Lorenz Bauer | 2af30f1 | 2020-09-21 13:12:17 +0100 | [diff] [blame] | 2035 | bool btf_id_set_contains(const struct btf_id_set *set, u32 id); |
Jiri Olsa | eae2e83 | 2020-08-25 21:21:19 +0200 | [diff] [blame] | 2036 | |
Alexei Starovoitov | 99c55f7 | 2014-09-26 00:16:57 -0700 | [diff] [blame] | 2037 | #endif /* _LINUX_BPF_H */ |