include/linux/bpf_local_storage.h

0001 /* SPDX-License-Identifier: GPL-2.0 */
0002 /*
0003  * Copyright (c) 2019 Facebook
0004  * Copyright 2020 Google LLC.
0005  */
0006
0007 #ifndef _BPF_LOCAL_STORAGE_H
0008 #define _BPF_LOCAL_STORAGE_H
0009
0010 #include <linux/bpf.h>
0011 #include <linux/filter.h>
0012 #include <linux/rculist.h>
0013 #include <linux/list.h>
0014 #include <linux/hash.h>
0015 #include <linux/types.h>
0016 #include <uapi/linux/btf.h>
0017
0018 #define BPF_LOCAL_STORAGE_CACHE_SIZE    16
0019
0020 #define bpf_rcu_lock_held()                                                    \
0021     (rcu_read_lock_held() || rcu_read_lock_trace_held() ||                 \
0022      rcu_read_lock_bh_held())
0023 struct bpf_local_storage_map_bucket {
0024     struct hlist_head list;
0025     raw_spinlock_t lock;
0026 };
0027
0028 /* Thp map is not the primary owner of a bpf_local_storage_elem.
0029  * Instead, the container object (eg. sk->sk_bpf_storage) is.
0030  *
0031  * The map (bpf_local_storage_map) is for two purposes
0032  * 1. Define the size of the "local storage".  It is
0033  *    the map's value_size.
0034  *
0035  * 2. Maintain a list to keep track of all elems such
0036  *    that they can be cleaned up during the map destruction.
0037  *
0038  * When a bpf local storage is being looked up for a
0039  * particular object,  the "bpf_map" pointer is actually used
0040  * as the "key" to search in the list of elem in
0041  * the respective bpf_local_storage owned by the object.
0042  *
0043  * e.g. sk->sk_bpf_storage is the mini-map with the "bpf_map" pointer
0044  * as the searching key.
0045  */
0046 struct bpf_local_storage_map {
0047     struct bpf_map map;
0048     /* Lookup elem does not require accessing the map.
0049      *
0050      * Updating/Deleting requires a bucket lock to
0051      * link/unlink the elem from the map.  Having
0052      * multiple buckets to improve contention.
0053      */
0054     struct bpf_local_storage_map_bucket *buckets;
0055     u32 bucket_log;
0056     u16 elem_size;
0057     u16 cache_idx;
0058 };
0059
0060 struct bpf_local_storage_data {
0061     /* smap is used as the searching key when looking up
0062      * from the object's bpf_local_storage.
0063      *
0064      * Put it in the same cacheline as the data to minimize
0065      * the number of cachelines accessed during the cache hit case.
0066      */
0067     struct bpf_local_storage_map __rcu *smap;
0068     u8 data[] __aligned(8);
0069 };
0070
0071 /* Linked to bpf_local_storage and bpf_local_storage_map */
0072 struct bpf_local_storage_elem {
0073     struct hlist_node map_node; /* Linked to bpf_local_storage_map */
0074     struct hlist_node snode;    /* Linked to bpf_local_storage */
0075     struct bpf_local_storage __rcu *local_storage;
0076     struct rcu_head rcu;
0077     /* 8 bytes hole */
0078     /* The data is stored in another cacheline to minimize
0079      * the number of cachelines access during a cache hit.
0080      */
0081     struct bpf_local_storage_data sdata ____cacheline_aligned;
0082 };
0083
0084 struct bpf_local_storage {
0085     struct bpf_local_storage_data __rcu *cache[BPF_LOCAL_STORAGE_CACHE_SIZE];
0086     struct hlist_head list; /* List of bpf_local_storage_elem */
0087     void *owner;        /* The object that owns the above "list" of
0088                  * bpf_local_storage_elem.
0089                  */
0090     struct rcu_head rcu;
0091     raw_spinlock_t lock;    /* Protect adding/removing from the "list" */
0092 };
0093
0094 /* U16_MAX is much more than enough for sk local storage
0095  * considering a tcp_sock is ~2k.
0096  */
0097 #define BPF_LOCAL_STORAGE_MAX_VALUE_SIZE                       \
0098     min_t(u32,                                                             \
0099           (KMALLOC_MAX_SIZE - MAX_BPF_STACK -                              \
0100            sizeof(struct bpf_local_storage_elem)),                         \
0101           (U16_MAX - sizeof(struct bpf_local_storage_elem)))
0102
0103 #define SELEM(_SDATA)                                                          \
0104     container_of((_SDATA), struct bpf_local_storage_elem, sdata)
0105 #define SDATA(_SELEM) (&(_SELEM)->sdata)
0106
0107 #define BPF_LOCAL_STORAGE_CACHE_SIZE    16
0108
0109 struct bpf_local_storage_cache {
0110     spinlock_t idx_lock;
0111     u64 idx_usage_counts[BPF_LOCAL_STORAGE_CACHE_SIZE];
0112 };
0113
0114 #define DEFINE_BPF_STORAGE_CACHE(name)              \
0115 static struct bpf_local_storage_cache name = {          \
0116     .idx_lock = __SPIN_LOCK_UNLOCKED(name.idx_lock),    \
0117 }
0118
0119 u16 bpf_local_storage_cache_idx_get(struct bpf_local_storage_cache *cache);
0120 void bpf_local_storage_cache_idx_free(struct bpf_local_storage_cache *cache,
0121                       u16 idx);
0122
0123 /* Helper functions for bpf_local_storage */
0124 int bpf_local_storage_map_alloc_check(union bpf_attr *attr);
0125
0126 struct bpf_local_storage_map *bpf_local_storage_map_alloc(union bpf_attr *attr);
0127
0128 struct bpf_local_storage_data *
0129 bpf_local_storage_lookup(struct bpf_local_storage *local_storage,
0130              struct bpf_local_storage_map *smap,
0131              bool cacheit_lockit);
0132
0133 void bpf_local_storage_map_free(struct bpf_local_storage_map *smap,
0134                 int __percpu *busy_counter);
0135
0136 int bpf_local_storage_map_check_btf(const struct bpf_map *map,
0137                     const struct btf *btf,
0138                     const struct btf_type *key_type,
0139                     const struct btf_type *value_type);
0140
0141 void bpf_selem_link_storage_nolock(struct bpf_local_storage *local_storage,
0142                    struct bpf_local_storage_elem *selem);
0143
0144 bool bpf_selem_unlink_storage_nolock(struct bpf_local_storage *local_storage,
0145                      struct bpf_local_storage_elem *selem,
0146                      bool uncharge_omem, bool use_trace_rcu);
0147
0148 void bpf_selem_unlink(struct bpf_local_storage_elem *selem, bool use_trace_rcu);
0149
0150 void bpf_selem_link_map(struct bpf_local_storage_map *smap,
0151             struct bpf_local_storage_elem *selem);
0152
0153 void bpf_selem_unlink_map(struct bpf_local_storage_elem *selem);
0154
0155 struct bpf_local_storage_elem *
0156 bpf_selem_alloc(struct bpf_local_storage_map *smap, void *owner, void *value,
0157         bool charge_mem, gfp_t gfp_flags);
0158
0159 int
0160 bpf_local_storage_alloc(void *owner,
0161             struct bpf_local_storage_map *smap,
0162             struct bpf_local_storage_elem *first_selem,
0163             gfp_t gfp_flags);
0164
0165 struct bpf_local_storage_data *
0166 bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap,
0167              void *value, u64 map_flags, gfp_t gfp_flags);
0168
0169 void bpf_local_storage_free_rcu(struct rcu_head *rcu);
0170
0171 #endif /* _BPF_LOCAL_STORAGE_H */