kernel/bpf/bpf_inode_storage.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * Copyright (c) 2019 Facebook
0004  * Copyright 2020 Google LLC.
0005  */
0006
0007 #include <linux/rculist.h>
0008 #include <linux/list.h>
0009 #include <linux/hash.h>
0010 #include <linux/types.h>
0011 #include <linux/spinlock.h>
0012 #include <linux/bpf.h>
0013 #include <linux/bpf_local_storage.h>
0014 #include <net/sock.h>
0015 #include <uapi/linux/sock_diag.h>
0016 #include <uapi/linux/btf.h>
0017 #include <linux/bpf_lsm.h>
0018 #include <linux/btf_ids.h>
0019 #include <linux/fdtable.h>
0020 #include <linux/rcupdate_trace.h>
0021
0022 DEFINE_BPF_STORAGE_CACHE(inode_cache);
0023
0024 static struct bpf_local_storage __rcu **
0025 inode_storage_ptr(void *owner)
0026 {
0027     struct inode *inode = owner;
0028     struct bpf_storage_blob *bsb;
0029
0030     bsb = bpf_inode(inode);
0031     if (!bsb)
0032         return NULL;
0033     return &bsb->storage;
0034 }
0035
0036 static struct bpf_local_storage_data *inode_storage_lookup(struct inode *inode,
0037                                struct bpf_map *map,
0038                                bool cacheit_lockit)
0039 {
0040     struct bpf_local_storage *inode_storage;
0041     struct bpf_local_storage_map *smap;
0042     struct bpf_storage_blob *bsb;
0043
0044     bsb = bpf_inode(inode);
0045     if (!bsb)
0046         return NULL;
0047
0048     inode_storage =
0049         rcu_dereference_check(bsb->storage, bpf_rcu_lock_held());
0050     if (!inode_storage)
0051         return NULL;
0052
0053     smap = (struct bpf_local_storage_map *)map;
0054     return bpf_local_storage_lookup(inode_storage, smap, cacheit_lockit);
0055 }
0056
0057 void bpf_inode_storage_free(struct inode *inode)
0058 {
0059     struct bpf_local_storage_elem *selem;
0060     struct bpf_local_storage *local_storage;
0061     bool free_inode_storage = false;
0062     struct bpf_storage_blob *bsb;
0063     struct hlist_node *n;
0064
0065     bsb = bpf_inode(inode);
0066     if (!bsb)
0067         return;
0068
0069     rcu_read_lock();
0070
0071     local_storage = rcu_dereference(bsb->storage);
0072     if (!local_storage) {
0073         rcu_read_unlock();
0074         return;
0075     }
0076
0077     /* Neither the bpf_prog nor the bpf-map's syscall
0078      * could be modifying the local_storage->list now.
0079      * Thus, no elem can be added-to or deleted-from the
0080      * local_storage->list by the bpf_prog or by the bpf-map's syscall.
0081      *
0082      * It is racing with bpf_local_storage_map_free() alone
0083      * when unlinking elem from the local_storage->list and
0084      * the map's bucket->list.
0085      */
0086     raw_spin_lock_bh(&local_storage->lock);
0087     hlist_for_each_entry_safe(selem, n, &local_storage->list, snode) {
0088         /* Always unlink from map before unlinking from
0089          * local_storage.
0090          */
0091         bpf_selem_unlink_map(selem);
0092         free_inode_storage = bpf_selem_unlink_storage_nolock(
0093             local_storage, selem, false, false);
0094     }
0095     raw_spin_unlock_bh(&local_storage->lock);
0096     rcu_read_unlock();
0097
0098     /* free_inoode_storage should always be true as long as
0099      * local_storage->list was non-empty.
0100      */
0101     if (free_inode_storage)
0102         kfree_rcu(local_storage, rcu);
0103 }
0104
0105 static void *bpf_fd_inode_storage_lookup_elem(struct bpf_map *map, void *key)
0106 {
0107     struct bpf_local_storage_data *sdata;
0108     struct file *f;
0109     int fd;
0110
0111     fd = *(int *)key;
0112     f = fget_raw(fd);
0113     if (!f)
0114         return ERR_PTR(-EBADF);
0115
0116     sdata = inode_storage_lookup(f->f_inode, map, true);
0117     fput(f);
0118     return sdata ? sdata->data : NULL;
0119 }
0120
0121 static int bpf_fd_inode_storage_update_elem(struct bpf_map *map, void *key,
0122                      void *value, u64 map_flags)
0123 {
0124     struct bpf_local_storage_data *sdata;
0125     struct file *f;
0126     int fd;
0127
0128     fd = *(int *)key;
0129     f = fget_raw(fd);
0130     if (!f)
0131         return -EBADF;
0132     if (!inode_storage_ptr(f->f_inode)) {
0133         fput(f);
0134         return -EBADF;
0135     }
0136
0137     sdata = bpf_local_storage_update(f->f_inode,
0138                      (struct bpf_local_storage_map *)map,
0139                      value, map_flags, GFP_ATOMIC);
0140     fput(f);
0141     return PTR_ERR_OR_ZERO(sdata);
0142 }
0143
0144 static int inode_storage_delete(struct inode *inode, struct bpf_map *map)
0145 {
0146     struct bpf_local_storage_data *sdata;
0147
0148     sdata = inode_storage_lookup(inode, map, false);
0149     if (!sdata)
0150         return -ENOENT;
0151
0152     bpf_selem_unlink(SELEM(sdata), true);
0153
0154     return 0;
0155 }
0156
0157 static int bpf_fd_inode_storage_delete_elem(struct bpf_map *map, void *key)
0158 {
0159     struct file *f;
0160     int fd, err;
0161
0162     fd = *(int *)key;
0163     f = fget_raw(fd);
0164     if (!f)
0165         return -EBADF;
0166
0167     err = inode_storage_delete(f->f_inode, map);
0168     fput(f);
0169     return err;
0170 }
0171
0172 /* *gfp_flags* is a hidden argument provided by the verifier */
0173 BPF_CALL_5(bpf_inode_storage_get, struct bpf_map *, map, struct inode *, inode,
0174        void *, value, u64, flags, gfp_t, gfp_flags)
0175 {
0176     struct bpf_local_storage_data *sdata;
0177
0178     WARN_ON_ONCE(!bpf_rcu_lock_held());
0179     if (flags & ~(BPF_LOCAL_STORAGE_GET_F_CREATE))
0180         return (unsigned long)NULL;
0181
0182     /* explicitly check that the inode_storage_ptr is not
0183      * NULL as inode_storage_lookup returns NULL in this case and
0184      * bpf_local_storage_update expects the owner to have a
0185      * valid storage pointer.
0186      */
0187     if (!inode || !inode_storage_ptr(inode))
0188         return (unsigned long)NULL;
0189
0190     sdata = inode_storage_lookup(inode, map, true);
0191     if (sdata)
0192         return (unsigned long)sdata->data;
0193
0194     /* This helper must only called from where the inode is guaranteed
0195      * to have a refcount and cannot be freed.
0196      */
0197     if (flags & BPF_LOCAL_STORAGE_GET_F_CREATE) {
0198         sdata = bpf_local_storage_update(
0199             inode, (struct bpf_local_storage_map *)map, value,
0200             BPF_NOEXIST, gfp_flags);
0201         return IS_ERR(sdata) ? (unsigned long)NULL :
0202                          (unsigned long)sdata->data;
0203     }
0204
0205     return (unsigned long)NULL;
0206 }
0207
0208 BPF_CALL_2(bpf_inode_storage_delete,
0209        struct bpf_map *, map, struct inode *, inode)
0210 {
0211     WARN_ON_ONCE(!bpf_rcu_lock_held());
0212     if (!inode)
0213         return -EINVAL;
0214
0215     /* This helper must only called from where the inode is guaranteed
0216      * to have a refcount and cannot be freed.
0217      */
0218     return inode_storage_delete(inode, map);
0219 }
0220
0221 static int notsupp_get_next_key(struct bpf_map *map, void *key,
0222                 void *next_key)
0223 {
0224     return -ENOTSUPP;
0225 }
0226
0227 static struct bpf_map *inode_storage_map_alloc(union bpf_attr *attr)
0228 {
0229     struct bpf_local_storage_map *smap;
0230
0231     smap = bpf_local_storage_map_alloc(attr);
0232     if (IS_ERR(smap))
0233         return ERR_CAST(smap);
0234
0235     smap->cache_idx = bpf_local_storage_cache_idx_get(&inode_cache);
0236     return &smap->map;
0237 }
0238
0239 static void inode_storage_map_free(struct bpf_map *map)
0240 {
0241     struct bpf_local_storage_map *smap;
0242
0243     smap = (struct bpf_local_storage_map *)map;
0244     bpf_local_storage_cache_idx_free(&inode_cache, smap->cache_idx);
0245     bpf_local_storage_map_free(smap, NULL);
0246 }
0247
0248 BTF_ID_LIST_SINGLE(inode_storage_map_btf_ids, struct,
0249            bpf_local_storage_map)
0250 const struct bpf_map_ops inode_storage_map_ops = {
0251     .map_meta_equal = bpf_map_meta_equal,
0252     .map_alloc_check = bpf_local_storage_map_alloc_check,
0253     .map_alloc = inode_storage_map_alloc,
0254     .map_free = inode_storage_map_free,
0255     .map_get_next_key = notsupp_get_next_key,
0256     .map_lookup_elem = bpf_fd_inode_storage_lookup_elem,
0257     .map_update_elem = bpf_fd_inode_storage_update_elem,
0258     .map_delete_elem = bpf_fd_inode_storage_delete_elem,
0259     .map_check_btf = bpf_local_storage_map_check_btf,
0260     .map_btf_id = &inode_storage_map_btf_ids[0],
0261     .map_owner_storage_ptr = inode_storage_ptr,
0262 };
0263
0264 BTF_ID_LIST_SINGLE(bpf_inode_storage_btf_ids, struct, inode)
0265
0266 const struct bpf_func_proto bpf_inode_storage_get_proto = {
0267     .func       = bpf_inode_storage_get,
0268     .gpl_only   = false,
0269     .ret_type   = RET_PTR_TO_MAP_VALUE_OR_NULL,
0270     .arg1_type  = ARG_CONST_MAP_PTR,
0271     .arg2_type  = ARG_PTR_TO_BTF_ID,
0272     .arg2_btf_id    = &bpf_inode_storage_btf_ids[0],
0273     .arg3_type  = ARG_PTR_TO_MAP_VALUE_OR_NULL,
0274     .arg4_type  = ARG_ANYTHING,
0275 };
0276
0277 const struct bpf_func_proto bpf_inode_storage_delete_proto = {
0278     .func       = bpf_inode_storage_delete,
0279     .gpl_only   = false,
0280     .ret_type   = RET_INTEGER,
0281     .arg1_type  = ARG_CONST_MAP_PTR,
0282     .arg2_type  = ARG_PTR_TO_BTF_ID,
0283     .arg2_btf_id    = &bpf_inode_storage_btf_ids[0],
0284 };