infiniband/core/uverbs_ioctl.c

0001 /*
0002  * Copyright (c) 2017, Mellanox Technologies inc.  All rights reserved.
0003  *
0004  * This software is available to you under a choice of one of two
0005  * licenses.  You may choose to be licensed under the terms of the GNU
0006  * General Public License (GPL) Version 2, available from the file
0007  * COPYING in the main directory of this source tree, or the
0008  * OpenIB.org BSD license below:
0009  *
0010  *     Redistribution and use in source and binary forms, with or
0011  *     without modification, are permitted provided that the following
0012  *     conditions are met:
0013  *
0014  *      - Redistributions of source code must retain the above
0015  *        copyright notice, this list of conditions and the following
0016  *        disclaimer.
0017  *
0018  *      - Redistributions in binary form must reproduce the above
0019  *        copyright notice, this list of conditions and the following
0020  *        disclaimer in the documentation and/or other materials
0021  *        provided with the distribution.
0022  *
0023  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
0024  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
0025  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
0026  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
0027  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
0028  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
0029  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
0030  * SOFTWARE.
0031  */
0032
0033 #include <rdma/rdma_user_ioctl.h>
0034 #include <rdma/uverbs_ioctl.h>
0035 #include "rdma_core.h"
0036 #include "uverbs.h"
0037
0038 struct bundle_alloc_head {
0039     struct bundle_alloc_head *next;
0040     u8 data[];
0041 };
0042
0043 struct bundle_priv {
0044     /* Must be first */
0045     struct bundle_alloc_head alloc_head;
0046     struct bundle_alloc_head *allocated_mem;
0047     size_t internal_avail;
0048     size_t internal_used;
0049
0050     struct radix_tree_root *radix;
0051     const struct uverbs_api_ioctl_method *method_elm;
0052     void __rcu **radix_slots;
0053     unsigned long radix_slots_len;
0054     u32 method_key;
0055
0056     struct ib_uverbs_attr __user *user_attrs;
0057     struct ib_uverbs_attr *uattrs;
0058
0059     DECLARE_BITMAP(uobj_finalize, UVERBS_API_ATTR_BKEY_LEN);
0060     DECLARE_BITMAP(spec_finalize, UVERBS_API_ATTR_BKEY_LEN);
0061     DECLARE_BITMAP(uobj_hw_obj_valid, UVERBS_API_ATTR_BKEY_LEN);
0062
0063     /*
0064      * Must be last. bundle ends in a flex array which overlaps
0065      * internal_buffer.
0066      */
0067     struct uverbs_attr_bundle bundle;
0068     u64 internal_buffer[32];
0069 };
0070
0071 /*
0072  * Each method has an absolute minimum amount of memory it needs to allocate,
0073  * precompute that amount and determine if the onstack memory can be used or
0074  * if allocation is need.
0075  */
0076 void uapi_compute_bundle_size(struct uverbs_api_ioctl_method *method_elm,
0077                   unsigned int num_attrs)
0078 {
0079     struct bundle_priv *pbundle;
0080     size_t bundle_size =
0081         offsetof(struct bundle_priv, internal_buffer) +
0082         sizeof(*pbundle->bundle.attrs) * method_elm->key_bitmap_len +
0083         sizeof(*pbundle->uattrs) * num_attrs;
0084
0085     method_elm->use_stack = bundle_size <= sizeof(*pbundle);
0086     method_elm->bundle_size =
0087         ALIGN(bundle_size + 256, sizeof(*pbundle->internal_buffer));
0088
0089     /* Do not want order-2 allocations for this. */
0090     WARN_ON_ONCE(method_elm->bundle_size > PAGE_SIZE);
0091 }
0092
0093 /**
0094  * _uverbs_alloc() - Quickly allocate memory for use with a bundle
0095  * @bundle: The bundle
0096  * @size: Number of bytes to allocate
0097  * @flags: Allocator flags
0098  *
0099  * The bundle allocator is intended for allocations that are connected with
0100  * processing the system call related to the bundle. The allocated memory is
0101  * always freed once the system call completes, and cannot be freed any other
0102  * way.
0103  *
0104  * This tries to use a small pool of pre-allocated memory for performance.
0105  */
0106 __malloc void *_uverbs_alloc(struct uverbs_attr_bundle *bundle, size_t size,
0107                  gfp_t flags)
0108 {
0109     struct bundle_priv *pbundle =
0110         container_of(bundle, struct bundle_priv, bundle);
0111     size_t new_used;
0112     void *res;
0113
0114     if (check_add_overflow(size, pbundle->internal_used, &new_used))
0115         return ERR_PTR(-EOVERFLOW);
0116
0117     if (new_used > pbundle->internal_avail) {
0118         struct bundle_alloc_head *buf;
0119
0120         buf = kvmalloc(struct_size(buf, data, size), flags);
0121         if (!buf)
0122             return ERR_PTR(-ENOMEM);
0123         buf->next = pbundle->allocated_mem;
0124         pbundle->allocated_mem = buf;
0125         return buf->data;
0126     }
0127
0128     res = (void *)pbundle->internal_buffer + pbundle->internal_used;
0129     pbundle->internal_used =
0130         ALIGN(new_used, sizeof(*pbundle->internal_buffer));
0131     if (want_init_on_alloc(flags))
0132         memset(res, 0, size);
0133     return res;
0134 }
0135 EXPORT_SYMBOL(_uverbs_alloc);
0136
0137 static bool uverbs_is_attr_cleared(const struct ib_uverbs_attr *uattr,
0138                    u16 len)
0139 {
0140     if (uattr->len > sizeof_field(struct ib_uverbs_attr, data))
0141         return ib_is_buffer_cleared(u64_to_user_ptr(uattr->data) + len,
0142                         uattr->len - len);
0143
0144     return !memchr_inv((const void *)&uattr->data + len,
0145                0, uattr->len - len);
0146 }
0147
0148 static int uverbs_set_output(const struct uverbs_attr_bundle *bundle,
0149                  const struct uverbs_attr *attr)
0150 {
0151     struct bundle_priv *pbundle =
0152         container_of(bundle, struct bundle_priv, bundle);
0153     u16 flags;
0154
0155     flags = pbundle->uattrs[attr->ptr_attr.uattr_idx].flags |
0156         UVERBS_ATTR_F_VALID_OUTPUT;
0157     if (put_user(flags,
0158              &pbundle->user_attrs[attr->ptr_attr.uattr_idx].flags))
0159         return -EFAULT;
0160     return 0;
0161 }
0162
0163 static int uverbs_process_idrs_array(struct bundle_priv *pbundle,
0164                      const struct uverbs_api_attr *attr_uapi,
0165                      struct uverbs_objs_arr_attr *attr,
0166                      struct ib_uverbs_attr *uattr,
0167                      u32 attr_bkey)
0168 {
0169     const struct uverbs_attr_spec *spec = &attr_uapi->spec;
0170     size_t array_len;
0171     u32 *idr_vals;
0172     int ret = 0;
0173     size_t i;
0174
0175     if (uattr->attr_data.reserved)
0176         return -EINVAL;
0177
0178     if (uattr->len % sizeof(u32))
0179         return -EINVAL;
0180
0181     array_len = uattr->len / sizeof(u32);
0182     if (array_len < spec->u2.objs_arr.min_len ||
0183         array_len > spec->u2.objs_arr.max_len)
0184         return -EINVAL;
0185
0186     attr->uobjects =
0187         uverbs_alloc(&pbundle->bundle,
0188                  array_size(array_len, sizeof(*attr->uobjects)));
0189     if (IS_ERR(attr->uobjects))
0190         return PTR_ERR(attr->uobjects);
0191
0192     /*
0193      * Since idr is 4B and *uobjects is >= 4B, we can use attr->uobjects
0194      * to store idrs array and avoid additional memory allocation. The
0195      * idrs array is offset to the end of the uobjects array so we will be
0196      * able to read idr and replace with a pointer.
0197      */
0198     idr_vals = (u32 *)(attr->uobjects + array_len) - array_len;
0199
0200     if (uattr->len > sizeof(uattr->data)) {
0201         ret = copy_from_user(idr_vals, u64_to_user_ptr(uattr->data),
0202                      uattr->len);
0203         if (ret)
0204             return -EFAULT;
0205     } else {
0206         memcpy(idr_vals, &uattr->data, uattr->len);
0207     }
0208
0209     for (i = 0; i != array_len; i++) {
0210         attr->uobjects[i] = uverbs_get_uobject_from_file(
0211             spec->u2.objs_arr.obj_type, spec->u2.objs_arr.access,
0212             idr_vals[i], &pbundle->bundle);
0213         if (IS_ERR(attr->uobjects[i])) {
0214             ret = PTR_ERR(attr->uobjects[i]);
0215             break;
0216         }
0217     }
0218
0219     attr->len = i;
0220     __set_bit(attr_bkey, pbundle->spec_finalize);
0221     return ret;
0222 }
0223
0224 static void uverbs_free_idrs_array(const struct uverbs_api_attr *attr_uapi,
0225                    struct uverbs_objs_arr_attr *attr,
0226                    bool commit,
0227                    struct uverbs_attr_bundle *attrs)
0228 {
0229     const struct uverbs_attr_spec *spec = &attr_uapi->spec;
0230     size_t i;
0231
0232     for (i = 0; i != attr->len; i++)
0233         uverbs_finalize_object(attr->uobjects[i],
0234                        spec->u2.objs_arr.access, false, commit,
0235                        attrs);
0236 }
0237
0238 static int uverbs_process_attr(struct bundle_priv *pbundle,
0239                    const struct uverbs_api_attr *attr_uapi,
0240                    struct ib_uverbs_attr *uattr, u32 attr_bkey)
0241 {
0242     const struct uverbs_attr_spec *spec = &attr_uapi->spec;
0243     struct uverbs_attr *e = &pbundle->bundle.attrs[attr_bkey];
0244     const struct uverbs_attr_spec *val_spec = spec;
0245     struct uverbs_obj_attr *o_attr;
0246
0247     switch (spec->type) {
0248     case UVERBS_ATTR_TYPE_ENUM_IN:
0249         if (uattr->attr_data.enum_data.elem_id >= spec->u.enum_def.num_elems)
0250             return -EOPNOTSUPP;
0251
0252         if (uattr->attr_data.enum_data.reserved)
0253             return -EINVAL;
0254
0255         val_spec = &spec->u2.enum_def.ids[uattr->attr_data.enum_data.elem_id];
0256
0257         /* Currently we only support PTR_IN based enums */
0258         if (val_spec->type != UVERBS_ATTR_TYPE_PTR_IN)
0259             return -EOPNOTSUPP;
0260
0261         e->ptr_attr.enum_id = uattr->attr_data.enum_data.elem_id;
0262         fallthrough;
0263     case UVERBS_ATTR_TYPE_PTR_IN:
0264         /* Ensure that any data provided by userspace beyond the known
0265          * struct is zero. Userspace that knows how to use some future
0266          * longer struct will fail here if used with an old kernel and
0267          * non-zero content, making ABI compat/discovery simpler.
0268          */
0269         if (uattr->len > val_spec->u.ptr.len &&
0270             val_spec->zero_trailing &&
0271             !uverbs_is_attr_cleared(uattr, val_spec->u.ptr.len))
0272             return -EOPNOTSUPP;
0273
0274         fallthrough;
0275     case UVERBS_ATTR_TYPE_PTR_OUT:
0276         if (uattr->len < val_spec->u.ptr.min_len ||
0277             (!val_spec->zero_trailing &&
0278              uattr->len > val_spec->u.ptr.len))
0279             return -EINVAL;
0280
0281         if (spec->type != UVERBS_ATTR_TYPE_ENUM_IN &&
0282             uattr->attr_data.reserved)
0283             return -EINVAL;
0284
0285         e->ptr_attr.uattr_idx = uattr - pbundle->uattrs;
0286         e->ptr_attr.len = uattr->len;
0287
0288         if (val_spec->alloc_and_copy && !uverbs_attr_ptr_is_inline(e)) {
0289             void *p;
0290
0291             p = uverbs_alloc(&pbundle->bundle, uattr->len);
0292             if (IS_ERR(p))
0293                 return PTR_ERR(p);
0294
0295             e->ptr_attr.ptr = p;
0296
0297             if (copy_from_user(p, u64_to_user_ptr(uattr->data),
0298                        uattr->len))
0299                 return -EFAULT;
0300         } else {
0301             e->ptr_attr.data = uattr->data;
0302         }
0303         break;
0304
0305     case UVERBS_ATTR_TYPE_IDR:
0306     case UVERBS_ATTR_TYPE_FD:
0307         if (uattr->attr_data.reserved)
0308             return -EINVAL;
0309
0310         if (uattr->len != 0)
0311             return -EINVAL;
0312
0313         o_attr = &e->obj_attr;
0314         o_attr->attr_elm = attr_uapi;
0315
0316         /*
0317          * The type of uattr->data is u64 for UVERBS_ATTR_TYPE_IDR and
0318          * s64 for UVERBS_ATTR_TYPE_FD. We can cast the u64 to s64
0319          * here without caring about truncation as we know that the
0320          * IDR implementation today rejects negative IDs
0321          */
0322         o_attr->uobject = uverbs_get_uobject_from_file(
0323             spec->u.obj.obj_type, spec->u.obj.access,
0324             uattr->data_s64, &pbundle->bundle);
0325         if (IS_ERR(o_attr->uobject))
0326             return PTR_ERR(o_attr->uobject);
0327         __set_bit(attr_bkey, pbundle->uobj_finalize);
0328
0329         if (spec->u.obj.access == UVERBS_ACCESS_NEW) {
0330             unsigned int uattr_idx = uattr - pbundle->uattrs;
0331             s64 id = o_attr->uobject->id;
0332
0333             /* Copy the allocated id to the user-space */
0334             if (put_user(id, &pbundle->user_attrs[uattr_idx].data))
0335                 return -EFAULT;
0336         }
0337
0338         break;
0339
0340     case UVERBS_ATTR_TYPE_IDRS_ARRAY:
0341         return uverbs_process_idrs_array(pbundle, attr_uapi,
0342                          &e->objs_arr_attr, uattr,
0343                          attr_bkey);
0344     default:
0345         return -EOPNOTSUPP;
0346     }
0347
0348     return 0;
0349 }
0350
0351 /*
0352  * We search the radix tree with the method prefix and now we want to fast
0353  * search the suffix bits to get a particular attribute pointer. It is not
0354  * totally clear to me if this breaks the radix tree encasulation or not, but
0355  * it uses the iter data to determine if the method iter points at the same
0356  * chunk that will store the attribute, if so it just derefs it directly. By
0357  * construction in most kernel configs the method and attrs will all fit in a
0358  * single radix chunk, so in most cases this will have no search. Other cases
0359  * this falls back to a full search.
0360  */
0361 static void __rcu **uapi_get_attr_for_method(struct bundle_priv *pbundle,
0362                          u32 attr_key)
0363 {
0364     void __rcu **slot;
0365
0366     if (likely(attr_key < pbundle->radix_slots_len)) {
0367         void *entry;
0368
0369         slot = pbundle->radix_slots + attr_key;
0370         entry = rcu_dereference_raw(*slot);
0371         if (likely(!radix_tree_is_internal_node(entry) && entry))
0372             return slot;
0373     }
0374
0375     return radix_tree_lookup_slot(pbundle->radix,
0376                       pbundle->method_key | attr_key);
0377 }
0378
0379 static int uverbs_set_attr(struct bundle_priv *pbundle,
0380                struct ib_uverbs_attr *uattr)
0381 {
0382     u32 attr_key = uapi_key_attr(uattr->attr_id);
0383     u32 attr_bkey = uapi_bkey_attr(attr_key);
0384     const struct uverbs_api_attr *attr;
0385     void __rcu **slot;
0386     int ret;
0387
0388     slot = uapi_get_attr_for_method(pbundle, attr_key);
0389     if (!slot) {
0390         /*
0391          * Kernel does not support the attribute but user-space says it
0392          * is mandatory
0393          */
0394         if (uattr->flags & UVERBS_ATTR_F_MANDATORY)
0395             return -EPROTONOSUPPORT;
0396         return 0;
0397     }
0398     attr = rcu_dereference_protected(*slot, true);
0399
0400     /* Reject duplicate attributes from user-space */
0401     if (test_bit(attr_bkey, pbundle->bundle.attr_present))
0402         return -EINVAL;
0403
0404     ret = uverbs_process_attr(pbundle, attr, uattr, attr_bkey);
0405     if (ret)
0406         return ret;
0407
0408     __set_bit(attr_bkey, pbundle->bundle.attr_present);
0409
0410     return 0;
0411 }
0412
0413 static int ib_uverbs_run_method(struct bundle_priv *pbundle,
0414                 unsigned int num_attrs)
0415 {
0416     int (*handler)(struct uverbs_attr_bundle *attrs);
0417     size_t uattrs_size = array_size(sizeof(*pbundle->uattrs), num_attrs);
0418     unsigned int destroy_bkey = pbundle->method_elm->destroy_bkey;
0419     unsigned int i;
0420     int ret;
0421
0422     /* See uverbs_disassociate_api() */
0423     handler = srcu_dereference(
0424         pbundle->method_elm->handler,
0425         &pbundle->bundle.ufile->device->disassociate_srcu);
0426     if (!handler)
0427         return -EIO;
0428
0429     pbundle->uattrs = uverbs_alloc(&pbundle->bundle, uattrs_size);
0430     if (IS_ERR(pbundle->uattrs))
0431         return PTR_ERR(pbundle->uattrs);
0432     if (copy_from_user(pbundle->uattrs, pbundle->user_attrs, uattrs_size))
0433         return -EFAULT;
0434
0435     for (i = 0; i != num_attrs; i++) {
0436         ret = uverbs_set_attr(pbundle, &pbundle->uattrs[i]);
0437         if (unlikely(ret))
0438             return ret;
0439     }
0440
0441     /* User space did not provide all the mandatory attributes */
0442     if (unlikely(!bitmap_subset(pbundle->method_elm->attr_mandatory,
0443                     pbundle->bundle.attr_present,
0444                     pbundle->method_elm->key_bitmap_len)))
0445         return -EINVAL;
0446
0447     if (pbundle->method_elm->has_udata)
0448         uverbs_fill_udata(&pbundle->bundle,
0449                   &pbundle->bundle.driver_udata,
0450                   UVERBS_ATTR_UHW_IN, UVERBS_ATTR_UHW_OUT);
0451     else
0452         pbundle->bundle.driver_udata = (struct ib_udata){};
0453
0454     if (destroy_bkey != UVERBS_API_ATTR_BKEY_LEN) {
0455         struct uverbs_obj_attr *destroy_attr =
0456             &pbundle->bundle.attrs[destroy_bkey].obj_attr;
0457
0458         ret = uobj_destroy(destroy_attr->uobject, &pbundle->bundle);
0459         if (ret)
0460             return ret;
0461         __clear_bit(destroy_bkey, pbundle->uobj_finalize);
0462
0463         ret = handler(&pbundle->bundle);
0464         uobj_put_destroy(destroy_attr->uobject);
0465     } else {
0466         ret = handler(&pbundle->bundle);
0467     }
0468
0469     /*
0470      * Until the drivers are revised to use the bundle directly we have to
0471      * assume that the driver wrote to its UHW_OUT and flag userspace
0472      * appropriately.
0473      */
0474     if (!ret && pbundle->method_elm->has_udata) {
0475         const struct uverbs_attr *attr =
0476             uverbs_attr_get(&pbundle->bundle, UVERBS_ATTR_UHW_OUT);
0477
0478         if (!IS_ERR(attr))
0479             ret = uverbs_set_output(&pbundle->bundle, attr);
0480     }
0481
0482     /*
0483      * EPROTONOSUPPORT is ONLY to be returned if the ioctl framework can
0484      * not invoke the method because the request is not supported.  No
0485      * other cases should return this code.
0486      */
0487     if (WARN_ON_ONCE(ret == -EPROTONOSUPPORT))
0488         return -EINVAL;
0489
0490     return ret;
0491 }
0492
0493 static void bundle_destroy(struct bundle_priv *pbundle, bool commit)
0494 {
0495     unsigned int key_bitmap_len = pbundle->method_elm->key_bitmap_len;
0496     struct bundle_alloc_head *memblock;
0497     unsigned int i;
0498
0499     /* fast path for simple uobjects */
0500     i = -1;
0501     while ((i = find_next_bit(pbundle->uobj_finalize, key_bitmap_len,
0502                   i + 1)) < key_bitmap_len) {
0503         struct uverbs_attr *attr = &pbundle->bundle.attrs[i];
0504
0505         uverbs_finalize_object(
0506             attr->obj_attr.uobject,
0507             attr->obj_attr.attr_elm->spec.u.obj.access,
0508             test_bit(i, pbundle->uobj_hw_obj_valid),
0509             commit,
0510             &pbundle->bundle);
0511     }
0512
0513     i = -1;
0514     while ((i = find_next_bit(pbundle->spec_finalize, key_bitmap_len,
0515                   i + 1)) < key_bitmap_len) {
0516         struct uverbs_attr *attr = &pbundle->bundle.attrs[i];
0517         const struct uverbs_api_attr *attr_uapi;
0518         void __rcu **slot;
0519
0520         slot = uapi_get_attr_for_method(
0521             pbundle,
0522             pbundle->method_key | uapi_bkey_to_key_attr(i));
0523         if (WARN_ON(!slot))
0524             continue;
0525
0526         attr_uapi = rcu_dereference_protected(*slot, true);
0527
0528         if (attr_uapi->spec.type == UVERBS_ATTR_TYPE_IDRS_ARRAY) {
0529             uverbs_free_idrs_array(attr_uapi, &attr->objs_arr_attr,
0530                            commit, &pbundle->bundle);
0531         }
0532     }
0533
0534     for (memblock = pbundle->allocated_mem; memblock;) {
0535         struct bundle_alloc_head *tmp = memblock;
0536
0537         memblock = memblock->next;
0538         kvfree(tmp);
0539     }
0540 }
0541
0542 static int ib_uverbs_cmd_verbs(struct ib_uverbs_file *ufile,
0543                    struct ib_uverbs_ioctl_hdr *hdr,
0544                    struct ib_uverbs_attr __user *user_attrs)
0545 {
0546     const struct uverbs_api_ioctl_method *method_elm;
0547     struct uverbs_api *uapi = ufile->device->uapi;
0548     struct radix_tree_iter attrs_iter;
0549     struct bundle_priv *pbundle;
0550     struct bundle_priv onstack;
0551     void __rcu **slot;
0552     int ret;
0553
0554     if (unlikely(hdr->driver_id != uapi->driver_id))
0555         return -EINVAL;
0556
0557     slot = radix_tree_iter_lookup(
0558         &uapi->radix, &attrs_iter,
0559         uapi_key_obj(hdr->object_id) |
0560             uapi_key_ioctl_method(hdr->method_id));
0561     if (unlikely(!slot))
0562         return -EPROTONOSUPPORT;
0563     method_elm = rcu_dereference_protected(*slot, true);
0564
0565     if (!method_elm->use_stack) {
0566         pbundle = kmalloc(method_elm->bundle_size, GFP_KERNEL);
0567         if (!pbundle)
0568             return -ENOMEM;
0569         pbundle->internal_avail =
0570             method_elm->bundle_size -
0571             offsetof(struct bundle_priv, internal_buffer);
0572         pbundle->alloc_head.next = NULL;
0573         pbundle->allocated_mem = &pbundle->alloc_head;
0574     } else {
0575         pbundle = &onstack;
0576         pbundle->internal_avail = sizeof(pbundle->internal_buffer);
0577         pbundle->allocated_mem = NULL;
0578     }
0579
0580     /* Space for the pbundle->bundle.attrs flex array */
0581     pbundle->method_elm = method_elm;
0582     pbundle->method_key = attrs_iter.index;
0583     pbundle->bundle.ufile = ufile;
0584     pbundle->bundle.context = NULL; /* only valid if bundle has uobject */
0585     pbundle->radix = &uapi->radix;
0586     pbundle->radix_slots = slot;
0587     pbundle->radix_slots_len = radix_tree_chunk_size(&attrs_iter);
0588     pbundle->user_attrs = user_attrs;
0589
0590     pbundle->internal_used = ALIGN(pbundle->method_elm->key_bitmap_len *
0591                            sizeof(*pbundle->bundle.attrs),
0592                        sizeof(*pbundle->internal_buffer));
0593     memset(pbundle->bundle.attr_present, 0,
0594            sizeof(pbundle->bundle.attr_present));
0595     memset(pbundle->uobj_finalize, 0, sizeof(pbundle->uobj_finalize));
0596     memset(pbundle->spec_finalize, 0, sizeof(pbundle->spec_finalize));
0597     memset(pbundle->uobj_hw_obj_valid, 0,
0598            sizeof(pbundle->uobj_hw_obj_valid));
0599
0600     ret = ib_uverbs_run_method(pbundle, hdr->num_attrs);
0601     bundle_destroy(pbundle, ret == 0);
0602     return ret;
0603 }
0604
0605 long ib_uverbs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
0606 {
0607     struct ib_uverbs_file *file = filp->private_data;
0608     struct ib_uverbs_ioctl_hdr __user *user_hdr =
0609         (struct ib_uverbs_ioctl_hdr __user *)arg;
0610     struct ib_uverbs_ioctl_hdr hdr;
0611     int srcu_key;
0612     int err;
0613
0614     if (unlikely(cmd != RDMA_VERBS_IOCTL))
0615         return -ENOIOCTLCMD;
0616
0617     err = copy_from_user(&hdr, user_hdr, sizeof(hdr));
0618     if (err)
0619         return -EFAULT;
0620
0621     if (hdr.length > PAGE_SIZE ||
0622         hdr.length != struct_size(&hdr, attrs, hdr.num_attrs))
0623         return -EINVAL;
0624
0625     if (hdr.reserved1 || hdr.reserved2)
0626         return -EPROTONOSUPPORT;
0627
0628     srcu_key = srcu_read_lock(&file->device->disassociate_srcu);
0629     err = ib_uverbs_cmd_verbs(file, &hdr, user_hdr->attrs);
0630     srcu_read_unlock(&file->device->disassociate_srcu, srcu_key);
0631     return err;
0632 }
0633
0634 int uverbs_get_flags64(u64 *to, const struct uverbs_attr_bundle *attrs_bundle,
0635                size_t idx, u64 allowed_bits)
0636 {
0637     const struct uverbs_attr *attr;
0638     u64 flags;
0639
0640     attr = uverbs_attr_get(attrs_bundle, idx);
0641     /* Missing attribute means 0 flags */
0642     if (IS_ERR(attr)) {
0643         *to = 0;
0644         return 0;
0645     }
0646
0647     /*
0648      * New userspace code should use 8 bytes to pass flags, but we
0649      * transparently support old userspaces that were using 4 bytes as
0650      * well.
0651      */
0652     if (attr->ptr_attr.len == 8)
0653         flags = attr->ptr_attr.data;
0654     else if (attr->ptr_attr.len == 4)
0655         flags = *(u32 *)&attr->ptr_attr.data;
0656     else
0657         return -EINVAL;
0658
0659     if (flags & ~allowed_bits)
0660         return -EINVAL;
0661
0662     *to = flags;
0663     return 0;
0664 }
0665 EXPORT_SYMBOL(uverbs_get_flags64);
0666
0667 int uverbs_get_flags32(u32 *to, const struct uverbs_attr_bundle *attrs_bundle,
0668                size_t idx, u64 allowed_bits)
0669 {
0670     u64 flags;
0671     int ret;
0672
0673     ret = uverbs_get_flags64(&flags, attrs_bundle, idx, allowed_bits);
0674     if (ret)
0675         return ret;
0676
0677     if (flags > U32_MAX)
0678         return -EINVAL;
0679     *to = flags;
0680
0681     return 0;
0682 }
0683 EXPORT_SYMBOL(uverbs_get_flags32);
0684
0685 /*
0686  * Fill a ib_udata struct (core or uhw) using the given attribute IDs.
0687  * This is primarily used to convert the UVERBS_ATTR_UHW() into the
0688  * ib_udata format used by the drivers.
0689  */
0690 void uverbs_fill_udata(struct uverbs_attr_bundle *bundle,
0691                struct ib_udata *udata, unsigned int attr_in,
0692                unsigned int attr_out)
0693 {
0694     struct bundle_priv *pbundle =
0695         container_of(bundle, struct bundle_priv, bundle);
0696     const struct uverbs_attr *in =
0697         uverbs_attr_get(&pbundle->bundle, attr_in);
0698     const struct uverbs_attr *out =
0699         uverbs_attr_get(&pbundle->bundle, attr_out);
0700
0701     if (!IS_ERR(in)) {
0702         udata->inlen = in->ptr_attr.len;
0703         if (uverbs_attr_ptr_is_inline(in))
0704             udata->inbuf =
0705                 &pbundle->user_attrs[in->ptr_attr.uattr_idx]
0706                      .data;
0707         else
0708             udata->inbuf = u64_to_user_ptr(in->ptr_attr.data);
0709     } else {
0710         udata->inbuf = NULL;
0711         udata->inlen = 0;
0712     }
0713
0714     if (!IS_ERR(out)) {
0715         udata->outbuf = u64_to_user_ptr(out->ptr_attr.data);
0716         udata->outlen = out->ptr_attr.len;
0717     } else {
0718         udata->outbuf = NULL;
0719         udata->outlen = 0;
0720     }
0721 }
0722
0723 int uverbs_copy_to(const struct uverbs_attr_bundle *bundle, size_t idx,
0724            const void *from, size_t size)
0725 {
0726     const struct uverbs_attr *attr = uverbs_attr_get(bundle, idx);
0727     size_t min_size;
0728
0729     if (IS_ERR(attr))
0730         return PTR_ERR(attr);
0731
0732     min_size = min_t(size_t, attr->ptr_attr.len, size);
0733     if (copy_to_user(u64_to_user_ptr(attr->ptr_attr.data), from, min_size))
0734         return -EFAULT;
0735
0736     return uverbs_set_output(bundle, attr);
0737 }
0738 EXPORT_SYMBOL(uverbs_copy_to);
0739
0740
0741 /*
0742  * This is only used if the caller has directly used copy_to_use to write the
0743  * data.  It signals to user space that the buffer is filled in.
0744  */
0745 int uverbs_output_written(const struct uverbs_attr_bundle *bundle, size_t idx)
0746 {
0747     const struct uverbs_attr *attr = uverbs_attr_get(bundle, idx);
0748
0749     if (IS_ERR(attr))
0750         return PTR_ERR(attr);
0751
0752     return uverbs_set_output(bundle, attr);
0753 }
0754
0755 int _uverbs_get_const_signed(s64 *to,
0756                  const struct uverbs_attr_bundle *attrs_bundle,
0757                  size_t idx, s64 lower_bound, u64 upper_bound,
0758                  s64  *def_val)
0759 {
0760     const struct uverbs_attr *attr;
0761
0762     attr = uverbs_attr_get(attrs_bundle, idx);
0763     if (IS_ERR(attr)) {
0764         if ((PTR_ERR(attr) != -ENOENT) || !def_val)
0765             return PTR_ERR(attr);
0766
0767         *to = *def_val;
0768     } else {
0769         *to = attr->ptr_attr.data;
0770     }
0771
0772     if (*to < lower_bound || (*to > 0 && (u64)*to > upper_bound))
0773         return -EINVAL;
0774
0775     return 0;
0776 }
0777 EXPORT_SYMBOL(_uverbs_get_const_signed);
0778
0779 int _uverbs_get_const_unsigned(u64 *to,
0780                    const struct uverbs_attr_bundle *attrs_bundle,
0781                    size_t idx, u64 upper_bound, u64 *def_val)
0782 {
0783     const struct uverbs_attr *attr;
0784
0785     attr = uverbs_attr_get(attrs_bundle, idx);
0786     if (IS_ERR(attr)) {
0787         if ((PTR_ERR(attr) != -ENOENT) || !def_val)
0788             return PTR_ERR(attr);
0789
0790         *to = *def_val;
0791     } else {
0792         *to = attr->ptr_attr.data;
0793     }
0794
0795     if (*to > upper_bound)
0796         return -EINVAL;
0797
0798     return 0;
0799 }
0800 EXPORT_SYMBOL(_uverbs_get_const_unsigned);
0801
0802 int uverbs_copy_to_struct_or_zero(const struct uverbs_attr_bundle *bundle,
0803                   size_t idx, const void *from, size_t size)
0804 {
0805     const struct uverbs_attr *attr = uverbs_attr_get(bundle, idx);
0806
0807     if (IS_ERR(attr))
0808         return PTR_ERR(attr);
0809
0810     if (size < attr->ptr_attr.len) {
0811         if (clear_user(u64_to_user_ptr(attr->ptr_attr.data) + size,
0812                    attr->ptr_attr.len - size))
0813             return -EFAULT;
0814     }
0815     return uverbs_copy_to(bundle, idx, from, size);
0816 }
0817 EXPORT_SYMBOL(uverbs_copy_to_struct_or_zero);
0818
0819 /* Once called an abort will call through to the type's destroy_hw() */
0820 void uverbs_finalize_uobj_create(const struct uverbs_attr_bundle *bundle,
0821                  u16 idx)
0822 {
0823     struct bundle_priv *pbundle =
0824         container_of(bundle, struct bundle_priv, bundle);
0825
0826     __set_bit(uapi_bkey_attr(uapi_key_attr(idx)),
0827           pbundle->uobj_hw_obj_valid);
0828 }
0829 EXPORT_SYMBOL(uverbs_finalize_uobj_create);