Back to home page

LXR

 
 

    


0001 #include <linux/export.h>
0002 #include <linux/bvec.h>
0003 #include <linux/uio.h>
0004 #include <linux/pagemap.h>
0005 #include <linux/slab.h>
0006 #include <linux/vmalloc.h>
0007 #include <linux/splice.h>
0008 #include <net/checksum.h>
0009 
0010 #define PIPE_PARANOIA /* for now */
0011 
0012 #define iterate_iovec(i, n, __v, __p, skip, STEP) { \
0013     size_t left;                    \
0014     size_t wanted = n;              \
0015     __p = i->iov;                   \
0016     __v.iov_len = min(n, __p->iov_len - skip);  \
0017     if (likely(__v.iov_len)) {          \
0018         __v.iov_base = __p->iov_base + skip;    \
0019         left = (STEP);              \
0020         __v.iov_len -= left;            \
0021         skip += __v.iov_len;            \
0022         n -= __v.iov_len;           \
0023     } else {                    \
0024         left = 0;               \
0025     }                       \
0026     while (unlikely(!left && n)) {          \
0027         __p++;                  \
0028         __v.iov_len = min(n, __p->iov_len); \
0029         if (unlikely(!__v.iov_len))     \
0030             continue;           \
0031         __v.iov_base = __p->iov_base;       \
0032         left = (STEP);              \
0033         __v.iov_len -= left;            \
0034         skip = __v.iov_len;         \
0035         n -= __v.iov_len;           \
0036     }                       \
0037     n = wanted - n;                 \
0038 }
0039 
0040 #define iterate_kvec(i, n, __v, __p, skip, STEP) {  \
0041     size_t wanted = n;              \
0042     __p = i->kvec;                  \
0043     __v.iov_len = min(n, __p->iov_len - skip);  \
0044     if (likely(__v.iov_len)) {          \
0045         __v.iov_base = __p->iov_base + skip;    \
0046         (void)(STEP);               \
0047         skip += __v.iov_len;            \
0048         n -= __v.iov_len;           \
0049     }                       \
0050     while (unlikely(n)) {               \
0051         __p++;                  \
0052         __v.iov_len = min(n, __p->iov_len); \
0053         if (unlikely(!__v.iov_len))     \
0054             continue;           \
0055         __v.iov_base = __p->iov_base;       \
0056         (void)(STEP);               \
0057         skip = __v.iov_len;         \
0058         n -= __v.iov_len;           \
0059     }                       \
0060     n = wanted;                 \
0061 }
0062 
0063 #define iterate_bvec(i, n, __v, __bi, skip, STEP) { \
0064     struct bvec_iter __start;           \
0065     __start.bi_size = n;                \
0066     __start.bi_bvec_done = skip;            \
0067     __start.bi_idx = 0;             \
0068     for_each_bvec(__v, i->bvec, __bi, __start) {    \
0069         if (!__v.bv_len)            \
0070             continue;           \
0071         (void)(STEP);               \
0072     }                       \
0073 }
0074 
0075 #define iterate_all_kinds(i, n, v, I, B, K) {           \
0076     if (likely(n)) {                    \
0077         size_t skip = i->iov_offset;            \
0078         if (unlikely(i->type & ITER_BVEC)) {        \
0079             struct bio_vec v;           \
0080             struct bvec_iter __bi;          \
0081             iterate_bvec(i, n, v, __bi, skip, (B))  \
0082         } else if (unlikely(i->type & ITER_KVEC)) { \
0083             const struct kvec *kvec;        \
0084             struct kvec v;              \
0085             iterate_kvec(i, n, v, kvec, skip, (K))  \
0086         } else {                    \
0087             const struct iovec *iov;        \
0088             struct iovec v;             \
0089             iterate_iovec(i, n, v, iov, skip, (I))  \
0090         }                       \
0091     }                           \
0092 }
0093 
0094 #define iterate_and_advance(i, n, v, I, B, K) {         \
0095     if (unlikely(i->count < n))             \
0096         n = i->count;                   \
0097     if (i->count) {                     \
0098         size_t skip = i->iov_offset;            \
0099         if (unlikely(i->type & ITER_BVEC)) {        \
0100             const struct bio_vec *bvec = i->bvec;   \
0101             struct bio_vec v;           \
0102             struct bvec_iter __bi;          \
0103             iterate_bvec(i, n, v, __bi, skip, (B))  \
0104             i->bvec = __bvec_iter_bvec(i->bvec, __bi);  \
0105             i->nr_segs -= i->bvec - bvec;       \
0106             skip = __bi.bi_bvec_done;       \
0107         } else if (unlikely(i->type & ITER_KVEC)) { \
0108             const struct kvec *kvec;        \
0109             struct kvec v;              \
0110             iterate_kvec(i, n, v, kvec, skip, (K))  \
0111             if (skip == kvec->iov_len) {        \
0112                 kvec++;             \
0113                 skip = 0;           \
0114             }                   \
0115             i->nr_segs -= kvec - i->kvec;       \
0116             i->kvec = kvec;             \
0117         } else {                    \
0118             const struct iovec *iov;        \
0119             struct iovec v;             \
0120             iterate_iovec(i, n, v, iov, skip, (I))  \
0121             if (skip == iov->iov_len) {     \
0122                 iov++;              \
0123                 skip = 0;           \
0124             }                   \
0125             i->nr_segs -= iov - i->iov;     \
0126             i->iov = iov;               \
0127         }                       \
0128         i->count -= n;                  \
0129         i->iov_offset = skip;               \
0130     }                           \
0131 }
0132 
0133 static size_t copy_page_to_iter_iovec(struct page *page, size_t offset, size_t bytes,
0134              struct iov_iter *i)
0135 {
0136     size_t skip, copy, left, wanted;
0137     const struct iovec *iov;
0138     char __user *buf;
0139     void *kaddr, *from;
0140 
0141     if (unlikely(bytes > i->count))
0142         bytes = i->count;
0143 
0144     if (unlikely(!bytes))
0145         return 0;
0146 
0147     wanted = bytes;
0148     iov = i->iov;
0149     skip = i->iov_offset;
0150     buf = iov->iov_base + skip;
0151     copy = min(bytes, iov->iov_len - skip);
0152 
0153     if (IS_ENABLED(CONFIG_HIGHMEM) && !fault_in_pages_writeable(buf, copy)) {
0154         kaddr = kmap_atomic(page);
0155         from = kaddr + offset;
0156 
0157         /* first chunk, usually the only one */
0158         left = __copy_to_user_inatomic(buf, from, copy);
0159         copy -= left;
0160         skip += copy;
0161         from += copy;
0162         bytes -= copy;
0163 
0164         while (unlikely(!left && bytes)) {
0165             iov++;
0166             buf = iov->iov_base;
0167             copy = min(bytes, iov->iov_len);
0168             left = __copy_to_user_inatomic(buf, from, copy);
0169             copy -= left;
0170             skip = copy;
0171             from += copy;
0172             bytes -= copy;
0173         }
0174         if (likely(!bytes)) {
0175             kunmap_atomic(kaddr);
0176             goto done;
0177         }
0178         offset = from - kaddr;
0179         buf += copy;
0180         kunmap_atomic(kaddr);
0181         copy = min(bytes, iov->iov_len - skip);
0182     }
0183     /* Too bad - revert to non-atomic kmap */
0184 
0185     kaddr = kmap(page);
0186     from = kaddr + offset;
0187     left = __copy_to_user(buf, from, copy);
0188     copy -= left;
0189     skip += copy;
0190     from += copy;
0191     bytes -= copy;
0192     while (unlikely(!left && bytes)) {
0193         iov++;
0194         buf = iov->iov_base;
0195         copy = min(bytes, iov->iov_len);
0196         left = __copy_to_user(buf, from, copy);
0197         copy -= left;
0198         skip = copy;
0199         from += copy;
0200         bytes -= copy;
0201     }
0202     kunmap(page);
0203 
0204 done:
0205     if (skip == iov->iov_len) {
0206         iov++;
0207         skip = 0;
0208     }
0209     i->count -= wanted - bytes;
0210     i->nr_segs -= iov - i->iov;
0211     i->iov = iov;
0212     i->iov_offset = skip;
0213     return wanted - bytes;
0214 }
0215 
0216 static size_t copy_page_from_iter_iovec(struct page *page, size_t offset, size_t bytes,
0217              struct iov_iter *i)
0218 {
0219     size_t skip, copy, left, wanted;
0220     const struct iovec *iov;
0221     char __user *buf;
0222     void *kaddr, *to;
0223 
0224     if (unlikely(bytes > i->count))
0225         bytes = i->count;
0226 
0227     if (unlikely(!bytes))
0228         return 0;
0229 
0230     wanted = bytes;
0231     iov = i->iov;
0232     skip = i->iov_offset;
0233     buf = iov->iov_base + skip;
0234     copy = min(bytes, iov->iov_len - skip);
0235 
0236     if (IS_ENABLED(CONFIG_HIGHMEM) && !fault_in_pages_readable(buf, copy)) {
0237         kaddr = kmap_atomic(page);
0238         to = kaddr + offset;
0239 
0240         /* first chunk, usually the only one */
0241         left = __copy_from_user_inatomic(to, buf, copy);
0242         copy -= left;
0243         skip += copy;
0244         to += copy;
0245         bytes -= copy;
0246 
0247         while (unlikely(!left && bytes)) {
0248             iov++;
0249             buf = iov->iov_base;
0250             copy = min(bytes, iov->iov_len);
0251             left = __copy_from_user_inatomic(to, buf, copy);
0252             copy -= left;
0253             skip = copy;
0254             to += copy;
0255             bytes -= copy;
0256         }
0257         if (likely(!bytes)) {
0258             kunmap_atomic(kaddr);
0259             goto done;
0260         }
0261         offset = to - kaddr;
0262         buf += copy;
0263         kunmap_atomic(kaddr);
0264         copy = min(bytes, iov->iov_len - skip);
0265     }
0266     /* Too bad - revert to non-atomic kmap */
0267 
0268     kaddr = kmap(page);
0269     to = kaddr + offset;
0270     left = __copy_from_user(to, buf, copy);
0271     copy -= left;
0272     skip += copy;
0273     to += copy;
0274     bytes -= copy;
0275     while (unlikely(!left && bytes)) {
0276         iov++;
0277         buf = iov->iov_base;
0278         copy = min(bytes, iov->iov_len);
0279         left = __copy_from_user(to, buf, copy);
0280         copy -= left;
0281         skip = copy;
0282         to += copy;
0283         bytes -= copy;
0284     }
0285     kunmap(page);
0286 
0287 done:
0288     if (skip == iov->iov_len) {
0289         iov++;
0290         skip = 0;
0291     }
0292     i->count -= wanted - bytes;
0293     i->nr_segs -= iov - i->iov;
0294     i->iov = iov;
0295     i->iov_offset = skip;
0296     return wanted - bytes;
0297 }
0298 
0299 #ifdef PIPE_PARANOIA
0300 static bool sanity(const struct iov_iter *i)
0301 {
0302     struct pipe_inode_info *pipe = i->pipe;
0303     int idx = i->idx;
0304     int next = pipe->curbuf + pipe->nrbufs;
0305     if (i->iov_offset) {
0306         struct pipe_buffer *p;
0307         if (unlikely(!pipe->nrbufs))
0308             goto Bad;   // pipe must be non-empty
0309         if (unlikely(idx != ((next - 1) & (pipe->buffers - 1))))
0310             goto Bad;   // must be at the last buffer...
0311 
0312         p = &pipe->bufs[idx];
0313         if (unlikely(p->offset + p->len != i->iov_offset))
0314             goto Bad;   // ... at the end of segment
0315     } else {
0316         if (idx != (next & (pipe->buffers - 1)))
0317             goto Bad;   // must be right after the last buffer
0318     }
0319     return true;
0320 Bad:
0321     printk(KERN_ERR "idx = %d, offset = %zd\n", i->idx, i->iov_offset);
0322     printk(KERN_ERR "curbuf = %d, nrbufs = %d, buffers = %d\n",
0323             pipe->curbuf, pipe->nrbufs, pipe->buffers);
0324     for (idx = 0; idx < pipe->buffers; idx++)
0325         printk(KERN_ERR "[%p %p %d %d]\n",
0326             pipe->bufs[idx].ops,
0327             pipe->bufs[idx].page,
0328             pipe->bufs[idx].offset,
0329             pipe->bufs[idx].len);
0330     WARN_ON(1);
0331     return false;
0332 }
0333 #else
0334 #define sanity(i) true
0335 #endif
0336 
0337 static inline int next_idx(int idx, struct pipe_inode_info *pipe)
0338 {
0339     return (idx + 1) & (pipe->buffers - 1);
0340 }
0341 
0342 static size_t copy_page_to_iter_pipe(struct page *page, size_t offset, size_t bytes,
0343              struct iov_iter *i)
0344 {
0345     struct pipe_inode_info *pipe = i->pipe;
0346     struct pipe_buffer *buf;
0347     size_t off;
0348     int idx;
0349 
0350     if (unlikely(bytes > i->count))
0351         bytes = i->count;
0352 
0353     if (unlikely(!bytes))
0354         return 0;
0355 
0356     if (!sanity(i))
0357         return 0;
0358 
0359     off = i->iov_offset;
0360     idx = i->idx;
0361     buf = &pipe->bufs[idx];
0362     if (off) {
0363         if (offset == off && buf->page == page) {
0364             /* merge with the last one */
0365             buf->len += bytes;
0366             i->iov_offset += bytes;
0367             goto out;
0368         }
0369         idx = next_idx(idx, pipe);
0370         buf = &pipe->bufs[idx];
0371     }
0372     if (idx == pipe->curbuf && pipe->nrbufs)
0373         return 0;
0374     pipe->nrbufs++;
0375     buf->ops = &page_cache_pipe_buf_ops;
0376     get_page(buf->page = page);
0377     buf->offset = offset;
0378     buf->len = bytes;
0379     i->iov_offset = offset + bytes;
0380     i->idx = idx;
0381 out:
0382     i->count -= bytes;
0383     return bytes;
0384 }
0385 
0386 /*
0387  * Fault in one or more iovecs of the given iov_iter, to a maximum length of
0388  * bytes.  For each iovec, fault in each page that constitutes the iovec.
0389  *
0390  * Return 0 on success, or non-zero if the memory could not be accessed (i.e.
0391  * because it is an invalid address).
0392  */
0393 int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes)
0394 {
0395     size_t skip = i->iov_offset;
0396     const struct iovec *iov;
0397     int err;
0398     struct iovec v;
0399 
0400     if (!(i->type & (ITER_BVEC|ITER_KVEC))) {
0401         iterate_iovec(i, bytes, v, iov, skip, ({
0402             err = fault_in_pages_readable(v.iov_base, v.iov_len);
0403             if (unlikely(err))
0404             return err;
0405         0;}))
0406     }
0407     return 0;
0408 }
0409 EXPORT_SYMBOL(iov_iter_fault_in_readable);
0410 
0411 void iov_iter_init(struct iov_iter *i, int direction,
0412             const struct iovec *iov, unsigned long nr_segs,
0413             size_t count)
0414 {
0415     /* It will get better.  Eventually... */
0416     if (segment_eq(get_fs(), KERNEL_DS)) {
0417         direction |= ITER_KVEC;
0418         i->type = direction;
0419         i->kvec = (struct kvec *)iov;
0420     } else {
0421         i->type = direction;
0422         i->iov = iov;
0423     }
0424     i->nr_segs = nr_segs;
0425     i->iov_offset = 0;
0426     i->count = count;
0427 }
0428 EXPORT_SYMBOL(iov_iter_init);
0429 
0430 static void memcpy_from_page(char *to, struct page *page, size_t offset, size_t len)
0431 {
0432     char *from = kmap_atomic(page);
0433     memcpy(to, from + offset, len);
0434     kunmap_atomic(from);
0435 }
0436 
0437 static void memcpy_to_page(struct page *page, size_t offset, const char *from, size_t len)
0438 {
0439     char *to = kmap_atomic(page);
0440     memcpy(to + offset, from, len);
0441     kunmap_atomic(to);
0442 }
0443 
0444 static void memzero_page(struct page *page, size_t offset, size_t len)
0445 {
0446     char *addr = kmap_atomic(page);
0447     memset(addr + offset, 0, len);
0448     kunmap_atomic(addr);
0449 }
0450 
0451 static inline bool allocated(struct pipe_buffer *buf)
0452 {
0453     return buf->ops == &default_pipe_buf_ops;
0454 }
0455 
0456 static inline void data_start(const struct iov_iter *i, int *idxp, size_t *offp)
0457 {
0458     size_t off = i->iov_offset;
0459     int idx = i->idx;
0460     if (off && (!allocated(&i->pipe->bufs[idx]) || off == PAGE_SIZE)) {
0461         idx = next_idx(idx, i->pipe);
0462         off = 0;
0463     }
0464     *idxp = idx;
0465     *offp = off;
0466 }
0467 
0468 static size_t push_pipe(struct iov_iter *i, size_t size,
0469             int *idxp, size_t *offp)
0470 {
0471     struct pipe_inode_info *pipe = i->pipe;
0472     size_t off;
0473     int idx;
0474     ssize_t left;
0475 
0476     if (unlikely(size > i->count))
0477         size = i->count;
0478     if (unlikely(!size))
0479         return 0;
0480 
0481     left = size;
0482     data_start(i, &idx, &off);
0483     *idxp = idx;
0484     *offp = off;
0485     if (off) {
0486         left -= PAGE_SIZE - off;
0487         if (left <= 0) {
0488             pipe->bufs[idx].len += size;
0489             return size;
0490         }
0491         pipe->bufs[idx].len = PAGE_SIZE;
0492         idx = next_idx(idx, pipe);
0493     }
0494     while (idx != pipe->curbuf || !pipe->nrbufs) {
0495         struct page *page = alloc_page(GFP_USER);
0496         if (!page)
0497             break;
0498         pipe->nrbufs++;
0499         pipe->bufs[idx].ops = &default_pipe_buf_ops;
0500         pipe->bufs[idx].page = page;
0501         pipe->bufs[idx].offset = 0;
0502         if (left <= PAGE_SIZE) {
0503             pipe->bufs[idx].len = left;
0504             return size;
0505         }
0506         pipe->bufs[idx].len = PAGE_SIZE;
0507         left -= PAGE_SIZE;
0508         idx = next_idx(idx, pipe);
0509     }
0510     return size - left;
0511 }
0512 
0513 static size_t copy_pipe_to_iter(const void *addr, size_t bytes,
0514                 struct iov_iter *i)
0515 {
0516     struct pipe_inode_info *pipe = i->pipe;
0517     size_t n, off;
0518     int idx;
0519 
0520     if (!sanity(i))
0521         return 0;
0522 
0523     bytes = n = push_pipe(i, bytes, &idx, &off);
0524     if (unlikely(!n))
0525         return 0;
0526     for ( ; n; idx = next_idx(idx, pipe), off = 0) {
0527         size_t chunk = min_t(size_t, n, PAGE_SIZE - off);
0528         memcpy_to_page(pipe->bufs[idx].page, off, addr, chunk);
0529         i->idx = idx;
0530         i->iov_offset = off + chunk;
0531         n -= chunk;
0532         addr += chunk;
0533     }
0534     i->count -= bytes;
0535     return bytes;
0536 }
0537 
0538 size_t copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i)
0539 {
0540     const char *from = addr;
0541     if (unlikely(i->type & ITER_PIPE))
0542         return copy_pipe_to_iter(addr, bytes, i);
0543     iterate_and_advance(i, bytes, v,
0544         __copy_to_user(v.iov_base, (from += v.iov_len) - v.iov_len,
0545                    v.iov_len),
0546         memcpy_to_page(v.bv_page, v.bv_offset,
0547                    (from += v.bv_len) - v.bv_len, v.bv_len),
0548         memcpy(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len)
0549     )
0550 
0551     return bytes;
0552 }
0553 EXPORT_SYMBOL(copy_to_iter);
0554 
0555 size_t copy_from_iter(void *addr, size_t bytes, struct iov_iter *i)
0556 {
0557     char *to = addr;
0558     if (unlikely(i->type & ITER_PIPE)) {
0559         WARN_ON(1);
0560         return 0;
0561     }
0562     iterate_and_advance(i, bytes, v,
0563         __copy_from_user((to += v.iov_len) - v.iov_len, v.iov_base,
0564                  v.iov_len),
0565         memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
0566                  v.bv_offset, v.bv_len),
0567         memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
0568     )
0569 
0570     return bytes;
0571 }
0572 EXPORT_SYMBOL(copy_from_iter);
0573 
0574 bool copy_from_iter_full(void *addr, size_t bytes, struct iov_iter *i)
0575 {
0576     char *to = addr;
0577     if (unlikely(i->type & ITER_PIPE)) {
0578         WARN_ON(1);
0579         return false;
0580     }
0581     if (unlikely(i->count < bytes))
0582         return false;
0583 
0584     iterate_all_kinds(i, bytes, v, ({
0585         if (__copy_from_user((to += v.iov_len) - v.iov_len,
0586                       v.iov_base, v.iov_len))
0587             return false;
0588         0;}),
0589         memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
0590                  v.bv_offset, v.bv_len),
0591         memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
0592     )
0593 
0594     iov_iter_advance(i, bytes);
0595     return true;
0596 }
0597 EXPORT_SYMBOL(copy_from_iter_full);
0598 
0599 size_t copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i)
0600 {
0601     char *to = addr;
0602     if (unlikely(i->type & ITER_PIPE)) {
0603         WARN_ON(1);
0604         return 0;
0605     }
0606     iterate_and_advance(i, bytes, v,
0607         __copy_from_user_nocache((to += v.iov_len) - v.iov_len,
0608                      v.iov_base, v.iov_len),
0609         memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
0610                  v.bv_offset, v.bv_len),
0611         memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
0612     )
0613 
0614     return bytes;
0615 }
0616 EXPORT_SYMBOL(copy_from_iter_nocache);
0617 
0618 bool copy_from_iter_full_nocache(void *addr, size_t bytes, struct iov_iter *i)
0619 {
0620     char *to = addr;
0621     if (unlikely(i->type & ITER_PIPE)) {
0622         WARN_ON(1);
0623         return false;
0624     }
0625     if (unlikely(i->count < bytes))
0626         return false;
0627     iterate_all_kinds(i, bytes, v, ({
0628         if (__copy_from_user_nocache((to += v.iov_len) - v.iov_len,
0629                          v.iov_base, v.iov_len))
0630             return false;
0631         0;}),
0632         memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
0633                  v.bv_offset, v.bv_len),
0634         memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
0635     )
0636 
0637     iov_iter_advance(i, bytes);
0638     return true;
0639 }
0640 EXPORT_SYMBOL(copy_from_iter_full_nocache);
0641 
0642 size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes,
0643              struct iov_iter *i)
0644 {
0645     if (i->type & (ITER_BVEC|ITER_KVEC)) {
0646         void *kaddr = kmap_atomic(page);
0647         size_t wanted = copy_to_iter(kaddr + offset, bytes, i);
0648         kunmap_atomic(kaddr);
0649         return wanted;
0650     } else if (likely(!(i->type & ITER_PIPE)))
0651         return copy_page_to_iter_iovec(page, offset, bytes, i);
0652     else
0653         return copy_page_to_iter_pipe(page, offset, bytes, i);
0654 }
0655 EXPORT_SYMBOL(copy_page_to_iter);
0656 
0657 size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes,
0658              struct iov_iter *i)
0659 {
0660     if (unlikely(i->type & ITER_PIPE)) {
0661         WARN_ON(1);
0662         return 0;
0663     }
0664     if (i->type & (ITER_BVEC|ITER_KVEC)) {
0665         void *kaddr = kmap_atomic(page);
0666         size_t wanted = copy_from_iter(kaddr + offset, bytes, i);
0667         kunmap_atomic(kaddr);
0668         return wanted;
0669     } else
0670         return copy_page_from_iter_iovec(page, offset, bytes, i);
0671 }
0672 EXPORT_SYMBOL(copy_page_from_iter);
0673 
0674 static size_t pipe_zero(size_t bytes, struct iov_iter *i)
0675 {
0676     struct pipe_inode_info *pipe = i->pipe;
0677     size_t n, off;
0678     int idx;
0679 
0680     if (!sanity(i))
0681         return 0;
0682 
0683     bytes = n = push_pipe(i, bytes, &idx, &off);
0684     if (unlikely(!n))
0685         return 0;
0686 
0687     for ( ; n; idx = next_idx(idx, pipe), off = 0) {
0688         size_t chunk = min_t(size_t, n, PAGE_SIZE - off);
0689         memzero_page(pipe->bufs[idx].page, off, chunk);
0690         i->idx = idx;
0691         i->iov_offset = off + chunk;
0692         n -= chunk;
0693     }
0694     i->count -= bytes;
0695     return bytes;
0696 }
0697 
0698 size_t iov_iter_zero(size_t bytes, struct iov_iter *i)
0699 {
0700     if (unlikely(i->type & ITER_PIPE))
0701         return pipe_zero(bytes, i);
0702     iterate_and_advance(i, bytes, v,
0703         __clear_user(v.iov_base, v.iov_len),
0704         memzero_page(v.bv_page, v.bv_offset, v.bv_len),
0705         memset(v.iov_base, 0, v.iov_len)
0706     )
0707 
0708     return bytes;
0709 }
0710 EXPORT_SYMBOL(iov_iter_zero);
0711 
0712 size_t iov_iter_copy_from_user_atomic(struct page *page,
0713         struct iov_iter *i, unsigned long offset, size_t bytes)
0714 {
0715     char *kaddr = kmap_atomic(page), *p = kaddr + offset;
0716     if (unlikely(i->type & ITER_PIPE)) {
0717         kunmap_atomic(kaddr);
0718         WARN_ON(1);
0719         return 0;
0720     }
0721     iterate_all_kinds(i, bytes, v,
0722         __copy_from_user_inatomic((p += v.iov_len) - v.iov_len,
0723                       v.iov_base, v.iov_len),
0724         memcpy_from_page((p += v.bv_len) - v.bv_len, v.bv_page,
0725                  v.bv_offset, v.bv_len),
0726         memcpy((p += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
0727     )
0728     kunmap_atomic(kaddr);
0729     return bytes;
0730 }
0731 EXPORT_SYMBOL(iov_iter_copy_from_user_atomic);
0732 
0733 static inline void pipe_truncate(struct iov_iter *i)
0734 {
0735     struct pipe_inode_info *pipe = i->pipe;
0736     if (pipe->nrbufs) {
0737         size_t off = i->iov_offset;
0738         int idx = i->idx;
0739         int nrbufs = (idx - pipe->curbuf) & (pipe->buffers - 1);
0740         if (off) {
0741             pipe->bufs[idx].len = off - pipe->bufs[idx].offset;
0742             idx = next_idx(idx, pipe);
0743             nrbufs++;
0744         }
0745         while (pipe->nrbufs > nrbufs) {
0746             pipe_buf_release(pipe, &pipe->bufs[idx]);
0747             idx = next_idx(idx, pipe);
0748             pipe->nrbufs--;
0749         }
0750     }
0751 }
0752 
0753 static void pipe_advance(struct iov_iter *i, size_t size)
0754 {
0755     struct pipe_inode_info *pipe = i->pipe;
0756     if (unlikely(i->count < size))
0757         size = i->count;
0758     if (size) {
0759         struct pipe_buffer *buf;
0760         size_t off = i->iov_offset, left = size;
0761         int idx = i->idx;
0762         if (off) /* make it relative to the beginning of buffer */
0763             left += off - pipe->bufs[idx].offset;
0764         while (1) {
0765             buf = &pipe->bufs[idx];
0766             if (left <= buf->len)
0767                 break;
0768             left -= buf->len;
0769             idx = next_idx(idx, pipe);
0770         }
0771         i->idx = idx;
0772         i->iov_offset = buf->offset + left;
0773     }
0774     i->count -= size;
0775     /* ... and discard everything past that point */
0776     pipe_truncate(i);
0777 }
0778 
0779 void iov_iter_advance(struct iov_iter *i, size_t size)
0780 {
0781     if (unlikely(i->type & ITER_PIPE)) {
0782         pipe_advance(i, size);
0783         return;
0784     }
0785     iterate_and_advance(i, size, v, 0, 0, 0)
0786 }
0787 EXPORT_SYMBOL(iov_iter_advance);
0788 
0789 /*
0790  * Return the count of just the current iov_iter segment.
0791  */
0792 size_t iov_iter_single_seg_count(const struct iov_iter *i)
0793 {
0794     if (unlikely(i->type & ITER_PIPE))
0795         return i->count;    // it is a silly place, anyway
0796     if (i->nr_segs == 1)
0797         return i->count;
0798     else if (i->type & ITER_BVEC)
0799         return min(i->count, i->bvec->bv_len - i->iov_offset);
0800     else
0801         return min(i->count, i->iov->iov_len - i->iov_offset);
0802 }
0803 EXPORT_SYMBOL(iov_iter_single_seg_count);
0804 
0805 void iov_iter_kvec(struct iov_iter *i, int direction,
0806             const struct kvec *kvec, unsigned long nr_segs,
0807             size_t count)
0808 {
0809     BUG_ON(!(direction & ITER_KVEC));
0810     i->type = direction;
0811     i->kvec = kvec;
0812     i->nr_segs = nr_segs;
0813     i->iov_offset = 0;
0814     i->count = count;
0815 }
0816 EXPORT_SYMBOL(iov_iter_kvec);
0817 
0818 void iov_iter_bvec(struct iov_iter *i, int direction,
0819             const struct bio_vec *bvec, unsigned long nr_segs,
0820             size_t count)
0821 {
0822     BUG_ON(!(direction & ITER_BVEC));
0823     i->type = direction;
0824     i->bvec = bvec;
0825     i->nr_segs = nr_segs;
0826     i->iov_offset = 0;
0827     i->count = count;
0828 }
0829 EXPORT_SYMBOL(iov_iter_bvec);
0830 
0831 void iov_iter_pipe(struct iov_iter *i, int direction,
0832             struct pipe_inode_info *pipe,
0833             size_t count)
0834 {
0835     BUG_ON(direction != ITER_PIPE);
0836     WARN_ON(pipe->nrbufs == pipe->buffers);
0837     i->type = direction;
0838     i->pipe = pipe;
0839     i->idx = (pipe->curbuf + pipe->nrbufs) & (pipe->buffers - 1);
0840     i->iov_offset = 0;
0841     i->count = count;
0842 }
0843 EXPORT_SYMBOL(iov_iter_pipe);
0844 
0845 unsigned long iov_iter_alignment(const struct iov_iter *i)
0846 {
0847     unsigned long res = 0;
0848     size_t size = i->count;
0849 
0850     if (unlikely(i->type & ITER_PIPE)) {
0851         if (size && i->iov_offset && allocated(&i->pipe->bufs[i->idx]))
0852             return size | i->iov_offset;
0853         return size;
0854     }
0855     iterate_all_kinds(i, size, v,
0856         (res |= (unsigned long)v.iov_base | v.iov_len, 0),
0857         res |= v.bv_offset | v.bv_len,
0858         res |= (unsigned long)v.iov_base | v.iov_len
0859     )
0860     return res;
0861 }
0862 EXPORT_SYMBOL(iov_iter_alignment);
0863 
0864 unsigned long iov_iter_gap_alignment(const struct iov_iter *i)
0865 {
0866     unsigned long res = 0;
0867     size_t size = i->count;
0868 
0869     if (unlikely(i->type & ITER_PIPE)) {
0870         WARN_ON(1);
0871         return ~0U;
0872     }
0873 
0874     iterate_all_kinds(i, size, v,
0875         (res |= (!res ? 0 : (unsigned long)v.iov_base) |
0876             (size != v.iov_len ? size : 0), 0),
0877         (res |= (!res ? 0 : (unsigned long)v.bv_offset) |
0878             (size != v.bv_len ? size : 0)),
0879         (res |= (!res ? 0 : (unsigned long)v.iov_base) |
0880             (size != v.iov_len ? size : 0))
0881         );
0882     return res;
0883 }
0884 EXPORT_SYMBOL(iov_iter_gap_alignment);
0885 
0886 static inline size_t __pipe_get_pages(struct iov_iter *i,
0887                 size_t maxsize,
0888                 struct page **pages,
0889                 int idx,
0890                 size_t *start)
0891 {
0892     struct pipe_inode_info *pipe = i->pipe;
0893     ssize_t n = push_pipe(i, maxsize, &idx, start);
0894     if (!n)
0895         return -EFAULT;
0896 
0897     maxsize = n;
0898     n += *start;
0899     while (n > 0) {
0900         get_page(*pages++ = pipe->bufs[idx].page);
0901         idx = next_idx(idx, pipe);
0902         n -= PAGE_SIZE;
0903     }
0904 
0905     return maxsize;
0906 }
0907 
0908 static ssize_t pipe_get_pages(struct iov_iter *i,
0909            struct page **pages, size_t maxsize, unsigned maxpages,
0910            size_t *start)
0911 {
0912     unsigned npages;
0913     size_t capacity;
0914     int idx;
0915 
0916     if (!maxsize)
0917         return 0;
0918 
0919     if (!sanity(i))
0920         return -EFAULT;
0921 
0922     data_start(i, &idx, start);
0923     /* some of this one + all after this one */
0924     npages = ((i->pipe->curbuf - idx - 1) & (i->pipe->buffers - 1)) + 1;
0925     capacity = min(npages,maxpages) * PAGE_SIZE - *start;
0926 
0927     return __pipe_get_pages(i, min(maxsize, capacity), pages, idx, start);
0928 }
0929 
0930 ssize_t iov_iter_get_pages(struct iov_iter *i,
0931            struct page **pages, size_t maxsize, unsigned maxpages,
0932            size_t *start)
0933 {
0934     if (maxsize > i->count)
0935         maxsize = i->count;
0936 
0937     if (unlikely(i->type & ITER_PIPE))
0938         return pipe_get_pages(i, pages, maxsize, maxpages, start);
0939     iterate_all_kinds(i, maxsize, v, ({
0940         unsigned long addr = (unsigned long)v.iov_base;
0941         size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1));
0942         int n;
0943         int res;
0944 
0945         if (len > maxpages * PAGE_SIZE)
0946             len = maxpages * PAGE_SIZE;
0947         addr &= ~(PAGE_SIZE - 1);
0948         n = DIV_ROUND_UP(len, PAGE_SIZE);
0949         res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, pages);
0950         if (unlikely(res < 0))
0951             return res;
0952         return (res == n ? len : res * PAGE_SIZE) - *start;
0953     0;}),({
0954         /* can't be more than PAGE_SIZE */
0955         *start = v.bv_offset;
0956         get_page(*pages = v.bv_page);
0957         return v.bv_len;
0958     }),({
0959         return -EFAULT;
0960     })
0961     )
0962     return 0;
0963 }
0964 EXPORT_SYMBOL(iov_iter_get_pages);
0965 
0966 static struct page **get_pages_array(size_t n)
0967 {
0968     struct page **p = kmalloc(n * sizeof(struct page *), GFP_KERNEL);
0969     if (!p)
0970         p = vmalloc(n * sizeof(struct page *));
0971     return p;
0972 }
0973 
0974 static ssize_t pipe_get_pages_alloc(struct iov_iter *i,
0975            struct page ***pages, size_t maxsize,
0976            size_t *start)
0977 {
0978     struct page **p;
0979     size_t n;
0980     int idx;
0981     int npages;
0982 
0983     if (!maxsize)
0984         return 0;
0985 
0986     if (!sanity(i))
0987         return -EFAULT;
0988 
0989     data_start(i, &idx, start);
0990     /* some of this one + all after this one */
0991     npages = ((i->pipe->curbuf - idx - 1) & (i->pipe->buffers - 1)) + 1;
0992     n = npages * PAGE_SIZE - *start;
0993     if (maxsize > n)
0994         maxsize = n;
0995     else
0996         npages = DIV_ROUND_UP(maxsize + *start, PAGE_SIZE);
0997     p = get_pages_array(npages);
0998     if (!p)
0999         return -ENOMEM;
1000     n = __pipe_get_pages(i, maxsize, p, idx, start);
1001     if (n > 0)
1002         *pages = p;
1003     else
1004         kvfree(p);
1005     return n;
1006 }
1007 
1008 ssize_t iov_iter_get_pages_alloc(struct iov_iter *i,
1009            struct page ***pages, size_t maxsize,
1010            size_t *start)
1011 {
1012     struct page **p;
1013 
1014     if (maxsize > i->count)
1015         maxsize = i->count;
1016 
1017     if (unlikely(i->type & ITER_PIPE))
1018         return pipe_get_pages_alloc(i, pages, maxsize, start);
1019     iterate_all_kinds(i, maxsize, v, ({
1020         unsigned long addr = (unsigned long)v.iov_base;
1021         size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1));
1022         int n;
1023         int res;
1024 
1025         addr &= ~(PAGE_SIZE - 1);
1026         n = DIV_ROUND_UP(len, PAGE_SIZE);
1027         p = get_pages_array(n);
1028         if (!p)
1029             return -ENOMEM;
1030         res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, p);
1031         if (unlikely(res < 0)) {
1032             kvfree(p);
1033             return res;
1034         }
1035         *pages = p;
1036         return (res == n ? len : res * PAGE_SIZE) - *start;
1037     0;}),({
1038         /* can't be more than PAGE_SIZE */
1039         *start = v.bv_offset;
1040         *pages = p = get_pages_array(1);
1041         if (!p)
1042             return -ENOMEM;
1043         get_page(*p = v.bv_page);
1044         return v.bv_len;
1045     }),({
1046         return -EFAULT;
1047     })
1048     )
1049     return 0;
1050 }
1051 EXPORT_SYMBOL(iov_iter_get_pages_alloc);
1052 
1053 size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum,
1054                    struct iov_iter *i)
1055 {
1056     char *to = addr;
1057     __wsum sum, next;
1058     size_t off = 0;
1059     sum = *csum;
1060     if (unlikely(i->type & ITER_PIPE)) {
1061         WARN_ON(1);
1062         return 0;
1063     }
1064     iterate_and_advance(i, bytes, v, ({
1065         int err = 0;
1066         next = csum_and_copy_from_user(v.iov_base,
1067                            (to += v.iov_len) - v.iov_len,
1068                            v.iov_len, 0, &err);
1069         if (!err) {
1070             sum = csum_block_add(sum, next, off);
1071             off += v.iov_len;
1072         }
1073         err ? v.iov_len : 0;
1074     }), ({
1075         char *p = kmap_atomic(v.bv_page);
1076         next = csum_partial_copy_nocheck(p + v.bv_offset,
1077                          (to += v.bv_len) - v.bv_len,
1078                          v.bv_len, 0);
1079         kunmap_atomic(p);
1080         sum = csum_block_add(sum, next, off);
1081         off += v.bv_len;
1082     }),({
1083         next = csum_partial_copy_nocheck(v.iov_base,
1084                          (to += v.iov_len) - v.iov_len,
1085                          v.iov_len, 0);
1086         sum = csum_block_add(sum, next, off);
1087         off += v.iov_len;
1088     })
1089     )
1090     *csum = sum;
1091     return bytes;
1092 }
1093 EXPORT_SYMBOL(csum_and_copy_from_iter);
1094 
1095 bool csum_and_copy_from_iter_full(void *addr, size_t bytes, __wsum *csum,
1096                    struct iov_iter *i)
1097 {
1098     char *to = addr;
1099     __wsum sum, next;
1100     size_t off = 0;
1101     sum = *csum;
1102     if (unlikely(i->type & ITER_PIPE)) {
1103         WARN_ON(1);
1104         return false;
1105     }
1106     if (unlikely(i->count < bytes))
1107         return false;
1108     iterate_all_kinds(i, bytes, v, ({
1109         int err = 0;
1110         next = csum_and_copy_from_user(v.iov_base,
1111                            (to += v.iov_len) - v.iov_len,
1112                            v.iov_len, 0, &err);
1113         if (err)
1114             return false;
1115         sum = csum_block_add(sum, next, off);
1116         off += v.iov_len;
1117         0;
1118     }), ({
1119         char *p = kmap_atomic(v.bv_page);
1120         next = csum_partial_copy_nocheck(p + v.bv_offset,
1121                          (to += v.bv_len) - v.bv_len,
1122                          v.bv_len, 0);
1123         kunmap_atomic(p);
1124         sum = csum_block_add(sum, next, off);
1125         off += v.bv_len;
1126     }),({
1127         next = csum_partial_copy_nocheck(v.iov_base,
1128                          (to += v.iov_len) - v.iov_len,
1129                          v.iov_len, 0);
1130         sum = csum_block_add(sum, next, off);
1131         off += v.iov_len;
1132     })
1133     )
1134     *csum = sum;
1135     iov_iter_advance(i, bytes);
1136     return true;
1137 }
1138 EXPORT_SYMBOL(csum_and_copy_from_iter_full);
1139 
1140 size_t csum_and_copy_to_iter(const void *addr, size_t bytes, __wsum *csum,
1141                  struct iov_iter *i)
1142 {
1143     const char *from = addr;
1144     __wsum sum, next;
1145     size_t off = 0;
1146     sum = *csum;
1147     if (unlikely(i->type & ITER_PIPE)) {
1148         WARN_ON(1); /* for now */
1149         return 0;
1150     }
1151     iterate_and_advance(i, bytes, v, ({
1152         int err = 0;
1153         next = csum_and_copy_to_user((from += v.iov_len) - v.iov_len,
1154                          v.iov_base,
1155                          v.iov_len, 0, &err);
1156         if (!err) {
1157             sum = csum_block_add(sum, next, off);
1158             off += v.iov_len;
1159         }
1160         err ? v.iov_len : 0;
1161     }), ({
1162         char *p = kmap_atomic(v.bv_page);
1163         next = csum_partial_copy_nocheck((from += v.bv_len) - v.bv_len,
1164                          p + v.bv_offset,
1165                          v.bv_len, 0);
1166         kunmap_atomic(p);
1167         sum = csum_block_add(sum, next, off);
1168         off += v.bv_len;
1169     }),({
1170         next = csum_partial_copy_nocheck((from += v.iov_len) - v.iov_len,
1171                          v.iov_base,
1172                          v.iov_len, 0);
1173         sum = csum_block_add(sum, next, off);
1174         off += v.iov_len;
1175     })
1176     )
1177     *csum = sum;
1178     return bytes;
1179 }
1180 EXPORT_SYMBOL(csum_and_copy_to_iter);
1181 
1182 int iov_iter_npages(const struct iov_iter *i, int maxpages)
1183 {
1184     size_t size = i->count;
1185     int npages = 0;
1186 
1187     if (!size)
1188         return 0;
1189 
1190     if (unlikely(i->type & ITER_PIPE)) {
1191         struct pipe_inode_info *pipe = i->pipe;
1192         size_t off;
1193         int idx;
1194 
1195         if (!sanity(i))
1196             return 0;
1197 
1198         data_start(i, &idx, &off);
1199         /* some of this one + all after this one */
1200         npages = ((pipe->curbuf - idx - 1) & (pipe->buffers - 1)) + 1;
1201         if (npages >= maxpages)
1202             return maxpages;
1203     } else iterate_all_kinds(i, size, v, ({
1204         unsigned long p = (unsigned long)v.iov_base;
1205         npages += DIV_ROUND_UP(p + v.iov_len, PAGE_SIZE)
1206             - p / PAGE_SIZE;
1207         if (npages >= maxpages)
1208             return maxpages;
1209     0;}),({
1210         npages++;
1211         if (npages >= maxpages)
1212             return maxpages;
1213     }),({
1214         unsigned long p = (unsigned long)v.iov_base;
1215         npages += DIV_ROUND_UP(p + v.iov_len, PAGE_SIZE)
1216             - p / PAGE_SIZE;
1217         if (npages >= maxpages)
1218             return maxpages;
1219     })
1220     )
1221     return npages;
1222 }
1223 EXPORT_SYMBOL(iov_iter_npages);
1224 
1225 const void *dup_iter(struct iov_iter *new, struct iov_iter *old, gfp_t flags)
1226 {
1227     *new = *old;
1228     if (unlikely(new->type & ITER_PIPE)) {
1229         WARN_ON(1);
1230         return NULL;
1231     }
1232     if (new->type & ITER_BVEC)
1233         return new->bvec = kmemdup(new->bvec,
1234                     new->nr_segs * sizeof(struct bio_vec),
1235                     flags);
1236     else
1237         /* iovec and kvec have identical layout */
1238         return new->iov = kmemdup(new->iov,
1239                    new->nr_segs * sizeof(struct iovec),
1240                    flags);
1241 }
1242 EXPORT_SYMBOL(dup_iter);
1243 
1244 /**
1245  * import_iovec() - Copy an array of &struct iovec from userspace
1246  *     into the kernel, check that it is valid, and initialize a new
1247  *     &struct iov_iter iterator to access it.
1248  *
1249  * @type: One of %READ or %WRITE.
1250  * @uvector: Pointer to the userspace array.
1251  * @nr_segs: Number of elements in userspace array.
1252  * @fast_segs: Number of elements in @iov.
1253  * @iov: (input and output parameter) Pointer to pointer to (usually small
1254  *     on-stack) kernel array.
1255  * @i: Pointer to iterator that will be initialized on success.
1256  *
1257  * If the array pointed to by *@iov is large enough to hold all @nr_segs,
1258  * then this function places %NULL in *@iov on return. Otherwise, a new
1259  * array will be allocated and the result placed in *@iov. This means that
1260  * the caller may call kfree() on *@iov regardless of whether the small
1261  * on-stack array was used or not (and regardless of whether this function
1262  * returns an error or not).
1263  *
1264  * Return: 0 on success or negative error code on error.
1265  */
1266 int import_iovec(int type, const struct iovec __user * uvector,
1267          unsigned nr_segs, unsigned fast_segs,
1268          struct iovec **iov, struct iov_iter *i)
1269 {
1270     ssize_t n;
1271     struct iovec *p;
1272     n = rw_copy_check_uvector(type, uvector, nr_segs, fast_segs,
1273                   *iov, &p);
1274     if (n < 0) {
1275         if (p != *iov)
1276             kfree(p);
1277         *iov = NULL;
1278         return n;
1279     }
1280     iov_iter_init(i, type, p, nr_segs, n);
1281     *iov = p == *iov ? NULL : p;
1282     return 0;
1283 }
1284 EXPORT_SYMBOL(import_iovec);
1285 
1286 #ifdef CONFIG_COMPAT
1287 #include <linux/compat.h>
1288 
1289 int compat_import_iovec(int type, const struct compat_iovec __user * uvector,
1290          unsigned nr_segs, unsigned fast_segs,
1291          struct iovec **iov, struct iov_iter *i)
1292 {
1293     ssize_t n;
1294     struct iovec *p;
1295     n = compat_rw_copy_check_uvector(type, uvector, nr_segs, fast_segs,
1296                   *iov, &p);
1297     if (n < 0) {
1298         if (p != *iov)
1299             kfree(p);
1300         *iov = NULL;
1301         return n;
1302     }
1303     iov_iter_init(i, type, p, nr_segs, n);
1304     *iov = p == *iov ? NULL : p;
1305     return 0;
1306 }
1307 #endif
1308 
1309 int import_single_range(int rw, void __user *buf, size_t len,
1310          struct iovec *iov, struct iov_iter *i)
1311 {
1312     if (len > MAX_RW_COUNT)
1313         len = MAX_RW_COUNT;
1314     if (unlikely(!access_ok(!rw, buf, len)))
1315         return -EFAULT;
1316 
1317     iov->iov_base = buf;
1318     iov->iov_len = len;
1319     iov_iter_init(i, rw, iov, 1, len);
1320     return 0;
1321 }
1322 EXPORT_SYMBOL(import_single_range);