Back to home page

OSCL-LXR
 
 

     

0001 /*
0002   FUSE: Filesystem in Userspace
0003   Copyright (C) 2001-2008  Miklos Szeredi <miklos@szeredi.hu>
0004 
0005   This program can be distributed under the terms of the GNU GPL.
0006   See the file COPYING.
0007 */
0008 
0009 #include "fuse_i.h"
0010 
0011 #include <linux/init.h>
0012 #include <linux/module.h>
0013 #include <linux/poll.h>
0014 #include <linux/sched/signal.h>
0015 #include <linux/uio.h>
0016 #include <linux/miscdevice.h>
0017 #include <linux/pagemap.h>
0018 #include <linux/file.h>
0019 #include <linux/slab.h>
0020 #include <linux/pipe_fs_i.h>
0021 #include <linux/swap.h>
0022 #include <linux/splice.h>
0023 #include <linux/sched.h>
0024 
0025 MODULE_ALIAS_MISCDEV(FUSE_MINOR);
0026 MODULE_ALIAS("devname:fuse");
0027 
0028 /* Ordinary requests have even IDs, while interrupts IDs are odd */
0029 #define FUSE_INT_REQ_BIT (1ULL << 0)
0030 #define FUSE_REQ_ID_STEP (1ULL << 1)
0031 
0032 static struct kmem_cache *fuse_req_cachep;
0033 
0034 static struct fuse_dev *fuse_get_dev(struct file *file)
0035 {
0036     /*
0037      * Lockless access is OK, because file->private data is set
0038      * once during mount and is valid until the file is released.
0039      */
0040     return READ_ONCE(file->private_data);
0041 }
0042 
0043 static void fuse_request_init(struct fuse_mount *fm, struct fuse_req *req)
0044 {
0045     INIT_LIST_HEAD(&req->list);
0046     INIT_LIST_HEAD(&req->intr_entry);
0047     init_waitqueue_head(&req->waitq);
0048     refcount_set(&req->count, 1);
0049     __set_bit(FR_PENDING, &req->flags);
0050     req->fm = fm;
0051 }
0052 
0053 static struct fuse_req *fuse_request_alloc(struct fuse_mount *fm, gfp_t flags)
0054 {
0055     struct fuse_req *req = kmem_cache_zalloc(fuse_req_cachep, flags);
0056     if (req)
0057         fuse_request_init(fm, req);
0058 
0059     return req;
0060 }
0061 
0062 static void fuse_request_free(struct fuse_req *req)
0063 {
0064     kmem_cache_free(fuse_req_cachep, req);
0065 }
0066 
0067 static void __fuse_get_request(struct fuse_req *req)
0068 {
0069     refcount_inc(&req->count);
0070 }
0071 
0072 /* Must be called with > 1 refcount */
0073 static void __fuse_put_request(struct fuse_req *req)
0074 {
0075     refcount_dec(&req->count);
0076 }
0077 
0078 void fuse_set_initialized(struct fuse_conn *fc)
0079 {
0080     /* Make sure stores before this are seen on another CPU */
0081     smp_wmb();
0082     fc->initialized = 1;
0083 }
0084 
0085 static bool fuse_block_alloc(struct fuse_conn *fc, bool for_background)
0086 {
0087     return !fc->initialized || (for_background && fc->blocked);
0088 }
0089 
0090 static void fuse_drop_waiting(struct fuse_conn *fc)
0091 {
0092     /*
0093      * lockess check of fc->connected is okay, because atomic_dec_and_test()
0094      * provides a memory barrier matched with the one in fuse_wait_aborted()
0095      * to ensure no wake-up is missed.
0096      */
0097     if (atomic_dec_and_test(&fc->num_waiting) &&
0098         !READ_ONCE(fc->connected)) {
0099         /* wake up aborters */
0100         wake_up_all(&fc->blocked_waitq);
0101     }
0102 }
0103 
0104 static void fuse_put_request(struct fuse_req *req);
0105 
0106 static struct fuse_req *fuse_get_req(struct fuse_mount *fm, bool for_background)
0107 {
0108     struct fuse_conn *fc = fm->fc;
0109     struct fuse_req *req;
0110     int err;
0111     atomic_inc(&fc->num_waiting);
0112 
0113     if (fuse_block_alloc(fc, for_background)) {
0114         err = -EINTR;
0115         if (wait_event_killable_exclusive(fc->blocked_waitq,
0116                 !fuse_block_alloc(fc, for_background)))
0117             goto out;
0118     }
0119     /* Matches smp_wmb() in fuse_set_initialized() */
0120     smp_rmb();
0121 
0122     err = -ENOTCONN;
0123     if (!fc->connected)
0124         goto out;
0125 
0126     err = -ECONNREFUSED;
0127     if (fc->conn_error)
0128         goto out;
0129 
0130     req = fuse_request_alloc(fm, GFP_KERNEL);
0131     err = -ENOMEM;
0132     if (!req) {
0133         if (for_background)
0134             wake_up(&fc->blocked_waitq);
0135         goto out;
0136     }
0137 
0138     req->in.h.uid = from_kuid(fc->user_ns, current_fsuid());
0139     req->in.h.gid = from_kgid(fc->user_ns, current_fsgid());
0140     req->in.h.pid = pid_nr_ns(task_pid(current), fc->pid_ns);
0141 
0142     __set_bit(FR_WAITING, &req->flags);
0143     if (for_background)
0144         __set_bit(FR_BACKGROUND, &req->flags);
0145 
0146     if (unlikely(req->in.h.uid == ((uid_t)-1) ||
0147              req->in.h.gid == ((gid_t)-1))) {
0148         fuse_put_request(req);
0149         return ERR_PTR(-EOVERFLOW);
0150     }
0151     return req;
0152 
0153  out:
0154     fuse_drop_waiting(fc);
0155     return ERR_PTR(err);
0156 }
0157 
0158 static void fuse_put_request(struct fuse_req *req)
0159 {
0160     struct fuse_conn *fc = req->fm->fc;
0161 
0162     if (refcount_dec_and_test(&req->count)) {
0163         if (test_bit(FR_BACKGROUND, &req->flags)) {
0164             /*
0165              * We get here in the unlikely case that a background
0166              * request was allocated but not sent
0167              */
0168             spin_lock(&fc->bg_lock);
0169             if (!fc->blocked)
0170                 wake_up(&fc->blocked_waitq);
0171             spin_unlock(&fc->bg_lock);
0172         }
0173 
0174         if (test_bit(FR_WAITING, &req->flags)) {
0175             __clear_bit(FR_WAITING, &req->flags);
0176             fuse_drop_waiting(fc);
0177         }
0178 
0179         fuse_request_free(req);
0180     }
0181 }
0182 
0183 unsigned int fuse_len_args(unsigned int numargs, struct fuse_arg *args)
0184 {
0185     unsigned nbytes = 0;
0186     unsigned i;
0187 
0188     for (i = 0; i < numargs; i++)
0189         nbytes += args[i].size;
0190 
0191     return nbytes;
0192 }
0193 EXPORT_SYMBOL_GPL(fuse_len_args);
0194 
0195 u64 fuse_get_unique(struct fuse_iqueue *fiq)
0196 {
0197     fiq->reqctr += FUSE_REQ_ID_STEP;
0198     return fiq->reqctr;
0199 }
0200 EXPORT_SYMBOL_GPL(fuse_get_unique);
0201 
0202 static unsigned int fuse_req_hash(u64 unique)
0203 {
0204     return hash_long(unique & ~FUSE_INT_REQ_BIT, FUSE_PQ_HASH_BITS);
0205 }
0206 
0207 /**
0208  * A new request is available, wake fiq->waitq
0209  */
0210 static void fuse_dev_wake_and_unlock(struct fuse_iqueue *fiq)
0211 __releases(fiq->lock)
0212 {
0213     wake_up(&fiq->waitq);
0214     kill_fasync(&fiq->fasync, SIGIO, POLL_IN);
0215     spin_unlock(&fiq->lock);
0216 }
0217 
0218 const struct fuse_iqueue_ops fuse_dev_fiq_ops = {
0219     .wake_forget_and_unlock     = fuse_dev_wake_and_unlock,
0220     .wake_interrupt_and_unlock  = fuse_dev_wake_and_unlock,
0221     .wake_pending_and_unlock    = fuse_dev_wake_and_unlock,
0222 };
0223 EXPORT_SYMBOL_GPL(fuse_dev_fiq_ops);
0224 
0225 static void queue_request_and_unlock(struct fuse_iqueue *fiq,
0226                      struct fuse_req *req)
0227 __releases(fiq->lock)
0228 {
0229     req->in.h.len = sizeof(struct fuse_in_header) +
0230         fuse_len_args(req->args->in_numargs,
0231                   (struct fuse_arg *) req->args->in_args);
0232     list_add_tail(&req->list, &fiq->pending);
0233     fiq->ops->wake_pending_and_unlock(fiq);
0234 }
0235 
0236 void fuse_queue_forget(struct fuse_conn *fc, struct fuse_forget_link *forget,
0237                u64 nodeid, u64 nlookup)
0238 {
0239     struct fuse_iqueue *fiq = &fc->iq;
0240 
0241     forget->forget_one.nodeid = nodeid;
0242     forget->forget_one.nlookup = nlookup;
0243 
0244     spin_lock(&fiq->lock);
0245     if (fiq->connected) {
0246         fiq->forget_list_tail->next = forget;
0247         fiq->forget_list_tail = forget;
0248         fiq->ops->wake_forget_and_unlock(fiq);
0249     } else {
0250         kfree(forget);
0251         spin_unlock(&fiq->lock);
0252     }
0253 }
0254 
0255 static void flush_bg_queue(struct fuse_conn *fc)
0256 {
0257     struct fuse_iqueue *fiq = &fc->iq;
0258 
0259     while (fc->active_background < fc->max_background &&
0260            !list_empty(&fc->bg_queue)) {
0261         struct fuse_req *req;
0262 
0263         req = list_first_entry(&fc->bg_queue, struct fuse_req, list);
0264         list_del(&req->list);
0265         fc->active_background++;
0266         spin_lock(&fiq->lock);
0267         req->in.h.unique = fuse_get_unique(fiq);
0268         queue_request_and_unlock(fiq, req);
0269     }
0270 }
0271 
0272 /*
0273  * This function is called when a request is finished.  Either a reply
0274  * has arrived or it was aborted (and not yet sent) or some error
0275  * occurred during communication with userspace, or the device file
0276  * was closed.  The requester thread is woken up (if still waiting),
0277  * the 'end' callback is called if given, else the reference to the
0278  * request is released
0279  */
0280 void fuse_request_end(struct fuse_req *req)
0281 {
0282     struct fuse_mount *fm = req->fm;
0283     struct fuse_conn *fc = fm->fc;
0284     struct fuse_iqueue *fiq = &fc->iq;
0285 
0286     if (test_and_set_bit(FR_FINISHED, &req->flags))
0287         goto put_request;
0288 
0289     /*
0290      * test_and_set_bit() implies smp_mb() between bit
0291      * changing and below FR_INTERRUPTED check. Pairs with
0292      * smp_mb() from queue_interrupt().
0293      */
0294     if (test_bit(FR_INTERRUPTED, &req->flags)) {
0295         spin_lock(&fiq->lock);
0296         list_del_init(&req->intr_entry);
0297         spin_unlock(&fiq->lock);
0298     }
0299     WARN_ON(test_bit(FR_PENDING, &req->flags));
0300     WARN_ON(test_bit(FR_SENT, &req->flags));
0301     if (test_bit(FR_BACKGROUND, &req->flags)) {
0302         spin_lock(&fc->bg_lock);
0303         clear_bit(FR_BACKGROUND, &req->flags);
0304         if (fc->num_background == fc->max_background) {
0305             fc->blocked = 0;
0306             wake_up(&fc->blocked_waitq);
0307         } else if (!fc->blocked) {
0308             /*
0309              * Wake up next waiter, if any.  It's okay to use
0310              * waitqueue_active(), as we've already synced up
0311              * fc->blocked with waiters with the wake_up() call
0312              * above.
0313              */
0314             if (waitqueue_active(&fc->blocked_waitq))
0315                 wake_up(&fc->blocked_waitq);
0316         }
0317 
0318         fc->num_background--;
0319         fc->active_background--;
0320         flush_bg_queue(fc);
0321         spin_unlock(&fc->bg_lock);
0322     } else {
0323         /* Wake up waiter sleeping in request_wait_answer() */
0324         wake_up(&req->waitq);
0325     }
0326 
0327     if (test_bit(FR_ASYNC, &req->flags))
0328         req->args->end(fm, req->args, req->out.h.error);
0329 put_request:
0330     fuse_put_request(req);
0331 }
0332 EXPORT_SYMBOL_GPL(fuse_request_end);
0333 
0334 static int queue_interrupt(struct fuse_req *req)
0335 {
0336     struct fuse_iqueue *fiq = &req->fm->fc->iq;
0337 
0338     spin_lock(&fiq->lock);
0339     /* Check for we've sent request to interrupt this req */
0340     if (unlikely(!test_bit(FR_INTERRUPTED, &req->flags))) {
0341         spin_unlock(&fiq->lock);
0342         return -EINVAL;
0343     }
0344 
0345     if (list_empty(&req->intr_entry)) {
0346         list_add_tail(&req->intr_entry, &fiq->interrupts);
0347         /*
0348          * Pairs with smp_mb() implied by test_and_set_bit()
0349          * from fuse_request_end().
0350          */
0351         smp_mb();
0352         if (test_bit(FR_FINISHED, &req->flags)) {
0353             list_del_init(&req->intr_entry);
0354             spin_unlock(&fiq->lock);
0355             return 0;
0356         }
0357         fiq->ops->wake_interrupt_and_unlock(fiq);
0358     } else {
0359         spin_unlock(&fiq->lock);
0360     }
0361     return 0;
0362 }
0363 
0364 static void request_wait_answer(struct fuse_req *req)
0365 {
0366     struct fuse_conn *fc = req->fm->fc;
0367     struct fuse_iqueue *fiq = &fc->iq;
0368     int err;
0369 
0370     if (!fc->no_interrupt) {
0371         /* Any signal may interrupt this */
0372         err = wait_event_interruptible(req->waitq,
0373                     test_bit(FR_FINISHED, &req->flags));
0374         if (!err)
0375             return;
0376 
0377         set_bit(FR_INTERRUPTED, &req->flags);
0378         /* matches barrier in fuse_dev_do_read() */
0379         smp_mb__after_atomic();
0380         if (test_bit(FR_SENT, &req->flags))
0381             queue_interrupt(req);
0382     }
0383 
0384     if (!test_bit(FR_FORCE, &req->flags)) {
0385         /* Only fatal signals may interrupt this */
0386         err = wait_event_killable(req->waitq,
0387                     test_bit(FR_FINISHED, &req->flags));
0388         if (!err)
0389             return;
0390 
0391         spin_lock(&fiq->lock);
0392         /* Request is not yet in userspace, bail out */
0393         if (test_bit(FR_PENDING, &req->flags)) {
0394             list_del(&req->list);
0395             spin_unlock(&fiq->lock);
0396             __fuse_put_request(req);
0397             req->out.h.error = -EINTR;
0398             return;
0399         }
0400         spin_unlock(&fiq->lock);
0401     }
0402 
0403     /*
0404      * Either request is already in userspace, or it was forced.
0405      * Wait it out.
0406      */
0407     wait_event(req->waitq, test_bit(FR_FINISHED, &req->flags));
0408 }
0409 
0410 static void __fuse_request_send(struct fuse_req *req)
0411 {
0412     struct fuse_iqueue *fiq = &req->fm->fc->iq;
0413 
0414     BUG_ON(test_bit(FR_BACKGROUND, &req->flags));
0415     spin_lock(&fiq->lock);
0416     if (!fiq->connected) {
0417         spin_unlock(&fiq->lock);
0418         req->out.h.error = -ENOTCONN;
0419     } else {
0420         req->in.h.unique = fuse_get_unique(fiq);
0421         /* acquire extra reference, since request is still needed
0422            after fuse_request_end() */
0423         __fuse_get_request(req);
0424         queue_request_and_unlock(fiq, req);
0425 
0426         request_wait_answer(req);
0427         /* Pairs with smp_wmb() in fuse_request_end() */
0428         smp_rmb();
0429     }
0430 }
0431 
0432 static void fuse_adjust_compat(struct fuse_conn *fc, struct fuse_args *args)
0433 {
0434     if (fc->minor < 4 && args->opcode == FUSE_STATFS)
0435         args->out_args[0].size = FUSE_COMPAT_STATFS_SIZE;
0436 
0437     if (fc->minor < 9) {
0438         switch (args->opcode) {
0439         case FUSE_LOOKUP:
0440         case FUSE_CREATE:
0441         case FUSE_MKNOD:
0442         case FUSE_MKDIR:
0443         case FUSE_SYMLINK:
0444         case FUSE_LINK:
0445             args->out_args[0].size = FUSE_COMPAT_ENTRY_OUT_SIZE;
0446             break;
0447         case FUSE_GETATTR:
0448         case FUSE_SETATTR:
0449             args->out_args[0].size = FUSE_COMPAT_ATTR_OUT_SIZE;
0450             break;
0451         }
0452     }
0453     if (fc->minor < 12) {
0454         switch (args->opcode) {
0455         case FUSE_CREATE:
0456             args->in_args[0].size = sizeof(struct fuse_open_in);
0457             break;
0458         case FUSE_MKNOD:
0459             args->in_args[0].size = FUSE_COMPAT_MKNOD_IN_SIZE;
0460             break;
0461         }
0462     }
0463 }
0464 
0465 static void fuse_force_creds(struct fuse_req *req)
0466 {
0467     struct fuse_conn *fc = req->fm->fc;
0468 
0469     req->in.h.uid = from_kuid_munged(fc->user_ns, current_fsuid());
0470     req->in.h.gid = from_kgid_munged(fc->user_ns, current_fsgid());
0471     req->in.h.pid = pid_nr_ns(task_pid(current), fc->pid_ns);
0472 }
0473 
0474 static void fuse_args_to_req(struct fuse_req *req, struct fuse_args *args)
0475 {
0476     req->in.h.opcode = args->opcode;
0477     req->in.h.nodeid = args->nodeid;
0478     req->args = args;
0479     if (args->end)
0480         __set_bit(FR_ASYNC, &req->flags);
0481 }
0482 
0483 ssize_t fuse_simple_request(struct fuse_mount *fm, struct fuse_args *args)
0484 {
0485     struct fuse_conn *fc = fm->fc;
0486     struct fuse_req *req;
0487     ssize_t ret;
0488 
0489     if (args->force) {
0490         atomic_inc(&fc->num_waiting);
0491         req = fuse_request_alloc(fm, GFP_KERNEL | __GFP_NOFAIL);
0492 
0493         if (!args->nocreds)
0494             fuse_force_creds(req);
0495 
0496         __set_bit(FR_WAITING, &req->flags);
0497         __set_bit(FR_FORCE, &req->flags);
0498     } else {
0499         WARN_ON(args->nocreds);
0500         req = fuse_get_req(fm, false);
0501         if (IS_ERR(req))
0502             return PTR_ERR(req);
0503     }
0504 
0505     /* Needs to be done after fuse_get_req() so that fc->minor is valid */
0506     fuse_adjust_compat(fc, args);
0507     fuse_args_to_req(req, args);
0508 
0509     if (!args->noreply)
0510         __set_bit(FR_ISREPLY, &req->flags);
0511     __fuse_request_send(req);
0512     ret = req->out.h.error;
0513     if (!ret && args->out_argvar) {
0514         BUG_ON(args->out_numargs == 0);
0515         ret = args->out_args[args->out_numargs - 1].size;
0516     }
0517     fuse_put_request(req);
0518 
0519     return ret;
0520 }
0521 
0522 static bool fuse_request_queue_background(struct fuse_req *req)
0523 {
0524     struct fuse_mount *fm = req->fm;
0525     struct fuse_conn *fc = fm->fc;
0526     bool queued = false;
0527 
0528     WARN_ON(!test_bit(FR_BACKGROUND, &req->flags));
0529     if (!test_bit(FR_WAITING, &req->flags)) {
0530         __set_bit(FR_WAITING, &req->flags);
0531         atomic_inc(&fc->num_waiting);
0532     }
0533     __set_bit(FR_ISREPLY, &req->flags);
0534     spin_lock(&fc->bg_lock);
0535     if (likely(fc->connected)) {
0536         fc->num_background++;
0537         if (fc->num_background == fc->max_background)
0538             fc->blocked = 1;
0539         list_add_tail(&req->list, &fc->bg_queue);
0540         flush_bg_queue(fc);
0541         queued = true;
0542     }
0543     spin_unlock(&fc->bg_lock);
0544 
0545     return queued;
0546 }
0547 
0548 int fuse_simple_background(struct fuse_mount *fm, struct fuse_args *args,
0549                 gfp_t gfp_flags)
0550 {
0551     struct fuse_req *req;
0552 
0553     if (args->force) {
0554         WARN_ON(!args->nocreds);
0555         req = fuse_request_alloc(fm, gfp_flags);
0556         if (!req)
0557             return -ENOMEM;
0558         __set_bit(FR_BACKGROUND, &req->flags);
0559     } else {
0560         WARN_ON(args->nocreds);
0561         req = fuse_get_req(fm, true);
0562         if (IS_ERR(req))
0563             return PTR_ERR(req);
0564     }
0565 
0566     fuse_args_to_req(req, args);
0567 
0568     if (!fuse_request_queue_background(req)) {
0569         fuse_put_request(req);
0570         return -ENOTCONN;
0571     }
0572 
0573     return 0;
0574 }
0575 EXPORT_SYMBOL_GPL(fuse_simple_background);
0576 
0577 static int fuse_simple_notify_reply(struct fuse_mount *fm,
0578                     struct fuse_args *args, u64 unique)
0579 {
0580     struct fuse_req *req;
0581     struct fuse_iqueue *fiq = &fm->fc->iq;
0582     int err = 0;
0583 
0584     req = fuse_get_req(fm, false);
0585     if (IS_ERR(req))
0586         return PTR_ERR(req);
0587 
0588     __clear_bit(FR_ISREPLY, &req->flags);
0589     req->in.h.unique = unique;
0590 
0591     fuse_args_to_req(req, args);
0592 
0593     spin_lock(&fiq->lock);
0594     if (fiq->connected) {
0595         queue_request_and_unlock(fiq, req);
0596     } else {
0597         err = -ENODEV;
0598         spin_unlock(&fiq->lock);
0599         fuse_put_request(req);
0600     }
0601 
0602     return err;
0603 }
0604 
0605 /*
0606  * Lock the request.  Up to the next unlock_request() there mustn't be
0607  * anything that could cause a page-fault.  If the request was already
0608  * aborted bail out.
0609  */
0610 static int lock_request(struct fuse_req *req)
0611 {
0612     int err = 0;
0613     if (req) {
0614         spin_lock(&req->waitq.lock);
0615         if (test_bit(FR_ABORTED, &req->flags))
0616             err = -ENOENT;
0617         else
0618             set_bit(FR_LOCKED, &req->flags);
0619         spin_unlock(&req->waitq.lock);
0620     }
0621     return err;
0622 }
0623 
0624 /*
0625  * Unlock request.  If it was aborted while locked, caller is responsible
0626  * for unlocking and ending the request.
0627  */
0628 static int unlock_request(struct fuse_req *req)
0629 {
0630     int err = 0;
0631     if (req) {
0632         spin_lock(&req->waitq.lock);
0633         if (test_bit(FR_ABORTED, &req->flags))
0634             err = -ENOENT;
0635         else
0636             clear_bit(FR_LOCKED, &req->flags);
0637         spin_unlock(&req->waitq.lock);
0638     }
0639     return err;
0640 }
0641 
0642 struct fuse_copy_state {
0643     int write;
0644     struct fuse_req *req;
0645     struct iov_iter *iter;
0646     struct pipe_buffer *pipebufs;
0647     struct pipe_buffer *currbuf;
0648     struct pipe_inode_info *pipe;
0649     unsigned long nr_segs;
0650     struct page *pg;
0651     unsigned len;
0652     unsigned offset;
0653     unsigned move_pages:1;
0654 };
0655 
0656 static void fuse_copy_init(struct fuse_copy_state *cs, int write,
0657                struct iov_iter *iter)
0658 {
0659     memset(cs, 0, sizeof(*cs));
0660     cs->write = write;
0661     cs->iter = iter;
0662 }
0663 
0664 /* Unmap and put previous page of userspace buffer */
0665 static void fuse_copy_finish(struct fuse_copy_state *cs)
0666 {
0667     if (cs->currbuf) {
0668         struct pipe_buffer *buf = cs->currbuf;
0669 
0670         if (cs->write)
0671             buf->len = PAGE_SIZE - cs->len;
0672         cs->currbuf = NULL;
0673     } else if (cs->pg) {
0674         if (cs->write) {
0675             flush_dcache_page(cs->pg);
0676             set_page_dirty_lock(cs->pg);
0677         }
0678         put_page(cs->pg);
0679     }
0680     cs->pg = NULL;
0681 }
0682 
0683 /*
0684  * Get another pagefull of userspace buffer, and map it to kernel
0685  * address space, and lock request
0686  */
0687 static int fuse_copy_fill(struct fuse_copy_state *cs)
0688 {
0689     struct page *page;
0690     int err;
0691 
0692     err = unlock_request(cs->req);
0693     if (err)
0694         return err;
0695 
0696     fuse_copy_finish(cs);
0697     if (cs->pipebufs) {
0698         struct pipe_buffer *buf = cs->pipebufs;
0699 
0700         if (!cs->write) {
0701             err = pipe_buf_confirm(cs->pipe, buf);
0702             if (err)
0703                 return err;
0704 
0705             BUG_ON(!cs->nr_segs);
0706             cs->currbuf = buf;
0707             cs->pg = buf->page;
0708             cs->offset = buf->offset;
0709             cs->len = buf->len;
0710             cs->pipebufs++;
0711             cs->nr_segs--;
0712         } else {
0713             if (cs->nr_segs >= cs->pipe->max_usage)
0714                 return -EIO;
0715 
0716             page = alloc_page(GFP_HIGHUSER);
0717             if (!page)
0718                 return -ENOMEM;
0719 
0720             buf->page = page;
0721             buf->offset = 0;
0722             buf->len = 0;
0723 
0724             cs->currbuf = buf;
0725             cs->pg = page;
0726             cs->offset = 0;
0727             cs->len = PAGE_SIZE;
0728             cs->pipebufs++;
0729             cs->nr_segs++;
0730         }
0731     } else {
0732         size_t off;
0733         err = iov_iter_get_pages2(cs->iter, &page, PAGE_SIZE, 1, &off);
0734         if (err < 0)
0735             return err;
0736         BUG_ON(!err);
0737         cs->len = err;
0738         cs->offset = off;
0739         cs->pg = page;
0740     }
0741 
0742     return lock_request(cs->req);
0743 }
0744 
0745 /* Do as much copy to/from userspace buffer as we can */
0746 static int fuse_copy_do(struct fuse_copy_state *cs, void **val, unsigned *size)
0747 {
0748     unsigned ncpy = min(*size, cs->len);
0749     if (val) {
0750         void *pgaddr = kmap_local_page(cs->pg);
0751         void *buf = pgaddr + cs->offset;
0752 
0753         if (cs->write)
0754             memcpy(buf, *val, ncpy);
0755         else
0756             memcpy(*val, buf, ncpy);
0757 
0758         kunmap_local(pgaddr);
0759         *val += ncpy;
0760     }
0761     *size -= ncpy;
0762     cs->len -= ncpy;
0763     cs->offset += ncpy;
0764     return ncpy;
0765 }
0766 
0767 static int fuse_check_page(struct page *page)
0768 {
0769     if (page_mapcount(page) ||
0770         page->mapping != NULL ||
0771         (page->flags & PAGE_FLAGS_CHECK_AT_PREP &
0772          ~(1 << PG_locked |
0773            1 << PG_referenced |
0774            1 << PG_uptodate |
0775            1 << PG_lru |
0776            1 << PG_active |
0777            1 << PG_workingset |
0778            1 << PG_reclaim |
0779            1 << PG_waiters))) {
0780         dump_page(page, "fuse: trying to steal weird page");
0781         return 1;
0782     }
0783     return 0;
0784 }
0785 
0786 static int fuse_try_move_page(struct fuse_copy_state *cs, struct page **pagep)
0787 {
0788     int err;
0789     struct page *oldpage = *pagep;
0790     struct page *newpage;
0791     struct pipe_buffer *buf = cs->pipebufs;
0792 
0793     get_page(oldpage);
0794     err = unlock_request(cs->req);
0795     if (err)
0796         goto out_put_old;
0797 
0798     fuse_copy_finish(cs);
0799 
0800     err = pipe_buf_confirm(cs->pipe, buf);
0801     if (err)
0802         goto out_put_old;
0803 
0804     BUG_ON(!cs->nr_segs);
0805     cs->currbuf = buf;
0806     cs->len = buf->len;
0807     cs->pipebufs++;
0808     cs->nr_segs--;
0809 
0810     if (cs->len != PAGE_SIZE)
0811         goto out_fallback;
0812 
0813     if (!pipe_buf_try_steal(cs->pipe, buf))
0814         goto out_fallback;
0815 
0816     newpage = buf->page;
0817 
0818     if (!PageUptodate(newpage))
0819         SetPageUptodate(newpage);
0820 
0821     ClearPageMappedToDisk(newpage);
0822 
0823     if (fuse_check_page(newpage) != 0)
0824         goto out_fallback_unlock;
0825 
0826     /*
0827      * This is a new and locked page, it shouldn't be mapped or
0828      * have any special flags on it
0829      */
0830     if (WARN_ON(page_mapped(oldpage)))
0831         goto out_fallback_unlock;
0832     if (WARN_ON(page_has_private(oldpage)))
0833         goto out_fallback_unlock;
0834     if (WARN_ON(PageDirty(oldpage) || PageWriteback(oldpage)))
0835         goto out_fallback_unlock;
0836     if (WARN_ON(PageMlocked(oldpage)))
0837         goto out_fallback_unlock;
0838 
0839     replace_page_cache_page(oldpage, newpage);
0840 
0841     get_page(newpage);
0842 
0843     if (!(buf->flags & PIPE_BUF_FLAG_LRU))
0844         lru_cache_add(newpage);
0845 
0846     /*
0847      * Release while we have extra ref on stolen page.  Otherwise
0848      * anon_pipe_buf_release() might think the page can be reused.
0849      */
0850     pipe_buf_release(cs->pipe, buf);
0851 
0852     err = 0;
0853     spin_lock(&cs->req->waitq.lock);
0854     if (test_bit(FR_ABORTED, &cs->req->flags))
0855         err = -ENOENT;
0856     else
0857         *pagep = newpage;
0858     spin_unlock(&cs->req->waitq.lock);
0859 
0860     if (err) {
0861         unlock_page(newpage);
0862         put_page(newpage);
0863         goto out_put_old;
0864     }
0865 
0866     unlock_page(oldpage);
0867     /* Drop ref for ap->pages[] array */
0868     put_page(oldpage);
0869     cs->len = 0;
0870 
0871     err = 0;
0872 out_put_old:
0873     /* Drop ref obtained in this function */
0874     put_page(oldpage);
0875     return err;
0876 
0877 out_fallback_unlock:
0878     unlock_page(newpage);
0879 out_fallback:
0880     cs->pg = buf->page;
0881     cs->offset = buf->offset;
0882 
0883     err = lock_request(cs->req);
0884     if (!err)
0885         err = 1;
0886 
0887     goto out_put_old;
0888 }
0889 
0890 static int fuse_ref_page(struct fuse_copy_state *cs, struct page *page,
0891              unsigned offset, unsigned count)
0892 {
0893     struct pipe_buffer *buf;
0894     int err;
0895 
0896     if (cs->nr_segs >= cs->pipe->max_usage)
0897         return -EIO;
0898 
0899     get_page(page);
0900     err = unlock_request(cs->req);
0901     if (err) {
0902         put_page(page);
0903         return err;
0904     }
0905 
0906     fuse_copy_finish(cs);
0907 
0908     buf = cs->pipebufs;
0909     buf->page = page;
0910     buf->offset = offset;
0911     buf->len = count;
0912 
0913     cs->pipebufs++;
0914     cs->nr_segs++;
0915     cs->len = 0;
0916 
0917     return 0;
0918 }
0919 
0920 /*
0921  * Copy a page in the request to/from the userspace buffer.  Must be
0922  * done atomically
0923  */
0924 static int fuse_copy_page(struct fuse_copy_state *cs, struct page **pagep,
0925               unsigned offset, unsigned count, int zeroing)
0926 {
0927     int err;
0928     struct page *page = *pagep;
0929 
0930     if (page && zeroing && count < PAGE_SIZE)
0931         clear_highpage(page);
0932 
0933     while (count) {
0934         if (cs->write && cs->pipebufs && page) {
0935             /*
0936              * Can't control lifetime of pipe buffers, so always
0937              * copy user pages.
0938              */
0939             if (cs->req->args->user_pages) {
0940                 err = fuse_copy_fill(cs);
0941                 if (err)
0942                     return err;
0943             } else {
0944                 return fuse_ref_page(cs, page, offset, count);
0945             }
0946         } else if (!cs->len) {
0947             if (cs->move_pages && page &&
0948                 offset == 0 && count == PAGE_SIZE) {
0949                 err = fuse_try_move_page(cs, pagep);
0950                 if (err <= 0)
0951                     return err;
0952             } else {
0953                 err = fuse_copy_fill(cs);
0954                 if (err)
0955                     return err;
0956             }
0957         }
0958         if (page) {
0959             void *mapaddr = kmap_local_page(page);
0960             void *buf = mapaddr + offset;
0961             offset += fuse_copy_do(cs, &buf, &count);
0962             kunmap_local(mapaddr);
0963         } else
0964             offset += fuse_copy_do(cs, NULL, &count);
0965     }
0966     if (page && !cs->write)
0967         flush_dcache_page(page);
0968     return 0;
0969 }
0970 
0971 /* Copy pages in the request to/from userspace buffer */
0972 static int fuse_copy_pages(struct fuse_copy_state *cs, unsigned nbytes,
0973                int zeroing)
0974 {
0975     unsigned i;
0976     struct fuse_req *req = cs->req;
0977     struct fuse_args_pages *ap = container_of(req->args, typeof(*ap), args);
0978 
0979 
0980     for (i = 0; i < ap->num_pages && (nbytes || zeroing); i++) {
0981         int err;
0982         unsigned int offset = ap->descs[i].offset;
0983         unsigned int count = min(nbytes, ap->descs[i].length);
0984 
0985         err = fuse_copy_page(cs, &ap->pages[i], offset, count, zeroing);
0986         if (err)
0987             return err;
0988 
0989         nbytes -= count;
0990     }
0991     return 0;
0992 }
0993 
0994 /* Copy a single argument in the request to/from userspace buffer */
0995 static int fuse_copy_one(struct fuse_copy_state *cs, void *val, unsigned size)
0996 {
0997     while (size) {
0998         if (!cs->len) {
0999             int err = fuse_copy_fill(cs);
1000             if (err)
1001                 return err;
1002         }
1003         fuse_copy_do(cs, &val, &size);
1004     }
1005     return 0;
1006 }
1007 
1008 /* Copy request arguments to/from userspace buffer */
1009 static int fuse_copy_args(struct fuse_copy_state *cs, unsigned numargs,
1010               unsigned argpages, struct fuse_arg *args,
1011               int zeroing)
1012 {
1013     int err = 0;
1014     unsigned i;
1015 
1016     for (i = 0; !err && i < numargs; i++)  {
1017         struct fuse_arg *arg = &args[i];
1018         if (i == numargs - 1 && argpages)
1019             err = fuse_copy_pages(cs, arg->size, zeroing);
1020         else
1021             err = fuse_copy_one(cs, arg->value, arg->size);
1022     }
1023     return err;
1024 }
1025 
1026 static int forget_pending(struct fuse_iqueue *fiq)
1027 {
1028     return fiq->forget_list_head.next != NULL;
1029 }
1030 
1031 static int request_pending(struct fuse_iqueue *fiq)
1032 {
1033     return !list_empty(&fiq->pending) || !list_empty(&fiq->interrupts) ||
1034         forget_pending(fiq);
1035 }
1036 
1037 /*
1038  * Transfer an interrupt request to userspace
1039  *
1040  * Unlike other requests this is assembled on demand, without a need
1041  * to allocate a separate fuse_req structure.
1042  *
1043  * Called with fiq->lock held, releases it
1044  */
1045 static int fuse_read_interrupt(struct fuse_iqueue *fiq,
1046                    struct fuse_copy_state *cs,
1047                    size_t nbytes, struct fuse_req *req)
1048 __releases(fiq->lock)
1049 {
1050     struct fuse_in_header ih;
1051     struct fuse_interrupt_in arg;
1052     unsigned reqsize = sizeof(ih) + sizeof(arg);
1053     int err;
1054 
1055     list_del_init(&req->intr_entry);
1056     memset(&ih, 0, sizeof(ih));
1057     memset(&arg, 0, sizeof(arg));
1058     ih.len = reqsize;
1059     ih.opcode = FUSE_INTERRUPT;
1060     ih.unique = (req->in.h.unique | FUSE_INT_REQ_BIT);
1061     arg.unique = req->in.h.unique;
1062 
1063     spin_unlock(&fiq->lock);
1064     if (nbytes < reqsize)
1065         return -EINVAL;
1066 
1067     err = fuse_copy_one(cs, &ih, sizeof(ih));
1068     if (!err)
1069         err = fuse_copy_one(cs, &arg, sizeof(arg));
1070     fuse_copy_finish(cs);
1071 
1072     return err ? err : reqsize;
1073 }
1074 
1075 struct fuse_forget_link *fuse_dequeue_forget(struct fuse_iqueue *fiq,
1076                          unsigned int max,
1077                          unsigned int *countp)
1078 {
1079     struct fuse_forget_link *head = fiq->forget_list_head.next;
1080     struct fuse_forget_link **newhead = &head;
1081     unsigned count;
1082 
1083     for (count = 0; *newhead != NULL && count < max; count++)
1084         newhead = &(*newhead)->next;
1085 
1086     fiq->forget_list_head.next = *newhead;
1087     *newhead = NULL;
1088     if (fiq->forget_list_head.next == NULL)
1089         fiq->forget_list_tail = &fiq->forget_list_head;
1090 
1091     if (countp != NULL)
1092         *countp = count;
1093 
1094     return head;
1095 }
1096 EXPORT_SYMBOL(fuse_dequeue_forget);
1097 
1098 static int fuse_read_single_forget(struct fuse_iqueue *fiq,
1099                    struct fuse_copy_state *cs,
1100                    size_t nbytes)
1101 __releases(fiq->lock)
1102 {
1103     int err;
1104     struct fuse_forget_link *forget = fuse_dequeue_forget(fiq, 1, NULL);
1105     struct fuse_forget_in arg = {
1106         .nlookup = forget->forget_one.nlookup,
1107     };
1108     struct fuse_in_header ih = {
1109         .opcode = FUSE_FORGET,
1110         .nodeid = forget->forget_one.nodeid,
1111         .unique = fuse_get_unique(fiq),
1112         .len = sizeof(ih) + sizeof(arg),
1113     };
1114 
1115     spin_unlock(&fiq->lock);
1116     kfree(forget);
1117     if (nbytes < ih.len)
1118         return -EINVAL;
1119 
1120     err = fuse_copy_one(cs, &ih, sizeof(ih));
1121     if (!err)
1122         err = fuse_copy_one(cs, &arg, sizeof(arg));
1123     fuse_copy_finish(cs);
1124 
1125     if (err)
1126         return err;
1127 
1128     return ih.len;
1129 }
1130 
1131 static int fuse_read_batch_forget(struct fuse_iqueue *fiq,
1132                    struct fuse_copy_state *cs, size_t nbytes)
1133 __releases(fiq->lock)
1134 {
1135     int err;
1136     unsigned max_forgets;
1137     unsigned count;
1138     struct fuse_forget_link *head;
1139     struct fuse_batch_forget_in arg = { .count = 0 };
1140     struct fuse_in_header ih = {
1141         .opcode = FUSE_BATCH_FORGET,
1142         .unique = fuse_get_unique(fiq),
1143         .len = sizeof(ih) + sizeof(arg),
1144     };
1145 
1146     if (nbytes < ih.len) {
1147         spin_unlock(&fiq->lock);
1148         return -EINVAL;
1149     }
1150 
1151     max_forgets = (nbytes - ih.len) / sizeof(struct fuse_forget_one);
1152     head = fuse_dequeue_forget(fiq, max_forgets, &count);
1153     spin_unlock(&fiq->lock);
1154 
1155     arg.count = count;
1156     ih.len += count * sizeof(struct fuse_forget_one);
1157     err = fuse_copy_one(cs, &ih, sizeof(ih));
1158     if (!err)
1159         err = fuse_copy_one(cs, &arg, sizeof(arg));
1160 
1161     while (head) {
1162         struct fuse_forget_link *forget = head;
1163 
1164         if (!err) {
1165             err = fuse_copy_one(cs, &forget->forget_one,
1166                         sizeof(forget->forget_one));
1167         }
1168         head = forget->next;
1169         kfree(forget);
1170     }
1171 
1172     fuse_copy_finish(cs);
1173 
1174     if (err)
1175         return err;
1176 
1177     return ih.len;
1178 }
1179 
1180 static int fuse_read_forget(struct fuse_conn *fc, struct fuse_iqueue *fiq,
1181                 struct fuse_copy_state *cs,
1182                 size_t nbytes)
1183 __releases(fiq->lock)
1184 {
1185     if (fc->minor < 16 || fiq->forget_list_head.next->next == NULL)
1186         return fuse_read_single_forget(fiq, cs, nbytes);
1187     else
1188         return fuse_read_batch_forget(fiq, cs, nbytes);
1189 }
1190 
1191 /*
1192  * Read a single request into the userspace filesystem's buffer.  This
1193  * function waits until a request is available, then removes it from
1194  * the pending list and copies request data to userspace buffer.  If
1195  * no reply is needed (FORGET) or request has been aborted or there
1196  * was an error during the copying then it's finished by calling
1197  * fuse_request_end().  Otherwise add it to the processing list, and set
1198  * the 'sent' flag.
1199  */
1200 static ssize_t fuse_dev_do_read(struct fuse_dev *fud, struct file *file,
1201                 struct fuse_copy_state *cs, size_t nbytes)
1202 {
1203     ssize_t err;
1204     struct fuse_conn *fc = fud->fc;
1205     struct fuse_iqueue *fiq = &fc->iq;
1206     struct fuse_pqueue *fpq = &fud->pq;
1207     struct fuse_req *req;
1208     struct fuse_args *args;
1209     unsigned reqsize;
1210     unsigned int hash;
1211 
1212     /*
1213      * Require sane minimum read buffer - that has capacity for fixed part
1214      * of any request header + negotiated max_write room for data.
1215      *
1216      * Historically libfuse reserves 4K for fixed header room, but e.g.
1217      * GlusterFS reserves only 80 bytes
1218      *
1219      *  = `sizeof(fuse_in_header) + sizeof(fuse_write_in)`
1220      *
1221      * which is the absolute minimum any sane filesystem should be using
1222      * for header room.
1223      */
1224     if (nbytes < max_t(size_t, FUSE_MIN_READ_BUFFER,
1225                sizeof(struct fuse_in_header) +
1226                sizeof(struct fuse_write_in) +
1227                fc->max_write))
1228         return -EINVAL;
1229 
1230  restart:
1231     for (;;) {
1232         spin_lock(&fiq->lock);
1233         if (!fiq->connected || request_pending(fiq))
1234             break;
1235         spin_unlock(&fiq->lock);
1236 
1237         if (file->f_flags & O_NONBLOCK)
1238             return -EAGAIN;
1239         err = wait_event_interruptible_exclusive(fiq->waitq,
1240                 !fiq->connected || request_pending(fiq));
1241         if (err)
1242             return err;
1243     }
1244 
1245     if (!fiq->connected) {
1246         err = fc->aborted ? -ECONNABORTED : -ENODEV;
1247         goto err_unlock;
1248     }
1249 
1250     if (!list_empty(&fiq->interrupts)) {
1251         req = list_entry(fiq->interrupts.next, struct fuse_req,
1252                  intr_entry);
1253         return fuse_read_interrupt(fiq, cs, nbytes, req);
1254     }
1255 
1256     if (forget_pending(fiq)) {
1257         if (list_empty(&fiq->pending) || fiq->forget_batch-- > 0)
1258             return fuse_read_forget(fc, fiq, cs, nbytes);
1259 
1260         if (fiq->forget_batch <= -8)
1261             fiq->forget_batch = 16;
1262     }
1263 
1264     req = list_entry(fiq->pending.next, struct fuse_req, list);
1265     clear_bit(FR_PENDING, &req->flags);
1266     list_del_init(&req->list);
1267     spin_unlock(&fiq->lock);
1268 
1269     args = req->args;
1270     reqsize = req->in.h.len;
1271 
1272     /* If request is too large, reply with an error and restart the read */
1273     if (nbytes < reqsize) {
1274         req->out.h.error = -EIO;
1275         /* SETXATTR is special, since it may contain too large data */
1276         if (args->opcode == FUSE_SETXATTR)
1277             req->out.h.error = -E2BIG;
1278         fuse_request_end(req);
1279         goto restart;
1280     }
1281     spin_lock(&fpq->lock);
1282     /*
1283      *  Must not put request on fpq->io queue after having been shut down by
1284      *  fuse_abort_conn()
1285      */
1286     if (!fpq->connected) {
1287         req->out.h.error = err = -ECONNABORTED;
1288         goto out_end;
1289 
1290     }
1291     list_add(&req->list, &fpq->io);
1292     spin_unlock(&fpq->lock);
1293     cs->req = req;
1294     err = fuse_copy_one(cs, &req->in.h, sizeof(req->in.h));
1295     if (!err)
1296         err = fuse_copy_args(cs, args->in_numargs, args->in_pages,
1297                      (struct fuse_arg *) args->in_args, 0);
1298     fuse_copy_finish(cs);
1299     spin_lock(&fpq->lock);
1300     clear_bit(FR_LOCKED, &req->flags);
1301     if (!fpq->connected) {
1302         err = fc->aborted ? -ECONNABORTED : -ENODEV;
1303         goto out_end;
1304     }
1305     if (err) {
1306         req->out.h.error = -EIO;
1307         goto out_end;
1308     }
1309     if (!test_bit(FR_ISREPLY, &req->flags)) {
1310         err = reqsize;
1311         goto out_end;
1312     }
1313     hash = fuse_req_hash(req->in.h.unique);
1314     list_move_tail(&req->list, &fpq->processing[hash]);
1315     __fuse_get_request(req);
1316     set_bit(FR_SENT, &req->flags);
1317     spin_unlock(&fpq->lock);
1318     /* matches barrier in request_wait_answer() */
1319     smp_mb__after_atomic();
1320     if (test_bit(FR_INTERRUPTED, &req->flags))
1321         queue_interrupt(req);
1322     fuse_put_request(req);
1323 
1324     return reqsize;
1325 
1326 out_end:
1327     if (!test_bit(FR_PRIVATE, &req->flags))
1328         list_del_init(&req->list);
1329     spin_unlock(&fpq->lock);
1330     fuse_request_end(req);
1331     return err;
1332 
1333  err_unlock:
1334     spin_unlock(&fiq->lock);
1335     return err;
1336 }
1337 
1338 static int fuse_dev_open(struct inode *inode, struct file *file)
1339 {
1340     /*
1341      * The fuse device's file's private_data is used to hold
1342      * the fuse_conn(ection) when it is mounted, and is used to
1343      * keep track of whether the file has been mounted already.
1344      */
1345     file->private_data = NULL;
1346     return 0;
1347 }
1348 
1349 static ssize_t fuse_dev_read(struct kiocb *iocb, struct iov_iter *to)
1350 {
1351     struct fuse_copy_state cs;
1352     struct file *file = iocb->ki_filp;
1353     struct fuse_dev *fud = fuse_get_dev(file);
1354 
1355     if (!fud)
1356         return -EPERM;
1357 
1358     if (!user_backed_iter(to))
1359         return -EINVAL;
1360 
1361     fuse_copy_init(&cs, 1, to);
1362 
1363     return fuse_dev_do_read(fud, file, &cs, iov_iter_count(to));
1364 }
1365 
1366 static ssize_t fuse_dev_splice_read(struct file *in, loff_t *ppos,
1367                     struct pipe_inode_info *pipe,
1368                     size_t len, unsigned int flags)
1369 {
1370     int total, ret;
1371     int page_nr = 0;
1372     struct pipe_buffer *bufs;
1373     struct fuse_copy_state cs;
1374     struct fuse_dev *fud = fuse_get_dev(in);
1375 
1376     if (!fud)
1377         return -EPERM;
1378 
1379     bufs = kvmalloc_array(pipe->max_usage, sizeof(struct pipe_buffer),
1380                   GFP_KERNEL);
1381     if (!bufs)
1382         return -ENOMEM;
1383 
1384     fuse_copy_init(&cs, 1, NULL);
1385     cs.pipebufs = bufs;
1386     cs.pipe = pipe;
1387     ret = fuse_dev_do_read(fud, in, &cs, len);
1388     if (ret < 0)
1389         goto out;
1390 
1391     if (pipe_occupancy(pipe->head, pipe->tail) + cs.nr_segs > pipe->max_usage) {
1392         ret = -EIO;
1393         goto out;
1394     }
1395 
1396     for (ret = total = 0; page_nr < cs.nr_segs; total += ret) {
1397         /*
1398          * Need to be careful about this.  Having buf->ops in module
1399          * code can Oops if the buffer persists after module unload.
1400          */
1401         bufs[page_nr].ops = &nosteal_pipe_buf_ops;
1402         bufs[page_nr].flags = 0;
1403         ret = add_to_pipe(pipe, &bufs[page_nr++]);
1404         if (unlikely(ret < 0))
1405             break;
1406     }
1407     if (total)
1408         ret = total;
1409 out:
1410     for (; page_nr < cs.nr_segs; page_nr++)
1411         put_page(bufs[page_nr].page);
1412 
1413     kvfree(bufs);
1414     return ret;
1415 }
1416 
1417 static int fuse_notify_poll(struct fuse_conn *fc, unsigned int size,
1418                 struct fuse_copy_state *cs)
1419 {
1420     struct fuse_notify_poll_wakeup_out outarg;
1421     int err = -EINVAL;
1422 
1423     if (size != sizeof(outarg))
1424         goto err;
1425 
1426     err = fuse_copy_one(cs, &outarg, sizeof(outarg));
1427     if (err)
1428         goto err;
1429 
1430     fuse_copy_finish(cs);
1431     return fuse_notify_poll_wakeup(fc, &outarg);
1432 
1433 err:
1434     fuse_copy_finish(cs);
1435     return err;
1436 }
1437 
1438 static int fuse_notify_inval_inode(struct fuse_conn *fc, unsigned int size,
1439                    struct fuse_copy_state *cs)
1440 {
1441     struct fuse_notify_inval_inode_out outarg;
1442     int err = -EINVAL;
1443 
1444     if (size != sizeof(outarg))
1445         goto err;
1446 
1447     err = fuse_copy_one(cs, &outarg, sizeof(outarg));
1448     if (err)
1449         goto err;
1450     fuse_copy_finish(cs);
1451 
1452     down_read(&fc->killsb);
1453     err = fuse_reverse_inval_inode(fc, outarg.ino,
1454                        outarg.off, outarg.len);
1455     up_read(&fc->killsb);
1456     return err;
1457 
1458 err:
1459     fuse_copy_finish(cs);
1460     return err;
1461 }
1462 
1463 static int fuse_notify_inval_entry(struct fuse_conn *fc, unsigned int size,
1464                    struct fuse_copy_state *cs)
1465 {
1466     struct fuse_notify_inval_entry_out outarg;
1467     int err = -ENOMEM;
1468     char *buf;
1469     struct qstr name;
1470 
1471     buf = kzalloc(FUSE_NAME_MAX + 1, GFP_KERNEL);
1472     if (!buf)
1473         goto err;
1474 
1475     err = -EINVAL;
1476     if (size < sizeof(outarg))
1477         goto err;
1478 
1479     err = fuse_copy_one(cs, &outarg, sizeof(outarg));
1480     if (err)
1481         goto err;
1482 
1483     err = -ENAMETOOLONG;
1484     if (outarg.namelen > FUSE_NAME_MAX)
1485         goto err;
1486 
1487     err = -EINVAL;
1488     if (size != sizeof(outarg) + outarg.namelen + 1)
1489         goto err;
1490 
1491     name.name = buf;
1492     name.len = outarg.namelen;
1493     err = fuse_copy_one(cs, buf, outarg.namelen + 1);
1494     if (err)
1495         goto err;
1496     fuse_copy_finish(cs);
1497     buf[outarg.namelen] = 0;
1498 
1499     down_read(&fc->killsb);
1500     err = fuse_reverse_inval_entry(fc, outarg.parent, 0, &name);
1501     up_read(&fc->killsb);
1502     kfree(buf);
1503     return err;
1504 
1505 err:
1506     kfree(buf);
1507     fuse_copy_finish(cs);
1508     return err;
1509 }
1510 
1511 static int fuse_notify_delete(struct fuse_conn *fc, unsigned int size,
1512                   struct fuse_copy_state *cs)
1513 {
1514     struct fuse_notify_delete_out outarg;
1515     int err = -ENOMEM;
1516     char *buf;
1517     struct qstr name;
1518 
1519     buf = kzalloc(FUSE_NAME_MAX + 1, GFP_KERNEL);
1520     if (!buf)
1521         goto err;
1522 
1523     err = -EINVAL;
1524     if (size < sizeof(outarg))
1525         goto err;
1526 
1527     err = fuse_copy_one(cs, &outarg, sizeof(outarg));
1528     if (err)
1529         goto err;
1530 
1531     err = -ENAMETOOLONG;
1532     if (outarg.namelen > FUSE_NAME_MAX)
1533         goto err;
1534 
1535     err = -EINVAL;
1536     if (size != sizeof(outarg) + outarg.namelen + 1)
1537         goto err;
1538 
1539     name.name = buf;
1540     name.len = outarg.namelen;
1541     err = fuse_copy_one(cs, buf, outarg.namelen + 1);
1542     if (err)
1543         goto err;
1544     fuse_copy_finish(cs);
1545     buf[outarg.namelen] = 0;
1546 
1547     down_read(&fc->killsb);
1548     err = fuse_reverse_inval_entry(fc, outarg.parent, outarg.child, &name);
1549     up_read(&fc->killsb);
1550     kfree(buf);
1551     return err;
1552 
1553 err:
1554     kfree(buf);
1555     fuse_copy_finish(cs);
1556     return err;
1557 }
1558 
1559 static int fuse_notify_store(struct fuse_conn *fc, unsigned int size,
1560                  struct fuse_copy_state *cs)
1561 {
1562     struct fuse_notify_store_out outarg;
1563     struct inode *inode;
1564     struct address_space *mapping;
1565     u64 nodeid;
1566     int err;
1567     pgoff_t index;
1568     unsigned int offset;
1569     unsigned int num;
1570     loff_t file_size;
1571     loff_t end;
1572 
1573     err = -EINVAL;
1574     if (size < sizeof(outarg))
1575         goto out_finish;
1576 
1577     err = fuse_copy_one(cs, &outarg, sizeof(outarg));
1578     if (err)
1579         goto out_finish;
1580 
1581     err = -EINVAL;
1582     if (size - sizeof(outarg) != outarg.size)
1583         goto out_finish;
1584 
1585     nodeid = outarg.nodeid;
1586 
1587     down_read(&fc->killsb);
1588 
1589     err = -ENOENT;
1590     inode = fuse_ilookup(fc, nodeid,  NULL);
1591     if (!inode)
1592         goto out_up_killsb;
1593 
1594     mapping = inode->i_mapping;
1595     index = outarg.offset >> PAGE_SHIFT;
1596     offset = outarg.offset & ~PAGE_MASK;
1597     file_size = i_size_read(inode);
1598     end = outarg.offset + outarg.size;
1599     if (end > file_size) {
1600         file_size = end;
1601         fuse_write_update_attr(inode, file_size, outarg.size);
1602     }
1603 
1604     num = outarg.size;
1605     while (num) {
1606         struct page *page;
1607         unsigned int this_num;
1608 
1609         err = -ENOMEM;
1610         page = find_or_create_page(mapping, index,
1611                        mapping_gfp_mask(mapping));
1612         if (!page)
1613             goto out_iput;
1614 
1615         this_num = min_t(unsigned, num, PAGE_SIZE - offset);
1616         err = fuse_copy_page(cs, &page, offset, this_num, 0);
1617         if (!err && offset == 0 &&
1618             (this_num == PAGE_SIZE || file_size == end))
1619             SetPageUptodate(page);
1620         unlock_page(page);
1621         put_page(page);
1622 
1623         if (err)
1624             goto out_iput;
1625 
1626         num -= this_num;
1627         offset = 0;
1628         index++;
1629     }
1630 
1631     err = 0;
1632 
1633 out_iput:
1634     iput(inode);
1635 out_up_killsb:
1636     up_read(&fc->killsb);
1637 out_finish:
1638     fuse_copy_finish(cs);
1639     return err;
1640 }
1641 
1642 struct fuse_retrieve_args {
1643     struct fuse_args_pages ap;
1644     struct fuse_notify_retrieve_in inarg;
1645 };
1646 
1647 static void fuse_retrieve_end(struct fuse_mount *fm, struct fuse_args *args,
1648                   int error)
1649 {
1650     struct fuse_retrieve_args *ra =
1651         container_of(args, typeof(*ra), ap.args);
1652 
1653     release_pages(ra->ap.pages, ra->ap.num_pages);
1654     kfree(ra);
1655 }
1656 
1657 static int fuse_retrieve(struct fuse_mount *fm, struct inode *inode,
1658              struct fuse_notify_retrieve_out *outarg)
1659 {
1660     int err;
1661     struct address_space *mapping = inode->i_mapping;
1662     pgoff_t index;
1663     loff_t file_size;
1664     unsigned int num;
1665     unsigned int offset;
1666     size_t total_len = 0;
1667     unsigned int num_pages;
1668     struct fuse_conn *fc = fm->fc;
1669     struct fuse_retrieve_args *ra;
1670     size_t args_size = sizeof(*ra);
1671     struct fuse_args_pages *ap;
1672     struct fuse_args *args;
1673 
1674     offset = outarg->offset & ~PAGE_MASK;
1675     file_size = i_size_read(inode);
1676 
1677     num = min(outarg->size, fc->max_write);
1678     if (outarg->offset > file_size)
1679         num = 0;
1680     else if (outarg->offset + num > file_size)
1681         num = file_size - outarg->offset;
1682 
1683     num_pages = (num + offset + PAGE_SIZE - 1) >> PAGE_SHIFT;
1684     num_pages = min(num_pages, fc->max_pages);
1685 
1686     args_size += num_pages * (sizeof(ap->pages[0]) + sizeof(ap->descs[0]));
1687 
1688     ra = kzalloc(args_size, GFP_KERNEL);
1689     if (!ra)
1690         return -ENOMEM;
1691 
1692     ap = &ra->ap;
1693     ap->pages = (void *) (ra + 1);
1694     ap->descs = (void *) (ap->pages + num_pages);
1695 
1696     args = &ap->args;
1697     args->nodeid = outarg->nodeid;
1698     args->opcode = FUSE_NOTIFY_REPLY;
1699     args->in_numargs = 2;
1700     args->in_pages = true;
1701     args->end = fuse_retrieve_end;
1702 
1703     index = outarg->offset >> PAGE_SHIFT;
1704 
1705     while (num && ap->num_pages < num_pages) {
1706         struct page *page;
1707         unsigned int this_num;
1708 
1709         page = find_get_page(mapping, index);
1710         if (!page)
1711             break;
1712 
1713         this_num = min_t(unsigned, num, PAGE_SIZE - offset);
1714         ap->pages[ap->num_pages] = page;
1715         ap->descs[ap->num_pages].offset = offset;
1716         ap->descs[ap->num_pages].length = this_num;
1717         ap->num_pages++;
1718 
1719         offset = 0;
1720         num -= this_num;
1721         total_len += this_num;
1722         index++;
1723     }
1724     ra->inarg.offset = outarg->offset;
1725     ra->inarg.size = total_len;
1726     args->in_args[0].size = sizeof(ra->inarg);
1727     args->in_args[0].value = &ra->inarg;
1728     args->in_args[1].size = total_len;
1729 
1730     err = fuse_simple_notify_reply(fm, args, outarg->notify_unique);
1731     if (err)
1732         fuse_retrieve_end(fm, args, err);
1733 
1734     return err;
1735 }
1736 
1737 static int fuse_notify_retrieve(struct fuse_conn *fc, unsigned int size,
1738                 struct fuse_copy_state *cs)
1739 {
1740     struct fuse_notify_retrieve_out outarg;
1741     struct fuse_mount *fm;
1742     struct inode *inode;
1743     u64 nodeid;
1744     int err;
1745 
1746     err = -EINVAL;
1747     if (size != sizeof(outarg))
1748         goto copy_finish;
1749 
1750     err = fuse_copy_one(cs, &outarg, sizeof(outarg));
1751     if (err)
1752         goto copy_finish;
1753 
1754     fuse_copy_finish(cs);
1755 
1756     down_read(&fc->killsb);
1757     err = -ENOENT;
1758     nodeid = outarg.nodeid;
1759 
1760     inode = fuse_ilookup(fc, nodeid, &fm);
1761     if (inode) {
1762         err = fuse_retrieve(fm, inode, &outarg);
1763         iput(inode);
1764     }
1765     up_read(&fc->killsb);
1766 
1767     return err;
1768 
1769 copy_finish:
1770     fuse_copy_finish(cs);
1771     return err;
1772 }
1773 
1774 static int fuse_notify(struct fuse_conn *fc, enum fuse_notify_code code,
1775                unsigned int size, struct fuse_copy_state *cs)
1776 {
1777     /* Don't try to move pages (yet) */
1778     cs->move_pages = 0;
1779 
1780     switch (code) {
1781     case FUSE_NOTIFY_POLL:
1782         return fuse_notify_poll(fc, size, cs);
1783 
1784     case FUSE_NOTIFY_INVAL_INODE:
1785         return fuse_notify_inval_inode(fc, size, cs);
1786 
1787     case FUSE_NOTIFY_INVAL_ENTRY:
1788         return fuse_notify_inval_entry(fc, size, cs);
1789 
1790     case FUSE_NOTIFY_STORE:
1791         return fuse_notify_store(fc, size, cs);
1792 
1793     case FUSE_NOTIFY_RETRIEVE:
1794         return fuse_notify_retrieve(fc, size, cs);
1795 
1796     case FUSE_NOTIFY_DELETE:
1797         return fuse_notify_delete(fc, size, cs);
1798 
1799     default:
1800         fuse_copy_finish(cs);
1801         return -EINVAL;
1802     }
1803 }
1804 
1805 /* Look up request on processing list by unique ID */
1806 static struct fuse_req *request_find(struct fuse_pqueue *fpq, u64 unique)
1807 {
1808     unsigned int hash = fuse_req_hash(unique);
1809     struct fuse_req *req;
1810 
1811     list_for_each_entry(req, &fpq->processing[hash], list) {
1812         if (req->in.h.unique == unique)
1813             return req;
1814     }
1815     return NULL;
1816 }
1817 
1818 static int copy_out_args(struct fuse_copy_state *cs, struct fuse_args *args,
1819              unsigned nbytes)
1820 {
1821     unsigned reqsize = sizeof(struct fuse_out_header);
1822 
1823     reqsize += fuse_len_args(args->out_numargs, args->out_args);
1824 
1825     if (reqsize < nbytes || (reqsize > nbytes && !args->out_argvar))
1826         return -EINVAL;
1827     else if (reqsize > nbytes) {
1828         struct fuse_arg *lastarg = &args->out_args[args->out_numargs-1];
1829         unsigned diffsize = reqsize - nbytes;
1830 
1831         if (diffsize > lastarg->size)
1832             return -EINVAL;
1833         lastarg->size -= diffsize;
1834     }
1835     return fuse_copy_args(cs, args->out_numargs, args->out_pages,
1836                   args->out_args, args->page_zeroing);
1837 }
1838 
1839 /*
1840  * Write a single reply to a request.  First the header is copied from
1841  * the write buffer.  The request is then searched on the processing
1842  * list by the unique ID found in the header.  If found, then remove
1843  * it from the list and copy the rest of the buffer to the request.
1844  * The request is finished by calling fuse_request_end().
1845  */
1846 static ssize_t fuse_dev_do_write(struct fuse_dev *fud,
1847                  struct fuse_copy_state *cs, size_t nbytes)
1848 {
1849     int err;
1850     struct fuse_conn *fc = fud->fc;
1851     struct fuse_pqueue *fpq = &fud->pq;
1852     struct fuse_req *req;
1853     struct fuse_out_header oh;
1854 
1855     err = -EINVAL;
1856     if (nbytes < sizeof(struct fuse_out_header))
1857         goto out;
1858 
1859     err = fuse_copy_one(cs, &oh, sizeof(oh));
1860     if (err)
1861         goto copy_finish;
1862 
1863     err = -EINVAL;
1864     if (oh.len != nbytes)
1865         goto copy_finish;
1866 
1867     /*
1868      * Zero oh.unique indicates unsolicited notification message
1869      * and error contains notification code.
1870      */
1871     if (!oh.unique) {
1872         err = fuse_notify(fc, oh.error, nbytes - sizeof(oh), cs);
1873         goto out;
1874     }
1875 
1876     err = -EINVAL;
1877     if (oh.error <= -512 || oh.error > 0)
1878         goto copy_finish;
1879 
1880     spin_lock(&fpq->lock);
1881     req = NULL;
1882     if (fpq->connected)
1883         req = request_find(fpq, oh.unique & ~FUSE_INT_REQ_BIT);
1884 
1885     err = -ENOENT;
1886     if (!req) {
1887         spin_unlock(&fpq->lock);
1888         goto copy_finish;
1889     }
1890 
1891     /* Is it an interrupt reply ID? */
1892     if (oh.unique & FUSE_INT_REQ_BIT) {
1893         __fuse_get_request(req);
1894         spin_unlock(&fpq->lock);
1895 
1896         err = 0;
1897         if (nbytes != sizeof(struct fuse_out_header))
1898             err = -EINVAL;
1899         else if (oh.error == -ENOSYS)
1900             fc->no_interrupt = 1;
1901         else if (oh.error == -EAGAIN)
1902             err = queue_interrupt(req);
1903 
1904         fuse_put_request(req);
1905 
1906         goto copy_finish;
1907     }
1908 
1909     clear_bit(FR_SENT, &req->flags);
1910     list_move(&req->list, &fpq->io);
1911     req->out.h = oh;
1912     set_bit(FR_LOCKED, &req->flags);
1913     spin_unlock(&fpq->lock);
1914     cs->req = req;
1915     if (!req->args->page_replace)
1916         cs->move_pages = 0;
1917 
1918     if (oh.error)
1919         err = nbytes != sizeof(oh) ? -EINVAL : 0;
1920     else
1921         err = copy_out_args(cs, req->args, nbytes);
1922     fuse_copy_finish(cs);
1923 
1924     spin_lock(&fpq->lock);
1925     clear_bit(FR_LOCKED, &req->flags);
1926     if (!fpq->connected)
1927         err = -ENOENT;
1928     else if (err)
1929         req->out.h.error = -EIO;
1930     if (!test_bit(FR_PRIVATE, &req->flags))
1931         list_del_init(&req->list);
1932     spin_unlock(&fpq->lock);
1933 
1934     fuse_request_end(req);
1935 out:
1936     return err ? err : nbytes;
1937 
1938 copy_finish:
1939     fuse_copy_finish(cs);
1940     goto out;
1941 }
1942 
1943 static ssize_t fuse_dev_write(struct kiocb *iocb, struct iov_iter *from)
1944 {
1945     struct fuse_copy_state cs;
1946     struct fuse_dev *fud = fuse_get_dev(iocb->ki_filp);
1947 
1948     if (!fud)
1949         return -EPERM;
1950 
1951     if (!user_backed_iter(from))
1952         return -EINVAL;
1953 
1954     fuse_copy_init(&cs, 0, from);
1955 
1956     return fuse_dev_do_write(fud, &cs, iov_iter_count(from));
1957 }
1958 
1959 static ssize_t fuse_dev_splice_write(struct pipe_inode_info *pipe,
1960                      struct file *out, loff_t *ppos,
1961                      size_t len, unsigned int flags)
1962 {
1963     unsigned int head, tail, mask, count;
1964     unsigned nbuf;
1965     unsigned idx;
1966     struct pipe_buffer *bufs;
1967     struct fuse_copy_state cs;
1968     struct fuse_dev *fud;
1969     size_t rem;
1970     ssize_t ret;
1971 
1972     fud = fuse_get_dev(out);
1973     if (!fud)
1974         return -EPERM;
1975 
1976     pipe_lock(pipe);
1977 
1978     head = pipe->head;
1979     tail = pipe->tail;
1980     mask = pipe->ring_size - 1;
1981     count = head - tail;
1982 
1983     bufs = kvmalloc_array(count, sizeof(struct pipe_buffer), GFP_KERNEL);
1984     if (!bufs) {
1985         pipe_unlock(pipe);
1986         return -ENOMEM;
1987     }
1988 
1989     nbuf = 0;
1990     rem = 0;
1991     for (idx = tail; idx != head && rem < len; idx++)
1992         rem += pipe->bufs[idx & mask].len;
1993 
1994     ret = -EINVAL;
1995     if (rem < len)
1996         goto out_free;
1997 
1998     rem = len;
1999     while (rem) {
2000         struct pipe_buffer *ibuf;
2001         struct pipe_buffer *obuf;
2002 
2003         if (WARN_ON(nbuf >= count || tail == head))
2004             goto out_free;
2005 
2006         ibuf = &pipe->bufs[tail & mask];
2007         obuf = &bufs[nbuf];
2008 
2009         if (rem >= ibuf->len) {
2010             *obuf = *ibuf;
2011             ibuf->ops = NULL;
2012             tail++;
2013             pipe->tail = tail;
2014         } else {
2015             if (!pipe_buf_get(pipe, ibuf))
2016                 goto out_free;
2017 
2018             *obuf = *ibuf;
2019             obuf->flags &= ~PIPE_BUF_FLAG_GIFT;
2020             obuf->len = rem;
2021             ibuf->offset += obuf->len;
2022             ibuf->len -= obuf->len;
2023         }
2024         nbuf++;
2025         rem -= obuf->len;
2026     }
2027     pipe_unlock(pipe);
2028 
2029     fuse_copy_init(&cs, 0, NULL);
2030     cs.pipebufs = bufs;
2031     cs.nr_segs = nbuf;
2032     cs.pipe = pipe;
2033 
2034     if (flags & SPLICE_F_MOVE)
2035         cs.move_pages = 1;
2036 
2037     ret = fuse_dev_do_write(fud, &cs, len);
2038 
2039     pipe_lock(pipe);
2040 out_free:
2041     for (idx = 0; idx < nbuf; idx++) {
2042         struct pipe_buffer *buf = &bufs[idx];
2043 
2044         if (buf->ops)
2045             pipe_buf_release(pipe, buf);
2046     }
2047     pipe_unlock(pipe);
2048 
2049     kvfree(bufs);
2050     return ret;
2051 }
2052 
2053 static __poll_t fuse_dev_poll(struct file *file, poll_table *wait)
2054 {
2055     __poll_t mask = EPOLLOUT | EPOLLWRNORM;
2056     struct fuse_iqueue *fiq;
2057     struct fuse_dev *fud = fuse_get_dev(file);
2058 
2059     if (!fud)
2060         return EPOLLERR;
2061 
2062     fiq = &fud->fc->iq;
2063     poll_wait(file, &fiq->waitq, wait);
2064 
2065     spin_lock(&fiq->lock);
2066     if (!fiq->connected)
2067         mask = EPOLLERR;
2068     else if (request_pending(fiq))
2069         mask |= EPOLLIN | EPOLLRDNORM;
2070     spin_unlock(&fiq->lock);
2071 
2072     return mask;
2073 }
2074 
2075 /* Abort all requests on the given list (pending or processing) */
2076 static void end_requests(struct list_head *head)
2077 {
2078     while (!list_empty(head)) {
2079         struct fuse_req *req;
2080         req = list_entry(head->next, struct fuse_req, list);
2081         req->out.h.error = -ECONNABORTED;
2082         clear_bit(FR_SENT, &req->flags);
2083         list_del_init(&req->list);
2084         fuse_request_end(req);
2085     }
2086 }
2087 
2088 static void end_polls(struct fuse_conn *fc)
2089 {
2090     struct rb_node *p;
2091 
2092     p = rb_first(&fc->polled_files);
2093 
2094     while (p) {
2095         struct fuse_file *ff;
2096         ff = rb_entry(p, struct fuse_file, polled_node);
2097         wake_up_interruptible_all(&ff->poll_wait);
2098 
2099         p = rb_next(p);
2100     }
2101 }
2102 
2103 /*
2104  * Abort all requests.
2105  *
2106  * Emergency exit in case of a malicious or accidental deadlock, or just a hung
2107  * filesystem.
2108  *
2109  * The same effect is usually achievable through killing the filesystem daemon
2110  * and all users of the filesystem.  The exception is the combination of an
2111  * asynchronous request and the tricky deadlock (see
2112  * Documentation/filesystems/fuse.rst).
2113  *
2114  * Aborting requests under I/O goes as follows: 1: Separate out unlocked
2115  * requests, they should be finished off immediately.  Locked requests will be
2116  * finished after unlock; see unlock_request(). 2: Finish off the unlocked
2117  * requests.  It is possible that some request will finish before we can.  This
2118  * is OK, the request will in that case be removed from the list before we touch
2119  * it.
2120  */
2121 void fuse_abort_conn(struct fuse_conn *fc)
2122 {
2123     struct fuse_iqueue *fiq = &fc->iq;
2124 
2125     spin_lock(&fc->lock);
2126     if (fc->connected) {
2127         struct fuse_dev *fud;
2128         struct fuse_req *req, *next;
2129         LIST_HEAD(to_end);
2130         unsigned int i;
2131 
2132         /* Background queuing checks fc->connected under bg_lock */
2133         spin_lock(&fc->bg_lock);
2134         fc->connected = 0;
2135         spin_unlock(&fc->bg_lock);
2136 
2137         fuse_set_initialized(fc);
2138         list_for_each_entry(fud, &fc->devices, entry) {
2139             struct fuse_pqueue *fpq = &fud->pq;
2140 
2141             spin_lock(&fpq->lock);
2142             fpq->connected = 0;
2143             list_for_each_entry_safe(req, next, &fpq->io, list) {
2144                 req->out.h.error = -ECONNABORTED;
2145                 spin_lock(&req->waitq.lock);
2146                 set_bit(FR_ABORTED, &req->flags);
2147                 if (!test_bit(FR_LOCKED, &req->flags)) {
2148                     set_bit(FR_PRIVATE, &req->flags);
2149                     __fuse_get_request(req);
2150                     list_move(&req->list, &to_end);
2151                 }
2152                 spin_unlock(&req->waitq.lock);
2153             }
2154             for (i = 0; i < FUSE_PQ_HASH_SIZE; i++)
2155                 list_splice_tail_init(&fpq->processing[i],
2156                               &to_end);
2157             spin_unlock(&fpq->lock);
2158         }
2159         spin_lock(&fc->bg_lock);
2160         fc->blocked = 0;
2161         fc->max_background = UINT_MAX;
2162         flush_bg_queue(fc);
2163         spin_unlock(&fc->bg_lock);
2164 
2165         spin_lock(&fiq->lock);
2166         fiq->connected = 0;
2167         list_for_each_entry(req, &fiq->pending, list)
2168             clear_bit(FR_PENDING, &req->flags);
2169         list_splice_tail_init(&fiq->pending, &to_end);
2170         while (forget_pending(fiq))
2171             kfree(fuse_dequeue_forget(fiq, 1, NULL));
2172         wake_up_all(&fiq->waitq);
2173         spin_unlock(&fiq->lock);
2174         kill_fasync(&fiq->fasync, SIGIO, POLL_IN);
2175         end_polls(fc);
2176         wake_up_all(&fc->blocked_waitq);
2177         spin_unlock(&fc->lock);
2178 
2179         end_requests(&to_end);
2180     } else {
2181         spin_unlock(&fc->lock);
2182     }
2183 }
2184 EXPORT_SYMBOL_GPL(fuse_abort_conn);
2185 
2186 void fuse_wait_aborted(struct fuse_conn *fc)
2187 {
2188     /* matches implicit memory barrier in fuse_drop_waiting() */
2189     smp_mb();
2190     wait_event(fc->blocked_waitq, atomic_read(&fc->num_waiting) == 0);
2191 }
2192 
2193 int fuse_dev_release(struct inode *inode, struct file *file)
2194 {
2195     struct fuse_dev *fud = fuse_get_dev(file);
2196 
2197     if (fud) {
2198         struct fuse_conn *fc = fud->fc;
2199         struct fuse_pqueue *fpq = &fud->pq;
2200         LIST_HEAD(to_end);
2201         unsigned int i;
2202 
2203         spin_lock(&fpq->lock);
2204         WARN_ON(!list_empty(&fpq->io));
2205         for (i = 0; i < FUSE_PQ_HASH_SIZE; i++)
2206             list_splice_init(&fpq->processing[i], &to_end);
2207         spin_unlock(&fpq->lock);
2208 
2209         end_requests(&to_end);
2210 
2211         /* Are we the last open device? */
2212         if (atomic_dec_and_test(&fc->dev_count)) {
2213             WARN_ON(fc->iq.fasync != NULL);
2214             fuse_abort_conn(fc);
2215         }
2216         fuse_dev_free(fud);
2217     }
2218     return 0;
2219 }
2220 EXPORT_SYMBOL_GPL(fuse_dev_release);
2221 
2222 static int fuse_dev_fasync(int fd, struct file *file, int on)
2223 {
2224     struct fuse_dev *fud = fuse_get_dev(file);
2225 
2226     if (!fud)
2227         return -EPERM;
2228 
2229     /* No locking - fasync_helper does its own locking */
2230     return fasync_helper(fd, file, on, &fud->fc->iq.fasync);
2231 }
2232 
2233 static int fuse_device_clone(struct fuse_conn *fc, struct file *new)
2234 {
2235     struct fuse_dev *fud;
2236 
2237     if (new->private_data)
2238         return -EINVAL;
2239 
2240     fud = fuse_dev_alloc_install(fc);
2241     if (!fud)
2242         return -ENOMEM;
2243 
2244     new->private_data = fud;
2245     atomic_inc(&fc->dev_count);
2246 
2247     return 0;
2248 }
2249 
2250 static long fuse_dev_ioctl(struct file *file, unsigned int cmd,
2251                unsigned long arg)
2252 {
2253     int res;
2254     int oldfd;
2255     struct fuse_dev *fud = NULL;
2256 
2257     switch (cmd) {
2258     case FUSE_DEV_IOC_CLONE:
2259         res = -EFAULT;
2260         if (!get_user(oldfd, (__u32 __user *)arg)) {
2261             struct file *old = fget(oldfd);
2262 
2263             res = -EINVAL;
2264             if (old) {
2265                 /*
2266                  * Check against file->f_op because CUSE
2267                  * uses the same ioctl handler.
2268                  */
2269                 if (old->f_op == file->f_op &&
2270                     old->f_cred->user_ns == file->f_cred->user_ns)
2271                     fud = fuse_get_dev(old);
2272 
2273                 if (fud) {
2274                     mutex_lock(&fuse_mutex);
2275                     res = fuse_device_clone(fud->fc, file);
2276                     mutex_unlock(&fuse_mutex);
2277                 }
2278                 fput(old);
2279             }
2280         }
2281         break;
2282     default:
2283         res = -ENOTTY;
2284         break;
2285     }
2286     return res;
2287 }
2288 
2289 const struct file_operations fuse_dev_operations = {
2290     .owner      = THIS_MODULE,
2291     .open       = fuse_dev_open,
2292     .llseek     = no_llseek,
2293     .read_iter  = fuse_dev_read,
2294     .splice_read    = fuse_dev_splice_read,
2295     .write_iter = fuse_dev_write,
2296     .splice_write   = fuse_dev_splice_write,
2297     .poll       = fuse_dev_poll,
2298     .release    = fuse_dev_release,
2299     .fasync     = fuse_dev_fasync,
2300     .unlocked_ioctl = fuse_dev_ioctl,
2301     .compat_ioctl   = compat_ptr_ioctl,
2302 };
2303 EXPORT_SYMBOL_GPL(fuse_dev_operations);
2304 
2305 static struct miscdevice fuse_miscdevice = {
2306     .minor = FUSE_MINOR,
2307     .name  = "fuse",
2308     .fops = &fuse_dev_operations,
2309 };
2310 
2311 int __init fuse_dev_init(void)
2312 {
2313     int err = -ENOMEM;
2314     fuse_req_cachep = kmem_cache_create("fuse_request",
2315                         sizeof(struct fuse_req),
2316                         0, 0, NULL);
2317     if (!fuse_req_cachep)
2318         goto out;
2319 
2320     err = misc_register(&fuse_miscdevice);
2321     if (err)
2322         goto out_cache_clean;
2323 
2324     return 0;
2325 
2326  out_cache_clean:
2327     kmem_cache_destroy(fuse_req_cachep);
2328  out:
2329     return err;
2330 }
2331 
2332 void fuse_dev_cleanup(void)
2333 {
2334     misc_deregister(&fuse_miscdevice);
2335     kmem_cache_destroy(fuse_req_cachep);
2336 }
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!