fs/netfs/io.c

0001 // SPDX-License-Identifier: GPL-2.0-or-later
0002 /* Network filesystem high-level read support.
0003  *
0004  * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved.
0005  * Written by David Howells (dhowells@redhat.com)
0006  */
0007
0008 #include <linux/module.h>
0009 #include <linux/export.h>
0010 #include <linux/fs.h>
0011 #include <linux/mm.h>
0012 #include <linux/pagemap.h>
0013 #include <linux/slab.h>
0014 #include <linux/uio.h>
0015 #include <linux/sched/mm.h>
0016 #include <linux/task_io_accounting_ops.h>
0017 #include "internal.h"
0018
0019 /*
0020  * Clear the unread part of an I/O request.
0021  */
0022 static void netfs_clear_unread(struct netfs_io_subrequest *subreq)
0023 {
0024     struct iov_iter iter;
0025
0026     iov_iter_xarray(&iter, READ, &subreq->rreq->mapping->i_pages,
0027             subreq->start + subreq->transferred,
0028             subreq->len   - subreq->transferred);
0029     iov_iter_zero(iov_iter_count(&iter), &iter);
0030 }
0031
0032 static void netfs_cache_read_terminated(void *priv, ssize_t transferred_or_error,
0033                     bool was_async)
0034 {
0035     struct netfs_io_subrequest *subreq = priv;
0036
0037     netfs_subreq_terminated(subreq, transferred_or_error, was_async);
0038 }
0039
0040 /*
0041  * Issue a read against the cache.
0042  * - Eats the caller's ref on subreq.
0043  */
0044 static void netfs_read_from_cache(struct netfs_io_request *rreq,
0045                   struct netfs_io_subrequest *subreq,
0046                   enum netfs_read_from_hole read_hole)
0047 {
0048     struct netfs_cache_resources *cres = &rreq->cache_resources;
0049     struct iov_iter iter;
0050
0051     netfs_stat(&netfs_n_rh_read);
0052     iov_iter_xarray(&iter, READ, &rreq->mapping->i_pages,
0053             subreq->start + subreq->transferred,
0054             subreq->len   - subreq->transferred);
0055
0056     cres->ops->read(cres, subreq->start, &iter, read_hole,
0057             netfs_cache_read_terminated, subreq);
0058 }
0059
0060 /*
0061  * Fill a subrequest region with zeroes.
0062  */
0063 static void netfs_fill_with_zeroes(struct netfs_io_request *rreq,
0064                    struct netfs_io_subrequest *subreq)
0065 {
0066     netfs_stat(&netfs_n_rh_zero);
0067     __set_bit(NETFS_SREQ_CLEAR_TAIL, &subreq->flags);
0068     netfs_subreq_terminated(subreq, 0, false);
0069 }
0070
0071 /*
0072  * Ask the netfs to issue a read request to the server for us.
0073  *
0074  * The netfs is expected to read from subreq->pos + subreq->transferred to
0075  * subreq->pos + subreq->len - 1.  It may not backtrack and write data into the
0076  * buffer prior to the transferred point as it might clobber dirty data
0077  * obtained from the cache.
0078  *
0079  * Alternatively, the netfs is allowed to indicate one of two things:
0080  *
0081  * - NETFS_SREQ_SHORT_READ: A short read - it will get called again to try and
0082  *   make progress.
0083  *
0084  * - NETFS_SREQ_CLEAR_TAIL: A short read - the rest of the buffer will be
0085  *   cleared.
0086  */
0087 static void netfs_read_from_server(struct netfs_io_request *rreq,
0088                    struct netfs_io_subrequest *subreq)
0089 {
0090     netfs_stat(&netfs_n_rh_download);
0091     rreq->netfs_ops->issue_read(subreq);
0092 }
0093
0094 /*
0095  * Release those waiting.
0096  */
0097 static void netfs_rreq_completed(struct netfs_io_request *rreq, bool was_async)
0098 {
0099     trace_netfs_rreq(rreq, netfs_rreq_trace_done);
0100     netfs_clear_subrequests(rreq, was_async);
0101     netfs_put_request(rreq, was_async, netfs_rreq_trace_put_complete);
0102 }
0103
0104 /*
0105  * Deal with the completion of writing the data to the cache.  We have to clear
0106  * the PG_fscache bits on the folios involved and release the caller's ref.
0107  *
0108  * May be called in softirq mode and we inherit a ref from the caller.
0109  */
0110 static void netfs_rreq_unmark_after_write(struct netfs_io_request *rreq,
0111                       bool was_async)
0112 {
0113     struct netfs_io_subrequest *subreq;
0114     struct folio *folio;
0115     pgoff_t unlocked = 0;
0116     bool have_unlocked = false;
0117
0118     rcu_read_lock();
0119
0120     list_for_each_entry(subreq, &rreq->subrequests, rreq_link) {
0121         XA_STATE(xas, &rreq->mapping->i_pages, subreq->start / PAGE_SIZE);
0122
0123         xas_for_each(&xas, folio, (subreq->start + subreq->len - 1) / PAGE_SIZE) {
0124             /* We might have multiple writes from the same huge
0125              * folio, but we mustn't unlock a folio more than once.
0126              */
0127             if (have_unlocked && folio_index(folio) <= unlocked)
0128                 continue;
0129             unlocked = folio_index(folio);
0130             folio_end_fscache(folio);
0131             have_unlocked = true;
0132         }
0133     }
0134
0135     rcu_read_unlock();
0136     netfs_rreq_completed(rreq, was_async);
0137 }
0138
0139 static void netfs_rreq_copy_terminated(void *priv, ssize_t transferred_or_error,
0140                        bool was_async)
0141 {
0142     struct netfs_io_subrequest *subreq = priv;
0143     struct netfs_io_request *rreq = subreq->rreq;
0144
0145     if (IS_ERR_VALUE(transferred_or_error)) {
0146         netfs_stat(&netfs_n_rh_write_failed);
0147         trace_netfs_failure(rreq, subreq, transferred_or_error,
0148                     netfs_fail_copy_to_cache);
0149     } else {
0150         netfs_stat(&netfs_n_rh_write_done);
0151     }
0152
0153     trace_netfs_sreq(subreq, netfs_sreq_trace_write_term);
0154
0155     /* If we decrement nr_copy_ops to 0, the ref belongs to us. */
0156     if (atomic_dec_and_test(&rreq->nr_copy_ops))
0157         netfs_rreq_unmark_after_write(rreq, was_async);
0158
0159     netfs_put_subrequest(subreq, was_async, netfs_sreq_trace_put_terminated);
0160 }
0161
0162 /*
0163  * Perform any outstanding writes to the cache.  We inherit a ref from the
0164  * caller.
0165  */
0166 static void netfs_rreq_do_write_to_cache(struct netfs_io_request *rreq)
0167 {
0168     struct netfs_cache_resources *cres = &rreq->cache_resources;
0169     struct netfs_io_subrequest *subreq, *next, *p;
0170     struct iov_iter iter;
0171     int ret;
0172
0173     trace_netfs_rreq(rreq, netfs_rreq_trace_copy);
0174
0175     /* We don't want terminating writes trying to wake us up whilst we're
0176      * still going through the list.
0177      */
0178     atomic_inc(&rreq->nr_copy_ops);
0179
0180     list_for_each_entry_safe(subreq, p, &rreq->subrequests, rreq_link) {
0181         if (!test_bit(NETFS_SREQ_COPY_TO_CACHE, &subreq->flags)) {
0182             list_del_init(&subreq->rreq_link);
0183             netfs_put_subrequest(subreq, false,
0184                          netfs_sreq_trace_put_no_copy);
0185         }
0186     }
0187
0188     list_for_each_entry(subreq, &rreq->subrequests, rreq_link) {
0189         /* Amalgamate adjacent writes */
0190         while (!list_is_last(&subreq->rreq_link, &rreq->subrequests)) {
0191             next = list_next_entry(subreq, rreq_link);
0192             if (next->start != subreq->start + subreq->len)
0193                 break;
0194             subreq->len += next->len;
0195             list_del_init(&next->rreq_link);
0196             netfs_put_subrequest(next, false,
0197                          netfs_sreq_trace_put_merged);
0198         }
0199
0200         ret = cres->ops->prepare_write(cres, &subreq->start, &subreq->len,
0201                            rreq->i_size, true);
0202         if (ret < 0) {
0203             trace_netfs_failure(rreq, subreq, ret, netfs_fail_prepare_write);
0204             trace_netfs_sreq(subreq, netfs_sreq_trace_write_skip);
0205             continue;
0206         }
0207
0208         iov_iter_xarray(&iter, WRITE, &rreq->mapping->i_pages,
0209                 subreq->start, subreq->len);
0210
0211         atomic_inc(&rreq->nr_copy_ops);
0212         netfs_stat(&netfs_n_rh_write);
0213         netfs_get_subrequest(subreq, netfs_sreq_trace_get_copy_to_cache);
0214         trace_netfs_sreq(subreq, netfs_sreq_trace_write);
0215         cres->ops->write(cres, subreq->start, &iter,
0216                  netfs_rreq_copy_terminated, subreq);
0217     }
0218
0219     /* If we decrement nr_copy_ops to 0, the usage ref belongs to us. */
0220     if (atomic_dec_and_test(&rreq->nr_copy_ops))
0221         netfs_rreq_unmark_after_write(rreq, false);
0222 }
0223
0224 static void netfs_rreq_write_to_cache_work(struct work_struct *work)
0225 {
0226     struct netfs_io_request *rreq =
0227         container_of(work, struct netfs_io_request, work);
0228
0229     netfs_rreq_do_write_to_cache(rreq);
0230 }
0231
0232 static void netfs_rreq_write_to_cache(struct netfs_io_request *rreq)
0233 {
0234     rreq->work.func = netfs_rreq_write_to_cache_work;
0235     if (!queue_work(system_unbound_wq, &rreq->work))
0236         BUG();
0237 }
0238
0239 /*
0240  * Handle a short read.
0241  */
0242 static void netfs_rreq_short_read(struct netfs_io_request *rreq,
0243                   struct netfs_io_subrequest *subreq)
0244 {
0245     __clear_bit(NETFS_SREQ_SHORT_IO, &subreq->flags);
0246     __set_bit(NETFS_SREQ_SEEK_DATA_READ, &subreq->flags);
0247
0248     netfs_stat(&netfs_n_rh_short_read);
0249     trace_netfs_sreq(subreq, netfs_sreq_trace_resubmit_short);
0250
0251     netfs_get_subrequest(subreq, netfs_sreq_trace_get_short_read);
0252     atomic_inc(&rreq->nr_outstanding);
0253     if (subreq->source == NETFS_READ_FROM_CACHE)
0254         netfs_read_from_cache(rreq, subreq, NETFS_READ_HOLE_CLEAR);
0255     else
0256         netfs_read_from_server(rreq, subreq);
0257 }
0258
0259 /*
0260  * Resubmit any short or failed operations.  Returns true if we got the rreq
0261  * ref back.
0262  */
0263 static bool netfs_rreq_perform_resubmissions(struct netfs_io_request *rreq)
0264 {
0265     struct netfs_io_subrequest *subreq;
0266
0267     WARN_ON(in_interrupt());
0268
0269     trace_netfs_rreq(rreq, netfs_rreq_trace_resubmit);
0270
0271     /* We don't want terminating submissions trying to wake us up whilst
0272      * we're still going through the list.
0273      */
0274     atomic_inc(&rreq->nr_outstanding);
0275
0276     __clear_bit(NETFS_RREQ_INCOMPLETE_IO, &rreq->flags);
0277     list_for_each_entry(subreq, &rreq->subrequests, rreq_link) {
0278         if (subreq->error) {
0279             if (subreq->source != NETFS_READ_FROM_CACHE)
0280                 break;
0281             subreq->source = NETFS_DOWNLOAD_FROM_SERVER;
0282             subreq->error = 0;
0283             netfs_stat(&netfs_n_rh_download_instead);
0284             trace_netfs_sreq(subreq, netfs_sreq_trace_download_instead);
0285             netfs_get_subrequest(subreq, netfs_sreq_trace_get_resubmit);
0286             atomic_inc(&rreq->nr_outstanding);
0287             netfs_read_from_server(rreq, subreq);
0288         } else if (test_bit(NETFS_SREQ_SHORT_IO, &subreq->flags)) {
0289             netfs_rreq_short_read(rreq, subreq);
0290         }
0291     }
0292
0293     /* If we decrement nr_outstanding to 0, the usage ref belongs to us. */
0294     if (atomic_dec_and_test(&rreq->nr_outstanding))
0295         return true;
0296
0297     wake_up_var(&rreq->nr_outstanding);
0298     return false;
0299 }
0300
0301 /*
0302  * Check to see if the data read is still valid.
0303  */
0304 static void netfs_rreq_is_still_valid(struct netfs_io_request *rreq)
0305 {
0306     struct netfs_io_subrequest *subreq;
0307
0308     if (!rreq->netfs_ops->is_still_valid ||
0309         rreq->netfs_ops->is_still_valid(rreq))
0310         return;
0311
0312     list_for_each_entry(subreq, &rreq->subrequests, rreq_link) {
0313         if (subreq->source == NETFS_READ_FROM_CACHE) {
0314             subreq->error = -ESTALE;
0315             __set_bit(NETFS_RREQ_INCOMPLETE_IO, &rreq->flags);
0316         }
0317     }
0318 }
0319
0320 /*
0321  * Assess the state of a read request and decide what to do next.
0322  *
0323  * Note that we could be in an ordinary kernel thread, on a workqueue or in
0324  * softirq context at this point.  We inherit a ref from the caller.
0325  */
0326 static void netfs_rreq_assess(struct netfs_io_request *rreq, bool was_async)
0327 {
0328     trace_netfs_rreq(rreq, netfs_rreq_trace_assess);
0329
0330 again:
0331     netfs_rreq_is_still_valid(rreq);
0332
0333     if (!test_bit(NETFS_RREQ_FAILED, &rreq->flags) &&
0334         test_bit(NETFS_RREQ_INCOMPLETE_IO, &rreq->flags)) {
0335         if (netfs_rreq_perform_resubmissions(rreq))
0336             goto again;
0337         return;
0338     }
0339
0340     netfs_rreq_unlock_folios(rreq);
0341
0342     clear_bit_unlock(NETFS_RREQ_IN_PROGRESS, &rreq->flags);
0343     wake_up_bit(&rreq->flags, NETFS_RREQ_IN_PROGRESS);
0344
0345     if (test_bit(NETFS_RREQ_COPY_TO_CACHE, &rreq->flags))
0346         return netfs_rreq_write_to_cache(rreq);
0347
0348     netfs_rreq_completed(rreq, was_async);
0349 }
0350
0351 static void netfs_rreq_work(struct work_struct *work)
0352 {
0353     struct netfs_io_request *rreq =
0354         container_of(work, struct netfs_io_request, work);
0355     netfs_rreq_assess(rreq, false);
0356 }
0357
0358 /*
0359  * Handle the completion of all outstanding I/O operations on a read request.
0360  * We inherit a ref from the caller.
0361  */
0362 static void netfs_rreq_terminated(struct netfs_io_request *rreq,
0363                   bool was_async)
0364 {
0365     if (test_bit(NETFS_RREQ_INCOMPLETE_IO, &rreq->flags) &&
0366         was_async) {
0367         if (!queue_work(system_unbound_wq, &rreq->work))
0368             BUG();
0369     } else {
0370         netfs_rreq_assess(rreq, was_async);
0371     }
0372 }
0373
0374 /**
0375  * netfs_subreq_terminated - Note the termination of an I/O operation.
0376  * @subreq: The I/O request that has terminated.
0377  * @transferred_or_error: The amount of data transferred or an error code.
0378  * @was_async: The termination was asynchronous
0379  *
0380  * This tells the read helper that a contributory I/O operation has terminated,
0381  * one way or another, and that it should integrate the results.
0382  *
0383  * The caller indicates in @transferred_or_error the outcome of the operation,
0384  * supplying a positive value to indicate the number of bytes transferred, 0 to
0385  * indicate a failure to transfer anything that should be retried or a negative
0386  * error code.  The helper will look after reissuing I/O operations as
0387  * appropriate and writing downloaded data to the cache.
0388  *
0389  * If @was_async is true, the caller might be running in softirq or interrupt
0390  * context and we can't sleep.
0391  */
0392 void netfs_subreq_terminated(struct netfs_io_subrequest *subreq,
0393                  ssize_t transferred_or_error,
0394                  bool was_async)
0395 {
0396     struct netfs_io_request *rreq = subreq->rreq;
0397     int u;
0398
0399     _enter("[%u]{%llx,%lx},%zd",
0400            subreq->debug_index, subreq->start, subreq->flags,
0401            transferred_or_error);
0402
0403     switch (subreq->source) {
0404     case NETFS_READ_FROM_CACHE:
0405         netfs_stat(&netfs_n_rh_read_done);
0406         break;
0407     case NETFS_DOWNLOAD_FROM_SERVER:
0408         netfs_stat(&netfs_n_rh_download_done);
0409         break;
0410     default:
0411         break;
0412     }
0413
0414     if (IS_ERR_VALUE(transferred_or_error)) {
0415         subreq->error = transferred_or_error;
0416         trace_netfs_failure(rreq, subreq, transferred_or_error,
0417                     netfs_fail_read);
0418         goto failed;
0419     }
0420
0421     if (WARN(transferred_or_error > subreq->len - subreq->transferred,
0422          "Subreq overread: R%x[%x] %zd > %zu - %zu",
0423          rreq->debug_id, subreq->debug_index,
0424          transferred_or_error, subreq->len, subreq->transferred))
0425         transferred_or_error = subreq->len - subreq->transferred;
0426
0427     subreq->error = 0;
0428     subreq->transferred += transferred_or_error;
0429     if (subreq->transferred < subreq->len)
0430         goto incomplete;
0431
0432 complete:
0433     __clear_bit(NETFS_SREQ_NO_PROGRESS, &subreq->flags);
0434     if (test_bit(NETFS_SREQ_COPY_TO_CACHE, &subreq->flags))
0435         set_bit(NETFS_RREQ_COPY_TO_CACHE, &rreq->flags);
0436
0437 out:
0438     trace_netfs_sreq(subreq, netfs_sreq_trace_terminated);
0439
0440     /* If we decrement nr_outstanding to 0, the ref belongs to us. */
0441     u = atomic_dec_return(&rreq->nr_outstanding);
0442     if (u == 0)
0443         netfs_rreq_terminated(rreq, was_async);
0444     else if (u == 1)
0445         wake_up_var(&rreq->nr_outstanding);
0446
0447     netfs_put_subrequest(subreq, was_async, netfs_sreq_trace_put_terminated);
0448     return;
0449
0450 incomplete:
0451     if (test_bit(NETFS_SREQ_CLEAR_TAIL, &subreq->flags)) {
0452         netfs_clear_unread(subreq);
0453         subreq->transferred = subreq->len;
0454         goto complete;
0455     }
0456
0457     if (transferred_or_error == 0) {
0458         if (__test_and_set_bit(NETFS_SREQ_NO_PROGRESS, &subreq->flags)) {
0459             subreq->error = -ENODATA;
0460             goto failed;
0461         }
0462     } else {
0463         __clear_bit(NETFS_SREQ_NO_PROGRESS, &subreq->flags);
0464     }
0465
0466     __set_bit(NETFS_SREQ_SHORT_IO, &subreq->flags);
0467     set_bit(NETFS_RREQ_INCOMPLETE_IO, &rreq->flags);
0468     goto out;
0469
0470 failed:
0471     if (subreq->source == NETFS_READ_FROM_CACHE) {
0472         netfs_stat(&netfs_n_rh_read_failed);
0473         set_bit(NETFS_RREQ_INCOMPLETE_IO, &rreq->flags);
0474     } else {
0475         netfs_stat(&netfs_n_rh_download_failed);
0476         set_bit(NETFS_RREQ_FAILED, &rreq->flags);
0477         rreq->error = subreq->error;
0478     }
0479     goto out;
0480 }
0481 EXPORT_SYMBOL(netfs_subreq_terminated);
0482
0483 static enum netfs_io_source netfs_cache_prepare_read(struct netfs_io_subrequest *subreq,
0484                                loff_t i_size)
0485 {
0486     struct netfs_io_request *rreq = subreq->rreq;
0487     struct netfs_cache_resources *cres = &rreq->cache_resources;
0488
0489     if (cres->ops)
0490         return cres->ops->prepare_read(subreq, i_size);
0491     if (subreq->start >= rreq->i_size)
0492         return NETFS_FILL_WITH_ZEROES;
0493     return NETFS_DOWNLOAD_FROM_SERVER;
0494 }
0495
0496 /*
0497  * Work out what sort of subrequest the next one will be.
0498  */
0499 static enum netfs_io_source
0500 netfs_rreq_prepare_read(struct netfs_io_request *rreq,
0501             struct netfs_io_subrequest *subreq)
0502 {
0503     enum netfs_io_source source;
0504
0505     _enter("%llx-%llx,%llx", subreq->start, subreq->start + subreq->len, rreq->i_size);
0506
0507     source = netfs_cache_prepare_read(subreq, rreq->i_size);
0508     if (source == NETFS_INVALID_READ)
0509         goto out;
0510
0511     if (source == NETFS_DOWNLOAD_FROM_SERVER) {
0512         /* Call out to the netfs to let it shrink the request to fit
0513          * its own I/O sizes and boundaries.  If it shinks it here, it
0514          * will be called again to make simultaneous calls; if it wants
0515          * to make serial calls, it can indicate a short read and then
0516          * we will call it again.
0517          */
0518         if (subreq->len > rreq->i_size - subreq->start)
0519             subreq->len = rreq->i_size - subreq->start;
0520
0521         if (rreq->netfs_ops->clamp_length &&
0522             !rreq->netfs_ops->clamp_length(subreq)) {
0523             source = NETFS_INVALID_READ;
0524             goto out;
0525         }
0526     }
0527
0528     if (WARN_ON(subreq->len == 0))
0529         source = NETFS_INVALID_READ;
0530
0531 out:
0532     subreq->source = source;
0533     trace_netfs_sreq(subreq, netfs_sreq_trace_prepare);
0534     return source;
0535 }
0536
0537 /*
0538  * Slice off a piece of a read request and submit an I/O request for it.
0539  */
0540 static bool netfs_rreq_submit_slice(struct netfs_io_request *rreq,
0541                     unsigned int *_debug_index)
0542 {
0543     struct netfs_io_subrequest *subreq;
0544     enum netfs_io_source source;
0545
0546     subreq = netfs_alloc_subrequest(rreq);
0547     if (!subreq)
0548         return false;
0549
0550     subreq->debug_index = (*_debug_index)++;
0551     subreq->start       = rreq->start + rreq->submitted;
0552     subreq->len     = rreq->len   - rreq->submitted;
0553
0554     _debug("slice %llx,%zx,%zx", subreq->start, subreq->len, rreq->submitted);
0555     list_add_tail(&subreq->rreq_link, &rreq->subrequests);
0556
0557     /* Call out to the cache to find out what it can do with the remaining
0558      * subset.  It tells us in subreq->flags what it decided should be done
0559      * and adjusts subreq->len down if the subset crosses a cache boundary.
0560      *
0561      * Then when we hand the subset, it can choose to take a subset of that
0562      * (the starts must coincide), in which case, we go around the loop
0563      * again and ask it to download the next piece.
0564      */
0565     source = netfs_rreq_prepare_read(rreq, subreq);
0566     if (source == NETFS_INVALID_READ)
0567         goto subreq_failed;
0568
0569     atomic_inc(&rreq->nr_outstanding);
0570
0571     rreq->submitted += subreq->len;
0572
0573     trace_netfs_sreq(subreq, netfs_sreq_trace_submit);
0574     switch (source) {
0575     case NETFS_FILL_WITH_ZEROES:
0576         netfs_fill_with_zeroes(rreq, subreq);
0577         break;
0578     case NETFS_DOWNLOAD_FROM_SERVER:
0579         netfs_read_from_server(rreq, subreq);
0580         break;
0581     case NETFS_READ_FROM_CACHE:
0582         netfs_read_from_cache(rreq, subreq, NETFS_READ_HOLE_IGNORE);
0583         break;
0584     default:
0585         BUG();
0586     }
0587
0588     return true;
0589
0590 subreq_failed:
0591     rreq->error = subreq->error;
0592     netfs_put_subrequest(subreq, false, netfs_sreq_trace_put_failed);
0593     return false;
0594 }
0595
0596 /*
0597  * Begin the process of reading in a chunk of data, where that data may be
0598  * stitched together from multiple sources, including multiple servers and the
0599  * local cache.
0600  */
0601 int netfs_begin_read(struct netfs_io_request *rreq, bool sync)
0602 {
0603     unsigned int debug_index = 0;
0604     int ret;
0605
0606     _enter("R=%x %llx-%llx",
0607            rreq->debug_id, rreq->start, rreq->start + rreq->len - 1);
0608
0609     if (rreq->len == 0) {
0610         pr_err("Zero-sized read [R=%x]\n", rreq->debug_id);
0611         netfs_put_request(rreq, false, netfs_rreq_trace_put_zero_len);
0612         return -EIO;
0613     }
0614
0615     INIT_WORK(&rreq->work, netfs_rreq_work);
0616
0617     if (sync)
0618         netfs_get_request(rreq, netfs_rreq_trace_get_hold);
0619
0620     /* Chop the read into slices according to what the cache and the netfs
0621      * want and submit each one.
0622      */
0623     atomic_set(&rreq->nr_outstanding, 1);
0624     do {
0625         if (!netfs_rreq_submit_slice(rreq, &debug_index))
0626             break;
0627
0628     } while (rreq->submitted < rreq->len);
0629
0630     if (sync) {
0631         /* Keep nr_outstanding incremented so that the ref always belongs to
0632          * us, and the service code isn't punted off to a random thread pool to
0633          * process.
0634          */
0635         for (;;) {
0636             wait_var_event(&rreq->nr_outstanding,
0637                        atomic_read(&rreq->nr_outstanding) == 1);
0638             netfs_rreq_assess(rreq, false);
0639             if (!test_bit(NETFS_RREQ_IN_PROGRESS, &rreq->flags))
0640                 break;
0641             cond_resched();
0642         }
0643
0644         ret = rreq->error;
0645         if (ret == 0 && rreq->submitted < rreq->len) {
0646             trace_netfs_failure(rreq, NULL, ret, netfs_fail_short_read);
0647             ret = -EIO;
0648         }
0649         netfs_put_request(rreq, false, netfs_rreq_trace_put_hold);
0650     } else {
0651         /* If we decrement nr_outstanding to 0, the ref belongs to us. */
0652         if (atomic_dec_and_test(&rreq->nr_outstanding))
0653             netfs_rreq_assess(rreq, false);
0654         ret = 0;
0655     }
0656     return ret;
0657 }