OSCL-LXR linux-6.0.1/​fs/​xfs/​xfs_aops.c	Source navigation Diff markup Identifier search General search
	Version: linux-6.0.1 spark-3.0.0 hadoop-3.2.0-src hadoop-3.3.0-src spark-2.4.7 linux-4.15.13 hadoop-2.7.4-src Revert   Change

 // SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (c) 2000-2005 Silicon Graphics, Inc.
  * Copyright (c) 2016-2018 Christoph Hellwig.
  * All Rights Reserved.
  */
 #include "xfs.h"
 #include "xfs_shared.h"
 #include "xfs_format.h"
 #include "xfs_log_format.h"
 #include "xfs_trans_resv.h"
 #include "xfs_mount.h"
 #include "xfs_inode.h"
 #include "xfs_trans.h"
 #include "xfs_iomap.h"
 #include "xfs_trace.h"
 #include "xfs_bmap.h"
 #include "xfs_bmap_util.h"
 #include "xfs_reflink.h"
 
 struct xfs_writepage_ctx {
     struct iomap_writepage_ctx ctx;
     unsigned int        data_seq;
     unsigned int        cow_seq;
 };
 
 static inline struct xfs_writepage_ctx *
 XFS_WPC(struct iomap_writepage_ctx *ctx)
 {
     return container_of(ctx, struct xfs_writepage_ctx, ctx);
 }
 
 /*
  * Fast and loose check if this write could update the on-disk inode size.
  */
 static inline bool xfs_ioend_is_append(struct iomap_ioend *ioend)
 {
     return ioend->io_offset + ioend->io_size >
         XFS_I(ioend->io_inode)->i_disk_size;
 }
 
 /*
  * Update on-disk file size now that data has been written to disk.
  */
 int
 xfs_setfilesize(
     struct xfs_inode    *ip,
     xfs_off_t       offset,
     size_t          size)
 {
     struct xfs_mount    *mp = ip->i_mount;
     struct xfs_trans    *tp;
     xfs_fsize_t     isize;
     int         error;
 
     error = xfs_trans_alloc(mp, &M_RES(mp)->tr_fsyncts, 0, 0, 0, &tp);
     if (error)
         return error;
 
     xfs_ilock(ip, XFS_ILOCK_EXCL);
     isize = xfs_new_eof(ip, offset + size);
     if (!isize) {
         xfs_iunlock(ip, XFS_ILOCK_EXCL);
         xfs_trans_cancel(tp);
         return 0;
     }
 
     trace_xfs_setfilesize(ip, offset, size);
 
     ip->i_disk_size = isize;
     xfs_trans_ijoin(tp, ip, XFS_ILOCK_EXCL);
     xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE);
 
     return xfs_trans_commit(tp);
 }
 
 /*
  * IO write completion.
  */
 STATIC void
 xfs_end_ioend(
     struct iomap_ioend  *ioend)
 {
     struct xfs_inode    *ip = XFS_I(ioend->io_inode);
     struct xfs_mount    *mp = ip->i_mount;
     xfs_off_t       offset = ioend->io_offset;
     size_t          size = ioend->io_size;
     unsigned int        nofs_flag;
     int         error;
 
     /*
      * We can allocate memory here while doing writeback on behalf of
      * memory reclaim.  To avoid memory allocation deadlocks set the
      * task-wide nofs context for the following operations.
      */
     nofs_flag = memalloc_nofs_save();
 
     /*
      * Just clean up the in-memory structures if the fs has been shut down.
      */
     if (xfs_is_shutdown(mp)) {
         error = -EIO;
         goto done;
     }
 
     /*
      * Clean up all COW blocks and underlying data fork delalloc blocks on
      * I/O error. The delalloc punch is required because this ioend was
      * mapped to blocks in the COW fork and the associated pages are no
      * longer dirty. If we don't remove delalloc blocks here, they become
      * stale and can corrupt free space accounting on unmount.
      */
     error = blk_status_to_errno(ioend->io_bio->bi_status);
     if (unlikely(error)) {
         if (ioend->io_flags & IOMAP_F_SHARED) {
             xfs_reflink_cancel_cow_range(ip, offset, size, true);
             xfs_bmap_punch_delalloc_range(ip,
                               XFS_B_TO_FSBT(mp, offset),
                               XFS_B_TO_FSB(mp, size));
         }
         goto done;
     }
 
     /*
      * Success: commit the COW or unwritten blocks if needed.
      */
     if (ioend->io_flags & IOMAP_F_SHARED)
         error = xfs_reflink_end_cow(ip, offset, size);
     else if (ioend->io_type == IOMAP_UNWRITTEN)
         error = xfs_iomap_write_unwritten(ip, offset, size, false);
 
     if (!error && xfs_ioend_is_append(ioend))
         error = xfs_setfilesize(ip, ioend->io_offset, ioend->io_size);
 done:
     iomap_finish_ioends(ioend, error);
     memalloc_nofs_restore(nofs_flag);
 }
 
 /*
  * Finish all pending IO completions that require transactional modifications.
  *
  * We try to merge physical and logically contiguous ioends before completion to
  * minimise the number of transactions we need to perform during IO completion.
  * Both unwritten extent conversion and COW remapping need to iterate and modify
  * one physical extent at a time, so we gain nothing by merging physically
  * discontiguous extents here.
  *
  * The ioend chain length that we can be processing here is largely unbound in
  * length and we may have to perform significant amounts of work on each ioend
  * to complete it. Hence we have to be careful about holding the CPU for too
  * long in this loop.
  */
 void
 xfs_end_io(
     struct work_struct  *work)
 {
     struct xfs_inode    *ip =
         container_of(work, struct xfs_inode, i_ioend_work);
     struct iomap_ioend  *ioend;
     struct list_head    tmp;
     unsigned long       flags;
 
     spin_lock_irqsave(&ip->i_ioend_lock, flags);
     list_replace_init(&ip->i_ioend_list, &tmp);
     spin_unlock_irqrestore(&ip->i_ioend_lock, flags);
 
     iomap_sort_ioends(&tmp);
     while ((ioend = list_first_entry_or_null(&tmp, struct iomap_ioend,
             io_list))) {
         list_del_init(&ioend->io_list);
         iomap_ioend_try_merge(ioend, &tmp);
         xfs_end_ioend(ioend);
         cond_resched();
     }
 }
 
 STATIC void
 xfs_end_bio(
     struct bio      *bio)
 {
     struct iomap_ioend  *ioend = bio->bi_private;
     struct xfs_inode    *ip = XFS_I(ioend->io_inode);
     unsigned long       flags;
 
     spin_lock_irqsave(&ip->i_ioend_lock, flags);
     if (list_empty(&ip->i_ioend_list))
         WARN_ON_ONCE(!queue_work(ip->i_mount->m_unwritten_workqueue,
                      &ip->i_ioend_work));
     list_add_tail(&ioend->io_list, &ip->i_ioend_list);
     spin_unlock_irqrestore(&ip->i_ioend_lock, flags);
 }
 
 /*
  * Fast revalidation of the cached writeback mapping. Return true if the current
  * mapping is valid, false otherwise.
  */
 static bool
 xfs_imap_valid(
     struct iomap_writepage_ctx  *wpc,
     struct xfs_inode        *ip,
     loff_t              offset)
 {
     if (offset < wpc->iomap.offset ||
         offset >= wpc->iomap.offset + wpc->iomap.length)
         return false;
     /*
      * If this is a COW mapping, it is sufficient to check that the mapping
      * covers the offset. Be careful to check this first because the caller
      * can revalidate a COW mapping without updating the data seqno.
      */
     if (wpc->iomap.flags & IOMAP_F_SHARED)
         return true;
 
     /*
      * This is not a COW mapping. Check the sequence number of the data fork
      * because concurrent changes could have invalidated the extent. Check
      * the COW fork because concurrent changes since the last time we
      * checked (and found nothing at this offset) could have added
      * overlapping blocks.
      */
     if (XFS_WPC(wpc)->data_seq != READ_ONCE(ip->i_df.if_seq))
         return false;
     if (xfs_inode_has_cow_data(ip) &&
         XFS_WPC(wpc)->cow_seq != READ_ONCE(ip->i_cowfp->if_seq))
         return false;
     return true;
 }
 
 /*
  * Pass in a dellalloc extent and convert it to real extents, return the real
  * extent that maps offset_fsb in wpc->iomap.
  *
  * The current page is held locked so nothing could have removed the block
  * backing offset_fsb, although it could have moved from the COW to the data
  * fork by another thread.
  */
 static int
 xfs_convert_blocks(
     struct iomap_writepage_ctx *wpc,
     struct xfs_inode    *ip,
     int         whichfork,
     loff_t          offset)
 {
     int         error;
     unsigned        *seq;
 
     if (whichfork == XFS_COW_FORK)
         seq = &XFS_WPC(wpc)->cow_seq;
     else
         seq = &XFS_WPC(wpc)->data_seq;
 
     /*
      * Attempt to allocate whatever delalloc extent currently backs offset
      * and put the result into wpc->iomap.  Allocate in a loop because it
      * may take several attempts to allocate real blocks for a contiguous
      * delalloc extent if free space is sufficiently fragmented.
      */
     do {
         error = xfs_bmapi_convert_delalloc(ip, whichfork, offset,
                 &wpc->iomap, seq);
         if (error)
             return error;
     } while (wpc->iomap.offset + wpc->iomap.length <= offset);
 
     return 0;
 }
 
 static int
 xfs_map_blocks(
     struct iomap_writepage_ctx *wpc,
     struct inode        *inode,
     loff_t          offset)
 {
     struct xfs_inode    *ip = XFS_I(inode);
     struct xfs_mount    *mp = ip->i_mount;
     ssize_t         count = i_blocksize(inode);
     xfs_fileoff_t       offset_fsb = XFS_B_TO_FSBT(mp, offset);
     xfs_fileoff_t       end_fsb = XFS_B_TO_FSB(mp, offset + count);
     xfs_fileoff_t       cow_fsb;
     int         whichfork;
     struct xfs_bmbt_irec    imap;
     struct xfs_iext_cursor  icur;
     int         retries = 0;
     int         error = 0;
 
     if (xfs_is_shutdown(mp))
         return -EIO;
 
     /*
      * COW fork blocks can overlap data fork blocks even if the blocks
      * aren't shared.  COW I/O always takes precedent, so we must always
      * check for overlap on reflink inodes unless the mapping is already a
      * COW one, or the COW fork hasn't changed from the last time we looked
      * at it.
      *
      * It's safe to check the COW fork if_seq here without the ILOCK because
      * we've indirectly protected against concurrent updates: writeback has
      * the page locked, which prevents concurrent invalidations by reflink
      * and directio and prevents concurrent buffered writes to the same
      * page.  Changes to if_seq always happen under i_lock, which protects
      * against concurrent updates and provides a memory barrier on the way
      * out that ensures that we always see the current value.
      */
     if (xfs_imap_valid(wpc, ip, offset))
         return 0;
 
     /*
      * If we don't have a valid map, now it's time to get a new one for this
      * offset.  This will convert delayed allocations (including COW ones)
      * into real extents.  If we return without a valid map, it means we
      * landed in a hole and we skip the block.
      */
 retry:
     cow_fsb = NULLFILEOFF;
     whichfork = XFS_DATA_FORK;
     xfs_ilock(ip, XFS_ILOCK_SHARED);
     ASSERT(!xfs_need_iread_extents(&ip->i_df));
 
     /*
      * Check if this is offset is covered by a COW extents, and if yes use
      * it directly instead of looking up anything in the data fork.
      */
     if (xfs_inode_has_cow_data(ip) &&
         xfs_iext_lookup_extent(ip, ip->i_cowfp, offset_fsb, &icur, &imap))
         cow_fsb = imap.br_startoff;
     if (cow_fsb != NULLFILEOFF && cow_fsb <= offset_fsb) {
         XFS_WPC(wpc)->cow_seq = READ_ONCE(ip->i_cowfp->if_seq);
         xfs_iunlock(ip, XFS_ILOCK_SHARED);
 
         whichfork = XFS_COW_FORK;
         goto allocate_blocks;
     }
 
     /*
      * No COW extent overlap. Revalidate now that we may have updated
      * ->cow_seq. If the data mapping is still valid, we're done.
      */
     if (xfs_imap_valid(wpc, ip, offset)) {
         xfs_iunlock(ip, XFS_ILOCK_SHARED);
         return 0;
     }
 
     /*
      * If we don't have a valid map, now it's time to get a new one for this
      * offset.  This will convert delayed allocations (including COW ones)
      * into real extents.
      */
     if (!xfs_iext_lookup_extent(ip, &ip->i_df, offset_fsb, &icur, &imap))
         imap.br_startoff = end_fsb; /* fake a hole past EOF */
     XFS_WPC(wpc)->data_seq = READ_ONCE(ip->i_df.if_seq);
     xfs_iunlock(ip, XFS_ILOCK_SHARED);
 
     /* landed in a hole or beyond EOF? */
     if (imap.br_startoff > offset_fsb) {
         imap.br_blockcount = imap.br_startoff - offset_fsb;
         imap.br_startoff = offset_fsb;
         imap.br_startblock = HOLESTARTBLOCK;
         imap.br_state = XFS_EXT_NORM;
     }
 
     /*
      * Truncate to the next COW extent if there is one.  This is the only
      * opportunity to do this because we can skip COW fork lookups for the
      * subsequent blocks in the mapping; however, the requirement to treat
      * the COW range separately remains.
      */
     if (cow_fsb != NULLFILEOFF &&
         cow_fsb < imap.br_startoff + imap.br_blockcount)
         imap.br_blockcount = cow_fsb - imap.br_startoff;
 
     /* got a delalloc extent? */
     if (imap.br_startblock != HOLESTARTBLOCK &&
         isnullstartblock(imap.br_startblock))
         goto allocate_blocks;
 
     xfs_bmbt_to_iomap(ip, &wpc->iomap, &imap, 0, 0);
     trace_xfs_map_blocks_found(ip, offset, count, whichfork, &imap);
     return 0;
 allocate_blocks:
     error = xfs_convert_blocks(wpc, ip, whichfork, offset);
     if (error) {
         /*
          * If we failed to find the extent in the COW fork we might have
          * raced with a COW to data fork conversion or truncate.
          * Restart the lookup to catch the extent in the data fork for
          * the former case, but prevent additional retries to avoid
          * looping forever for the latter case.
          */
         if (error == -EAGAIN && whichfork == XFS_COW_FORK && !retries++)
             goto retry;
         ASSERT(error != -EAGAIN);
         return error;
     }
 
     /*
      * Due to merging the return real extent might be larger than the
      * original delalloc one.  Trim the return extent to the next COW
      * boundary again to force a re-lookup.
      */
     if (whichfork != XFS_COW_FORK && cow_fsb != NULLFILEOFF) {
         loff_t      cow_offset = XFS_FSB_TO_B(mp, cow_fsb);
 
         if (cow_offset < wpc->iomap.offset + wpc->iomap.length)
             wpc->iomap.length = cow_offset - wpc->iomap.offset;
     }
 
     ASSERT(wpc->iomap.offset <= offset);
     ASSERT(wpc->iomap.offset + wpc->iomap.length > offset);
     trace_xfs_map_blocks_alloc(ip, offset, count, whichfork, &imap);
     return 0;
 }
 
 static int
 xfs_prepare_ioend(
     struct iomap_ioend  *ioend,
     int         status)
 {
     unsigned int        nofs_flag;
 
     /*
      * We can allocate memory here while doing writeback on behalf of
      * memory reclaim.  To avoid memory allocation deadlocks set the
      * task-wide nofs context for the following operations.
      */
     nofs_flag = memalloc_nofs_save();
 
     /* Convert CoW extents to regular */
     if (!status && (ioend->io_flags & IOMAP_F_SHARED)) {
         status = xfs_reflink_convert_cow(XFS_I(ioend->io_inode),
                 ioend->io_offset, ioend->io_size);
     }
 
     memalloc_nofs_restore(nofs_flag);
 
     /* send ioends that might require a transaction to the completion wq */
     if (xfs_ioend_is_append(ioend) || ioend->io_type == IOMAP_UNWRITTEN ||
         (ioend->io_flags & IOMAP_F_SHARED))
         ioend->io_bio->bi_end_io = xfs_end_bio;
     return status;
 }
 
 /*
  * If the page has delalloc blocks on it, we need to punch them out before we
  * invalidate the page.  If we don't, we leave a stale delalloc mapping on the
  * inode that can trip up a later direct I/O read operation on the same region.
  *
  * We prevent this by truncating away the delalloc regions on the page.  Because
  * they are delalloc, we can do this without needing a transaction. Indeed - if
  * we get ENOSPC errors, we have to be able to do this truncation without a
  * transaction as there is no space left for block reservation (typically why we
  * see a ENOSPC in writeback).
  */
 static void
 xfs_discard_folio(
     struct folio        *folio,
     loff_t          pos)
 {
     struct inode        *inode = folio->mapping->host;
     struct xfs_inode    *ip = XFS_I(inode);
     struct xfs_mount    *mp = ip->i_mount;
     size_t          offset = offset_in_folio(folio, pos);
     xfs_fileoff_t       start_fsb = XFS_B_TO_FSBT(mp, pos);
     xfs_fileoff_t       pageoff_fsb = XFS_B_TO_FSBT(mp, offset);
     int         error;
 
     if (xfs_is_shutdown(mp))
         return;
 
     xfs_alert_ratelimited(mp,
         "page discard on page "PTR_FMT", inode 0x%llx, pos %llu.",
             folio, ip->i_ino, pos);
 
     error = xfs_bmap_punch_delalloc_range(ip, start_fsb,
             i_blocks_per_folio(inode, folio) - pageoff_fsb);
     if (error && !xfs_is_shutdown(mp))
         xfs_alert(mp, "page discard unable to remove delalloc mapping.");
 }
 
 static const struct iomap_writeback_ops xfs_writeback_ops = {
     .map_blocks     = xfs_map_blocks,
     .prepare_ioend      = xfs_prepare_ioend,
     .discard_folio      = xfs_discard_folio,
 };
 
 STATIC int
 xfs_vm_writepages(
     struct address_space    *mapping,
     struct writeback_control *wbc)
 {
     struct xfs_writepage_ctx wpc = { };
 
     /*
      * Writing back data in a transaction context can result in recursive
      * transactions. This is bad, so issue a warning and get out of here.
      */
     if (WARN_ON_ONCE(current->journal_info))
         return 0;
 
     xfs_iflags_clear(XFS_I(mapping->host), XFS_ITRUNCATED);
     return iomap_writepages(mapping, wbc, &wpc.ctx, &xfs_writeback_ops);
 }
 
 STATIC int
 xfs_dax_writepages(
     struct address_space    *mapping,
     struct writeback_control *wbc)
 {
     struct xfs_inode    *ip = XFS_I(mapping->host);
 
     xfs_iflags_clear(ip, XFS_ITRUNCATED);
     return dax_writeback_mapping_range(mapping,
             xfs_inode_buftarg(ip)->bt_daxdev, wbc);
 }
 
 STATIC sector_t
 xfs_vm_bmap(
     struct address_space    *mapping,
     sector_t        block)
 {
     struct xfs_inode    *ip = XFS_I(mapping->host);
 
     trace_xfs_vm_bmap(ip);
 
     /*
      * The swap code (ab-)uses ->bmap to get a block mapping and then
      * bypasses the file system for actual I/O.  We really can't allow
      * that on reflinks inodes, so we have to skip out here.  And yes,
      * 0 is the magic code for a bmap error.
      *
      * Since we don't pass back blockdev info, we can't return bmap
      * information for rt files either.
      */
     if (xfs_is_cow_inode(ip) || XFS_IS_REALTIME_INODE(ip))
         return 0;
     return iomap_bmap(mapping, block, &xfs_read_iomap_ops);
 }
 
 STATIC int
 xfs_vm_read_folio(
     struct file     *unused,
     struct folio        *folio)
 {
     return iomap_read_folio(folio, &xfs_read_iomap_ops);
 }
 
 STATIC void
 xfs_vm_readahead(
     struct readahead_control    *rac)
 {
     iomap_readahead(rac, &xfs_read_iomap_ops);
 }
 
 static int
 xfs_iomap_swapfile_activate(
     struct swap_info_struct     *sis,
     struct file         *swap_file,
     sector_t            *span)
 {
     sis->bdev = xfs_inode_buftarg(XFS_I(file_inode(swap_file)))->bt_bdev;
     return iomap_swapfile_activate(sis, swap_file, span,
             &xfs_read_iomap_ops);
 }
 
 const struct address_space_operations xfs_address_space_operations = {
     .read_folio     = xfs_vm_read_folio,
     .readahead      = xfs_vm_readahead,
     .writepages     = xfs_vm_writepages,
     .dirty_folio        = filemap_dirty_folio,
     .release_folio      = iomap_release_folio,
     .invalidate_folio   = iomap_invalidate_folio,
     .bmap           = xfs_vm_bmap,
     .direct_IO      = noop_direct_IO,
     .migrate_folio      = filemap_migrate_folio,
     .is_partially_uptodate  = iomap_is_partially_uptodate,
     .error_remove_page  = generic_error_remove_page,
     .swap_activate      = xfs_iomap_swapfile_activate,
 };
 
 const struct address_space_operations xfs_dax_aops = {
     .writepages     = xfs_dax_writepages,
     .direct_IO      = noop_direct_IO,
     .dirty_folio        = noop_dirty_folio,
     .swap_activate      = xfs_iomap_swapfile_activate,
 };

This page was automatically generated by the 2.1.0 LXR engine. The LXR team