fs/nilfs2/page.c

0001 // SPDX-License-Identifier: GPL-2.0+
0002 /*
0003  * Buffer/page management specific to NILFS
0004  *
0005  * Copyright (C) 2005-2008 Nippon Telegraph and Telephone Corporation.
0006  *
0007  * Written by Ryusuke Konishi and Seiji Kihara.
0008  */
0009
0010 #include <linux/pagemap.h>
0011 #include <linux/writeback.h>
0012 #include <linux/swap.h>
0013 #include <linux/bitops.h>
0014 #include <linux/page-flags.h>
0015 #include <linux/list.h>
0016 #include <linux/highmem.h>
0017 #include <linux/pagevec.h>
0018 #include <linux/gfp.h>
0019 #include "nilfs.h"
0020 #include "page.h"
0021 #include "mdt.h"
0022
0023
0024 #define NILFS_BUFFER_INHERENT_BITS                  \
0025     (BIT(BH_Uptodate) | BIT(BH_Mapped) | BIT(BH_NILFS_Node) |   \
0026      BIT(BH_NILFS_Volatile) | BIT(BH_NILFS_Checked))
0027
0028 static struct buffer_head *
0029 __nilfs_get_page_block(struct page *page, unsigned long block, pgoff_t index,
0030                int blkbits, unsigned long b_state)
0031
0032 {
0033     unsigned long first_block;
0034     struct buffer_head *bh;
0035
0036     if (!page_has_buffers(page))
0037         create_empty_buffers(page, 1 << blkbits, b_state);
0038
0039     first_block = (unsigned long)index << (PAGE_SHIFT - blkbits);
0040     bh = nilfs_page_get_nth_block(page, block - first_block);
0041
0042     touch_buffer(bh);
0043     wait_on_buffer(bh);
0044     return bh;
0045 }
0046
0047 struct buffer_head *nilfs_grab_buffer(struct inode *inode,
0048                       struct address_space *mapping,
0049                       unsigned long blkoff,
0050                       unsigned long b_state)
0051 {
0052     int blkbits = inode->i_blkbits;
0053     pgoff_t index = blkoff >> (PAGE_SHIFT - blkbits);
0054     struct page *page;
0055     struct buffer_head *bh;
0056
0057     page = grab_cache_page(mapping, index);
0058     if (unlikely(!page))
0059         return NULL;
0060
0061     bh = __nilfs_get_page_block(page, blkoff, index, blkbits, b_state);
0062     if (unlikely(!bh)) {
0063         unlock_page(page);
0064         put_page(page);
0065         return NULL;
0066     }
0067     return bh;
0068 }
0069
0070 /**
0071  * nilfs_forget_buffer - discard dirty state
0072  * @bh: buffer head of the buffer to be discarded
0073  */
0074 void nilfs_forget_buffer(struct buffer_head *bh)
0075 {
0076     struct page *page = bh->b_page;
0077     const unsigned long clear_bits =
0078         (BIT(BH_Uptodate) | BIT(BH_Dirty) | BIT(BH_Mapped) |
0079          BIT(BH_Async_Write) | BIT(BH_NILFS_Volatile) |
0080          BIT(BH_NILFS_Checked) | BIT(BH_NILFS_Redirected));
0081
0082     lock_buffer(bh);
0083     set_mask_bits(&bh->b_state, clear_bits, 0);
0084     if (nilfs_page_buffers_clean(page))
0085         __nilfs_clear_page_dirty(page);
0086
0087     bh->b_blocknr = -1;
0088     ClearPageUptodate(page);
0089     ClearPageMappedToDisk(page);
0090     unlock_buffer(bh);
0091     brelse(bh);
0092 }
0093
0094 /**
0095  * nilfs_copy_buffer -- copy buffer data and flags
0096  * @dbh: destination buffer
0097  * @sbh: source buffer
0098  */
0099 void nilfs_copy_buffer(struct buffer_head *dbh, struct buffer_head *sbh)
0100 {
0101     void *kaddr0, *kaddr1;
0102     unsigned long bits;
0103     struct page *spage = sbh->b_page, *dpage = dbh->b_page;
0104     struct buffer_head *bh;
0105
0106     kaddr0 = kmap_atomic(spage);
0107     kaddr1 = kmap_atomic(dpage);
0108     memcpy(kaddr1 + bh_offset(dbh), kaddr0 + bh_offset(sbh), sbh->b_size);
0109     kunmap_atomic(kaddr1);
0110     kunmap_atomic(kaddr0);
0111
0112     dbh->b_state = sbh->b_state & NILFS_BUFFER_INHERENT_BITS;
0113     dbh->b_blocknr = sbh->b_blocknr;
0114     dbh->b_bdev = sbh->b_bdev;
0115
0116     bh = dbh;
0117     bits = sbh->b_state & (BIT(BH_Uptodate) | BIT(BH_Mapped));
0118     while ((bh = bh->b_this_page) != dbh) {
0119         lock_buffer(bh);
0120         bits &= bh->b_state;
0121         unlock_buffer(bh);
0122     }
0123     if (bits & BIT(BH_Uptodate))
0124         SetPageUptodate(dpage);
0125     else
0126         ClearPageUptodate(dpage);
0127     if (bits & BIT(BH_Mapped))
0128         SetPageMappedToDisk(dpage);
0129     else
0130         ClearPageMappedToDisk(dpage);
0131 }
0132
0133 /**
0134  * nilfs_page_buffers_clean - check if a page has dirty buffers or not.
0135  * @page: page to be checked
0136  *
0137  * nilfs_page_buffers_clean() returns zero if the page has dirty buffers.
0138  * Otherwise, it returns non-zero value.
0139  */
0140 int nilfs_page_buffers_clean(struct page *page)
0141 {
0142     struct buffer_head *bh, *head;
0143
0144     bh = head = page_buffers(page);
0145     do {
0146         if (buffer_dirty(bh))
0147             return 0;
0148         bh = bh->b_this_page;
0149     } while (bh != head);
0150     return 1;
0151 }
0152
0153 void nilfs_page_bug(struct page *page)
0154 {
0155     struct address_space *m;
0156     unsigned long ino;
0157
0158     if (unlikely(!page)) {
0159         printk(KERN_CRIT "NILFS_PAGE_BUG(NULL)\n");
0160         return;
0161     }
0162
0163     m = page->mapping;
0164     ino = m ? m->host->i_ino : 0;
0165
0166     printk(KERN_CRIT "NILFS_PAGE_BUG(%p): cnt=%d index#=%llu flags=0x%lx "
0167            "mapping=%p ino=%lu\n",
0168            page, page_ref_count(page),
0169            (unsigned long long)page->index, page->flags, m, ino);
0170
0171     if (page_has_buffers(page)) {
0172         struct buffer_head *bh, *head;
0173         int i = 0;
0174
0175         bh = head = page_buffers(page);
0176         do {
0177             printk(KERN_CRIT
0178                    " BH[%d] %p: cnt=%d block#=%llu state=0x%lx\n",
0179                    i++, bh, atomic_read(&bh->b_count),
0180                    (unsigned long long)bh->b_blocknr, bh->b_state);
0181             bh = bh->b_this_page;
0182         } while (bh != head);
0183     }
0184 }
0185
0186 /**
0187  * nilfs_copy_page -- copy the page with buffers
0188  * @dst: destination page
0189  * @src: source page
0190  * @copy_dirty: flag whether to copy dirty states on the page's buffer heads.
0191  *
0192  * This function is for both data pages and btnode pages.  The dirty flag
0193  * should be treated by caller.  The page must not be under i/o.
0194  * Both src and dst page must be locked
0195  */
0196 static void nilfs_copy_page(struct page *dst, struct page *src, int copy_dirty)
0197 {
0198     struct buffer_head *dbh, *dbufs, *sbh;
0199     unsigned long mask = NILFS_BUFFER_INHERENT_BITS;
0200
0201     BUG_ON(PageWriteback(dst));
0202
0203     sbh = page_buffers(src);
0204     if (!page_has_buffers(dst))
0205         create_empty_buffers(dst, sbh->b_size, 0);
0206
0207     if (copy_dirty)
0208         mask |= BIT(BH_Dirty);
0209
0210     dbh = dbufs = page_buffers(dst);
0211     do {
0212         lock_buffer(sbh);
0213         lock_buffer(dbh);
0214         dbh->b_state = sbh->b_state & mask;
0215         dbh->b_blocknr = sbh->b_blocknr;
0216         dbh->b_bdev = sbh->b_bdev;
0217         sbh = sbh->b_this_page;
0218         dbh = dbh->b_this_page;
0219     } while (dbh != dbufs);
0220
0221     copy_highpage(dst, src);
0222
0223     if (PageUptodate(src) && !PageUptodate(dst))
0224         SetPageUptodate(dst);
0225     else if (!PageUptodate(src) && PageUptodate(dst))
0226         ClearPageUptodate(dst);
0227     if (PageMappedToDisk(src) && !PageMappedToDisk(dst))
0228         SetPageMappedToDisk(dst);
0229     else if (!PageMappedToDisk(src) && PageMappedToDisk(dst))
0230         ClearPageMappedToDisk(dst);
0231
0232     do {
0233         unlock_buffer(sbh);
0234         unlock_buffer(dbh);
0235         sbh = sbh->b_this_page;
0236         dbh = dbh->b_this_page;
0237     } while (dbh != dbufs);
0238 }
0239
0240 int nilfs_copy_dirty_pages(struct address_space *dmap,
0241                struct address_space *smap)
0242 {
0243     struct pagevec pvec;
0244     unsigned int i;
0245     pgoff_t index = 0;
0246     int err = 0;
0247
0248     pagevec_init(&pvec);
0249 repeat:
0250     if (!pagevec_lookup_tag(&pvec, smap, &index, PAGECACHE_TAG_DIRTY))
0251         return 0;
0252
0253     for (i = 0; i < pagevec_count(&pvec); i++) {
0254         struct page *page = pvec.pages[i], *dpage;
0255
0256         lock_page(page);
0257         if (unlikely(!PageDirty(page)))
0258             NILFS_PAGE_BUG(page, "inconsistent dirty state");
0259
0260         dpage = grab_cache_page(dmap, page->index);
0261         if (unlikely(!dpage)) {
0262             /* No empty page is added to the page cache */
0263             err = -ENOMEM;
0264             unlock_page(page);
0265             break;
0266         }
0267         if (unlikely(!page_has_buffers(page)))
0268             NILFS_PAGE_BUG(page,
0269                        "found empty page in dat page cache");
0270
0271         nilfs_copy_page(dpage, page, 1);
0272         __set_page_dirty_nobuffers(dpage);
0273
0274         unlock_page(dpage);
0275         put_page(dpage);
0276         unlock_page(page);
0277     }
0278     pagevec_release(&pvec);
0279     cond_resched();
0280
0281     if (likely(!err))
0282         goto repeat;
0283     return err;
0284 }
0285
0286 /**
0287  * nilfs_copy_back_pages -- copy back pages to original cache from shadow cache
0288  * @dmap: destination page cache
0289  * @smap: source page cache
0290  *
0291  * No pages must be added to the cache during this process.
0292  * This must be ensured by the caller.
0293  */
0294 void nilfs_copy_back_pages(struct address_space *dmap,
0295                struct address_space *smap)
0296 {
0297     struct folio_batch fbatch;
0298     unsigned int i, n;
0299     pgoff_t start = 0;
0300
0301     folio_batch_init(&fbatch);
0302 repeat:
0303     n = filemap_get_folios(smap, &start, ~0UL, &fbatch);
0304     if (!n)
0305         return;
0306
0307     for (i = 0; i < folio_batch_count(&fbatch); i++) {
0308         struct folio *folio = fbatch.folios[i], *dfolio;
0309         pgoff_t index = folio->index;
0310
0311         folio_lock(folio);
0312         dfolio = filemap_lock_folio(dmap, index);
0313         if (dfolio) {
0314             /* overwrite existing folio in the destination cache */
0315             WARN_ON(folio_test_dirty(dfolio));
0316             nilfs_copy_page(&dfolio->page, &folio->page, 0);
0317             folio_unlock(dfolio);
0318             folio_put(dfolio);
0319             /* Do we not need to remove folio from smap here? */
0320         } else {
0321             struct folio *f;
0322
0323             /* move the folio to the destination cache */
0324             xa_lock_irq(&smap->i_pages);
0325             f = __xa_erase(&smap->i_pages, index);
0326             WARN_ON(folio != f);
0327             smap->nrpages--;
0328             xa_unlock_irq(&smap->i_pages);
0329
0330             xa_lock_irq(&dmap->i_pages);
0331             f = __xa_store(&dmap->i_pages, index, folio, GFP_NOFS);
0332             if (unlikely(f)) {
0333                 /* Probably -ENOMEM */
0334                 folio->mapping = NULL;
0335                 folio_put(folio);
0336             } else {
0337                 folio->mapping = dmap;
0338                 dmap->nrpages++;
0339                 if (folio_test_dirty(folio))
0340                     __xa_set_mark(&dmap->i_pages, index,
0341                             PAGECACHE_TAG_DIRTY);
0342             }
0343             xa_unlock_irq(&dmap->i_pages);
0344         }
0345         folio_unlock(folio);
0346     }
0347     folio_batch_release(&fbatch);
0348     cond_resched();
0349
0350     goto repeat;
0351 }
0352
0353 /**
0354  * nilfs_clear_dirty_pages - discard dirty pages in address space
0355  * @mapping: address space with dirty pages for discarding
0356  * @silent: suppress [true] or print [false] warning messages
0357  */
0358 void nilfs_clear_dirty_pages(struct address_space *mapping, bool silent)
0359 {
0360     struct pagevec pvec;
0361     unsigned int i;
0362     pgoff_t index = 0;
0363
0364     pagevec_init(&pvec);
0365
0366     while (pagevec_lookup_tag(&pvec, mapping, &index,
0367                     PAGECACHE_TAG_DIRTY)) {
0368         for (i = 0; i < pagevec_count(&pvec); i++) {
0369             struct page *page = pvec.pages[i];
0370
0371             lock_page(page);
0372             nilfs_clear_dirty_page(page, silent);
0373             unlock_page(page);
0374         }
0375         pagevec_release(&pvec);
0376         cond_resched();
0377     }
0378 }
0379
0380 /**
0381  * nilfs_clear_dirty_page - discard dirty page
0382  * @page: dirty page that will be discarded
0383  * @silent: suppress [true] or print [false] warning messages
0384  */
0385 void nilfs_clear_dirty_page(struct page *page, bool silent)
0386 {
0387     struct inode *inode = page->mapping->host;
0388     struct super_block *sb = inode->i_sb;
0389
0390     BUG_ON(!PageLocked(page));
0391
0392     if (!silent)
0393         nilfs_warn(sb, "discard dirty page: offset=%lld, ino=%lu",
0394                page_offset(page), inode->i_ino);
0395
0396     ClearPageUptodate(page);
0397     ClearPageMappedToDisk(page);
0398
0399     if (page_has_buffers(page)) {
0400         struct buffer_head *bh, *head;
0401         const unsigned long clear_bits =
0402             (BIT(BH_Uptodate) | BIT(BH_Dirty) | BIT(BH_Mapped) |
0403              BIT(BH_Async_Write) | BIT(BH_NILFS_Volatile) |
0404              BIT(BH_NILFS_Checked) | BIT(BH_NILFS_Redirected));
0405
0406         bh = head = page_buffers(page);
0407         do {
0408             lock_buffer(bh);
0409             if (!silent)
0410                 nilfs_warn(sb,
0411                        "discard dirty block: blocknr=%llu, size=%zu",
0412                        (u64)bh->b_blocknr, bh->b_size);
0413
0414             set_mask_bits(&bh->b_state, clear_bits, 0);
0415             unlock_buffer(bh);
0416         } while (bh = bh->b_this_page, bh != head);
0417     }
0418
0419     __nilfs_clear_page_dirty(page);
0420 }
0421
0422 unsigned int nilfs_page_count_clean_buffers(struct page *page,
0423                         unsigned int from, unsigned int to)
0424 {
0425     unsigned int block_start, block_end;
0426     struct buffer_head *bh, *head;
0427     unsigned int nc = 0;
0428
0429     for (bh = head = page_buffers(page), block_start = 0;
0430          bh != head || !block_start;
0431          block_start = block_end, bh = bh->b_this_page) {
0432         block_end = block_start + bh->b_size;
0433         if (block_end > from && block_start < to && !buffer_dirty(bh))
0434             nc++;
0435     }
0436     return nc;
0437 }
0438
0439 /*
0440  * NILFS2 needs clear_page_dirty() in the following two cases:
0441  *
0442  * 1) For B-tree node pages and data pages of DAT file, NILFS2 clears dirty
0443  *    flag of pages when it copies back pages from shadow cache to the
0444  *    original cache.
0445  *
0446  * 2) Some B-tree operations like insertion or deletion may dispose buffers
0447  *    in dirty state, and this needs to cancel the dirty state of their pages.
0448  */
0449 int __nilfs_clear_page_dirty(struct page *page)
0450 {
0451     struct address_space *mapping = page->mapping;
0452
0453     if (mapping) {
0454         xa_lock_irq(&mapping->i_pages);
0455         if (test_bit(PG_dirty, &page->flags)) {
0456             __xa_clear_mark(&mapping->i_pages, page_index(page),
0457                          PAGECACHE_TAG_DIRTY);
0458             xa_unlock_irq(&mapping->i_pages);
0459             return clear_page_dirty_for_io(page);
0460         }
0461         xa_unlock_irq(&mapping->i_pages);
0462         return 0;
0463     }
0464     return TestClearPageDirty(page);
0465 }
0466
0467 /**
0468  * nilfs_find_uncommitted_extent - find extent of uncommitted data
0469  * @inode: inode
0470  * @start_blk: start block offset (in)
0471  * @blkoff: start offset of the found extent (out)
0472  *
0473  * This function searches an extent of buffers marked "delayed" which
0474  * starts from a block offset equal to or larger than @start_blk.  If
0475  * such an extent was found, this will store the start offset in
0476  * @blkoff and return its length in blocks.  Otherwise, zero is
0477  * returned.
0478  */
0479 unsigned long nilfs_find_uncommitted_extent(struct inode *inode,
0480                         sector_t start_blk,
0481                         sector_t *blkoff)
0482 {
0483     unsigned int i;
0484     pgoff_t index;
0485     unsigned int nblocks_in_page;
0486     unsigned long length = 0;
0487     sector_t b;
0488     struct pagevec pvec;
0489     struct page *page;
0490
0491     if (inode->i_mapping->nrpages == 0)
0492         return 0;
0493
0494     index = start_blk >> (PAGE_SHIFT - inode->i_blkbits);
0495     nblocks_in_page = 1U << (PAGE_SHIFT - inode->i_blkbits);
0496
0497     pagevec_init(&pvec);
0498
0499 repeat:
0500     pvec.nr = find_get_pages_contig(inode->i_mapping, index, PAGEVEC_SIZE,
0501                     pvec.pages);
0502     if (pvec.nr == 0)
0503         return length;
0504
0505     if (length > 0 && pvec.pages[0]->index > index)
0506         goto out;
0507
0508     b = pvec.pages[0]->index << (PAGE_SHIFT - inode->i_blkbits);
0509     i = 0;
0510     do {
0511         page = pvec.pages[i];
0512
0513         lock_page(page);
0514         if (page_has_buffers(page)) {
0515             struct buffer_head *bh, *head;
0516
0517             bh = head = page_buffers(page);
0518             do {
0519                 if (b < start_blk)
0520                     continue;
0521                 if (buffer_delay(bh)) {
0522                     if (length == 0)
0523                         *blkoff = b;
0524                     length++;
0525                 } else if (length > 0) {
0526                     goto out_locked;
0527                 }
0528             } while (++b, bh = bh->b_this_page, bh != head);
0529         } else {
0530             if (length > 0)
0531                 goto out_locked;
0532
0533             b += nblocks_in_page;
0534         }
0535         unlock_page(page);
0536
0537     } while (++i < pagevec_count(&pvec));
0538
0539     index = page->index + 1;
0540     pagevec_release(&pvec);
0541     cond_resched();
0542     goto repeat;
0543
0544 out_locked:
0545     unlock_page(page);
0546 out:
0547     pagevec_release(&pvec);
0548     return length;
0549 }