Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0
0002 
0003 #include <linux/slab.h>
0004 #include "ctree.h"
0005 #include "subpage.h"
0006 #include "btrfs_inode.h"
0007 
0008 /*
0009  * Subpage (sectorsize < PAGE_SIZE) support overview:
0010  *
0011  * Limitations:
0012  *
0013  * - Only support 64K page size for now
0014  *   This is to make metadata handling easier, as 64K page would ensure
0015  *   all nodesize would fit inside one page, thus we don't need to handle
0016  *   cases where a tree block crosses several pages.
0017  *
0018  * - Only metadata read-write for now
0019  *   The data read-write part is in development.
0020  *
0021  * - Metadata can't cross 64K page boundary
0022  *   btrfs-progs and kernel have done that for a while, thus only ancient
0023  *   filesystems could have such problem.  For such case, do a graceful
0024  *   rejection.
0025  *
0026  * Special behavior:
0027  *
0028  * - Metadata
0029  *   Metadata read is fully supported.
0030  *   Meaning when reading one tree block will only trigger the read for the
0031  *   needed range, other unrelated range in the same page will not be touched.
0032  *
0033  *   Metadata write support is partial.
0034  *   The writeback is still for the full page, but we will only submit
0035  *   the dirty extent buffers in the page.
0036  *
0037  *   This means, if we have a metadata page like this:
0038  *
0039  *   Page offset
0040  *   0         16K         32K         48K        64K
0041  *   |/////////|           |///////////|
0042  *        \- Tree block A        \- Tree block B
0043  *
0044  *   Even if we just want to writeback tree block A, we will also writeback
0045  *   tree block B if it's also dirty.
0046  *
0047  *   This may cause extra metadata writeback which results more COW.
0048  *
0049  * Implementation:
0050  *
0051  * - Common
0052  *   Both metadata and data will use a new structure, btrfs_subpage, to
0053  *   record the status of each sector inside a page.  This provides the extra
0054  *   granularity needed.
0055  *
0056  * - Metadata
0057  *   Since we have multiple tree blocks inside one page, we can't rely on page
0058  *   locking anymore, or we will have greatly reduced concurrency or even
0059  *   deadlocks (hold one tree lock while trying to lock another tree lock in
0060  *   the same page).
0061  *
0062  *   Thus for metadata locking, subpage support relies on io_tree locking only.
0063  *   This means a slightly higher tree locking latency.
0064  */
0065 
0066 bool btrfs_is_subpage(const struct btrfs_fs_info *fs_info, struct page *page)
0067 {
0068     if (fs_info->sectorsize >= PAGE_SIZE)
0069         return false;
0070 
0071     /*
0072      * Only data pages (either through DIO or compression) can have no
0073      * mapping. And if page->mapping->host is data inode, it's subpage.
0074      * As we have ruled our sectorsize >= PAGE_SIZE case already.
0075      */
0076     if (!page->mapping || !page->mapping->host ||
0077         is_data_inode(page->mapping->host))
0078         return true;
0079 
0080     /*
0081      * Now the only remaining case is metadata, which we only go subpage
0082      * routine if nodesize < PAGE_SIZE.
0083      */
0084     if (fs_info->nodesize < PAGE_SIZE)
0085         return true;
0086     return false;
0087 }
0088 
0089 void btrfs_init_subpage_info(struct btrfs_subpage_info *subpage_info, u32 sectorsize)
0090 {
0091     unsigned int cur = 0;
0092     unsigned int nr_bits;
0093 
0094     ASSERT(IS_ALIGNED(PAGE_SIZE, sectorsize));
0095 
0096     nr_bits = PAGE_SIZE / sectorsize;
0097     subpage_info->bitmap_nr_bits = nr_bits;
0098 
0099     subpage_info->uptodate_offset = cur;
0100     cur += nr_bits;
0101 
0102     subpage_info->error_offset = cur;
0103     cur += nr_bits;
0104 
0105     subpage_info->dirty_offset = cur;
0106     cur += nr_bits;
0107 
0108     subpage_info->writeback_offset = cur;
0109     cur += nr_bits;
0110 
0111     subpage_info->ordered_offset = cur;
0112     cur += nr_bits;
0113 
0114     subpage_info->checked_offset = cur;
0115     cur += nr_bits;
0116 
0117     subpage_info->total_nr_bits = cur;
0118 }
0119 
0120 int btrfs_attach_subpage(const struct btrfs_fs_info *fs_info,
0121              struct page *page, enum btrfs_subpage_type type)
0122 {
0123     struct btrfs_subpage *subpage;
0124 
0125     /*
0126      * We have cases like a dummy extent buffer page, which is not mapped
0127      * and doesn't need to be locked.
0128      */
0129     if (page->mapping)
0130         ASSERT(PageLocked(page));
0131 
0132     /* Either not subpage, or the page already has private attached */
0133     if (!btrfs_is_subpage(fs_info, page) || PagePrivate(page))
0134         return 0;
0135 
0136     subpage = btrfs_alloc_subpage(fs_info, type);
0137     if (IS_ERR(subpage))
0138         return  PTR_ERR(subpage);
0139 
0140     attach_page_private(page, subpage);
0141     return 0;
0142 }
0143 
0144 void btrfs_detach_subpage(const struct btrfs_fs_info *fs_info,
0145               struct page *page)
0146 {
0147     struct btrfs_subpage *subpage;
0148 
0149     /* Either not subpage, or already detached */
0150     if (!btrfs_is_subpage(fs_info, page) || !PagePrivate(page))
0151         return;
0152 
0153     subpage = detach_page_private(page);
0154     ASSERT(subpage);
0155     btrfs_free_subpage(subpage);
0156 }
0157 
0158 struct btrfs_subpage *btrfs_alloc_subpage(const struct btrfs_fs_info *fs_info,
0159                       enum btrfs_subpage_type type)
0160 {
0161     struct btrfs_subpage *ret;
0162     unsigned int real_size;
0163 
0164     ASSERT(fs_info->sectorsize < PAGE_SIZE);
0165 
0166     real_size = struct_size(ret, bitmaps,
0167             BITS_TO_LONGS(fs_info->subpage_info->total_nr_bits));
0168     ret = kzalloc(real_size, GFP_NOFS);
0169     if (!ret)
0170         return ERR_PTR(-ENOMEM);
0171 
0172     spin_lock_init(&ret->lock);
0173     if (type == BTRFS_SUBPAGE_METADATA) {
0174         atomic_set(&ret->eb_refs, 0);
0175     } else {
0176         atomic_set(&ret->readers, 0);
0177         atomic_set(&ret->writers, 0);
0178     }
0179     return ret;
0180 }
0181 
0182 void btrfs_free_subpage(struct btrfs_subpage *subpage)
0183 {
0184     kfree(subpage);
0185 }
0186 
0187 /*
0188  * Increase the eb_refs of current subpage.
0189  *
0190  * This is important for eb allocation, to prevent race with last eb freeing
0191  * of the same page.
0192  * With the eb_refs increased before the eb inserted into radix tree,
0193  * detach_extent_buffer_page() won't detach the page private while we're still
0194  * allocating the extent buffer.
0195  */
0196 void btrfs_page_inc_eb_refs(const struct btrfs_fs_info *fs_info,
0197                 struct page *page)
0198 {
0199     struct btrfs_subpage *subpage;
0200 
0201     if (!btrfs_is_subpage(fs_info, page))
0202         return;
0203 
0204     ASSERT(PagePrivate(page) && page->mapping);
0205     lockdep_assert_held(&page->mapping->private_lock);
0206 
0207     subpage = (struct btrfs_subpage *)page->private;
0208     atomic_inc(&subpage->eb_refs);
0209 }
0210 
0211 void btrfs_page_dec_eb_refs(const struct btrfs_fs_info *fs_info,
0212                 struct page *page)
0213 {
0214     struct btrfs_subpage *subpage;
0215 
0216     if (!btrfs_is_subpage(fs_info, page))
0217         return;
0218 
0219     ASSERT(PagePrivate(page) && page->mapping);
0220     lockdep_assert_held(&page->mapping->private_lock);
0221 
0222     subpage = (struct btrfs_subpage *)page->private;
0223     ASSERT(atomic_read(&subpage->eb_refs));
0224     atomic_dec(&subpage->eb_refs);
0225 }
0226 
0227 static void btrfs_subpage_assert(const struct btrfs_fs_info *fs_info,
0228         struct page *page, u64 start, u32 len)
0229 {
0230     /* Basic checks */
0231     ASSERT(PagePrivate(page) && page->private);
0232     ASSERT(IS_ALIGNED(start, fs_info->sectorsize) &&
0233            IS_ALIGNED(len, fs_info->sectorsize));
0234     /*
0235      * The range check only works for mapped page, we can still have
0236      * unmapped page like dummy extent buffer pages.
0237      */
0238     if (page->mapping)
0239         ASSERT(page_offset(page) <= start &&
0240                start + len <= page_offset(page) + PAGE_SIZE);
0241 }
0242 
0243 void btrfs_subpage_start_reader(const struct btrfs_fs_info *fs_info,
0244         struct page *page, u64 start, u32 len)
0245 {
0246     struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
0247     const int nbits = len >> fs_info->sectorsize_bits;
0248 
0249     btrfs_subpage_assert(fs_info, page, start, len);
0250 
0251     atomic_add(nbits, &subpage->readers);
0252 }
0253 
0254 void btrfs_subpage_end_reader(const struct btrfs_fs_info *fs_info,
0255         struct page *page, u64 start, u32 len)
0256 {
0257     struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
0258     const int nbits = len >> fs_info->sectorsize_bits;
0259     bool is_data;
0260     bool last;
0261 
0262     btrfs_subpage_assert(fs_info, page, start, len);
0263     is_data = is_data_inode(page->mapping->host);
0264     ASSERT(atomic_read(&subpage->readers) >= nbits);
0265     last = atomic_sub_and_test(nbits, &subpage->readers);
0266 
0267     /*
0268      * For data we need to unlock the page if the last read has finished.
0269      *
0270      * And please don't replace @last with atomic_sub_and_test() call
0271      * inside if () condition.
0272      * As we want the atomic_sub_and_test() to be always executed.
0273      */
0274     if (is_data && last)
0275         unlock_page(page);
0276 }
0277 
0278 static void btrfs_subpage_clamp_range(struct page *page, u64 *start, u32 *len)
0279 {
0280     u64 orig_start = *start;
0281     u32 orig_len = *len;
0282 
0283     *start = max_t(u64, page_offset(page), orig_start);
0284     /*
0285      * For certain call sites like btrfs_drop_pages(), we may have pages
0286      * beyond the target range. In that case, just set @len to 0, subpage
0287      * helpers can handle @len == 0 without any problem.
0288      */
0289     if (page_offset(page) >= orig_start + orig_len)
0290         *len = 0;
0291     else
0292         *len = min_t(u64, page_offset(page) + PAGE_SIZE,
0293                  orig_start + orig_len) - *start;
0294 }
0295 
0296 void btrfs_subpage_start_writer(const struct btrfs_fs_info *fs_info,
0297         struct page *page, u64 start, u32 len)
0298 {
0299     struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
0300     const int nbits = (len >> fs_info->sectorsize_bits);
0301     int ret;
0302 
0303     btrfs_subpage_assert(fs_info, page, start, len);
0304 
0305     ASSERT(atomic_read(&subpage->readers) == 0);
0306     ret = atomic_add_return(nbits, &subpage->writers);
0307     ASSERT(ret == nbits);
0308 }
0309 
0310 bool btrfs_subpage_end_and_test_writer(const struct btrfs_fs_info *fs_info,
0311         struct page *page, u64 start, u32 len)
0312 {
0313     struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
0314     const int nbits = (len >> fs_info->sectorsize_bits);
0315 
0316     btrfs_subpage_assert(fs_info, page, start, len);
0317 
0318     /*
0319      * We have call sites passing @lock_page into
0320      * extent_clear_unlock_delalloc() for compression path.
0321      *
0322      * This @locked_page is locked by plain lock_page(), thus its
0323      * subpage::writers is 0.  Handle them in a special way.
0324      */
0325     if (atomic_read(&subpage->writers) == 0)
0326         return true;
0327 
0328     ASSERT(atomic_read(&subpage->writers) >= nbits);
0329     return atomic_sub_and_test(nbits, &subpage->writers);
0330 }
0331 
0332 /*
0333  * Lock a page for delalloc page writeback.
0334  *
0335  * Return -EAGAIN if the page is not properly initialized.
0336  * Return 0 with the page locked, and writer counter updated.
0337  *
0338  * Even with 0 returned, the page still need extra check to make sure
0339  * it's really the correct page, as the caller is using
0340  * find_get_pages_contig(), which can race with page invalidating.
0341  */
0342 int btrfs_page_start_writer_lock(const struct btrfs_fs_info *fs_info,
0343         struct page *page, u64 start, u32 len)
0344 {
0345     if (unlikely(!fs_info) || !btrfs_is_subpage(fs_info, page)) {
0346         lock_page(page);
0347         return 0;
0348     }
0349     lock_page(page);
0350     if (!PagePrivate(page) || !page->private) {
0351         unlock_page(page);
0352         return -EAGAIN;
0353     }
0354     btrfs_subpage_clamp_range(page, &start, &len);
0355     btrfs_subpage_start_writer(fs_info, page, start, len);
0356     return 0;
0357 }
0358 
0359 void btrfs_page_end_writer_lock(const struct btrfs_fs_info *fs_info,
0360         struct page *page, u64 start, u32 len)
0361 {
0362     if (unlikely(!fs_info) || !btrfs_is_subpage(fs_info, page))
0363         return unlock_page(page);
0364     btrfs_subpage_clamp_range(page, &start, &len);
0365     if (btrfs_subpage_end_and_test_writer(fs_info, page, start, len))
0366         unlock_page(page);
0367 }
0368 
0369 static bool bitmap_test_range_all_set(unsigned long *addr, unsigned int start,
0370                       unsigned int nbits)
0371 {
0372     unsigned int found_zero;
0373 
0374     found_zero = find_next_zero_bit(addr, start + nbits, start);
0375     if (found_zero == start + nbits)
0376         return true;
0377     return false;
0378 }
0379 
0380 static bool bitmap_test_range_all_zero(unsigned long *addr, unsigned int start,
0381                        unsigned int nbits)
0382 {
0383     unsigned int found_set;
0384 
0385     found_set = find_next_bit(addr, start + nbits, start);
0386     if (found_set == start + nbits)
0387         return true;
0388     return false;
0389 }
0390 
0391 #define subpage_calc_start_bit(fs_info, page, name, start, len)     \
0392 ({                                  \
0393     unsigned int start_bit;                     \
0394                                     \
0395     btrfs_subpage_assert(fs_info, page, start, len);        \
0396     start_bit = offset_in_page(start) >> fs_info->sectorsize_bits;  \
0397     start_bit += fs_info->subpage_info->name##_offset;      \
0398     start_bit;                          \
0399 })
0400 
0401 #define subpage_test_bitmap_all_set(fs_info, subpage, name)     \
0402     bitmap_test_range_all_set(subpage->bitmaps,         \
0403             fs_info->subpage_info->name##_offset,       \
0404             fs_info->subpage_info->bitmap_nr_bits)
0405 
0406 #define subpage_test_bitmap_all_zero(fs_info, subpage, name)        \
0407     bitmap_test_range_all_zero(subpage->bitmaps,            \
0408             fs_info->subpage_info->name##_offset,       \
0409             fs_info->subpage_info->bitmap_nr_bits)
0410 
0411 void btrfs_subpage_set_uptodate(const struct btrfs_fs_info *fs_info,
0412         struct page *page, u64 start, u32 len)
0413 {
0414     struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
0415     unsigned int start_bit = subpage_calc_start_bit(fs_info, page,
0416                             uptodate, start, len);
0417     unsigned long flags;
0418 
0419     spin_lock_irqsave(&subpage->lock, flags);
0420     bitmap_set(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);
0421     if (subpage_test_bitmap_all_set(fs_info, subpage, uptodate))
0422         SetPageUptodate(page);
0423     spin_unlock_irqrestore(&subpage->lock, flags);
0424 }
0425 
0426 void btrfs_subpage_clear_uptodate(const struct btrfs_fs_info *fs_info,
0427         struct page *page, u64 start, u32 len)
0428 {
0429     struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
0430     unsigned int start_bit = subpage_calc_start_bit(fs_info, page,
0431                             uptodate, start, len);
0432     unsigned long flags;
0433 
0434     spin_lock_irqsave(&subpage->lock, flags);
0435     bitmap_clear(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);
0436     ClearPageUptodate(page);
0437     spin_unlock_irqrestore(&subpage->lock, flags);
0438 }
0439 
0440 void btrfs_subpage_set_error(const struct btrfs_fs_info *fs_info,
0441         struct page *page, u64 start, u32 len)
0442 {
0443     struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
0444     unsigned int start_bit = subpage_calc_start_bit(fs_info, page,
0445                             error, start, len);
0446     unsigned long flags;
0447 
0448     spin_lock_irqsave(&subpage->lock, flags);
0449     bitmap_set(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);
0450     SetPageError(page);
0451     spin_unlock_irqrestore(&subpage->lock, flags);
0452 }
0453 
0454 void btrfs_subpage_clear_error(const struct btrfs_fs_info *fs_info,
0455         struct page *page, u64 start, u32 len)
0456 {
0457     struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
0458     unsigned int start_bit = subpage_calc_start_bit(fs_info, page,
0459                             error, start, len);
0460     unsigned long flags;
0461 
0462     spin_lock_irqsave(&subpage->lock, flags);
0463     bitmap_clear(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);
0464     if (subpage_test_bitmap_all_zero(fs_info, subpage, error))
0465         ClearPageError(page);
0466     spin_unlock_irqrestore(&subpage->lock, flags);
0467 }
0468 
0469 void btrfs_subpage_set_dirty(const struct btrfs_fs_info *fs_info,
0470         struct page *page, u64 start, u32 len)
0471 {
0472     struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
0473     unsigned int start_bit = subpage_calc_start_bit(fs_info, page,
0474                             dirty, start, len);
0475     unsigned long flags;
0476 
0477     spin_lock_irqsave(&subpage->lock, flags);
0478     bitmap_set(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);
0479     spin_unlock_irqrestore(&subpage->lock, flags);
0480     set_page_dirty(page);
0481 }
0482 
0483 /*
0484  * Extra clear_and_test function for subpage dirty bitmap.
0485  *
0486  * Return true if we're the last bits in the dirty_bitmap and clear the
0487  * dirty_bitmap.
0488  * Return false otherwise.
0489  *
0490  * NOTE: Callers should manually clear page dirty for true case, as we have
0491  * extra handling for tree blocks.
0492  */
0493 bool btrfs_subpage_clear_and_test_dirty(const struct btrfs_fs_info *fs_info,
0494         struct page *page, u64 start, u32 len)
0495 {
0496     struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
0497     unsigned int start_bit = subpage_calc_start_bit(fs_info, page,
0498                             dirty, start, len);
0499     unsigned long flags;
0500     bool last = false;
0501 
0502     spin_lock_irqsave(&subpage->lock, flags);
0503     bitmap_clear(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);
0504     if (subpage_test_bitmap_all_zero(fs_info, subpage, dirty))
0505         last = true;
0506     spin_unlock_irqrestore(&subpage->lock, flags);
0507     return last;
0508 }
0509 
0510 void btrfs_subpage_clear_dirty(const struct btrfs_fs_info *fs_info,
0511         struct page *page, u64 start, u32 len)
0512 {
0513     bool last;
0514 
0515     last = btrfs_subpage_clear_and_test_dirty(fs_info, page, start, len);
0516     if (last)
0517         clear_page_dirty_for_io(page);
0518 }
0519 
0520 void btrfs_subpage_set_writeback(const struct btrfs_fs_info *fs_info,
0521         struct page *page, u64 start, u32 len)
0522 {
0523     struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
0524     unsigned int start_bit = subpage_calc_start_bit(fs_info, page,
0525                             writeback, start, len);
0526     unsigned long flags;
0527 
0528     spin_lock_irqsave(&subpage->lock, flags);
0529     bitmap_set(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);
0530     set_page_writeback(page);
0531     spin_unlock_irqrestore(&subpage->lock, flags);
0532 }
0533 
0534 void btrfs_subpage_clear_writeback(const struct btrfs_fs_info *fs_info,
0535         struct page *page, u64 start, u32 len)
0536 {
0537     struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
0538     unsigned int start_bit = subpage_calc_start_bit(fs_info, page,
0539                             writeback, start, len);
0540     unsigned long flags;
0541 
0542     spin_lock_irqsave(&subpage->lock, flags);
0543     bitmap_clear(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);
0544     if (subpage_test_bitmap_all_zero(fs_info, subpage, writeback)) {
0545         ASSERT(PageWriteback(page));
0546         end_page_writeback(page);
0547     }
0548     spin_unlock_irqrestore(&subpage->lock, flags);
0549 }
0550 
0551 void btrfs_subpage_set_ordered(const struct btrfs_fs_info *fs_info,
0552         struct page *page, u64 start, u32 len)
0553 {
0554     struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
0555     unsigned int start_bit = subpage_calc_start_bit(fs_info, page,
0556                             ordered, start, len);
0557     unsigned long flags;
0558 
0559     spin_lock_irqsave(&subpage->lock, flags);
0560     bitmap_set(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);
0561     SetPageOrdered(page);
0562     spin_unlock_irqrestore(&subpage->lock, flags);
0563 }
0564 
0565 void btrfs_subpage_clear_ordered(const struct btrfs_fs_info *fs_info,
0566         struct page *page, u64 start, u32 len)
0567 {
0568     struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
0569     unsigned int start_bit = subpage_calc_start_bit(fs_info, page,
0570                             ordered, start, len);
0571     unsigned long flags;
0572 
0573     spin_lock_irqsave(&subpage->lock, flags);
0574     bitmap_clear(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);
0575     if (subpage_test_bitmap_all_zero(fs_info, subpage, ordered))
0576         ClearPageOrdered(page);
0577     spin_unlock_irqrestore(&subpage->lock, flags);
0578 }
0579 
0580 void btrfs_subpage_set_checked(const struct btrfs_fs_info *fs_info,
0581                    struct page *page, u64 start, u32 len)
0582 {
0583     struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
0584     unsigned int start_bit = subpage_calc_start_bit(fs_info, page,
0585                             checked, start, len);
0586     unsigned long flags;
0587 
0588     spin_lock_irqsave(&subpage->lock, flags);
0589     bitmap_set(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);
0590     if (subpage_test_bitmap_all_set(fs_info, subpage, checked))
0591         SetPageChecked(page);
0592     spin_unlock_irqrestore(&subpage->lock, flags);
0593 }
0594 
0595 void btrfs_subpage_clear_checked(const struct btrfs_fs_info *fs_info,
0596                  struct page *page, u64 start, u32 len)
0597 {
0598     struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
0599     unsigned int start_bit = subpage_calc_start_bit(fs_info, page,
0600                             checked, start, len);
0601     unsigned long flags;
0602 
0603     spin_lock_irqsave(&subpage->lock, flags);
0604     bitmap_clear(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);
0605     ClearPageChecked(page);
0606     spin_unlock_irqrestore(&subpage->lock, flags);
0607 }
0608 
0609 /*
0610  * Unlike set/clear which is dependent on each page status, for test all bits
0611  * are tested in the same way.
0612  */
0613 #define IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(name)               \
0614 bool btrfs_subpage_test_##name(const struct btrfs_fs_info *fs_info, \
0615         struct page *page, u64 start, u32 len)          \
0616 {                                   \
0617     struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; \
0618     unsigned int start_bit = subpage_calc_start_bit(fs_info, page,  \
0619                         name, start, len);  \
0620     unsigned long flags;                        \
0621     bool ret;                           \
0622                                     \
0623     spin_lock_irqsave(&subpage->lock, flags);           \
0624     ret = bitmap_test_range_all_set(subpage->bitmaps, start_bit,    \
0625                 len >> fs_info->sectorsize_bits);   \
0626     spin_unlock_irqrestore(&subpage->lock, flags);          \
0627     return ret;                         \
0628 }
0629 IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(uptodate);
0630 IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(error);
0631 IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(dirty);
0632 IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(writeback);
0633 IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(ordered);
0634 IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(checked);
0635 
0636 /*
0637  * Note that, in selftests (extent-io-tests), we can have empty fs_info passed
0638  * in.  We only test sectorsize == PAGE_SIZE cases so far, thus we can fall
0639  * back to regular sectorsize branch.
0640  */
0641 #define IMPLEMENT_BTRFS_PAGE_OPS(name, set_page_func, clear_page_func,  \
0642                    test_page_func)              \
0643 void btrfs_page_set_##name(const struct btrfs_fs_info *fs_info,     \
0644         struct page *page, u64 start, u32 len)          \
0645 {                                   \
0646     if (unlikely(!fs_info) || !btrfs_is_subpage(fs_info, page)) {   \
0647         set_page_func(page);                    \
0648         return;                         \
0649     }                               \
0650     btrfs_subpage_set_##name(fs_info, page, start, len);        \
0651 }                                   \
0652 void btrfs_page_clear_##name(const struct btrfs_fs_info *fs_info,   \
0653         struct page *page, u64 start, u32 len)          \
0654 {                                   \
0655     if (unlikely(!fs_info) || !btrfs_is_subpage(fs_info, page)) {   \
0656         clear_page_func(page);                  \
0657         return;                         \
0658     }                               \
0659     btrfs_subpage_clear_##name(fs_info, page, start, len);      \
0660 }                                   \
0661 bool btrfs_page_test_##name(const struct btrfs_fs_info *fs_info,    \
0662         struct page *page, u64 start, u32 len)          \
0663 {                                   \
0664     if (unlikely(!fs_info) || !btrfs_is_subpage(fs_info, page)) \
0665         return test_page_func(page);                \
0666     return btrfs_subpage_test_##name(fs_info, page, start, len);    \
0667 }                                   \
0668 void btrfs_page_clamp_set_##name(const struct btrfs_fs_info *fs_info,   \
0669         struct page *page, u64 start, u32 len)          \
0670 {                                   \
0671     if (unlikely(!fs_info) || !btrfs_is_subpage(fs_info, page)) {   \
0672         set_page_func(page);                    \
0673         return;                         \
0674     }                               \
0675     btrfs_subpage_clamp_range(page, &start, &len);          \
0676     btrfs_subpage_set_##name(fs_info, page, start, len);        \
0677 }                                   \
0678 void btrfs_page_clamp_clear_##name(const struct btrfs_fs_info *fs_info, \
0679         struct page *page, u64 start, u32 len)          \
0680 {                                   \
0681     if (unlikely(!fs_info) || !btrfs_is_subpage(fs_info, page)) {   \
0682         clear_page_func(page);                  \
0683         return;                         \
0684     }                               \
0685     btrfs_subpage_clamp_range(page, &start, &len);          \
0686     btrfs_subpage_clear_##name(fs_info, page, start, len);      \
0687 }                                   \
0688 bool btrfs_page_clamp_test_##name(const struct btrfs_fs_info *fs_info,  \
0689         struct page *page, u64 start, u32 len)          \
0690 {                                   \
0691     if (unlikely(!fs_info) || !btrfs_is_subpage(fs_info, page)) \
0692         return test_page_func(page);                \
0693     btrfs_subpage_clamp_range(page, &start, &len);          \
0694     return btrfs_subpage_test_##name(fs_info, page, start, len);    \
0695 }
0696 IMPLEMENT_BTRFS_PAGE_OPS(uptodate, SetPageUptodate, ClearPageUptodate,
0697              PageUptodate);
0698 IMPLEMENT_BTRFS_PAGE_OPS(error, SetPageError, ClearPageError, PageError);
0699 IMPLEMENT_BTRFS_PAGE_OPS(dirty, set_page_dirty, clear_page_dirty_for_io,
0700              PageDirty);
0701 IMPLEMENT_BTRFS_PAGE_OPS(writeback, set_page_writeback, end_page_writeback,
0702              PageWriteback);
0703 IMPLEMENT_BTRFS_PAGE_OPS(ordered, SetPageOrdered, ClearPageOrdered,
0704              PageOrdered);
0705 IMPLEMENT_BTRFS_PAGE_OPS(checked, SetPageChecked, ClearPageChecked, PageChecked);
0706 
0707 /*
0708  * Make sure not only the page dirty bit is cleared, but also subpage dirty bit
0709  * is cleared.
0710  */
0711 void btrfs_page_assert_not_dirty(const struct btrfs_fs_info *fs_info,
0712                  struct page *page)
0713 {
0714     struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
0715 
0716     if (!IS_ENABLED(CONFIG_BTRFS_ASSERT))
0717         return;
0718 
0719     ASSERT(!PageDirty(page));
0720     if (!btrfs_is_subpage(fs_info, page))
0721         return;
0722 
0723     ASSERT(PagePrivate(page) && page->private);
0724     ASSERT(subpage_test_bitmap_all_zero(fs_info, subpage, dirty));
0725 }
0726 
0727 /*
0728  * Handle different locked pages with different page sizes:
0729  *
0730  * - Page locked by plain lock_page()
0731  *   It should not have any subpage::writers count.
0732  *   Can be unlocked by unlock_page().
0733  *   This is the most common locked page for __extent_writepage() called
0734  *   inside extent_write_cache_pages().
0735  *   Rarer cases include the @locked_page from extent_write_locked_range().
0736  *
0737  * - Page locked by lock_delalloc_pages()
0738  *   There is only one caller, all pages except @locked_page for
0739  *   extent_write_locked_range().
0740  *   In this case, we have to call subpage helper to handle the case.
0741  */
0742 void btrfs_page_unlock_writer(struct btrfs_fs_info *fs_info, struct page *page,
0743                   u64 start, u32 len)
0744 {
0745     struct btrfs_subpage *subpage;
0746 
0747     ASSERT(PageLocked(page));
0748     /* For non-subpage case, we just unlock the page */
0749     if (!btrfs_is_subpage(fs_info, page))
0750         return unlock_page(page);
0751 
0752     ASSERT(PagePrivate(page) && page->private);
0753     subpage = (struct btrfs_subpage *)page->private;
0754 
0755     /*
0756      * For subpage case, there are two types of locked page.  With or
0757      * without writers number.
0758      *
0759      * Since we own the page lock, no one else could touch subpage::writers
0760      * and we are safe to do several atomic operations without spinlock.
0761      */
0762     if (atomic_read(&subpage->writers) == 0)
0763         /* No writers, locked by plain lock_page() */
0764         return unlock_page(page);
0765 
0766     /* Have writers, use proper subpage helper to end it */
0767     btrfs_page_end_writer_lock(fs_info, page, start, len);
0768 }
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!