fs/erofs/utils.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * Copyright (C) 2018 HUAWEI, Inc.
0004  *             https://www.huawei.com/
0005  */
0006 #include "internal.h"
0007 #include <linux/pagevec.h>
0008
0009 struct page *erofs_allocpage(struct page **pagepool, gfp_t gfp)
0010 {
0011     struct page *page = *pagepool;
0012
0013     if (page) {
0014         DBG_BUGON(page_ref_count(page) != 1);
0015         *pagepool = (struct page *)page_private(page);
0016     } else {
0017         page = alloc_page(gfp);
0018     }
0019     return page;
0020 }
0021
0022 void erofs_release_pages(struct page **pagepool)
0023 {
0024     while (*pagepool) {
0025         struct page *page = *pagepool;
0026
0027         *pagepool = (struct page *)page_private(page);
0028         put_page(page);
0029     }
0030 }
0031
0032 #ifdef CONFIG_EROFS_FS_ZIP
0033 /* global shrink count (for all mounted EROFS instances) */
0034 static atomic_long_t erofs_global_shrink_cnt;
0035
0036 static int erofs_workgroup_get(struct erofs_workgroup *grp)
0037 {
0038     int o;
0039
0040 repeat:
0041     o = erofs_wait_on_workgroup_freezed(grp);
0042     if (o <= 0)
0043         return -1;
0044
0045     if (atomic_cmpxchg(&grp->refcount, o, o + 1) != o)
0046         goto repeat;
0047
0048     /* decrease refcount paired by erofs_workgroup_put */
0049     if (o == 1)
0050         atomic_long_dec(&erofs_global_shrink_cnt);
0051     return 0;
0052 }
0053
0054 struct erofs_workgroup *erofs_find_workgroup(struct super_block *sb,
0055                          pgoff_t index)
0056 {
0057     struct erofs_sb_info *sbi = EROFS_SB(sb);
0058     struct erofs_workgroup *grp;
0059
0060 repeat:
0061     rcu_read_lock();
0062     grp = xa_load(&sbi->managed_pslots, index);
0063     if (grp) {
0064         if (erofs_workgroup_get(grp)) {
0065             /* prefer to relax rcu read side */
0066             rcu_read_unlock();
0067             goto repeat;
0068         }
0069
0070         DBG_BUGON(index != grp->index);
0071     }
0072     rcu_read_unlock();
0073     return grp;
0074 }
0075
0076 struct erofs_workgroup *erofs_insert_workgroup(struct super_block *sb,
0077                            struct erofs_workgroup *grp)
0078 {
0079     struct erofs_sb_info *const sbi = EROFS_SB(sb);
0080     struct erofs_workgroup *pre;
0081
0082     /*
0083      * Bump up a reference count before making this visible
0084      * to others for the XArray in order to avoid potential
0085      * UAF without serialized by xa_lock.
0086      */
0087     atomic_inc(&grp->refcount);
0088
0089 repeat:
0090     xa_lock(&sbi->managed_pslots);
0091     pre = __xa_cmpxchg(&sbi->managed_pslots, grp->index,
0092                NULL, grp, GFP_NOFS);
0093     if (pre) {
0094         if (xa_is_err(pre)) {
0095             pre = ERR_PTR(xa_err(pre));
0096         } else if (erofs_workgroup_get(pre)) {
0097             /* try to legitimize the current in-tree one */
0098             xa_unlock(&sbi->managed_pslots);
0099             cond_resched();
0100             goto repeat;
0101         }
0102         atomic_dec(&grp->refcount);
0103         grp = pre;
0104     }
0105     xa_unlock(&sbi->managed_pslots);
0106     return grp;
0107 }
0108
0109 static void  __erofs_workgroup_free(struct erofs_workgroup *grp)
0110 {
0111     atomic_long_dec(&erofs_global_shrink_cnt);
0112     erofs_workgroup_free_rcu(grp);
0113 }
0114
0115 int erofs_workgroup_put(struct erofs_workgroup *grp)
0116 {
0117     int count = atomic_dec_return(&grp->refcount);
0118
0119     if (count == 1)
0120         atomic_long_inc(&erofs_global_shrink_cnt);
0121     else if (!count)
0122         __erofs_workgroup_free(grp);
0123     return count;
0124 }
0125
0126 static bool erofs_try_to_release_workgroup(struct erofs_sb_info *sbi,
0127                        struct erofs_workgroup *grp)
0128 {
0129     /*
0130      * If managed cache is on, refcount of workgroups
0131      * themselves could be < 0 (freezed). In other words,
0132      * there is no guarantee that all refcounts > 0.
0133      */
0134     if (!erofs_workgroup_try_to_freeze(grp, 1))
0135         return false;
0136
0137     /*
0138      * Note that all cached pages should be unattached
0139      * before deleted from the XArray. Otherwise some
0140      * cached pages could be still attached to the orphan
0141      * old workgroup when the new one is available in the tree.
0142      */
0143     if (erofs_try_to_free_all_cached_pages(sbi, grp)) {
0144         erofs_workgroup_unfreeze(grp, 1);
0145         return false;
0146     }
0147
0148     /*
0149      * It's impossible to fail after the workgroup is freezed,
0150      * however in order to avoid some race conditions, add a
0151      * DBG_BUGON to observe this in advance.
0152      */
0153     DBG_BUGON(__xa_erase(&sbi->managed_pslots, grp->index) != grp);
0154
0155     /* last refcount should be connected with its managed pslot.  */
0156     erofs_workgroup_unfreeze(grp, 0);
0157     __erofs_workgroup_free(grp);
0158     return true;
0159 }
0160
0161 static unsigned long erofs_shrink_workstation(struct erofs_sb_info *sbi,
0162                           unsigned long nr_shrink)
0163 {
0164     struct erofs_workgroup *grp;
0165     unsigned int freed = 0;
0166     unsigned long index;
0167
0168     xa_lock(&sbi->managed_pslots);
0169     xa_for_each(&sbi->managed_pslots, index, grp) {
0170         /* try to shrink each valid workgroup */
0171         if (!erofs_try_to_release_workgroup(sbi, grp))
0172             continue;
0173         xa_unlock(&sbi->managed_pslots);
0174
0175         ++freed;
0176         if (!--nr_shrink)
0177             return freed;
0178         xa_lock(&sbi->managed_pslots);
0179     }
0180     xa_unlock(&sbi->managed_pslots);
0181     return freed;
0182 }
0183
0184 /* protected by 'erofs_sb_list_lock' */
0185 static unsigned int shrinker_run_no;
0186
0187 /* protects the mounted 'erofs_sb_list' */
0188 static DEFINE_SPINLOCK(erofs_sb_list_lock);
0189 static LIST_HEAD(erofs_sb_list);
0190
0191 void erofs_shrinker_register(struct super_block *sb)
0192 {
0193     struct erofs_sb_info *sbi = EROFS_SB(sb);
0194
0195     mutex_init(&sbi->umount_mutex);
0196
0197     spin_lock(&erofs_sb_list_lock);
0198     list_add(&sbi->list, &erofs_sb_list);
0199     spin_unlock(&erofs_sb_list_lock);
0200 }
0201
0202 void erofs_shrinker_unregister(struct super_block *sb)
0203 {
0204     struct erofs_sb_info *const sbi = EROFS_SB(sb);
0205
0206     mutex_lock(&sbi->umount_mutex);
0207     /* clean up all remaining workgroups in memory */
0208     erofs_shrink_workstation(sbi, ~0UL);
0209
0210     spin_lock(&erofs_sb_list_lock);
0211     list_del(&sbi->list);
0212     spin_unlock(&erofs_sb_list_lock);
0213     mutex_unlock(&sbi->umount_mutex);
0214 }
0215
0216 static unsigned long erofs_shrink_count(struct shrinker *shrink,
0217                     struct shrink_control *sc)
0218 {
0219     return atomic_long_read(&erofs_global_shrink_cnt);
0220 }
0221
0222 static unsigned long erofs_shrink_scan(struct shrinker *shrink,
0223                        struct shrink_control *sc)
0224 {
0225     struct erofs_sb_info *sbi;
0226     struct list_head *p;
0227
0228     unsigned long nr = sc->nr_to_scan;
0229     unsigned int run_no;
0230     unsigned long freed = 0;
0231
0232     spin_lock(&erofs_sb_list_lock);
0233     do {
0234         run_no = ++shrinker_run_no;
0235     } while (run_no == 0);
0236
0237     /* Iterate over all mounted superblocks and try to shrink them */
0238     p = erofs_sb_list.next;
0239     while (p != &erofs_sb_list) {
0240         sbi = list_entry(p, struct erofs_sb_info, list);
0241
0242         /*
0243          * We move the ones we do to the end of the list, so we stop
0244          * when we see one we have already done.
0245          */
0246         if (sbi->shrinker_run_no == run_no)
0247             break;
0248
0249         if (!mutex_trylock(&sbi->umount_mutex)) {
0250             p = p->next;
0251             continue;
0252         }
0253
0254         spin_unlock(&erofs_sb_list_lock);
0255         sbi->shrinker_run_no = run_no;
0256
0257         freed += erofs_shrink_workstation(sbi, nr - freed);
0258
0259         spin_lock(&erofs_sb_list_lock);
0260         /* Get the next list element before we move this one */
0261         p = p->next;
0262
0263         /*
0264          * Move this one to the end of the list to provide some
0265          * fairness.
0266          */
0267         list_move_tail(&sbi->list, &erofs_sb_list);
0268         mutex_unlock(&sbi->umount_mutex);
0269
0270         if (freed >= nr)
0271             break;
0272     }
0273     spin_unlock(&erofs_sb_list_lock);
0274     return freed;
0275 }
0276
0277 static struct shrinker erofs_shrinker_info = {
0278     .scan_objects = erofs_shrink_scan,
0279     .count_objects = erofs_shrink_count,
0280     .seeks = DEFAULT_SEEKS,
0281 };
0282
0283 int __init erofs_init_shrinker(void)
0284 {
0285     return register_shrinker(&erofs_shrinker_info, "erofs-shrinker");
0286 }
0287
0288 void erofs_exit_shrinker(void)
0289 {
0290     unregister_shrinker(&erofs_shrinker_info);
0291 }
0292 #endif  /* !CONFIG_EROFS_FS_ZIP */