fs/fscache/volume.c

0001 // SPDX-License-Identifier: GPL-2.0-or-later
0002 /* Volume-level cache cookie handling.
0003  *
0004  * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved.
0005  * Written by David Howells (dhowells@redhat.com)
0006  */
0007
0008 #define FSCACHE_DEBUG_LEVEL COOKIE
0009 #include <linux/export.h>
0010 #include <linux/slab.h>
0011 #include "internal.h"
0012
0013 #define fscache_volume_hash_shift 10
0014 static struct hlist_bl_head fscache_volume_hash[1 << fscache_volume_hash_shift];
0015 static atomic_t fscache_volume_debug_id;
0016 static LIST_HEAD(fscache_volumes);
0017
0018 static void fscache_create_volume_work(struct work_struct *work);
0019
0020 struct fscache_volume *fscache_get_volume(struct fscache_volume *volume,
0021                       enum fscache_volume_trace where)
0022 {
0023     int ref;
0024
0025     __refcount_inc(&volume->ref, &ref);
0026     trace_fscache_volume(volume->debug_id, ref + 1, where);
0027     return volume;
0028 }
0029
0030 static void fscache_see_volume(struct fscache_volume *volume,
0031                    enum fscache_volume_trace where)
0032 {
0033     int ref = refcount_read(&volume->ref);
0034
0035     trace_fscache_volume(volume->debug_id, ref, where);
0036 }
0037
0038 /*
0039  * Pin the cache behind a volume so that we can access it.
0040  */
0041 static void __fscache_begin_volume_access(struct fscache_volume *volume,
0042                       struct fscache_cookie *cookie,
0043                       enum fscache_access_trace why)
0044 {
0045     int n_accesses;
0046
0047     n_accesses = atomic_inc_return(&volume->n_accesses);
0048     smp_mb__after_atomic();
0049     trace_fscache_access_volume(volume->debug_id, cookie ? cookie->debug_id : 0,
0050                     refcount_read(&volume->ref),
0051                     n_accesses, why);
0052 }
0053
0054 /**
0055  * fscache_begin_volume_access - Pin a cache so a volume can be accessed
0056  * @volume: The volume cookie
0057  * @cookie: A datafile cookie for a tracing reference (or NULL)
0058  * @why: An indication of the circumstances of the access for tracing
0059  *
0060  * Attempt to pin the cache to prevent it from going away whilst we're
0061  * accessing a volume and returns true if successful.  This works as follows:
0062  *
0063  *  (1) If the cache tests as not live (state is not FSCACHE_CACHE_IS_ACTIVE),
0064  *      then we return false to indicate access was not permitted.
0065  *
0066  *  (2) If the cache tests as live, then we increment the volume's n_accesses
0067  *      count and then recheck the cache liveness, ending the access if it
0068  *      ceased to be live.
0069  *
0070  *  (3) When we end the access, we decrement the volume's n_accesses and wake
0071  *      up the any waiters if it reaches 0.
0072  *
0073  *  (4) Whilst the cache is caching, the volume's n_accesses is kept
0074  *      artificially incremented to prevent wakeups from happening.
0075  *
0076  *  (5) When the cache is taken offline, the state is changed to prevent new
0077  *      accesses, the volume's n_accesses is decremented and we wait for it to
0078  *      become 0.
0079  *
0080  * The datafile @cookie and the @why indicator are merely provided for tracing
0081  * purposes.
0082  */
0083 bool fscache_begin_volume_access(struct fscache_volume *volume,
0084                  struct fscache_cookie *cookie,
0085                  enum fscache_access_trace why)
0086 {
0087     if (!fscache_cache_is_live(volume->cache))
0088         return false;
0089     __fscache_begin_volume_access(volume, cookie, why);
0090     if (!fscache_cache_is_live(volume->cache)) {
0091         fscache_end_volume_access(volume, cookie, fscache_access_unlive);
0092         return false;
0093     }
0094     return true;
0095 }
0096
0097 /**
0098  * fscache_end_volume_access - Unpin a cache at the end of an access.
0099  * @volume: The volume cookie
0100  * @cookie: A datafile cookie for a tracing reference (or NULL)
0101  * @why: An indication of the circumstances of the access for tracing
0102  *
0103  * Unpin a cache volume after we've accessed it.  The datafile @cookie and the
0104  * @why indicator are merely provided for tracing purposes.
0105  */
0106 void fscache_end_volume_access(struct fscache_volume *volume,
0107                    struct fscache_cookie *cookie,
0108                    enum fscache_access_trace why)
0109 {
0110     int n_accesses;
0111
0112     smp_mb__before_atomic();
0113     n_accesses = atomic_dec_return(&volume->n_accesses);
0114     trace_fscache_access_volume(volume->debug_id, cookie ? cookie->debug_id : 0,
0115                     refcount_read(&volume->ref),
0116                     n_accesses, why);
0117     if (n_accesses == 0)
0118         wake_up_var(&volume->n_accesses);
0119 }
0120 EXPORT_SYMBOL(fscache_end_volume_access);
0121
0122 static bool fscache_volume_same(const struct fscache_volume *a,
0123                 const struct fscache_volume *b)
0124 {
0125     size_t klen;
0126
0127     if (a->key_hash != b->key_hash ||
0128         a->cache    != b->cache ||
0129         a->key[0]   != b->key[0])
0130         return false;
0131
0132     klen = round_up(a->key[0] + 1, sizeof(__le32));
0133     return memcmp(a->key, b->key, klen) == 0;
0134 }
0135
0136 static bool fscache_is_acquire_pending(struct fscache_volume *volume)
0137 {
0138     return test_bit(FSCACHE_VOLUME_ACQUIRE_PENDING, &volume->flags);
0139 }
0140
0141 static void fscache_wait_on_volume_collision(struct fscache_volume *candidate,
0142                          unsigned int collidee_debug_id)
0143 {
0144     wait_var_event_timeout(&candidate->flags,
0145                    !fscache_is_acquire_pending(candidate), 20 * HZ);
0146     if (fscache_is_acquire_pending(candidate)) {
0147         pr_notice("Potential volume collision new=%08x old=%08x",
0148               candidate->debug_id, collidee_debug_id);
0149         fscache_stat(&fscache_n_volumes_collision);
0150         wait_var_event(&candidate->flags, !fscache_is_acquire_pending(candidate));
0151     }
0152 }
0153
0154 /*
0155  * Attempt to insert the new volume into the hash.  If there's a collision, we
0156  * wait for the old volume to complete if it's being relinquished and an error
0157  * otherwise.
0158  */
0159 static bool fscache_hash_volume(struct fscache_volume *candidate)
0160 {
0161     struct fscache_volume *cursor;
0162     struct hlist_bl_head *h;
0163     struct hlist_bl_node *p;
0164     unsigned int bucket, collidee_debug_id = 0;
0165
0166     bucket = candidate->key_hash & (ARRAY_SIZE(fscache_volume_hash) - 1);
0167     h = &fscache_volume_hash[bucket];
0168
0169     hlist_bl_lock(h);
0170     hlist_bl_for_each_entry(cursor, p, h, hash_link) {
0171         if (fscache_volume_same(candidate, cursor)) {
0172             if (!test_bit(FSCACHE_VOLUME_RELINQUISHED, &cursor->flags))
0173                 goto collision;
0174             fscache_see_volume(cursor, fscache_volume_get_hash_collision);
0175             set_bit(FSCACHE_VOLUME_COLLIDED_WITH, &cursor->flags);
0176             set_bit(FSCACHE_VOLUME_ACQUIRE_PENDING, &candidate->flags);
0177             collidee_debug_id = cursor->debug_id;
0178             break;
0179         }
0180     }
0181
0182     hlist_bl_add_head(&candidate->hash_link, h);
0183     hlist_bl_unlock(h);
0184
0185     if (fscache_is_acquire_pending(candidate))
0186         fscache_wait_on_volume_collision(candidate, collidee_debug_id);
0187     return true;
0188
0189 collision:
0190     fscache_see_volume(cursor, fscache_volume_collision);
0191     hlist_bl_unlock(h);
0192     return false;
0193 }
0194
0195 /*
0196  * Allocate and initialise a volume representation cookie.
0197  */
0198 static struct fscache_volume *fscache_alloc_volume(const char *volume_key,
0199                            const char *cache_name,
0200                            const void *coherency_data,
0201                            size_t coherency_len)
0202 {
0203     struct fscache_volume *volume;
0204     struct fscache_cache *cache;
0205     size_t klen, hlen;
0206     char *key;
0207
0208     if (!coherency_data)
0209         coherency_len = 0;
0210
0211     cache = fscache_lookup_cache(cache_name, false);
0212     if (IS_ERR(cache))
0213         return NULL;
0214
0215     volume = kzalloc(struct_size(volume, coherency, coherency_len),
0216              GFP_KERNEL);
0217     if (!volume)
0218         goto err_cache;
0219
0220     volume->cache = cache;
0221     volume->coherency_len = coherency_len;
0222     if (coherency_data)
0223         memcpy(volume->coherency, coherency_data, coherency_len);
0224     INIT_LIST_HEAD(&volume->proc_link);
0225     INIT_WORK(&volume->work, fscache_create_volume_work);
0226     refcount_set(&volume->ref, 1);
0227     spin_lock_init(&volume->lock);
0228
0229     /* Stick the length on the front of the key and pad it out to make
0230      * hashing easier.
0231      */
0232     klen = strlen(volume_key);
0233     hlen = round_up(1 + klen + 1, sizeof(__le32));
0234     key = kzalloc(hlen, GFP_KERNEL);
0235     if (!key)
0236         goto err_vol;
0237     key[0] = klen;
0238     memcpy(key + 1, volume_key, klen);
0239
0240     volume->key = key;
0241     volume->key_hash = fscache_hash(0, key, hlen);
0242
0243     volume->debug_id = atomic_inc_return(&fscache_volume_debug_id);
0244     down_write(&fscache_addremove_sem);
0245     atomic_inc(&cache->n_volumes);
0246     list_add_tail(&volume->proc_link, &fscache_volumes);
0247     fscache_see_volume(volume, fscache_volume_new_acquire);
0248     fscache_stat(&fscache_n_volumes);
0249     up_write(&fscache_addremove_sem);
0250     _leave(" = v=%x", volume->debug_id);
0251     return volume;
0252
0253 err_vol:
0254     kfree(volume);
0255 err_cache:
0256     fscache_put_cache(cache, fscache_cache_put_alloc_volume);
0257     fscache_stat(&fscache_n_volumes_nomem);
0258     return NULL;
0259 }
0260
0261 /*
0262  * Create a volume's representation on disk.  Have a volume ref and a cache
0263  * access we have to release.
0264  */
0265 static void fscache_create_volume_work(struct work_struct *work)
0266 {
0267     const struct fscache_cache_ops *ops;
0268     struct fscache_volume *volume =
0269         container_of(work, struct fscache_volume, work);
0270
0271     fscache_see_volume(volume, fscache_volume_see_create_work);
0272
0273     ops = volume->cache->ops;
0274     if (ops->acquire_volume)
0275         ops->acquire_volume(volume);
0276     fscache_end_cache_access(volume->cache,
0277                  fscache_access_acquire_volume_end);
0278
0279     clear_bit_unlock(FSCACHE_VOLUME_CREATING, &volume->flags);
0280     wake_up_bit(&volume->flags, FSCACHE_VOLUME_CREATING);
0281     fscache_put_volume(volume, fscache_volume_put_create_work);
0282 }
0283
0284 /*
0285  * Dispatch a worker thread to create a volume's representation on disk.
0286  */
0287 void fscache_create_volume(struct fscache_volume *volume, bool wait)
0288 {
0289     if (test_and_set_bit(FSCACHE_VOLUME_CREATING, &volume->flags))
0290         goto maybe_wait;
0291     if (volume->cache_priv)
0292         goto no_wait; /* We raced */
0293     if (!fscache_begin_cache_access(volume->cache,
0294                     fscache_access_acquire_volume))
0295         goto no_wait;
0296
0297     fscache_get_volume(volume, fscache_volume_get_create_work);
0298     if (!schedule_work(&volume->work))
0299         fscache_put_volume(volume, fscache_volume_put_create_work);
0300
0301 maybe_wait:
0302     if (wait) {
0303         fscache_see_volume(volume, fscache_volume_wait_create_work);
0304         wait_on_bit(&volume->flags, FSCACHE_VOLUME_CREATING,
0305                 TASK_UNINTERRUPTIBLE);
0306     }
0307     return;
0308 no_wait:
0309     clear_bit_unlock(FSCACHE_VOLUME_CREATING, &volume->flags);
0310     wake_up_bit(&volume->flags, FSCACHE_VOLUME_CREATING);
0311 }
0312
0313 /*
0314  * Acquire a volume representation cookie and link it to a (proposed) cache.
0315  */
0316 struct fscache_volume *__fscache_acquire_volume(const char *volume_key,
0317                         const char *cache_name,
0318                         const void *coherency_data,
0319                         size_t coherency_len)
0320 {
0321     struct fscache_volume *volume;
0322
0323     volume = fscache_alloc_volume(volume_key, cache_name,
0324                       coherency_data, coherency_len);
0325     if (!volume)
0326         return ERR_PTR(-ENOMEM);
0327
0328     if (!fscache_hash_volume(volume)) {
0329         fscache_put_volume(volume, fscache_volume_put_hash_collision);
0330         return ERR_PTR(-EBUSY);
0331     }
0332
0333     fscache_create_volume(volume, false);
0334     return volume;
0335 }
0336 EXPORT_SYMBOL(__fscache_acquire_volume);
0337
0338 static void fscache_wake_pending_volume(struct fscache_volume *volume,
0339                     struct hlist_bl_head *h)
0340 {
0341     struct fscache_volume *cursor;
0342     struct hlist_bl_node *p;
0343
0344     hlist_bl_for_each_entry(cursor, p, h, hash_link) {
0345         if (fscache_volume_same(cursor, volume)) {
0346             fscache_see_volume(cursor, fscache_volume_see_hash_wake);
0347             clear_bit(FSCACHE_VOLUME_ACQUIRE_PENDING, &cursor->flags);
0348             wake_up_bit(&cursor->flags, FSCACHE_VOLUME_ACQUIRE_PENDING);
0349             return;
0350         }
0351     }
0352 }
0353
0354 /*
0355  * Remove a volume cookie from the hash table.
0356  */
0357 static void fscache_unhash_volume(struct fscache_volume *volume)
0358 {
0359     struct hlist_bl_head *h;
0360     unsigned int bucket;
0361
0362     bucket = volume->key_hash & (ARRAY_SIZE(fscache_volume_hash) - 1);
0363     h = &fscache_volume_hash[bucket];
0364
0365     hlist_bl_lock(h);
0366     hlist_bl_del(&volume->hash_link);
0367     if (test_bit(FSCACHE_VOLUME_COLLIDED_WITH, &volume->flags))
0368         fscache_wake_pending_volume(volume, h);
0369     hlist_bl_unlock(h);
0370 }
0371
0372 /*
0373  * Drop a cache's volume attachments.
0374  */
0375 static void fscache_free_volume(struct fscache_volume *volume)
0376 {
0377     struct fscache_cache *cache = volume->cache;
0378
0379     if (volume->cache_priv) {
0380         __fscache_begin_volume_access(volume, NULL,
0381                           fscache_access_relinquish_volume);
0382         if (volume->cache_priv)
0383             cache->ops->free_volume(volume);
0384         fscache_end_volume_access(volume, NULL,
0385                       fscache_access_relinquish_volume_end);
0386     }
0387
0388     down_write(&fscache_addremove_sem);
0389     list_del_init(&volume->proc_link);
0390     atomic_dec(&volume->cache->n_volumes);
0391     up_write(&fscache_addremove_sem);
0392
0393     if (!hlist_bl_unhashed(&volume->hash_link))
0394         fscache_unhash_volume(volume);
0395
0396     trace_fscache_volume(volume->debug_id, 0, fscache_volume_free);
0397     kfree(volume->key);
0398     kfree(volume);
0399     fscache_stat_d(&fscache_n_volumes);
0400     fscache_put_cache(cache, fscache_cache_put_volume);
0401 }
0402
0403 /*
0404  * Drop a reference to a volume cookie.
0405  */
0406 void fscache_put_volume(struct fscache_volume *volume,
0407             enum fscache_volume_trace where)
0408 {
0409     if (volume) {
0410         unsigned int debug_id = volume->debug_id;
0411         bool zero;
0412         int ref;
0413
0414         zero = __refcount_dec_and_test(&volume->ref, &ref);
0415         trace_fscache_volume(debug_id, ref - 1, where);
0416         if (zero)
0417             fscache_free_volume(volume);
0418     }
0419 }
0420
0421 /*
0422  * Relinquish a volume representation cookie.
0423  */
0424 void __fscache_relinquish_volume(struct fscache_volume *volume,
0425                  const void *coherency_data,
0426                  bool invalidate)
0427 {
0428     if (WARN_ON(test_and_set_bit(FSCACHE_VOLUME_RELINQUISHED, &volume->flags)))
0429         return;
0430
0431     if (invalidate) {
0432         set_bit(FSCACHE_VOLUME_INVALIDATE, &volume->flags);
0433     } else if (coherency_data) {
0434         memcpy(volume->coherency, coherency_data, volume->coherency_len);
0435     }
0436
0437     fscache_put_volume(volume, fscache_volume_put_relinquish);
0438 }
0439 EXPORT_SYMBOL(__fscache_relinquish_volume);
0440
0441 /**
0442  * fscache_withdraw_volume - Withdraw a volume from being cached
0443  * @volume: Volume cookie
0444  *
0445  * Withdraw a cache volume from service, waiting for all accesses to complete
0446  * before returning.
0447  */
0448 void fscache_withdraw_volume(struct fscache_volume *volume)
0449 {
0450     int n_accesses;
0451
0452     _debug("withdraw V=%x", volume->debug_id);
0453
0454     /* Allow wakeups on dec-to-0 */
0455     n_accesses = atomic_dec_return(&volume->n_accesses);
0456     trace_fscache_access_volume(volume->debug_id, 0,
0457                     refcount_read(&volume->ref),
0458                     n_accesses, fscache_access_cache_unpin);
0459
0460     wait_var_event(&volume->n_accesses,
0461                atomic_read(&volume->n_accesses) == 0);
0462 }
0463 EXPORT_SYMBOL(fscache_withdraw_volume);
0464
0465 #ifdef CONFIG_PROC_FS
0466 /*
0467  * Generate a list of volumes in /proc/fs/fscache/volumes
0468  */
0469 static int fscache_volumes_seq_show(struct seq_file *m, void *v)
0470 {
0471     struct fscache_volume *volume;
0472
0473     if (v == &fscache_volumes) {
0474         seq_puts(m,
0475              "VOLUME   REF   nCOOK ACC FL CACHE           KEY\n"
0476              "======== ===== ===== === == =============== ================\n");
0477         return 0;
0478     }
0479
0480     volume = list_entry(v, struct fscache_volume, proc_link);
0481     seq_printf(m,
0482            "%08x %5d %5d %3d %02lx %-15.15s %s\n",
0483            volume->debug_id,
0484            refcount_read(&volume->ref),
0485            atomic_read(&volume->n_cookies),
0486            atomic_read(&volume->n_accesses),
0487            volume->flags,
0488            volume->cache->name ?: "-",
0489            volume->key + 1);
0490     return 0;
0491 }
0492
0493 static void *fscache_volumes_seq_start(struct seq_file *m, loff_t *_pos)
0494     __acquires(&fscache_addremove_sem)
0495 {
0496     down_read(&fscache_addremove_sem);
0497     return seq_list_start_head(&fscache_volumes, *_pos);
0498 }
0499
0500 static void *fscache_volumes_seq_next(struct seq_file *m, void *v, loff_t *_pos)
0501 {
0502     return seq_list_next(v, &fscache_volumes, _pos);
0503 }
0504
0505 static void fscache_volumes_seq_stop(struct seq_file *m, void *v)
0506     __releases(&fscache_addremove_sem)
0507 {
0508     up_read(&fscache_addremove_sem);
0509 }
0510
0511 const struct seq_operations fscache_volumes_seq_ops = {
0512     .start  = fscache_volumes_seq_start,
0513     .next   = fscache_volumes_seq_next,
0514     .stop   = fscache_volumes_seq_stop,
0515     .show   = fscache_volumes_seq_show,
0516 };
0517 #endif /* CONFIG_PROC_FS */