OSCL-LXR linux-6.0.1/​fs/​fscache/​cookie.c	Source navigation Diff markup Identifier search General search
	Version: linux-6.0.1 spark-3.0.0 hadoop-3.2.0-src hadoop-3.3.0-src spark-2.4.7 linux-4.15.13 hadoop-2.7.4-src Revert   Change

 // SPDX-License-Identifier: GPL-2.0-or-later
 /* netfs cookie management
  *
  * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved.
  * Written by David Howells (dhowells@redhat.com)
  *
  * See Documentation/filesystems/caching/netfs-api.rst for more information on
  * the netfs API.
  */
 
 #define FSCACHE_DEBUG_LEVEL COOKIE
 #include <linux/module.h>
 #include <linux/slab.h>
 #include "internal.h"
 
 struct kmem_cache *fscache_cookie_jar;
 
 static void fscache_cookie_lru_timed_out(struct timer_list *timer);
 static void fscache_cookie_lru_worker(struct work_struct *work);
 static void fscache_cookie_worker(struct work_struct *work);
 static void fscache_unhash_cookie(struct fscache_cookie *cookie);
 static void fscache_perform_invalidation(struct fscache_cookie *cookie);
 
 #define fscache_cookie_hash_shift 15
 static struct hlist_bl_head fscache_cookie_hash[1 << fscache_cookie_hash_shift];
 static LIST_HEAD(fscache_cookies);
 static DEFINE_RWLOCK(fscache_cookies_lock);
 static LIST_HEAD(fscache_cookie_lru);
 static DEFINE_SPINLOCK(fscache_cookie_lru_lock);
 DEFINE_TIMER(fscache_cookie_lru_timer, fscache_cookie_lru_timed_out);
 static DECLARE_WORK(fscache_cookie_lru_work, fscache_cookie_lru_worker);
 static const char fscache_cookie_states[FSCACHE_COOKIE_STATE__NR] = "-LCAIFUWRD";
 static unsigned int fscache_lru_cookie_timeout = 10 * HZ;
 
 void fscache_print_cookie(struct fscache_cookie *cookie, char prefix)
 {
     const u8 *k;
 
     pr_err("%c-cookie c=%08x [fl=%lx na=%u nA=%u s=%c]\n",
            prefix,
            cookie->debug_id,
            cookie->flags,
            atomic_read(&cookie->n_active),
            atomic_read(&cookie->n_accesses),
            fscache_cookie_states[cookie->state]);
     pr_err("%c-cookie V=%08x [%s]\n",
            prefix,
            cookie->volume->debug_id,
            cookie->volume->key);
 
     k = (cookie->key_len <= sizeof(cookie->inline_key)) ?
         cookie->inline_key : cookie->key;
     pr_err("%c-key=[%u] '%*phN'\n", prefix, cookie->key_len, cookie->key_len, k);
 }
 
 static void fscache_free_cookie(struct fscache_cookie *cookie)
 {
     if (WARN_ON_ONCE(!list_empty(&cookie->commit_link))) {
         spin_lock(&fscache_cookie_lru_lock);
         list_del_init(&cookie->commit_link);
         spin_unlock(&fscache_cookie_lru_lock);
         fscache_stat_d(&fscache_n_cookies_lru);
         fscache_stat(&fscache_n_cookies_lru_removed);
     }
 
     if (WARN_ON_ONCE(test_bit(FSCACHE_COOKIE_IS_HASHED, &cookie->flags))) {
         fscache_print_cookie(cookie, 'F');
         return;
     }
 
     write_lock(&fscache_cookies_lock);
     list_del(&cookie->proc_link);
     write_unlock(&fscache_cookies_lock);
     if (cookie->aux_len > sizeof(cookie->inline_aux))
         kfree(cookie->aux);
     if (cookie->key_len > sizeof(cookie->inline_key))
         kfree(cookie->key);
     fscache_stat_d(&fscache_n_cookies);
     kmem_cache_free(fscache_cookie_jar, cookie);
 }
 
 static void __fscache_queue_cookie(struct fscache_cookie *cookie)
 {
     if (!queue_work(fscache_wq, &cookie->work))
         fscache_put_cookie(cookie, fscache_cookie_put_over_queued);
 }
 
 static void fscache_queue_cookie(struct fscache_cookie *cookie,
                  enum fscache_cookie_trace where)
 {
     fscache_get_cookie(cookie, where);
     __fscache_queue_cookie(cookie);
 }
 
 /*
  * Initialise the access gate on a cookie by setting a flag to prevent the
  * state machine from being queued when the access counter transitions to 0.
  * We're only interested in this when we withdraw caching services from the
  * cookie.
  */
 static void fscache_init_access_gate(struct fscache_cookie *cookie)
 {
     int n_accesses;
 
     n_accesses = atomic_read(&cookie->n_accesses);
     trace_fscache_access(cookie->debug_id, refcount_read(&cookie->ref),
                  n_accesses, fscache_access_cache_pin);
     set_bit(FSCACHE_COOKIE_NO_ACCESS_WAKE, &cookie->flags);
 }
 
 /**
  * fscache_end_cookie_access - Unpin a cache at the end of an access.
  * @cookie: A data file cookie
  * @why: An indication of the circumstances of the access for tracing
  *
  * Unpin a cache cookie after we've accessed it and bring a deferred
  * relinquishment or withdrawal state into effect.
  *
  * The @why indicator is provided for tracing purposes.
  */
 void fscache_end_cookie_access(struct fscache_cookie *cookie,
                    enum fscache_access_trace why)
 {
     int n_accesses;
 
     smp_mb__before_atomic();
     n_accesses = atomic_dec_return(&cookie->n_accesses);
     trace_fscache_access(cookie->debug_id, refcount_read(&cookie->ref),
                  n_accesses, why);
     if (n_accesses == 0 &&
         !test_bit(FSCACHE_COOKIE_NO_ACCESS_WAKE, &cookie->flags))
         fscache_queue_cookie(cookie, fscache_cookie_get_end_access);
 }
 EXPORT_SYMBOL(fscache_end_cookie_access);
 
 /*
  * Pin the cache behind a cookie so that we can access it.
  */
 static void __fscache_begin_cookie_access(struct fscache_cookie *cookie,
                       enum fscache_access_trace why)
 {
     int n_accesses;
 
     n_accesses = atomic_inc_return(&cookie->n_accesses);
     smp_mb__after_atomic(); /* (Future) read state after is-caching.
                  * Reread n_accesses after is-caching
                  */
     trace_fscache_access(cookie->debug_id, refcount_read(&cookie->ref),
                  n_accesses, why);
 }
 
 /**
  * fscache_begin_cookie_access - Pin a cache so data can be accessed
  * @cookie: A data file cookie
  * @why: An indication of the circumstances of the access for tracing
  *
  * Attempt to pin the cache to prevent it from going away whilst we're
  * accessing data and returns true if successful.  This works as follows:
  *
  *  (1) If the cookie is not being cached (ie. FSCACHE_COOKIE_IS_CACHING is not
  *      set), we return false to indicate access was not permitted.
  *
  *  (2) If the cookie is being cached, we increment its n_accesses count and
  *      then recheck the IS_CACHING flag, ending the access if it got cleared.
  *
  *  (3) When we end the access, we decrement the cookie's n_accesses and wake
  *      up the any waiters if it reaches 0.
  *
  *  (4) Whilst the cookie is actively being cached, its n_accesses is kept
  *      artificially incremented to prevent wakeups from happening.
  *
  *  (5) When the cache is taken offline or if the cookie is culled, the flag is
  *      cleared to prevent new accesses, the cookie's n_accesses is decremented
  *      and we wait for it to become 0.
  *
  * The @why indicator are merely provided for tracing purposes.
  */
 bool fscache_begin_cookie_access(struct fscache_cookie *cookie,
                  enum fscache_access_trace why)
 {
     if (!test_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags))
         return false;
     __fscache_begin_cookie_access(cookie, why);
     if (!test_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags) ||
         !fscache_cache_is_live(cookie->volume->cache)) {
         fscache_end_cookie_access(cookie, fscache_access_unlive);
         return false;
     }
     return true;
 }
 
 static inline void wake_up_cookie_state(struct fscache_cookie *cookie)
 {
     /* Use a barrier to ensure that waiters see the state variable
      * change, as spin_unlock doesn't guarantee a barrier.
      *
      * See comments over wake_up_bit() and waitqueue_active().
      */
     smp_mb();
     wake_up_var(&cookie->state);
 }
 
 /*
  * Change the state a cookie is at and wake up anyone waiting for that.  Impose
  * an ordering between the stuff stored in the cookie and the state member.
  * Paired with fscache_cookie_state().
  */
 static void __fscache_set_cookie_state(struct fscache_cookie *cookie,
                        enum fscache_cookie_state state)
 {
     smp_store_release(&cookie->state, state);
 }
 
 static void fscache_set_cookie_state(struct fscache_cookie *cookie,
                      enum fscache_cookie_state state)
 {
     spin_lock(&cookie->lock);
     __fscache_set_cookie_state(cookie, state);
     spin_unlock(&cookie->lock);
     wake_up_cookie_state(cookie);
 }
 
 /**
  * fscache_cookie_lookup_negative - Note negative lookup
  * @cookie: The cookie that was being looked up
  *
  * Note that some part of the metadata path in the cache doesn't exist and so
  * we can release any waiting readers in the certain knowledge that there's
  * nothing for them to actually read.
  *
  * This function uses no locking and must only be called from the state machine.
  */
 void fscache_cookie_lookup_negative(struct fscache_cookie *cookie)
 {
     set_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags);
     fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_CREATING);
 }
 EXPORT_SYMBOL(fscache_cookie_lookup_negative);
 
 /**
  * fscache_resume_after_invalidation - Allow I/O to resume after invalidation
  * @cookie: The cookie that was invalidated
  *
  * Tell fscache that invalidation is sufficiently complete that I/O can be
  * allowed again.
  */
 void fscache_resume_after_invalidation(struct fscache_cookie *cookie)
 {
     fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_ACTIVE);
 }
 EXPORT_SYMBOL(fscache_resume_after_invalidation);
 
 /**
  * fscache_caching_failed - Report that a failure stopped caching on a cookie
  * @cookie: The cookie that was affected
  *
  * Tell fscache that caching on a cookie needs to be stopped due to some sort
  * of failure.
  *
  * This function uses no locking and must only be called from the state machine.
  */
 void fscache_caching_failed(struct fscache_cookie *cookie)
 {
     clear_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags);
     fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_FAILED);
     trace_fscache_cookie(cookie->debug_id, refcount_read(&cookie->ref),
                 fscache_cookie_failed);
 }
 EXPORT_SYMBOL(fscache_caching_failed);
 
 /*
  * Set the index key in a cookie.  The cookie struct has space for a 16-byte
  * key plus length and hash, but if that's not big enough, it's instead a
  * pointer to a buffer containing 3 bytes of hash, 1 byte of length and then
  * the key data.
  */
 static int fscache_set_key(struct fscache_cookie *cookie,
                const void *index_key, size_t index_key_len)
 {
     void *buf;
     size_t buf_size;
 
     buf_size = round_up(index_key_len, sizeof(__le32));
 
     if (index_key_len > sizeof(cookie->inline_key)) {
         buf = kzalloc(buf_size, GFP_KERNEL);
         if (!buf)
             return -ENOMEM;
         cookie->key = buf;
     } else {
         buf = cookie->inline_key;
     }
 
     memcpy(buf, index_key, index_key_len);
     cookie->key_hash = fscache_hash(cookie->volume->key_hash,
                     buf, buf_size);
     return 0;
 }
 
 static bool fscache_cookie_same(const struct fscache_cookie *a,
                 const struct fscache_cookie *b)
 {
     const void *ka, *kb;
 
     if (a->key_hash != b->key_hash ||
         a->volume   != b->volume ||
         a->key_len  != b->key_len)
         return false;
 
     if (a->key_len <= sizeof(a->inline_key)) {
         ka = &a->inline_key;
         kb = &b->inline_key;
     } else {
         ka = a->key;
         kb = b->key;
     }
     return memcmp(ka, kb, a->key_len) == 0;
 }
 
 static atomic_t fscache_cookie_debug_id = ATOMIC_INIT(1);
 
 /*
  * Allocate a cookie.
  */
 static struct fscache_cookie *fscache_alloc_cookie(
     struct fscache_volume *volume,
     u8 advice,
     const void *index_key, size_t index_key_len,
     const void *aux_data, size_t aux_data_len,
     loff_t object_size)
 {
     struct fscache_cookie *cookie;
 
     /* allocate and initialise a cookie */
     cookie = kmem_cache_zalloc(fscache_cookie_jar, GFP_KERNEL);
     if (!cookie)
         return NULL;
     fscache_stat(&fscache_n_cookies);
 
     cookie->volume      = volume;
     cookie->advice      = advice;
     cookie->key_len     = index_key_len;
     cookie->aux_len     = aux_data_len;
     cookie->object_size = object_size;
     if (object_size == 0)
         __set_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags);
 
     if (fscache_set_key(cookie, index_key, index_key_len) < 0)
         goto nomem;
 
     if (cookie->aux_len <= sizeof(cookie->inline_aux)) {
         memcpy(cookie->inline_aux, aux_data, cookie->aux_len);
     } else {
         cookie->aux = kmemdup(aux_data, cookie->aux_len, GFP_KERNEL);
         if (!cookie->aux)
             goto nomem;
     }
 
     refcount_set(&cookie->ref, 1);
     cookie->debug_id = atomic_inc_return(&fscache_cookie_debug_id);
     spin_lock_init(&cookie->lock);
     INIT_LIST_HEAD(&cookie->commit_link);
     INIT_WORK(&cookie->work, fscache_cookie_worker);
     __fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_QUIESCENT);
 
     write_lock(&fscache_cookies_lock);
     list_add_tail(&cookie->proc_link, &fscache_cookies);
     write_unlock(&fscache_cookies_lock);
     fscache_see_cookie(cookie, fscache_cookie_new_acquire);
     return cookie;
 
 nomem:
     fscache_free_cookie(cookie);
     return NULL;
 }
 
 static inline bool fscache_cookie_is_dropped(struct fscache_cookie *cookie)
 {
     return READ_ONCE(cookie->state) == FSCACHE_COOKIE_STATE_DROPPED;
 }
 
 static void fscache_wait_on_collision(struct fscache_cookie *candidate,
                       struct fscache_cookie *wait_for)
 {
     enum fscache_cookie_state *statep = &wait_for->state;
 
     wait_var_event_timeout(statep, fscache_cookie_is_dropped(wait_for),
                    20 * HZ);
     if (!fscache_cookie_is_dropped(wait_for)) {
         pr_notice("Potential collision c=%08x old: c=%08x",
               candidate->debug_id, wait_for->debug_id);
         wait_var_event(statep, fscache_cookie_is_dropped(wait_for));
     }
 }
 
 /*
  * Attempt to insert the new cookie into the hash.  If there's a collision, we
  * wait for the old cookie to complete if it's being relinquished and an error
  * otherwise.
  */
 static bool fscache_hash_cookie(struct fscache_cookie *candidate)
 {
     struct fscache_cookie *cursor, *wait_for = NULL;
     struct hlist_bl_head *h;
     struct hlist_bl_node *p;
     unsigned int bucket;
 
     bucket = candidate->key_hash & (ARRAY_SIZE(fscache_cookie_hash) - 1);
     h = &fscache_cookie_hash[bucket];
 
     hlist_bl_lock(h);
     hlist_bl_for_each_entry(cursor, p, h, hash_link) {
         if (fscache_cookie_same(candidate, cursor)) {
             if (!test_bit(FSCACHE_COOKIE_RELINQUISHED, &cursor->flags))
                 goto collision;
             wait_for = fscache_get_cookie(cursor,
                               fscache_cookie_get_hash_collision);
             break;
         }
     }
 
     fscache_get_volume(candidate->volume, fscache_volume_get_cookie);
     atomic_inc(&candidate->volume->n_cookies);
     hlist_bl_add_head(&candidate->hash_link, h);
     set_bit(FSCACHE_COOKIE_IS_HASHED, &candidate->flags);
     hlist_bl_unlock(h);
 
     if (wait_for) {
         fscache_wait_on_collision(candidate, wait_for);
         fscache_put_cookie(wait_for, fscache_cookie_put_hash_collision);
     }
     return true;
 
 collision:
     trace_fscache_cookie(cursor->debug_id, refcount_read(&cursor->ref),
                  fscache_cookie_collision);
     pr_err("Duplicate cookie detected\n");
     fscache_print_cookie(cursor, 'O');
     fscache_print_cookie(candidate, 'N');
     hlist_bl_unlock(h);
     return false;
 }
 
 /*
  * Request a cookie to represent a data storage object within a volume.
  *
  * We never let on to the netfs about errors.  We may set a negative cookie
  * pointer, but that's okay
  */
 struct fscache_cookie *__fscache_acquire_cookie(
     struct fscache_volume *volume,
     u8 advice,
     const void *index_key, size_t index_key_len,
     const void *aux_data, size_t aux_data_len,
     loff_t object_size)
 {
     struct fscache_cookie *cookie;
 
     _enter("V=%x", volume->debug_id);
 
     if (!index_key || !index_key_len || index_key_len > 255 || aux_data_len > 255)
         return NULL;
     if (!aux_data || !aux_data_len) {
         aux_data = NULL;
         aux_data_len = 0;
     }
 
     fscache_stat(&fscache_n_acquires);
 
     cookie = fscache_alloc_cookie(volume, advice,
                       index_key, index_key_len,
                       aux_data, aux_data_len,
                       object_size);
     if (!cookie) {
         fscache_stat(&fscache_n_acquires_oom);
         return NULL;
     }
 
     if (!fscache_hash_cookie(cookie)) {
         fscache_see_cookie(cookie, fscache_cookie_discard);
         fscache_free_cookie(cookie);
         return NULL;
     }
 
     trace_fscache_acquire(cookie);
     fscache_stat(&fscache_n_acquires_ok);
     _leave(" = c=%08x", cookie->debug_id);
     return cookie;
 }
 EXPORT_SYMBOL(__fscache_acquire_cookie);
 
 /*
  * Prepare a cache object to be written to.
  */
 static void fscache_prepare_to_write(struct fscache_cookie *cookie)
 {
     cookie->volume->cache->ops->prepare_to_write(cookie);
 }
 
 /*
  * Look up a cookie in the cache.
  */
 static void fscache_perform_lookup(struct fscache_cookie *cookie)
 {
     enum fscache_access_trace trace = fscache_access_lookup_cookie_end_failed;
     bool need_withdraw = false;
 
     _enter("");
 
     if (!cookie->volume->cache_priv) {
         fscache_create_volume(cookie->volume, true);
         if (!cookie->volume->cache_priv) {
             fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_QUIESCENT);
             goto out;
         }
     }
 
     if (!cookie->volume->cache->ops->lookup_cookie(cookie)) {
         if (cookie->state != FSCACHE_COOKIE_STATE_FAILED)
             fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_QUIESCENT);
         need_withdraw = true;
         _leave(" [fail]");
         goto out;
     }
 
     fscache_see_cookie(cookie, fscache_cookie_see_active);
     spin_lock(&cookie->lock);
     if (test_and_clear_bit(FSCACHE_COOKIE_DO_INVALIDATE, &cookie->flags))
         __fscache_set_cookie_state(cookie,
                        FSCACHE_COOKIE_STATE_INVALIDATING);
     else
         __fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_ACTIVE);
     spin_unlock(&cookie->lock);
     wake_up_cookie_state(cookie);
     trace = fscache_access_lookup_cookie_end;
 
 out:
     fscache_end_cookie_access(cookie, trace);
     if (need_withdraw)
         fscache_withdraw_cookie(cookie);
     fscache_end_volume_access(cookie->volume, cookie, trace);
 }
 
 /*
  * Begin the process of looking up a cookie.  We offload the actual process to
  * a worker thread.
  */
 static bool fscache_begin_lookup(struct fscache_cookie *cookie, bool will_modify)
 {
     if (will_modify) {
         set_bit(FSCACHE_COOKIE_LOCAL_WRITE, &cookie->flags);
         set_bit(FSCACHE_COOKIE_DO_PREP_TO_WRITE, &cookie->flags);
     }
     if (!fscache_begin_volume_access(cookie->volume, cookie,
                      fscache_access_lookup_cookie))
         return false;
 
     __fscache_begin_cookie_access(cookie, fscache_access_lookup_cookie);
     __fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_LOOKING_UP);
     set_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags);
     set_bit(FSCACHE_COOKIE_HAS_BEEN_CACHED, &cookie->flags);
     return true;
 }
 
 /*
  * Start using the cookie for I/O.  This prevents the backing object from being
  * reaped by VM pressure.
  */
 void __fscache_use_cookie(struct fscache_cookie *cookie, bool will_modify)
 {
     enum fscache_cookie_state state;
     bool queue = false;
     int n_active;
 
     _enter("c=%08x", cookie->debug_id);
 
     if (WARN(test_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags),
          "Trying to use relinquished cookie\n"))
         return;
 
     spin_lock(&cookie->lock);
 
     n_active = atomic_inc_return(&cookie->n_active);
     trace_fscache_active(cookie->debug_id, refcount_read(&cookie->ref),
                  n_active, atomic_read(&cookie->n_accesses),
                  will_modify ?
                  fscache_active_use_modify : fscache_active_use);
 
 again:
     state = fscache_cookie_state(cookie);
     switch (state) {
     case FSCACHE_COOKIE_STATE_QUIESCENT:
         queue = fscache_begin_lookup(cookie, will_modify);
         break;
 
     case FSCACHE_COOKIE_STATE_LOOKING_UP:
     case FSCACHE_COOKIE_STATE_CREATING:
         if (will_modify)
             set_bit(FSCACHE_COOKIE_LOCAL_WRITE, &cookie->flags);
         break;
     case FSCACHE_COOKIE_STATE_ACTIVE:
     case FSCACHE_COOKIE_STATE_INVALIDATING:
         if (will_modify &&
             !test_and_set_bit(FSCACHE_COOKIE_LOCAL_WRITE, &cookie->flags)) {
             set_bit(FSCACHE_COOKIE_DO_PREP_TO_WRITE, &cookie->flags);
             queue = true;
         }
         break;
 
     case FSCACHE_COOKIE_STATE_FAILED:
     case FSCACHE_COOKIE_STATE_WITHDRAWING:
         break;
 
     case FSCACHE_COOKIE_STATE_LRU_DISCARDING:
         spin_unlock(&cookie->lock);
         wait_var_event(&cookie->state,
                    fscache_cookie_state(cookie) !=
                    FSCACHE_COOKIE_STATE_LRU_DISCARDING);
         spin_lock(&cookie->lock);
         goto again;
 
     case FSCACHE_COOKIE_STATE_DROPPED:
     case FSCACHE_COOKIE_STATE_RELINQUISHING:
         WARN(1, "Can't use cookie in state %u\n", state);
         break;
     }
 
     spin_unlock(&cookie->lock);
     if (queue)
         fscache_queue_cookie(cookie, fscache_cookie_get_use_work);
     _leave("");
 }
 EXPORT_SYMBOL(__fscache_use_cookie);
 
 static void fscache_unuse_cookie_locked(struct fscache_cookie *cookie)
 {
     clear_bit(FSCACHE_COOKIE_DISABLED, &cookie->flags);
     if (!test_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags))
         return;
 
     cookie->unused_at = jiffies;
     spin_lock(&fscache_cookie_lru_lock);
     if (list_empty(&cookie->commit_link)) {
         fscache_get_cookie(cookie, fscache_cookie_get_lru);
         fscache_stat(&fscache_n_cookies_lru);
     }
     list_move_tail(&cookie->commit_link, &fscache_cookie_lru);
 
     spin_unlock(&fscache_cookie_lru_lock);
     timer_reduce(&fscache_cookie_lru_timer,
              jiffies + fscache_lru_cookie_timeout);
 }
 
 /*
  * Stop using the cookie for I/O.
  */
 void __fscache_unuse_cookie(struct fscache_cookie *cookie,
                 const void *aux_data, const loff_t *object_size)
 {
     unsigned int debug_id = cookie->debug_id;
     unsigned int r = refcount_read(&cookie->ref);
     unsigned int a = atomic_read(&cookie->n_accesses);
     unsigned int c;
 
     if (aux_data || object_size)
         __fscache_update_cookie(cookie, aux_data, object_size);
 
     /* Subtract 1 from counter unless that drops it to 0 (ie. it was 1) */
     c = atomic_fetch_add_unless(&cookie->n_active, -1, 1);
     if (c != 1) {
         trace_fscache_active(debug_id, r, c - 1, a, fscache_active_unuse);
         return;
     }
 
     spin_lock(&cookie->lock);
     r = refcount_read(&cookie->ref);
     a = atomic_read(&cookie->n_accesses);
     c = atomic_dec_return(&cookie->n_active);
     trace_fscache_active(debug_id, r, c, a, fscache_active_unuse);
     if (c == 0)
         fscache_unuse_cookie_locked(cookie);
     spin_unlock(&cookie->lock);
 }
 EXPORT_SYMBOL(__fscache_unuse_cookie);
 
 /*
  * Perform work upon the cookie, such as committing its cache state,
  * relinquishing it or withdrawing the backing cache.  We're protected from the
  * cache going away under us as object withdrawal must come through this
  * non-reentrant work item.
  */
 static void fscache_cookie_state_machine(struct fscache_cookie *cookie)
 {
     enum fscache_cookie_state state;
     bool wake = false;
 
     _enter("c=%x", cookie->debug_id);
 
 again:
     spin_lock(&cookie->lock);
 again_locked:
     state = cookie->state;
     switch (state) {
     case FSCACHE_COOKIE_STATE_QUIESCENT:
         /* The QUIESCENT state is jumped to the LOOKING_UP state by
          * fscache_use_cookie().
          */
 
         if (atomic_read(&cookie->n_accesses) == 0 &&
             test_bit(FSCACHE_COOKIE_DO_RELINQUISH, &cookie->flags)) {
             __fscache_set_cookie_state(cookie,
                            FSCACHE_COOKIE_STATE_RELINQUISHING);
             wake = true;
             goto again_locked;
         }
         break;
 
     case FSCACHE_COOKIE_STATE_LOOKING_UP:
         spin_unlock(&cookie->lock);
         fscache_init_access_gate(cookie);
         fscache_perform_lookup(cookie);
         goto again;
 
     case FSCACHE_COOKIE_STATE_INVALIDATING:
         spin_unlock(&cookie->lock);
         fscache_perform_invalidation(cookie);
         goto again;
 
     case FSCACHE_COOKIE_STATE_ACTIVE:
         if (test_and_clear_bit(FSCACHE_COOKIE_DO_PREP_TO_WRITE, &cookie->flags)) {
             spin_unlock(&cookie->lock);
             fscache_prepare_to_write(cookie);
             spin_lock(&cookie->lock);
         }
         if (test_bit(FSCACHE_COOKIE_DO_LRU_DISCARD, &cookie->flags)) {
             __fscache_set_cookie_state(cookie,
                            FSCACHE_COOKIE_STATE_LRU_DISCARDING);
             wake = true;
             goto again_locked;
         }
         fallthrough;
 
     case FSCACHE_COOKIE_STATE_FAILED:
         if (test_and_clear_bit(FSCACHE_COOKIE_DO_INVALIDATE, &cookie->flags))
             fscache_end_cookie_access(cookie, fscache_access_invalidate_cookie_end);
 
         if (atomic_read(&cookie->n_accesses) != 0)
             break;
         if (test_bit(FSCACHE_COOKIE_DO_RELINQUISH, &cookie->flags)) {
             __fscache_set_cookie_state(cookie,
                            FSCACHE_COOKIE_STATE_RELINQUISHING);
             wake = true;
             goto again_locked;
         }
         if (test_bit(FSCACHE_COOKIE_DO_WITHDRAW, &cookie->flags)) {
             __fscache_set_cookie_state(cookie,
                            FSCACHE_COOKIE_STATE_WITHDRAWING);
             wake = true;
             goto again_locked;
         }
         break;
 
     case FSCACHE_COOKIE_STATE_LRU_DISCARDING:
     case FSCACHE_COOKIE_STATE_RELINQUISHING:
     case FSCACHE_COOKIE_STATE_WITHDRAWING:
         if (cookie->cache_priv) {
             spin_unlock(&cookie->lock);
             cookie->volume->cache->ops->withdraw_cookie(cookie);
             spin_lock(&cookie->lock);
         }
 
         if (test_and_clear_bit(FSCACHE_COOKIE_DO_INVALIDATE, &cookie->flags))
             fscache_end_cookie_access(cookie, fscache_access_invalidate_cookie_end);
 
         switch (state) {
         case FSCACHE_COOKIE_STATE_RELINQUISHING:
             fscache_see_cookie(cookie, fscache_cookie_see_relinquish);
             fscache_unhash_cookie(cookie);
             __fscache_set_cookie_state(cookie,
                            FSCACHE_COOKIE_STATE_DROPPED);
             wake = true;
             goto out;
         case FSCACHE_COOKIE_STATE_LRU_DISCARDING:
             fscache_see_cookie(cookie, fscache_cookie_see_lru_discard);
             break;
         case FSCACHE_COOKIE_STATE_WITHDRAWING:
             fscache_see_cookie(cookie, fscache_cookie_see_withdraw);
             break;
         default:
             BUG();
         }
 
         clear_bit(FSCACHE_COOKIE_NEEDS_UPDATE, &cookie->flags);
         clear_bit(FSCACHE_COOKIE_DO_WITHDRAW, &cookie->flags);
         clear_bit(FSCACHE_COOKIE_DO_LRU_DISCARD, &cookie->flags);
         clear_bit(FSCACHE_COOKIE_DO_PREP_TO_WRITE, &cookie->flags);
         set_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags);
         __fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_QUIESCENT);
         wake = true;
         goto again_locked;
 
     case FSCACHE_COOKIE_STATE_DROPPED:
         break;
 
     default:
         WARN_ONCE(1, "Cookie %x in unexpected state %u\n",
               cookie->debug_id, state);
         break;
     }
 
 out:
     spin_unlock(&cookie->lock);
     if (wake)
         wake_up_cookie_state(cookie);
     _leave("");
 }
 
 static void fscache_cookie_worker(struct work_struct *work)
 {
     struct fscache_cookie *cookie = container_of(work, struct fscache_cookie, work);
 
     fscache_see_cookie(cookie, fscache_cookie_see_work);
     fscache_cookie_state_machine(cookie);
     fscache_put_cookie(cookie, fscache_cookie_put_work);
 }
 
 /*
  * Wait for the object to become inactive.  The cookie's work item will be
  * scheduled when someone transitions n_accesses to 0 - but if someone's
  * already done that, schedule it anyway.
  */
 static void __fscache_withdraw_cookie(struct fscache_cookie *cookie)
 {
     int n_accesses;
     bool unpinned;
 
     unpinned = test_and_clear_bit(FSCACHE_COOKIE_NO_ACCESS_WAKE, &cookie->flags);
 
     /* Need to read the access count after unpinning */
     n_accesses = atomic_read(&cookie->n_accesses);
     if (unpinned)
         trace_fscache_access(cookie->debug_id, refcount_read(&cookie->ref),
                      n_accesses, fscache_access_cache_unpin);
     if (n_accesses == 0)
         fscache_queue_cookie(cookie, fscache_cookie_get_end_access);
 }
 
 static void fscache_cookie_lru_do_one(struct fscache_cookie *cookie)
 {
     fscache_see_cookie(cookie, fscache_cookie_see_lru_do_one);
 
     spin_lock(&cookie->lock);
     if (cookie->state != FSCACHE_COOKIE_STATE_ACTIVE ||
         time_before(jiffies, cookie->unused_at + fscache_lru_cookie_timeout) ||
         atomic_read(&cookie->n_active) > 0) {
         spin_unlock(&cookie->lock);
         fscache_stat(&fscache_n_cookies_lru_removed);
     } else {
         set_bit(FSCACHE_COOKIE_DO_LRU_DISCARD, &cookie->flags);
         spin_unlock(&cookie->lock);
         fscache_stat(&fscache_n_cookies_lru_expired);
         _debug("lru c=%x", cookie->debug_id);
         __fscache_withdraw_cookie(cookie);
     }
 
     fscache_put_cookie(cookie, fscache_cookie_put_lru);
 }
 
 static void fscache_cookie_lru_worker(struct work_struct *work)
 {
     struct fscache_cookie *cookie;
     unsigned long unused_at;
 
     spin_lock(&fscache_cookie_lru_lock);
 
     while (!list_empty(&fscache_cookie_lru)) {
         cookie = list_first_entry(&fscache_cookie_lru,
                       struct fscache_cookie, commit_link);
         unused_at = cookie->unused_at + fscache_lru_cookie_timeout;
         if (time_before(jiffies, unused_at)) {
             timer_reduce(&fscache_cookie_lru_timer, unused_at);
             break;
         }
 
         list_del_init(&cookie->commit_link);
         fscache_stat_d(&fscache_n_cookies_lru);
         spin_unlock(&fscache_cookie_lru_lock);
         fscache_cookie_lru_do_one(cookie);
         spin_lock(&fscache_cookie_lru_lock);
     }
 
     spin_unlock(&fscache_cookie_lru_lock);
 }
 
 static void fscache_cookie_lru_timed_out(struct timer_list *timer)
 {
     queue_work(fscache_wq, &fscache_cookie_lru_work);
 }
 
 static void fscache_cookie_drop_from_lru(struct fscache_cookie *cookie)
 {
     bool need_put = false;
 
     if (!list_empty(&cookie->commit_link)) {
         spin_lock(&fscache_cookie_lru_lock);
         if (!list_empty(&cookie->commit_link)) {
             list_del_init(&cookie->commit_link);
             fscache_stat_d(&fscache_n_cookies_lru);
             fscache_stat(&fscache_n_cookies_lru_dropped);
             need_put = true;
         }
         spin_unlock(&fscache_cookie_lru_lock);
         if (need_put)
             fscache_put_cookie(cookie, fscache_cookie_put_lru);
     }
 }
 
 /*
  * Remove a cookie from the hash table.
  */
 static void fscache_unhash_cookie(struct fscache_cookie *cookie)
 {
     struct hlist_bl_head *h;
     unsigned int bucket;
 
     bucket = cookie->key_hash & (ARRAY_SIZE(fscache_cookie_hash) - 1);
     h = &fscache_cookie_hash[bucket];
 
     hlist_bl_lock(h);
     hlist_bl_del(&cookie->hash_link);
     clear_bit(FSCACHE_COOKIE_IS_HASHED, &cookie->flags);
     hlist_bl_unlock(h);
     fscache_stat(&fscache_n_relinquishes_dropped);
 }
 
 static void fscache_drop_withdraw_cookie(struct fscache_cookie *cookie)
 {
     fscache_cookie_drop_from_lru(cookie);
     __fscache_withdraw_cookie(cookie);
 }
 
 /**
  * fscache_withdraw_cookie - Mark a cookie for withdrawal
  * @cookie: The cookie to be withdrawn.
  *
  * Allow the cache backend to withdraw the backing for a cookie for its own
  * reasons, even if that cookie is in active use.
  */
 void fscache_withdraw_cookie(struct fscache_cookie *cookie)
 {
     set_bit(FSCACHE_COOKIE_DO_WITHDRAW, &cookie->flags);
     fscache_drop_withdraw_cookie(cookie);
 }
 EXPORT_SYMBOL(fscache_withdraw_cookie);
 
 /*
  * Allow the netfs to release a cookie back to the cache.
  * - the object will be marked as recyclable on disk if retire is true
  */
 void __fscache_relinquish_cookie(struct fscache_cookie *cookie, bool retire)
 {
     fscache_stat(&fscache_n_relinquishes);
     if (retire)
         fscache_stat(&fscache_n_relinquishes_retire);
 
     _enter("c=%08x{%d},%d",
            cookie->debug_id, atomic_read(&cookie->n_active), retire);
 
     if (WARN(test_and_set_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags),
          "Cookie c=%x already relinquished\n", cookie->debug_id))
         return;
 
     if (retire)
         set_bit(FSCACHE_COOKIE_RETIRED, &cookie->flags);
     trace_fscache_relinquish(cookie, retire);
 
     ASSERTCMP(atomic_read(&cookie->n_active), ==, 0);
     ASSERTCMP(atomic_read(&cookie->volume->n_cookies), >, 0);
     atomic_dec(&cookie->volume->n_cookies);
 
     if (test_bit(FSCACHE_COOKIE_HAS_BEEN_CACHED, &cookie->flags)) {
         set_bit(FSCACHE_COOKIE_DO_RELINQUISH, &cookie->flags);
         fscache_drop_withdraw_cookie(cookie);
     } else {
         fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_DROPPED);
         fscache_unhash_cookie(cookie);
     }
     fscache_put_cookie(cookie, fscache_cookie_put_relinquish);
 }
 EXPORT_SYMBOL(__fscache_relinquish_cookie);
 
 /*
  * Drop a reference to a cookie.
  */
 void fscache_put_cookie(struct fscache_cookie *cookie,
             enum fscache_cookie_trace where)
 {
     struct fscache_volume *volume = cookie->volume;
     unsigned int cookie_debug_id = cookie->debug_id;
     bool zero;
     int ref;
 
     zero = __refcount_dec_and_test(&cookie->ref, &ref);
     trace_fscache_cookie(cookie_debug_id, ref - 1, where);
     if (zero) {
         fscache_free_cookie(cookie);
         fscache_put_volume(volume, fscache_volume_put_cookie);
     }
 }
 EXPORT_SYMBOL(fscache_put_cookie);
 
 /*
  * Get a reference to a cookie.
  */
 struct fscache_cookie *fscache_get_cookie(struct fscache_cookie *cookie,
                       enum fscache_cookie_trace where)
 {
     int ref;
 
     __refcount_inc(&cookie->ref, &ref);
     trace_fscache_cookie(cookie->debug_id, ref + 1, where);
     return cookie;
 }
 EXPORT_SYMBOL(fscache_get_cookie);
 
 /*
  * Ask the cache to effect invalidation of a cookie.
  */
 static void fscache_perform_invalidation(struct fscache_cookie *cookie)
 {
     if (!cookie->volume->cache->ops->invalidate_cookie(cookie))
         fscache_caching_failed(cookie);
     fscache_end_cookie_access(cookie, fscache_access_invalidate_cookie_end);
 }
 
 /*
  * Invalidate an object.
  */
 void __fscache_invalidate(struct fscache_cookie *cookie,
               const void *aux_data, loff_t new_size,
               unsigned int flags)
 {
     bool is_caching;
 
     _enter("c=%x", cookie->debug_id);
 
     fscache_stat(&fscache_n_invalidates);
 
     if (WARN(test_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags),
          "Trying to invalidate relinquished cookie\n"))
         return;
 
     if ((flags & FSCACHE_INVAL_DIO_WRITE) &&
         test_and_set_bit(FSCACHE_COOKIE_DISABLED, &cookie->flags))
         return;
 
     spin_lock(&cookie->lock);
     set_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags);
     fscache_update_aux(cookie, aux_data, &new_size);
     cookie->inval_counter++;
     trace_fscache_invalidate(cookie, new_size);
 
     switch (cookie->state) {
     case FSCACHE_COOKIE_STATE_INVALIDATING: /* is_still_valid will catch it */
     default:
         spin_unlock(&cookie->lock);
         _leave(" [no %u]", cookie->state);
         return;
 
     case FSCACHE_COOKIE_STATE_LOOKING_UP:
         if (!test_and_set_bit(FSCACHE_COOKIE_DO_INVALIDATE, &cookie->flags))
             __fscache_begin_cookie_access(cookie, fscache_access_invalidate_cookie);
         fallthrough;
     case FSCACHE_COOKIE_STATE_CREATING:
         spin_unlock(&cookie->lock);
         _leave(" [look %x]", cookie->inval_counter);
         return;
 
     case FSCACHE_COOKIE_STATE_ACTIVE:
         is_caching = fscache_begin_cookie_access(
             cookie, fscache_access_invalidate_cookie);
         if (is_caching)
             __fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_INVALIDATING);
         spin_unlock(&cookie->lock);
         wake_up_cookie_state(cookie);
 
         if (is_caching)
             fscache_queue_cookie(cookie, fscache_cookie_get_inval_work);
         _leave(" [inv]");
         return;
     }
 }
 EXPORT_SYMBOL(__fscache_invalidate);
 
 #ifdef CONFIG_PROC_FS
 /*
  * Generate a list of extant cookies in /proc/fs/fscache/cookies
  */
 static int fscache_cookies_seq_show(struct seq_file *m, void *v)
 {
     struct fscache_cookie *cookie;
     unsigned int keylen = 0, auxlen = 0;
     u8 *p;
 
     if (v == &fscache_cookies) {
         seq_puts(m,
              "COOKIE   VOLUME   REF ACT ACC S FL DEF             \n"
              "======== ======== === === === = == ================\n"
              );
         return 0;
     }
 
     cookie = list_entry(v, struct fscache_cookie, proc_link);
 
     seq_printf(m,
            "%08x %08x %3d %3d %3d %c %02lx",
            cookie->debug_id,
            cookie->volume->debug_id,
            refcount_read(&cookie->ref),
            atomic_read(&cookie->n_active),
            atomic_read(&cookie->n_accesses),
            fscache_cookie_states[cookie->state],
            cookie->flags);
 
     keylen = cookie->key_len;
     auxlen = cookie->aux_len;
 
     if (keylen > 0 || auxlen > 0) {
         seq_puts(m, " ");
         p = keylen <= sizeof(cookie->inline_key) ?
             cookie->inline_key : cookie->key;
         for (; keylen > 0; keylen--)
             seq_printf(m, "%02x", *p++);
         if (auxlen > 0) {
             seq_puts(m, ", ");
             p = auxlen <= sizeof(cookie->inline_aux) ?
                 cookie->inline_aux : cookie->aux;
             for (; auxlen > 0; auxlen--)
                 seq_printf(m, "%02x", *p++);
         }
     }
 
     seq_puts(m, "\n");
     return 0;
 }
 
 static void *fscache_cookies_seq_start(struct seq_file *m, loff_t *_pos)
     __acquires(fscache_cookies_lock)
 {
     read_lock(&fscache_cookies_lock);
     return seq_list_start_head(&fscache_cookies, *_pos);
 }
 
 static void *fscache_cookies_seq_next(struct seq_file *m, void *v, loff_t *_pos)
 {
     return seq_list_next(v, &fscache_cookies, _pos);
 }
 
 static void fscache_cookies_seq_stop(struct seq_file *m, void *v)
     __releases(rcu)
 {
     read_unlock(&fscache_cookies_lock);
 }
 
 
 const struct seq_operations fscache_cookies_seq_ops = {
     .start  = fscache_cookies_seq_start,
     .next   = fscache_cookies_seq_next,
     .stop   = fscache_cookies_seq_stop,
     .show   = fscache_cookies_seq_show,
 };
 #endif

This page was automatically generated by the 2.1.0 LXR engine. The LXR team