kernel/livepatch/shadow.c

0001 // SPDX-License-Identifier: GPL-2.0-or-later
0002 /*
0003  * shadow.c - Shadow Variables
0004  *
0005  * Copyright (C) 2014 Josh Poimboeuf <jpoimboe@redhat.com>
0006  * Copyright (C) 2014 Seth Jennings <sjenning@redhat.com>
0007  * Copyright (C) 2017 Joe Lawrence <joe.lawrence@redhat.com>
0008  */
0009
0010 /**
0011  * DOC: Shadow variable API concurrency notes:
0012  *
0013  * The shadow variable API provides a simple relationship between an
0014  * <obj, id> pair and a pointer value.  It is the responsibility of the
0015  * caller to provide any mutual exclusion required of the shadow data.
0016  *
0017  * Once a shadow variable is attached to its parent object via the
0018  * klp_shadow_*alloc() API calls, it is considered live: any subsequent
0019  * call to klp_shadow_get() may then return the shadow variable's data
0020  * pointer.  Callers of klp_shadow_*alloc() should prepare shadow data
0021  * accordingly.
0022  *
0023  * The klp_shadow_*alloc() API calls may allocate memory for new shadow
0024  * variable structures.  Their implementation does not call kmalloc
0025  * inside any spinlocks, but API callers should pass GFP flags according
0026  * to their specific needs.
0027  *
0028  * The klp_shadow_hash is an RCU-enabled hashtable and is safe against
0029  * concurrent klp_shadow_free() and klp_shadow_get() operations.
0030  */
0031
0032 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
0033
0034 #include <linux/hashtable.h>
0035 #include <linux/slab.h>
0036 #include <linux/livepatch.h>
0037
0038 static DEFINE_HASHTABLE(klp_shadow_hash, 12);
0039
0040 /*
0041  * klp_shadow_lock provides exclusive access to the klp_shadow_hash and
0042  * the shadow variables it references.
0043  */
0044 static DEFINE_SPINLOCK(klp_shadow_lock);
0045
0046 /**
0047  * struct klp_shadow - shadow variable structure
0048  * @node:   klp_shadow_hash hash table node
0049  * @rcu_head:   RCU is used to safely free this structure
0050  * @obj:    pointer to parent object
0051  * @id:     data identifier
0052  * @data:   data area
0053  */
0054 struct klp_shadow {
0055     struct hlist_node node;
0056     struct rcu_head rcu_head;
0057     void *obj;
0058     unsigned long id;
0059     char data[];
0060 };
0061
0062 /**
0063  * klp_shadow_match() - verify a shadow variable matches given <obj, id>
0064  * @shadow: shadow variable to match
0065  * @obj:    pointer to parent object
0066  * @id:     data identifier
0067  *
0068  * Return: true if the shadow variable matches.
0069  */
0070 static inline bool klp_shadow_match(struct klp_shadow *shadow, void *obj,
0071                 unsigned long id)
0072 {
0073     return shadow->obj == obj && shadow->id == id;
0074 }
0075
0076 /**
0077  * klp_shadow_get() - retrieve a shadow variable data pointer
0078  * @obj:    pointer to parent object
0079  * @id:     data identifier
0080  *
0081  * Return: the shadow variable data element, NULL on failure.
0082  */
0083 void *klp_shadow_get(void *obj, unsigned long id)
0084 {
0085     struct klp_shadow *shadow;
0086
0087     rcu_read_lock();
0088
0089     hash_for_each_possible_rcu(klp_shadow_hash, shadow, node,
0090                    (unsigned long)obj) {
0091
0092         if (klp_shadow_match(shadow, obj, id)) {
0093             rcu_read_unlock();
0094             return shadow->data;
0095         }
0096     }
0097
0098     rcu_read_unlock();
0099
0100     return NULL;
0101 }
0102 EXPORT_SYMBOL_GPL(klp_shadow_get);
0103
0104 static void *__klp_shadow_get_or_alloc(void *obj, unsigned long id,
0105                        size_t size, gfp_t gfp_flags,
0106                        klp_shadow_ctor_t ctor, void *ctor_data,
0107                        bool warn_on_exist)
0108 {
0109     struct klp_shadow *new_shadow;
0110     void *shadow_data;
0111     unsigned long flags;
0112
0113     /* Check if the shadow variable already exists */
0114     shadow_data = klp_shadow_get(obj, id);
0115     if (shadow_data)
0116         goto exists;
0117
0118     /*
0119      * Allocate a new shadow variable.  Fill it with zeroes by default.
0120      * More complex setting can be done by @ctor function.  But it is
0121      * called only when the buffer is really used (under klp_shadow_lock).
0122      */
0123     new_shadow = kzalloc(size + sizeof(*new_shadow), gfp_flags);
0124     if (!new_shadow)
0125         return NULL;
0126
0127     /* Look for <obj, id> again under the lock */
0128     spin_lock_irqsave(&klp_shadow_lock, flags);
0129     shadow_data = klp_shadow_get(obj, id);
0130     if (unlikely(shadow_data)) {
0131         /*
0132          * Shadow variable was found, throw away speculative
0133          * allocation.
0134          */
0135         spin_unlock_irqrestore(&klp_shadow_lock, flags);
0136         kfree(new_shadow);
0137         goto exists;
0138     }
0139
0140     new_shadow->obj = obj;
0141     new_shadow->id = id;
0142
0143     if (ctor) {
0144         int err;
0145
0146         err = ctor(obj, new_shadow->data, ctor_data);
0147         if (err) {
0148             spin_unlock_irqrestore(&klp_shadow_lock, flags);
0149             kfree(new_shadow);
0150             pr_err("Failed to construct shadow variable <%p, %lx> (%d)\n",
0151                    obj, id, err);
0152             return NULL;
0153         }
0154     }
0155
0156     /* No <obj, id> found, so attach the newly allocated one */
0157     hash_add_rcu(klp_shadow_hash, &new_shadow->node,
0158              (unsigned long)new_shadow->obj);
0159     spin_unlock_irqrestore(&klp_shadow_lock, flags);
0160
0161     return new_shadow->data;
0162
0163 exists:
0164     if (warn_on_exist) {
0165         WARN(1, "Duplicate shadow variable <%p, %lx>\n", obj, id);
0166         return NULL;
0167     }
0168
0169     return shadow_data;
0170 }
0171
0172 /**
0173  * klp_shadow_alloc() - allocate and add a new shadow variable
0174  * @obj:    pointer to parent object
0175  * @id:     data identifier
0176  * @size:   size of attached data
0177  * @gfp_flags:  GFP mask for allocation
0178  * @ctor:   custom constructor to initialize the shadow data (optional)
0179  * @ctor_data:  pointer to any data needed by @ctor (optional)
0180  *
0181  * Allocates @size bytes for new shadow variable data using @gfp_flags.
0182  * The data are zeroed by default.  They are further initialized by @ctor
0183  * function if it is not NULL.  The new shadow variable is then added
0184  * to the global hashtable.
0185  *
0186  * If an existing <obj, id> shadow variable can be found, this routine will
0187  * issue a WARN, exit early and return NULL.
0188  *
0189  * This function guarantees that the constructor function is called only when
0190  * the variable did not exist before.  The cost is that @ctor is called
0191  * in atomic context under a spin lock.
0192  *
0193  * Return: the shadow variable data element, NULL on duplicate or
0194  * failure.
0195  */
0196 void *klp_shadow_alloc(void *obj, unsigned long id,
0197                size_t size, gfp_t gfp_flags,
0198                klp_shadow_ctor_t ctor, void *ctor_data)
0199 {
0200     return __klp_shadow_get_or_alloc(obj, id, size, gfp_flags,
0201                      ctor, ctor_data, true);
0202 }
0203 EXPORT_SYMBOL_GPL(klp_shadow_alloc);
0204
0205 /**
0206  * klp_shadow_get_or_alloc() - get existing or allocate a new shadow variable
0207  * @obj:    pointer to parent object
0208  * @id:     data identifier
0209  * @size:   size of attached data
0210  * @gfp_flags:  GFP mask for allocation
0211  * @ctor:   custom constructor to initialize the shadow data (optional)
0212  * @ctor_data:  pointer to any data needed by @ctor (optional)
0213  *
0214  * Returns a pointer to existing shadow data if an <obj, id> shadow
0215  * variable is already present.  Otherwise, it creates a new shadow
0216  * variable like klp_shadow_alloc().
0217  *
0218  * This function guarantees that only one shadow variable exists with the given
0219  * @id for the given @obj.  It also guarantees that the constructor function
0220  * will be called only when the variable did not exist before.  The cost is
0221  * that @ctor is called in atomic context under a spin lock.
0222  *
0223  * Return: the shadow variable data element, NULL on failure.
0224  */
0225 void *klp_shadow_get_or_alloc(void *obj, unsigned long id,
0226                   size_t size, gfp_t gfp_flags,
0227                   klp_shadow_ctor_t ctor, void *ctor_data)
0228 {
0229     return __klp_shadow_get_or_alloc(obj, id, size, gfp_flags,
0230                      ctor, ctor_data, false);
0231 }
0232 EXPORT_SYMBOL_GPL(klp_shadow_get_or_alloc);
0233
0234 static void klp_shadow_free_struct(struct klp_shadow *shadow,
0235                    klp_shadow_dtor_t dtor)
0236 {
0237     hash_del_rcu(&shadow->node);
0238     if (dtor)
0239         dtor(shadow->obj, shadow->data);
0240     kfree_rcu(shadow, rcu_head);
0241 }
0242
0243 /**
0244  * klp_shadow_free() - detach and free a <obj, id> shadow variable
0245  * @obj:    pointer to parent object
0246  * @id:     data identifier
0247  * @dtor:   custom callback that can be used to unregister the variable
0248  *      and/or free data that the shadow variable points to (optional)
0249  *
0250  * This function releases the memory for this <obj, id> shadow variable
0251  * instance, callers should stop referencing it accordingly.
0252  */
0253 void klp_shadow_free(void *obj, unsigned long id, klp_shadow_dtor_t dtor)
0254 {
0255     struct klp_shadow *shadow;
0256     unsigned long flags;
0257
0258     spin_lock_irqsave(&klp_shadow_lock, flags);
0259
0260     /* Delete <obj, id> from hash */
0261     hash_for_each_possible(klp_shadow_hash, shadow, node,
0262                    (unsigned long)obj) {
0263
0264         if (klp_shadow_match(shadow, obj, id)) {
0265             klp_shadow_free_struct(shadow, dtor);
0266             break;
0267         }
0268     }
0269
0270     spin_unlock_irqrestore(&klp_shadow_lock, flags);
0271 }
0272 EXPORT_SYMBOL_GPL(klp_shadow_free);
0273
0274 /**
0275  * klp_shadow_free_all() - detach and free all <_, id> shadow variables
0276  * @id:     data identifier
0277  * @dtor:   custom callback that can be used to unregister the variable
0278  *      and/or free data that the shadow variable points to (optional)
0279  *
0280  * This function releases the memory for all <_, id> shadow variable
0281  * instances, callers should stop referencing them accordingly.
0282  */
0283 void klp_shadow_free_all(unsigned long id, klp_shadow_dtor_t dtor)
0284 {
0285     struct klp_shadow *shadow;
0286     unsigned long flags;
0287     int i;
0288
0289     spin_lock_irqsave(&klp_shadow_lock, flags);
0290
0291     /* Delete all <_, id> from hash */
0292     hash_for_each(klp_shadow_hash, i, shadow, node) {
0293         if (klp_shadow_match(shadow, shadow->obj, id))
0294             klp_shadow_free_struct(shadow, dtor);
0295     }
0296
0297     spin_unlock_irqrestore(&klp_shadow_lock, flags);
0298 }
0299 EXPORT_SYMBOL_GPL(klp_shadow_free_all);