the-tree/mm/swap_cgroup.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 #include <linux/swap_cgroup.h>
0003 #include <linux/vmalloc.h>
0004 #include <linux/mm.h>
0005
0006 #include <linux/swapops.h> /* depends on mm.h include */
0007
0008 static DEFINE_MUTEX(swap_cgroup_mutex);
0009 struct swap_cgroup_ctrl {
0010     struct page **map;
0011     unsigned long length;
0012     spinlock_t  lock;
0013 };
0014
0015 static struct swap_cgroup_ctrl swap_cgroup_ctrl[MAX_SWAPFILES];
0016
0017 struct swap_cgroup {
0018     unsigned short      id;
0019 };
0020 #define SC_PER_PAGE (PAGE_SIZE/sizeof(struct swap_cgroup))
0021
0022 /*
0023  * SwapCgroup implements "lookup" and "exchange" operations.
0024  * In typical usage, this swap_cgroup is accessed via memcg's charge/uncharge
0025  * against SwapCache. At swap_free(), this is accessed directly from swap.
0026  *
0027  * This means,
0028  *  - we have no race in "exchange" when we're accessed via SwapCache because
0029  *    SwapCache(and its swp_entry) is under lock.
0030  *  - When called via swap_free(), there is no user of this entry and no race.
0031  * Then, we don't need lock around "exchange".
0032  *
0033  * TODO: we can push these buffers out to HIGHMEM.
0034  */
0035
0036 /*
0037  * allocate buffer for swap_cgroup.
0038  */
0039 static int swap_cgroup_prepare(int type)
0040 {
0041     struct page *page;
0042     struct swap_cgroup_ctrl *ctrl;
0043     unsigned long idx, max;
0044
0045     ctrl = &swap_cgroup_ctrl[type];
0046
0047     for (idx = 0; idx < ctrl->length; idx++) {
0048         page = alloc_page(GFP_KERNEL | __GFP_ZERO);
0049         if (!page)
0050             goto not_enough_page;
0051         ctrl->map[idx] = page;
0052
0053         if (!(idx % SWAP_CLUSTER_MAX))
0054             cond_resched();
0055     }
0056     return 0;
0057 not_enough_page:
0058     max = idx;
0059     for (idx = 0; idx < max; idx++)
0060         __free_page(ctrl->map[idx]);
0061
0062     return -ENOMEM;
0063 }
0064
0065 static struct swap_cgroup *__lookup_swap_cgroup(struct swap_cgroup_ctrl *ctrl,
0066                         pgoff_t offset)
0067 {
0068     struct page *mappage;
0069     struct swap_cgroup *sc;
0070
0071     mappage = ctrl->map[offset / SC_PER_PAGE];
0072     sc = page_address(mappage);
0073     return sc + offset % SC_PER_PAGE;
0074 }
0075
0076 static struct swap_cgroup *lookup_swap_cgroup(swp_entry_t ent,
0077                     struct swap_cgroup_ctrl **ctrlp)
0078 {
0079     pgoff_t offset = swp_offset(ent);
0080     struct swap_cgroup_ctrl *ctrl;
0081
0082     ctrl = &swap_cgroup_ctrl[swp_type(ent)];
0083     if (ctrlp)
0084         *ctrlp = ctrl;
0085     return __lookup_swap_cgroup(ctrl, offset);
0086 }
0087
0088 /**
0089  * swap_cgroup_cmpxchg - cmpxchg mem_cgroup's id for this swp_entry.
0090  * @ent: swap entry to be cmpxchged
0091  * @old: old id
0092  * @new: new id
0093  *
0094  * Returns old id at success, 0 at failure.
0095  * (There is no mem_cgroup using 0 as its id)
0096  */
0097 unsigned short swap_cgroup_cmpxchg(swp_entry_t ent,
0098                     unsigned short old, unsigned short new)
0099 {
0100     struct swap_cgroup_ctrl *ctrl;
0101     struct swap_cgroup *sc;
0102     unsigned long flags;
0103     unsigned short retval;
0104
0105     sc = lookup_swap_cgroup(ent, &ctrl);
0106
0107     spin_lock_irqsave(&ctrl->lock, flags);
0108     retval = sc->id;
0109     if (retval == old)
0110         sc->id = new;
0111     else
0112         retval = 0;
0113     spin_unlock_irqrestore(&ctrl->lock, flags);
0114     return retval;
0115 }
0116
0117 /**
0118  * swap_cgroup_record - record mem_cgroup for a set of swap entries
0119  * @ent: the first swap entry to be recorded into
0120  * @id: mem_cgroup to be recorded
0121  * @nr_ents: number of swap entries to be recorded
0122  *
0123  * Returns old value at success, 0 at failure.
0124  * (Of course, old value can be 0.)
0125  */
0126 unsigned short swap_cgroup_record(swp_entry_t ent, unsigned short id,
0127                   unsigned int nr_ents)
0128 {
0129     struct swap_cgroup_ctrl *ctrl;
0130     struct swap_cgroup *sc;
0131     unsigned short old;
0132     unsigned long flags;
0133     pgoff_t offset = swp_offset(ent);
0134     pgoff_t end = offset + nr_ents;
0135
0136     sc = lookup_swap_cgroup(ent, &ctrl);
0137
0138     spin_lock_irqsave(&ctrl->lock, flags);
0139     old = sc->id;
0140     for (;;) {
0141         VM_BUG_ON(sc->id != old);
0142         sc->id = id;
0143         offset++;
0144         if (offset == end)
0145             break;
0146         if (offset % SC_PER_PAGE)
0147             sc++;
0148         else
0149             sc = __lookup_swap_cgroup(ctrl, offset);
0150     }
0151     spin_unlock_irqrestore(&ctrl->lock, flags);
0152
0153     return old;
0154 }
0155
0156 /**
0157  * lookup_swap_cgroup_id - lookup mem_cgroup id tied to swap entry
0158  * @ent: swap entry to be looked up.
0159  *
0160  * Returns ID of mem_cgroup at success. 0 at failure. (0 is invalid ID)
0161  */
0162 unsigned short lookup_swap_cgroup_id(swp_entry_t ent)
0163 {
0164     return lookup_swap_cgroup(ent, NULL)->id;
0165 }
0166
0167 int swap_cgroup_swapon(int type, unsigned long max_pages)
0168 {
0169     void *array;
0170     unsigned long length;
0171     struct swap_cgroup_ctrl *ctrl;
0172
0173     length = DIV_ROUND_UP(max_pages, SC_PER_PAGE);
0174
0175     array = vcalloc(length, sizeof(void *));
0176     if (!array)
0177         goto nomem;
0178
0179     ctrl = &swap_cgroup_ctrl[type];
0180     mutex_lock(&swap_cgroup_mutex);
0181     ctrl->length = length;
0182     ctrl->map = array;
0183     spin_lock_init(&ctrl->lock);
0184     if (swap_cgroup_prepare(type)) {
0185         /* memory shortage */
0186         ctrl->map = NULL;
0187         ctrl->length = 0;
0188         mutex_unlock(&swap_cgroup_mutex);
0189         vfree(array);
0190         goto nomem;
0191     }
0192     mutex_unlock(&swap_cgroup_mutex);
0193
0194     return 0;
0195 nomem:
0196     pr_info("couldn't allocate enough memory for swap_cgroup\n");
0197     pr_info("swap_cgroup can be disabled by swapaccount=0 boot option\n");
0198     return -ENOMEM;
0199 }
0200
0201 void swap_cgroup_swapoff(int type)
0202 {
0203     struct page **map;
0204     unsigned long i, length;
0205     struct swap_cgroup_ctrl *ctrl;
0206
0207     mutex_lock(&swap_cgroup_mutex);
0208     ctrl = &swap_cgroup_ctrl[type];
0209     map = ctrl->map;
0210     length = ctrl->length;
0211     ctrl->map = NULL;
0212     ctrl->length = 0;
0213     mutex_unlock(&swap_cgroup_mutex);
0214
0215     if (map) {
0216         for (i = 0; i < length; i++) {
0217             struct page *page = map[i];
0218             if (page)
0219                 __free_page(page);
0220             if (!(i % SWAP_CLUSTER_MAX))
0221                 cond_resched();
0222         }
0223         vfree(map);
0224     }
0225 }