Back to home page

LXR

 
 

    


0001 /* memcontrol.c - Memory Controller
0002  *
0003  * Copyright IBM Corporation, 2007
0004  * Author Balbir Singh <balbir@linux.vnet.ibm.com>
0005  *
0006  * Copyright 2007 OpenVZ SWsoft Inc
0007  * Author: Pavel Emelianov <xemul@openvz.org>
0008  *
0009  * Memory thresholds
0010  * Copyright (C) 2009 Nokia Corporation
0011  * Author: Kirill A. Shutemov
0012  *
0013  * Kernel Memory Controller
0014  * Copyright (C) 2012 Parallels Inc. and Google Inc.
0015  * Authors: Glauber Costa and Suleiman Souhlal
0016  *
0017  * Native page reclaim
0018  * Charge lifetime sanitation
0019  * Lockless page tracking & accounting
0020  * Unified hierarchy configuration model
0021  * Copyright (C) 2015 Red Hat, Inc., Johannes Weiner
0022  *
0023  * This program is free software; you can redistribute it and/or modify
0024  * it under the terms of the GNU General Public License as published by
0025  * the Free Software Foundation; either version 2 of the License, or
0026  * (at your option) any later version.
0027  *
0028  * This program is distributed in the hope that it will be useful,
0029  * but WITHOUT ANY WARRANTY; without even the implied warranty of
0030  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
0031  * GNU General Public License for more details.
0032  */
0033 
0034 #include <linux/page_counter.h>
0035 #include <linux/memcontrol.h>
0036 #include <linux/cgroup.h>
0037 #include <linux/mm.h>
0038 #include <linux/hugetlb.h>
0039 #include <linux/pagemap.h>
0040 #include <linux/smp.h>
0041 #include <linux/page-flags.h>
0042 #include <linux/backing-dev.h>
0043 #include <linux/bit_spinlock.h>
0044 #include <linux/rcupdate.h>
0045 #include <linux/limits.h>
0046 #include <linux/export.h>
0047 #include <linux/mutex.h>
0048 #include <linux/rbtree.h>
0049 #include <linux/slab.h>
0050 #include <linux/swap.h>
0051 #include <linux/swapops.h>
0052 #include <linux/spinlock.h>
0053 #include <linux/eventfd.h>
0054 #include <linux/poll.h>
0055 #include <linux/sort.h>
0056 #include <linux/fs.h>
0057 #include <linux/seq_file.h>
0058 #include <linux/vmpressure.h>
0059 #include <linux/mm_inline.h>
0060 #include <linux/swap_cgroup.h>
0061 #include <linux/cpu.h>
0062 #include <linux/oom.h>
0063 #include <linux/lockdep.h>
0064 #include <linux/file.h>
0065 #include <linux/tracehook.h>
0066 #include "internal.h"
0067 #include <net/sock.h>
0068 #include <net/ip.h>
0069 #include "slab.h"
0070 
0071 #include <linux/uaccess.h>
0072 
0073 #include <trace/events/vmscan.h>
0074 
0075 struct cgroup_subsys memory_cgrp_subsys __read_mostly;
0076 EXPORT_SYMBOL(memory_cgrp_subsys);
0077 
0078 struct mem_cgroup *root_mem_cgroup __read_mostly;
0079 
0080 #define MEM_CGROUP_RECLAIM_RETRIES  5
0081 
0082 /* Socket memory accounting disabled? */
0083 static bool cgroup_memory_nosocket;
0084 
0085 /* Kernel memory accounting disabled? */
0086 static bool cgroup_memory_nokmem;
0087 
0088 /* Whether the swap controller is active */
0089 #ifdef CONFIG_MEMCG_SWAP
0090 int do_swap_account __read_mostly;
0091 #else
0092 #define do_swap_account     0
0093 #endif
0094 
0095 /* Whether legacy memory+swap accounting is active */
0096 static bool do_memsw_account(void)
0097 {
0098     return !cgroup_subsys_on_dfl(memory_cgrp_subsys) && do_swap_account;
0099 }
0100 
0101 static const char * const mem_cgroup_stat_names[] = {
0102     "cache",
0103     "rss",
0104     "rss_huge",
0105     "mapped_file",
0106     "dirty",
0107     "writeback",
0108     "swap",
0109 };
0110 
0111 static const char * const mem_cgroup_events_names[] = {
0112     "pgpgin",
0113     "pgpgout",
0114     "pgfault",
0115     "pgmajfault",
0116 };
0117 
0118 static const char * const mem_cgroup_lru_names[] = {
0119     "inactive_anon",
0120     "active_anon",
0121     "inactive_file",
0122     "active_file",
0123     "unevictable",
0124 };
0125 
0126 #define THRESHOLDS_EVENTS_TARGET 128
0127 #define SOFTLIMIT_EVENTS_TARGET 1024
0128 #define NUMAINFO_EVENTS_TARGET  1024
0129 
0130 /*
0131  * Cgroups above their limits are maintained in a RB-Tree, independent of
0132  * their hierarchy representation
0133  */
0134 
0135 struct mem_cgroup_tree_per_node {
0136     struct rb_root rb_root;
0137     spinlock_t lock;
0138 };
0139 
0140 struct mem_cgroup_tree {
0141     struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES];
0142 };
0143 
0144 static struct mem_cgroup_tree soft_limit_tree __read_mostly;
0145 
0146 /* for OOM */
0147 struct mem_cgroup_eventfd_list {
0148     struct list_head list;
0149     struct eventfd_ctx *eventfd;
0150 };
0151 
0152 /*
0153  * cgroup_event represents events which userspace want to receive.
0154  */
0155 struct mem_cgroup_event {
0156     /*
0157      * memcg which the event belongs to.
0158      */
0159     struct mem_cgroup *memcg;
0160     /*
0161      * eventfd to signal userspace about the event.
0162      */
0163     struct eventfd_ctx *eventfd;
0164     /*
0165      * Each of these stored in a list by the cgroup.
0166      */
0167     struct list_head list;
0168     /*
0169      * register_event() callback will be used to add new userspace
0170      * waiter for changes related to this event.  Use eventfd_signal()
0171      * on eventfd to send notification to userspace.
0172      */
0173     int (*register_event)(struct mem_cgroup *memcg,
0174                   struct eventfd_ctx *eventfd, const char *args);
0175     /*
0176      * unregister_event() callback will be called when userspace closes
0177      * the eventfd or on cgroup removing.  This callback must be set,
0178      * if you want provide notification functionality.
0179      */
0180     void (*unregister_event)(struct mem_cgroup *memcg,
0181                  struct eventfd_ctx *eventfd);
0182     /*
0183      * All fields below needed to unregister event when
0184      * userspace closes eventfd.
0185      */
0186     poll_table pt;
0187     wait_queue_head_t *wqh;
0188     wait_queue_t wait;
0189     struct work_struct remove;
0190 };
0191 
0192 static void mem_cgroup_threshold(struct mem_cgroup *memcg);
0193 static void mem_cgroup_oom_notify(struct mem_cgroup *memcg);
0194 
0195 /* Stuffs for move charges at task migration. */
0196 /*
0197  * Types of charges to be moved.
0198  */
0199 #define MOVE_ANON   0x1U
0200 #define MOVE_FILE   0x2U
0201 #define MOVE_MASK   (MOVE_ANON | MOVE_FILE)
0202 
0203 /* "mc" and its members are protected by cgroup_mutex */
0204 static struct move_charge_struct {
0205     spinlock_t    lock; /* for from, to */
0206     struct mm_struct  *mm;
0207     struct mem_cgroup *from;
0208     struct mem_cgroup *to;
0209     unsigned long flags;
0210     unsigned long precharge;
0211     unsigned long moved_charge;
0212     unsigned long moved_swap;
0213     struct task_struct *moving_task;    /* a task moving charges */
0214     wait_queue_head_t waitq;        /* a waitq for other context */
0215 } mc = {
0216     .lock = __SPIN_LOCK_UNLOCKED(mc.lock),
0217     .waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq),
0218 };
0219 
0220 /*
0221  * Maximum loops in mem_cgroup_hierarchical_reclaim(), used for soft
0222  * limit reclaim to prevent infinite loops, if they ever occur.
0223  */
0224 #define MEM_CGROUP_MAX_RECLAIM_LOOPS        100
0225 #define MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS 2
0226 
0227 enum charge_type {
0228     MEM_CGROUP_CHARGE_TYPE_CACHE = 0,
0229     MEM_CGROUP_CHARGE_TYPE_ANON,
0230     MEM_CGROUP_CHARGE_TYPE_SWAPOUT, /* for accounting swapcache */
0231     MEM_CGROUP_CHARGE_TYPE_DROP,    /* a page was unused swap cache */
0232     NR_CHARGE_TYPE,
0233 };
0234 
0235 /* for encoding cft->private value on file */
0236 enum res_type {
0237     _MEM,
0238     _MEMSWAP,
0239     _OOM_TYPE,
0240     _KMEM,
0241     _TCP,
0242 };
0243 
0244 #define MEMFILE_PRIVATE(x, val) ((x) << 16 | (val))
0245 #define MEMFILE_TYPE(val)   ((val) >> 16 & 0xffff)
0246 #define MEMFILE_ATTR(val)   ((val) & 0xffff)
0247 /* Used for OOM nofiier */
0248 #define OOM_CONTROL     (0)
0249 
0250 /* Some nice accessors for the vmpressure. */
0251 struct vmpressure *memcg_to_vmpressure(struct mem_cgroup *memcg)
0252 {
0253     if (!memcg)
0254         memcg = root_mem_cgroup;
0255     return &memcg->vmpressure;
0256 }
0257 
0258 struct cgroup_subsys_state *vmpressure_to_css(struct vmpressure *vmpr)
0259 {
0260     return &container_of(vmpr, struct mem_cgroup, vmpressure)->css;
0261 }
0262 
0263 static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg)
0264 {
0265     return (memcg == root_mem_cgroup);
0266 }
0267 
0268 #ifndef CONFIG_SLOB
0269 /*
0270  * This will be the memcg's index in each cache's ->memcg_params.memcg_caches.
0271  * The main reason for not using cgroup id for this:
0272  *  this works better in sparse environments, where we have a lot of memcgs,
0273  *  but only a few kmem-limited. Or also, if we have, for instance, 200
0274  *  memcgs, and none but the 200th is kmem-limited, we'd have to have a
0275  *  200 entry array for that.
0276  *
0277  * The current size of the caches array is stored in memcg_nr_cache_ids. It
0278  * will double each time we have to increase it.
0279  */
0280 static DEFINE_IDA(memcg_cache_ida);
0281 int memcg_nr_cache_ids;
0282 
0283 /* Protects memcg_nr_cache_ids */
0284 static DECLARE_RWSEM(memcg_cache_ids_sem);
0285 
0286 void memcg_get_cache_ids(void)
0287 {
0288     down_read(&memcg_cache_ids_sem);
0289 }
0290 
0291 void memcg_put_cache_ids(void)
0292 {
0293     up_read(&memcg_cache_ids_sem);
0294 }
0295 
0296 /*
0297  * MIN_SIZE is different than 1, because we would like to avoid going through
0298  * the alloc/free process all the time. In a small machine, 4 kmem-limited
0299  * cgroups is a reasonable guess. In the future, it could be a parameter or
0300  * tunable, but that is strictly not necessary.
0301  *
0302  * MAX_SIZE should be as large as the number of cgrp_ids. Ideally, we could get
0303  * this constant directly from cgroup, but it is understandable that this is
0304  * better kept as an internal representation in cgroup.c. In any case, the
0305  * cgrp_id space is not getting any smaller, and we don't have to necessarily
0306  * increase ours as well if it increases.
0307  */
0308 #define MEMCG_CACHES_MIN_SIZE 4
0309 #define MEMCG_CACHES_MAX_SIZE MEM_CGROUP_ID_MAX
0310 
0311 /*
0312  * A lot of the calls to the cache allocation functions are expected to be
0313  * inlined by the compiler. Since the calls to memcg_kmem_get_cache are
0314  * conditional to this static branch, we'll have to allow modules that does
0315  * kmem_cache_alloc and the such to see this symbol as well
0316  */
0317 DEFINE_STATIC_KEY_FALSE(memcg_kmem_enabled_key);
0318 EXPORT_SYMBOL(memcg_kmem_enabled_key);
0319 
0320 #endif /* !CONFIG_SLOB */
0321 
0322 /**
0323  * mem_cgroup_css_from_page - css of the memcg associated with a page
0324  * @page: page of interest
0325  *
0326  * If memcg is bound to the default hierarchy, css of the memcg associated
0327  * with @page is returned.  The returned css remains associated with @page
0328  * until it is released.
0329  *
0330  * If memcg is bound to a traditional hierarchy, the css of root_mem_cgroup
0331  * is returned.
0332  */
0333 struct cgroup_subsys_state *mem_cgroup_css_from_page(struct page *page)
0334 {
0335     struct mem_cgroup *memcg;
0336 
0337     memcg = page->mem_cgroup;
0338 
0339     if (!memcg || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
0340         memcg = root_mem_cgroup;
0341 
0342     return &memcg->css;
0343 }
0344 
0345 /**
0346  * page_cgroup_ino - return inode number of the memcg a page is charged to
0347  * @page: the page
0348  *
0349  * Look up the closest online ancestor of the memory cgroup @page is charged to
0350  * and return its inode number or 0 if @page is not charged to any cgroup. It
0351  * is safe to call this function without holding a reference to @page.
0352  *
0353  * Note, this function is inherently racy, because there is nothing to prevent
0354  * the cgroup inode from getting torn down and potentially reallocated a moment
0355  * after page_cgroup_ino() returns, so it only should be used by callers that
0356  * do not care (such as procfs interfaces).
0357  */
0358 ino_t page_cgroup_ino(struct page *page)
0359 {
0360     struct mem_cgroup *memcg;
0361     unsigned long ino = 0;
0362 
0363     rcu_read_lock();
0364     memcg = READ_ONCE(page->mem_cgroup);
0365     while (memcg && !(memcg->css.flags & CSS_ONLINE))
0366         memcg = parent_mem_cgroup(memcg);
0367     if (memcg)
0368         ino = cgroup_ino(memcg->css.cgroup);
0369     rcu_read_unlock();
0370     return ino;
0371 }
0372 
0373 static struct mem_cgroup_per_node *
0374 mem_cgroup_page_nodeinfo(struct mem_cgroup *memcg, struct page *page)
0375 {
0376     int nid = page_to_nid(page);
0377 
0378     return memcg->nodeinfo[nid];
0379 }
0380 
0381 static struct mem_cgroup_tree_per_node *
0382 soft_limit_tree_node(int nid)
0383 {
0384     return soft_limit_tree.rb_tree_per_node[nid];
0385 }
0386 
0387 static struct mem_cgroup_tree_per_node *
0388 soft_limit_tree_from_page(struct page *page)
0389 {
0390     int nid = page_to_nid(page);
0391 
0392     return soft_limit_tree.rb_tree_per_node[nid];
0393 }
0394 
0395 static void __mem_cgroup_insert_exceeded(struct mem_cgroup_per_node *mz,
0396                      struct mem_cgroup_tree_per_node *mctz,
0397                      unsigned long new_usage_in_excess)
0398 {
0399     struct rb_node **p = &mctz->rb_root.rb_node;
0400     struct rb_node *parent = NULL;
0401     struct mem_cgroup_per_node *mz_node;
0402 
0403     if (mz->on_tree)
0404         return;
0405 
0406     mz->usage_in_excess = new_usage_in_excess;
0407     if (!mz->usage_in_excess)
0408         return;
0409     while (*p) {
0410         parent = *p;
0411         mz_node = rb_entry(parent, struct mem_cgroup_per_node,
0412                     tree_node);
0413         if (mz->usage_in_excess < mz_node->usage_in_excess)
0414             p = &(*p)->rb_left;
0415         /*
0416          * We can't avoid mem cgroups that are over their soft
0417          * limit by the same amount
0418          */
0419         else if (mz->usage_in_excess >= mz_node->usage_in_excess)
0420             p = &(*p)->rb_right;
0421     }
0422     rb_link_node(&mz->tree_node, parent, p);
0423     rb_insert_color(&mz->tree_node, &mctz->rb_root);
0424     mz->on_tree = true;
0425 }
0426 
0427 static void __mem_cgroup_remove_exceeded(struct mem_cgroup_per_node *mz,
0428                      struct mem_cgroup_tree_per_node *mctz)
0429 {
0430     if (!mz->on_tree)
0431         return;
0432     rb_erase(&mz->tree_node, &mctz->rb_root);
0433     mz->on_tree = false;
0434 }
0435 
0436 static void mem_cgroup_remove_exceeded(struct mem_cgroup_per_node *mz,
0437                        struct mem_cgroup_tree_per_node *mctz)
0438 {
0439     unsigned long flags;
0440 
0441     spin_lock_irqsave(&mctz->lock, flags);
0442     __mem_cgroup_remove_exceeded(mz, mctz);
0443     spin_unlock_irqrestore(&mctz->lock, flags);
0444 }
0445 
0446 static unsigned long soft_limit_excess(struct mem_cgroup *memcg)
0447 {
0448     unsigned long nr_pages = page_counter_read(&memcg->memory);
0449     unsigned long soft_limit = READ_ONCE(memcg->soft_limit);
0450     unsigned long excess = 0;
0451 
0452     if (nr_pages > soft_limit)
0453         excess = nr_pages - soft_limit;
0454 
0455     return excess;
0456 }
0457 
0458 static void mem_cgroup_update_tree(struct mem_cgroup *memcg, struct page *page)
0459 {
0460     unsigned long excess;
0461     struct mem_cgroup_per_node *mz;
0462     struct mem_cgroup_tree_per_node *mctz;
0463 
0464     mctz = soft_limit_tree_from_page(page);
0465     /*
0466      * Necessary to update all ancestors when hierarchy is used.
0467      * because their event counter is not touched.
0468      */
0469     for (; memcg; memcg = parent_mem_cgroup(memcg)) {
0470         mz = mem_cgroup_page_nodeinfo(memcg, page);
0471         excess = soft_limit_excess(memcg);
0472         /*
0473          * We have to update the tree if mz is on RB-tree or
0474          * mem is over its softlimit.
0475          */
0476         if (excess || mz->on_tree) {
0477             unsigned long flags;
0478 
0479             spin_lock_irqsave(&mctz->lock, flags);
0480             /* if on-tree, remove it */
0481             if (mz->on_tree)
0482                 __mem_cgroup_remove_exceeded(mz, mctz);
0483             /*
0484              * Insert again. mz->usage_in_excess will be updated.
0485              * If excess is 0, no tree ops.
0486              */
0487             __mem_cgroup_insert_exceeded(mz, mctz, excess);
0488             spin_unlock_irqrestore(&mctz->lock, flags);
0489         }
0490     }
0491 }
0492 
0493 static void mem_cgroup_remove_from_trees(struct mem_cgroup *memcg)
0494 {
0495     struct mem_cgroup_tree_per_node *mctz;
0496     struct mem_cgroup_per_node *mz;
0497     int nid;
0498 
0499     for_each_node(nid) {
0500         mz = mem_cgroup_nodeinfo(memcg, nid);
0501         mctz = soft_limit_tree_node(nid);
0502         mem_cgroup_remove_exceeded(mz, mctz);
0503     }
0504 }
0505 
0506 static struct mem_cgroup_per_node *
0507 __mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz)
0508 {
0509     struct rb_node *rightmost = NULL;
0510     struct mem_cgroup_per_node *mz;
0511 
0512 retry:
0513     mz = NULL;
0514     rightmost = rb_last(&mctz->rb_root);
0515     if (!rightmost)
0516         goto done;      /* Nothing to reclaim from */
0517 
0518     mz = rb_entry(rightmost, struct mem_cgroup_per_node, tree_node);
0519     /*
0520      * Remove the node now but someone else can add it back,
0521      * we will to add it back at the end of reclaim to its correct
0522      * position in the tree.
0523      */
0524     __mem_cgroup_remove_exceeded(mz, mctz);
0525     if (!soft_limit_excess(mz->memcg) ||
0526         !css_tryget_online(&mz->memcg->css))
0527         goto retry;
0528 done:
0529     return mz;
0530 }
0531 
0532 static struct mem_cgroup_per_node *
0533 mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz)
0534 {
0535     struct mem_cgroup_per_node *mz;
0536 
0537     spin_lock_irq(&mctz->lock);
0538     mz = __mem_cgroup_largest_soft_limit_node(mctz);
0539     spin_unlock_irq(&mctz->lock);
0540     return mz;
0541 }
0542 
0543 /*
0544  * Return page count for single (non recursive) @memcg.
0545  *
0546  * Implementation Note: reading percpu statistics for memcg.
0547  *
0548  * Both of vmstat[] and percpu_counter has threshold and do periodic
0549  * synchronization to implement "quick" read. There are trade-off between
0550  * reading cost and precision of value. Then, we may have a chance to implement
0551  * a periodic synchronization of counter in memcg's counter.
0552  *
0553  * But this _read() function is used for user interface now. The user accounts
0554  * memory usage by memory cgroup and he _always_ requires exact value because
0555  * he accounts memory. Even if we provide quick-and-fuzzy read, we always
0556  * have to visit all online cpus and make sum. So, for now, unnecessary
0557  * synchronization is not implemented. (just implemented for cpu hotplug)
0558  *
0559  * If there are kernel internal actions which can make use of some not-exact
0560  * value, and reading all cpu value can be performance bottleneck in some
0561  * common workload, threshold and synchronization as vmstat[] should be
0562  * implemented.
0563  */
0564 static unsigned long
0565 mem_cgroup_read_stat(struct mem_cgroup *memcg, enum mem_cgroup_stat_index idx)
0566 {
0567     long val = 0;
0568     int cpu;
0569 
0570     /* Per-cpu values can be negative, use a signed accumulator */
0571     for_each_possible_cpu(cpu)
0572         val += per_cpu(memcg->stat->count[idx], cpu);
0573     /*
0574      * Summing races with updates, so val may be negative.  Avoid exposing
0575      * transient negative values.
0576      */
0577     if (val < 0)
0578         val = 0;
0579     return val;
0580 }
0581 
0582 static unsigned long mem_cgroup_read_events(struct mem_cgroup *memcg,
0583                         enum mem_cgroup_events_index idx)
0584 {
0585     unsigned long val = 0;
0586     int cpu;
0587 
0588     for_each_possible_cpu(cpu)
0589         val += per_cpu(memcg->stat->events[idx], cpu);
0590     return val;
0591 }
0592 
0593 static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg,
0594                      struct page *page,
0595                      bool compound, int nr_pages)
0596 {
0597     /*
0598      * Here, RSS means 'mapped anon' and anon's SwapCache. Shmem/tmpfs is
0599      * counted as CACHE even if it's on ANON LRU.
0600      */
0601     if (PageAnon(page))
0602         __this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_RSS],
0603                 nr_pages);
0604     else
0605         __this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_CACHE],
0606                 nr_pages);
0607 
0608     if (compound) {
0609         VM_BUG_ON_PAGE(!PageTransHuge(page), page);
0610         __this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_RSS_HUGE],
0611                 nr_pages);
0612     }
0613 
0614     /* pagein of a big page is an event. So, ignore page size */
0615     if (nr_pages > 0)
0616         __this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGIN]);
0617     else {
0618         __this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGOUT]);
0619         nr_pages = -nr_pages; /* for event */
0620     }
0621 
0622     __this_cpu_add(memcg->stat->nr_page_events, nr_pages);
0623 }
0624 
0625 unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
0626                        int nid, unsigned int lru_mask)
0627 {
0628     struct lruvec *lruvec = mem_cgroup_lruvec(NODE_DATA(nid), memcg);
0629     unsigned long nr = 0;
0630     enum lru_list lru;
0631 
0632     VM_BUG_ON((unsigned)nid >= nr_node_ids);
0633 
0634     for_each_lru(lru) {
0635         if (!(BIT(lru) & lru_mask))
0636             continue;
0637         nr += mem_cgroup_get_lru_size(lruvec, lru);
0638     }
0639     return nr;
0640 }
0641 
0642 static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *memcg,
0643             unsigned int lru_mask)
0644 {
0645     unsigned long nr = 0;
0646     int nid;
0647 
0648     for_each_node_state(nid, N_MEMORY)
0649         nr += mem_cgroup_node_nr_lru_pages(memcg, nid, lru_mask);
0650     return nr;
0651 }
0652 
0653 static bool mem_cgroup_event_ratelimit(struct mem_cgroup *memcg,
0654                        enum mem_cgroup_events_target target)
0655 {
0656     unsigned long val, next;
0657 
0658     val = __this_cpu_read(memcg->stat->nr_page_events);
0659     next = __this_cpu_read(memcg->stat->targets[target]);
0660     /* from time_after() in jiffies.h */
0661     if ((long)next - (long)val < 0) {
0662         switch (target) {
0663         case MEM_CGROUP_TARGET_THRESH:
0664             next = val + THRESHOLDS_EVENTS_TARGET;
0665             break;
0666         case MEM_CGROUP_TARGET_SOFTLIMIT:
0667             next = val + SOFTLIMIT_EVENTS_TARGET;
0668             break;
0669         case MEM_CGROUP_TARGET_NUMAINFO:
0670             next = val + NUMAINFO_EVENTS_TARGET;
0671             break;
0672         default:
0673             break;
0674         }
0675         __this_cpu_write(memcg->stat->targets[target], next);
0676         return true;
0677     }
0678     return false;
0679 }
0680 
0681 /*
0682  * Check events in order.
0683  *
0684  */
0685 static void memcg_check_events(struct mem_cgroup *memcg, struct page *page)
0686 {
0687     /* threshold event is triggered in finer grain than soft limit */
0688     if (unlikely(mem_cgroup_event_ratelimit(memcg,
0689                         MEM_CGROUP_TARGET_THRESH))) {
0690         bool do_softlimit;
0691         bool do_numainfo __maybe_unused;
0692 
0693         do_softlimit = mem_cgroup_event_ratelimit(memcg,
0694                         MEM_CGROUP_TARGET_SOFTLIMIT);
0695 #if MAX_NUMNODES > 1
0696         do_numainfo = mem_cgroup_event_ratelimit(memcg,
0697                         MEM_CGROUP_TARGET_NUMAINFO);
0698 #endif
0699         mem_cgroup_threshold(memcg);
0700         if (unlikely(do_softlimit))
0701             mem_cgroup_update_tree(memcg, page);
0702 #if MAX_NUMNODES > 1
0703         if (unlikely(do_numainfo))
0704             atomic_inc(&memcg->numainfo_events);
0705 #endif
0706     }
0707 }
0708 
0709 struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
0710 {
0711     /*
0712      * mm_update_next_owner() may clear mm->owner to NULL
0713      * if it races with swapoff, page migration, etc.
0714      * So this can be called with p == NULL.
0715      */
0716     if (unlikely(!p))
0717         return NULL;
0718 
0719     return mem_cgroup_from_css(task_css(p, memory_cgrp_id));
0720 }
0721 EXPORT_SYMBOL(mem_cgroup_from_task);
0722 
0723 static struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm)
0724 {
0725     struct mem_cgroup *memcg = NULL;
0726 
0727     rcu_read_lock();
0728     do {
0729         /*
0730          * Page cache insertions can happen withou an
0731          * actual mm context, e.g. during disk probing
0732          * on boot, loopback IO, acct() writes etc.
0733          */
0734         if (unlikely(!mm))
0735             memcg = root_mem_cgroup;
0736         else {
0737             memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
0738             if (unlikely(!memcg))
0739                 memcg = root_mem_cgroup;
0740         }
0741     } while (!css_tryget_online(&memcg->css));
0742     rcu_read_unlock();
0743     return memcg;
0744 }
0745 
0746 /**
0747  * mem_cgroup_iter - iterate over memory cgroup hierarchy
0748  * @root: hierarchy root
0749  * @prev: previously returned memcg, NULL on first invocation
0750  * @reclaim: cookie for shared reclaim walks, NULL for full walks
0751  *
0752  * Returns references to children of the hierarchy below @root, or
0753  * @root itself, or %NULL after a full round-trip.
0754  *
0755  * Caller must pass the return value in @prev on subsequent
0756  * invocations for reference counting, or use mem_cgroup_iter_break()
0757  * to cancel a hierarchy walk before the round-trip is complete.
0758  *
0759  * Reclaimers can specify a zone and a priority level in @reclaim to
0760  * divide up the memcgs in the hierarchy among all concurrent
0761  * reclaimers operating on the same zone and priority.
0762  */
0763 struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *root,
0764                    struct mem_cgroup *prev,
0765                    struct mem_cgroup_reclaim_cookie *reclaim)
0766 {
0767     struct mem_cgroup_reclaim_iter *uninitialized_var(iter);
0768     struct cgroup_subsys_state *css = NULL;
0769     struct mem_cgroup *memcg = NULL;
0770     struct mem_cgroup *pos = NULL;
0771 
0772     if (mem_cgroup_disabled())
0773         return NULL;
0774 
0775     if (!root)
0776         root = root_mem_cgroup;
0777 
0778     if (prev && !reclaim)
0779         pos = prev;
0780 
0781     if (!root->use_hierarchy && root != root_mem_cgroup) {
0782         if (prev)
0783             goto out;
0784         return root;
0785     }
0786 
0787     rcu_read_lock();
0788 
0789     if (reclaim) {
0790         struct mem_cgroup_per_node *mz;
0791 
0792         mz = mem_cgroup_nodeinfo(root, reclaim->pgdat->node_id);
0793         iter = &mz->iter[reclaim->priority];
0794 
0795         if (prev && reclaim->generation != iter->generation)
0796             goto out_unlock;
0797 
0798         while (1) {
0799             pos = READ_ONCE(iter->position);
0800             if (!pos || css_tryget(&pos->css))
0801                 break;
0802             /*
0803              * css reference reached zero, so iter->position will
0804              * be cleared by ->css_released. However, we should not
0805              * rely on this happening soon, because ->css_released
0806              * is called from a work queue, and by busy-waiting we
0807              * might block it. So we clear iter->position right
0808              * away.
0809              */
0810             (void)cmpxchg(&iter->position, pos, NULL);
0811         }
0812     }
0813 
0814     if (pos)
0815         css = &pos->css;
0816 
0817     for (;;) {
0818         css = css_next_descendant_pre(css, &root->css);
0819         if (!css) {
0820             /*
0821              * Reclaimers share the hierarchy walk, and a
0822              * new one might jump in right at the end of
0823              * the hierarchy - make sure they see at least
0824              * one group and restart from the beginning.
0825              */
0826             if (!prev)
0827                 continue;
0828             break;
0829         }
0830 
0831         /*
0832          * Verify the css and acquire a reference.  The root
0833          * is provided by the caller, so we know it's alive
0834          * and kicking, and don't take an extra reference.
0835          */
0836         memcg = mem_cgroup_from_css(css);
0837 
0838         if (css == &root->css)
0839             break;
0840 
0841         if (css_tryget(css))
0842             break;
0843 
0844         memcg = NULL;
0845     }
0846 
0847     if (reclaim) {
0848         /*
0849          * The position could have already been updated by a competing
0850          * thread, so check that the value hasn't changed since we read
0851          * it to avoid reclaiming from the same cgroup twice.
0852          */
0853         (void)cmpxchg(&iter->position, pos, memcg);
0854 
0855         if (pos)
0856             css_put(&pos->css);
0857 
0858         if (!memcg)
0859             iter->generation++;
0860         else if (!prev)
0861             reclaim->generation = iter->generation;
0862     }
0863 
0864 out_unlock:
0865     rcu_read_unlock();
0866 out:
0867     if (prev && prev != root)
0868         css_put(&prev->css);
0869 
0870     return memcg;
0871 }
0872 
0873 /**
0874  * mem_cgroup_iter_break - abort a hierarchy walk prematurely
0875  * @root: hierarchy root
0876  * @prev: last visited hierarchy member as returned by mem_cgroup_iter()
0877  */
0878 void mem_cgroup_iter_break(struct mem_cgroup *root,
0879                struct mem_cgroup *prev)
0880 {
0881     if (!root)
0882         root = root_mem_cgroup;
0883     if (prev && prev != root)
0884         css_put(&prev->css);
0885 }
0886 
0887 static void invalidate_reclaim_iterators(struct mem_cgroup *dead_memcg)
0888 {
0889     struct mem_cgroup *memcg = dead_memcg;
0890     struct mem_cgroup_reclaim_iter *iter;
0891     struct mem_cgroup_per_node *mz;
0892     int nid;
0893     int i;
0894 
0895     while ((memcg = parent_mem_cgroup(memcg))) {
0896         for_each_node(nid) {
0897             mz = mem_cgroup_nodeinfo(memcg, nid);
0898             for (i = 0; i <= DEF_PRIORITY; i++) {
0899                 iter = &mz->iter[i];
0900                 cmpxchg(&iter->position,
0901                     dead_memcg, NULL);
0902             }
0903         }
0904     }
0905 }
0906 
0907 /*
0908  * Iteration constructs for visiting all cgroups (under a tree).  If
0909  * loops are exited prematurely (break), mem_cgroup_iter_break() must
0910  * be used for reference counting.
0911  */
0912 #define for_each_mem_cgroup_tree(iter, root)        \
0913     for (iter = mem_cgroup_iter(root, NULL, NULL);  \
0914          iter != NULL;              \
0915          iter = mem_cgroup_iter(root, iter, NULL))
0916 
0917 #define for_each_mem_cgroup(iter)           \
0918     for (iter = mem_cgroup_iter(NULL, NULL, NULL);  \
0919          iter != NULL;              \
0920          iter = mem_cgroup_iter(NULL, iter, NULL))
0921 
0922 /**
0923  * mem_cgroup_scan_tasks - iterate over tasks of a memory cgroup hierarchy
0924  * @memcg: hierarchy root
0925  * @fn: function to call for each task
0926  * @arg: argument passed to @fn
0927  *
0928  * This function iterates over tasks attached to @memcg or to any of its
0929  * descendants and calls @fn for each task. If @fn returns a non-zero
0930  * value, the function breaks the iteration loop and returns the value.
0931  * Otherwise, it will iterate over all tasks and return 0.
0932  *
0933  * This function must not be called for the root memory cgroup.
0934  */
0935 int mem_cgroup_scan_tasks(struct mem_cgroup *memcg,
0936               int (*fn)(struct task_struct *, void *), void *arg)
0937 {
0938     struct mem_cgroup *iter;
0939     int ret = 0;
0940 
0941     BUG_ON(memcg == root_mem_cgroup);
0942 
0943     for_each_mem_cgroup_tree(iter, memcg) {
0944         struct css_task_iter it;
0945         struct task_struct *task;
0946 
0947         css_task_iter_start(&iter->css, &it);
0948         while (!ret && (task = css_task_iter_next(&it)))
0949             ret = fn(task, arg);
0950         css_task_iter_end(&it);
0951         if (ret) {
0952             mem_cgroup_iter_break(memcg, iter);
0953             break;
0954         }
0955     }
0956     return ret;
0957 }
0958 
0959 /**
0960  * mem_cgroup_page_lruvec - return lruvec for isolating/putting an LRU page
0961  * @page: the page
0962  * @zone: zone of the page
0963  *
0964  * This function is only safe when following the LRU page isolation
0965  * and putback protocol: the LRU lock must be held, and the page must
0966  * either be PageLRU() or the caller must have isolated/allocated it.
0967  */
0968 struct lruvec *mem_cgroup_page_lruvec(struct page *page, struct pglist_data *pgdat)
0969 {
0970     struct mem_cgroup_per_node *mz;
0971     struct mem_cgroup *memcg;
0972     struct lruvec *lruvec;
0973 
0974     if (mem_cgroup_disabled()) {
0975         lruvec = &pgdat->lruvec;
0976         goto out;
0977     }
0978 
0979     memcg = page->mem_cgroup;
0980     /*
0981      * Swapcache readahead pages are added to the LRU - and
0982      * possibly migrated - before they are charged.
0983      */
0984     if (!memcg)
0985         memcg = root_mem_cgroup;
0986 
0987     mz = mem_cgroup_page_nodeinfo(memcg, page);
0988     lruvec = &mz->lruvec;
0989 out:
0990     /*
0991      * Since a node can be onlined after the mem_cgroup was created,
0992      * we have to be prepared to initialize lruvec->zone here;
0993      * and if offlined then reonlined, we need to reinitialize it.
0994      */
0995     if (unlikely(lruvec->pgdat != pgdat))
0996         lruvec->pgdat = pgdat;
0997     return lruvec;
0998 }
0999 
1000 /**
1001  * mem_cgroup_update_lru_size - account for adding or removing an lru page
1002  * @lruvec: mem_cgroup per zone lru vector
1003  * @lru: index of lru list the page is sitting on
1004  * @zid: zone id of the accounted pages
1005  * @nr_pages: positive when adding or negative when removing
1006  *
1007  * This function must be called under lru_lock, just before a page is added
1008  * to or just after a page is removed from an lru list (that ordering being
1009  * so as to allow it to check that lru_size 0 is consistent with list_empty).
1010  */
1011 void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
1012                 int zid, int nr_pages)
1013 {
1014     struct mem_cgroup_per_node *mz;
1015     unsigned long *lru_size;
1016     long size;
1017 
1018     if (mem_cgroup_disabled())
1019         return;
1020 
1021     mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
1022     lru_size = &mz->lru_zone_size[zid][lru];
1023 
1024     if (nr_pages < 0)
1025         *lru_size += nr_pages;
1026 
1027     size = *lru_size;
1028     if (WARN_ONCE(size < 0,
1029         "%s(%p, %d, %d): lru_size %ld\n",
1030         __func__, lruvec, lru, nr_pages, size)) {
1031         VM_BUG_ON(1);
1032         *lru_size = 0;
1033     }
1034 
1035     if (nr_pages > 0)
1036         *lru_size += nr_pages;
1037 }
1038 
1039 bool task_in_mem_cgroup(struct task_struct *task, struct mem_cgroup *memcg)
1040 {
1041     struct mem_cgroup *task_memcg;
1042     struct task_struct *p;
1043     bool ret;
1044 
1045     p = find_lock_task_mm(task);
1046     if (p) {
1047         task_memcg = get_mem_cgroup_from_mm(p->mm);
1048         task_unlock(p);
1049     } else {
1050         /*
1051          * All threads may have already detached their mm's, but the oom
1052          * killer still needs to detect if they have already been oom
1053          * killed to prevent needlessly killing additional tasks.
1054          */
1055         rcu_read_lock();
1056         task_memcg = mem_cgroup_from_task(task);
1057         css_get(&task_memcg->css);
1058         rcu_read_unlock();
1059     }
1060     ret = mem_cgroup_is_descendant(task_memcg, memcg);
1061     css_put(&task_memcg->css);
1062     return ret;
1063 }
1064 
1065 /**
1066  * mem_cgroup_margin - calculate chargeable space of a memory cgroup
1067  * @memcg: the memory cgroup
1068  *
1069  * Returns the maximum amount of memory @mem can be charged with, in
1070  * pages.
1071  */
1072 static unsigned long mem_cgroup_margin(struct mem_cgroup *memcg)
1073 {
1074     unsigned long margin = 0;
1075     unsigned long count;
1076     unsigned long limit;
1077 
1078     count = page_counter_read(&memcg->memory);
1079     limit = READ_ONCE(memcg->memory.limit);
1080     if (count < limit)
1081         margin = limit - count;
1082 
1083     if (do_memsw_account()) {
1084         count = page_counter_read(&memcg->memsw);
1085         limit = READ_ONCE(memcg->memsw.limit);
1086         if (count <= limit)
1087             margin = min(margin, limit - count);
1088         else
1089             margin = 0;
1090     }
1091 
1092     return margin;
1093 }
1094 
1095 /*
1096  * A routine for checking "mem" is under move_account() or not.
1097  *
1098  * Checking a cgroup is mc.from or mc.to or under hierarchy of
1099  * moving cgroups. This is for waiting at high-memory pressure
1100  * caused by "move".
1101  */
1102 static bool mem_cgroup_under_move(struct mem_cgroup *memcg)
1103 {
1104     struct mem_cgroup *from;
1105     struct mem_cgroup *to;
1106     bool ret = false;
1107     /*
1108      * Unlike task_move routines, we access mc.to, mc.from not under
1109      * mutual exclusion by cgroup_mutex. Here, we take spinlock instead.
1110      */
1111     spin_lock(&mc.lock);
1112     from = mc.from;
1113     to = mc.to;
1114     if (!from)
1115         goto unlock;
1116 
1117     ret = mem_cgroup_is_descendant(from, memcg) ||
1118         mem_cgroup_is_descendant(to, memcg);
1119 unlock:
1120     spin_unlock(&mc.lock);
1121     return ret;
1122 }
1123 
1124 static bool mem_cgroup_wait_acct_move(struct mem_cgroup *memcg)
1125 {
1126     if (mc.moving_task && current != mc.moving_task) {
1127         if (mem_cgroup_under_move(memcg)) {
1128             DEFINE_WAIT(wait);
1129             prepare_to_wait(&mc.waitq, &wait, TASK_INTERRUPTIBLE);
1130             /* moving charge context might have finished. */
1131             if (mc.moving_task)
1132                 schedule();
1133             finish_wait(&mc.waitq, &wait);
1134             return true;
1135         }
1136     }
1137     return false;
1138 }
1139 
1140 #define K(x) ((x) << (PAGE_SHIFT-10))
1141 /**
1142  * mem_cgroup_print_oom_info: Print OOM information relevant to memory controller.
1143  * @memcg: The memory cgroup that went over limit
1144  * @p: Task that is going to be killed
1145  *
1146  * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is
1147  * enabled
1148  */
1149 void mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p)
1150 {
1151     struct mem_cgroup *iter;
1152     unsigned int i;
1153 
1154     rcu_read_lock();
1155 
1156     if (p) {
1157         pr_info("Task in ");
1158         pr_cont_cgroup_path(task_cgroup(p, memory_cgrp_id));
1159         pr_cont(" killed as a result of limit of ");
1160     } else {
1161         pr_info("Memory limit reached of cgroup ");
1162     }
1163 
1164     pr_cont_cgroup_path(memcg->css.cgroup);
1165     pr_cont("\n");
1166 
1167     rcu_read_unlock();
1168 
1169     pr_info("memory: usage %llukB, limit %llukB, failcnt %lu\n",
1170         K((u64)page_counter_read(&memcg->memory)),
1171         K((u64)memcg->memory.limit), memcg->memory.failcnt);
1172     pr_info("memory+swap: usage %llukB, limit %llukB, failcnt %lu\n",
1173         K((u64)page_counter_read(&memcg->memsw)),
1174         K((u64)memcg->memsw.limit), memcg->memsw.failcnt);
1175     pr_info("kmem: usage %llukB, limit %llukB, failcnt %lu\n",
1176         K((u64)page_counter_read(&memcg->kmem)),
1177         K((u64)memcg->kmem.limit), memcg->kmem.failcnt);
1178 
1179     for_each_mem_cgroup_tree(iter, memcg) {
1180         pr_info("Memory cgroup stats for ");
1181         pr_cont_cgroup_path(iter->css.cgroup);
1182         pr_cont(":");
1183 
1184         for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) {
1185             if (i == MEM_CGROUP_STAT_SWAP && !do_swap_account)
1186                 continue;
1187             pr_cont(" %s:%luKB", mem_cgroup_stat_names[i],
1188                 K(mem_cgroup_read_stat(iter, i)));
1189         }
1190 
1191         for (i = 0; i < NR_LRU_LISTS; i++)
1192             pr_cont(" %s:%luKB", mem_cgroup_lru_names[i],
1193                 K(mem_cgroup_nr_lru_pages(iter, BIT(i))));
1194 
1195         pr_cont("\n");
1196     }
1197 }
1198 
1199 /*
1200  * This function returns the number of memcg under hierarchy tree. Returns
1201  * 1(self count) if no children.
1202  */
1203 static int mem_cgroup_count_children(struct mem_cgroup *memcg)
1204 {
1205     int num = 0;
1206     struct mem_cgroup *iter;
1207 
1208     for_each_mem_cgroup_tree(iter, memcg)
1209         num++;
1210     return num;
1211 }
1212 
1213 /*
1214  * Return the memory (and swap, if configured) limit for a memcg.
1215  */
1216 unsigned long mem_cgroup_get_limit(struct mem_cgroup *memcg)
1217 {
1218     unsigned long limit;
1219 
1220     limit = memcg->memory.limit;
1221     if (mem_cgroup_swappiness(memcg)) {
1222         unsigned long memsw_limit;
1223         unsigned long swap_limit;
1224 
1225         memsw_limit = memcg->memsw.limit;
1226         swap_limit = memcg->swap.limit;
1227         swap_limit = min(swap_limit, (unsigned long)total_swap_pages);
1228         limit = min(limit + swap_limit, memsw_limit);
1229     }
1230     return limit;
1231 }
1232 
1233 static bool mem_cgroup_out_of_memory(struct mem_cgroup *memcg, gfp_t gfp_mask,
1234                      int order)
1235 {
1236     struct oom_control oc = {
1237         .zonelist = NULL,
1238         .nodemask = NULL,
1239         .memcg = memcg,
1240         .gfp_mask = gfp_mask,
1241         .order = order,
1242     };
1243     bool ret;
1244 
1245     mutex_lock(&oom_lock);
1246     ret = out_of_memory(&oc);
1247     mutex_unlock(&oom_lock);
1248     return ret;
1249 }
1250 
1251 #if MAX_NUMNODES > 1
1252 
1253 /**
1254  * test_mem_cgroup_node_reclaimable
1255  * @memcg: the target memcg
1256  * @nid: the node ID to be checked.
1257  * @noswap : specify true here if the user wants flle only information.
1258  *
1259  * This function returns whether the specified memcg contains any
1260  * reclaimable pages on a node. Returns true if there are any reclaimable
1261  * pages in the node.
1262  */
1263 static bool test_mem_cgroup_node_reclaimable(struct mem_cgroup *memcg,
1264         int nid, bool noswap)
1265 {
1266     if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_FILE))
1267         return true;
1268     if (noswap || !total_swap_pages)
1269         return false;
1270     if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_ANON))
1271         return true;
1272     return false;
1273 
1274 }
1275 
1276 /*
1277  * Always updating the nodemask is not very good - even if we have an empty
1278  * list or the wrong list here, we can start from some node and traverse all
1279  * nodes based on the zonelist. So update the list loosely once per 10 secs.
1280  *
1281  */
1282 static void mem_cgroup_may_update_nodemask(struct mem_cgroup *memcg)
1283 {
1284     int nid;
1285     /*
1286      * numainfo_events > 0 means there was at least NUMAINFO_EVENTS_TARGET
1287      * pagein/pageout changes since the last update.
1288      */
1289     if (!atomic_read(&memcg->numainfo_events))
1290         return;
1291     if (atomic_inc_return(&memcg->numainfo_updating) > 1)
1292         return;
1293 
1294     /* make a nodemask where this memcg uses memory from */
1295     memcg->scan_nodes = node_states[N_MEMORY];
1296 
1297     for_each_node_mask(nid, node_states[N_MEMORY]) {
1298 
1299         if (!test_mem_cgroup_node_reclaimable(memcg, nid, false))
1300             node_clear(nid, memcg->scan_nodes);
1301     }
1302 
1303     atomic_set(&memcg->numainfo_events, 0);
1304     atomic_set(&memcg->numainfo_updating, 0);
1305 }
1306 
1307 /*
1308  * Selecting a node where we start reclaim from. Because what we need is just
1309  * reducing usage counter, start from anywhere is O,K. Considering
1310  * memory reclaim from current node, there are pros. and cons.
1311  *
1312  * Freeing memory from current node means freeing memory from a node which
1313  * we'll use or we've used. So, it may make LRU bad. And if several threads
1314  * hit limits, it will see a contention on a node. But freeing from remote
1315  * node means more costs for memory reclaim because of memory latency.
1316  *
1317  * Now, we use round-robin. Better algorithm is welcomed.
1318  */
1319 int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)
1320 {
1321     int node;
1322 
1323     mem_cgroup_may_update_nodemask(memcg);
1324     node = memcg->last_scanned_node;
1325 
1326     node = next_node_in(node, memcg->scan_nodes);
1327     /*
1328      * mem_cgroup_may_update_nodemask might have seen no reclaimmable pages
1329      * last time it really checked all the LRUs due to rate limiting.
1330      * Fallback to the current node in that case for simplicity.
1331      */
1332     if (unlikely(node == MAX_NUMNODES))
1333         node = numa_node_id();
1334 
1335     memcg->last_scanned_node = node;
1336     return node;
1337 }
1338 #else
1339 int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)
1340 {
1341     return 0;
1342 }
1343 #endif
1344 
1345 static int mem_cgroup_soft_reclaim(struct mem_cgroup *root_memcg,
1346                    pg_data_t *pgdat,
1347                    gfp_t gfp_mask,
1348                    unsigned long *total_scanned)
1349 {
1350     struct mem_cgroup *victim = NULL;
1351     int total = 0;
1352     int loop = 0;
1353     unsigned long excess;
1354     unsigned long nr_scanned;
1355     struct mem_cgroup_reclaim_cookie reclaim = {
1356         .pgdat = pgdat,
1357         .priority = 0,
1358     };
1359 
1360     excess = soft_limit_excess(root_memcg);
1361 
1362     while (1) {
1363         victim = mem_cgroup_iter(root_memcg, victim, &reclaim);
1364         if (!victim) {
1365             loop++;
1366             if (loop >= 2) {
1367                 /*
1368                  * If we have not been able to reclaim
1369                  * anything, it might because there are
1370                  * no reclaimable pages under this hierarchy
1371                  */
1372                 if (!total)
1373                     break;
1374                 /*
1375                  * We want to do more targeted reclaim.
1376                  * excess >> 2 is not to excessive so as to
1377                  * reclaim too much, nor too less that we keep
1378                  * coming back to reclaim from this cgroup
1379                  */
1380                 if (total >= (excess >> 2) ||
1381                     (loop > MEM_CGROUP_MAX_RECLAIM_LOOPS))
1382                     break;
1383             }
1384             continue;
1385         }
1386         total += mem_cgroup_shrink_node(victim, gfp_mask, false,
1387                     pgdat, &nr_scanned);
1388         *total_scanned += nr_scanned;
1389         if (!soft_limit_excess(root_memcg))
1390             break;
1391     }
1392     mem_cgroup_iter_break(root_memcg, victim);
1393     return total;
1394 }
1395 
1396 #ifdef CONFIG_LOCKDEP
1397 static struct lockdep_map memcg_oom_lock_dep_map = {
1398     .name = "memcg_oom_lock",
1399 };
1400 #endif
1401 
1402 static DEFINE_SPINLOCK(memcg_oom_lock);
1403 
1404 /*
1405  * Check OOM-Killer is already running under our hierarchy.
1406  * If someone is running, return false.
1407  */
1408 static bool mem_cgroup_oom_trylock(struct mem_cgroup *memcg)
1409 {
1410     struct mem_cgroup *iter, *failed = NULL;
1411 
1412     spin_lock(&memcg_oom_lock);
1413 
1414     for_each_mem_cgroup_tree(iter, memcg) {
1415         if (iter->oom_lock) {
1416             /*
1417              * this subtree of our hierarchy is already locked
1418              * so we cannot give a lock.
1419              */
1420             failed = iter;
1421             mem_cgroup_iter_break(memcg, iter);
1422             break;
1423         } else
1424             iter->oom_lock = true;
1425     }
1426 
1427     if (failed) {
1428         /*
1429          * OK, we failed to lock the whole subtree so we have
1430          * to clean up what we set up to the failing subtree
1431          */
1432         for_each_mem_cgroup_tree(iter, memcg) {
1433             if (iter == failed) {
1434                 mem_cgroup_iter_break(memcg, iter);
1435                 break;
1436             }
1437             iter->oom_lock = false;
1438         }
1439     } else
1440         mutex_acquire(&memcg_oom_lock_dep_map, 0, 1, _RET_IP_);
1441 
1442     spin_unlock(&memcg_oom_lock);
1443 
1444     return !failed;
1445 }
1446 
1447 static void mem_cgroup_oom_unlock(struct mem_cgroup *memcg)
1448 {
1449     struct mem_cgroup *iter;
1450 
1451     spin_lock(&memcg_oom_lock);
1452     mutex_release(&memcg_oom_lock_dep_map, 1, _RET_IP_);
1453     for_each_mem_cgroup_tree(iter, memcg)
1454         iter->oom_lock = false;
1455     spin_unlock(&memcg_oom_lock);
1456 }
1457 
1458 static void mem_cgroup_mark_under_oom(struct mem_cgroup *memcg)
1459 {
1460     struct mem_cgroup *iter;
1461 
1462     spin_lock(&memcg_oom_lock);
1463     for_each_mem_cgroup_tree(iter, memcg)
1464         iter->under_oom++;
1465     spin_unlock(&memcg_oom_lock);
1466 }
1467 
1468 static void mem_cgroup_unmark_under_oom(struct mem_cgroup *memcg)
1469 {
1470     struct mem_cgroup *iter;
1471 
1472     /*
1473      * When a new child is created while the hierarchy is under oom,
1474      * mem_cgroup_oom_lock() may not be called. Watch for underflow.
1475      */
1476     spin_lock(&memcg_oom_lock);
1477     for_each_mem_cgroup_tree(iter, memcg)
1478         if (iter->under_oom > 0)
1479             iter->under_oom--;
1480     spin_unlock(&memcg_oom_lock);
1481 }
1482 
1483 static DECLARE_WAIT_QUEUE_HEAD(memcg_oom_waitq);
1484 
1485 struct oom_wait_info {
1486     struct mem_cgroup *memcg;
1487     wait_queue_t    wait;
1488 };
1489 
1490 static int memcg_oom_wake_function(wait_queue_t *wait,
1491     unsigned mode, int sync, void *arg)
1492 {
1493     struct mem_cgroup *wake_memcg = (struct mem_cgroup *)arg;
1494     struct mem_cgroup *oom_wait_memcg;
1495     struct oom_wait_info *oom_wait_info;
1496 
1497     oom_wait_info = container_of(wait, struct oom_wait_info, wait);
1498     oom_wait_memcg = oom_wait_info->memcg;
1499 
1500     if (!mem_cgroup_is_descendant(wake_memcg, oom_wait_memcg) &&
1501         !mem_cgroup_is_descendant(oom_wait_memcg, wake_memcg))
1502         return 0;
1503     return autoremove_wake_function(wait, mode, sync, arg);
1504 }
1505 
1506 static void memcg_oom_recover(struct mem_cgroup *memcg)
1507 {
1508     /*
1509      * For the following lockless ->under_oom test, the only required
1510      * guarantee is that it must see the state asserted by an OOM when
1511      * this function is called as a result of userland actions
1512      * triggered by the notification of the OOM.  This is trivially
1513      * achieved by invoking mem_cgroup_mark_under_oom() before
1514      * triggering notification.
1515      */
1516     if (memcg && memcg->under_oom)
1517         __wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, memcg);
1518 }
1519 
1520 static void mem_cgroup_oom(struct mem_cgroup *memcg, gfp_t mask, int order)
1521 {
1522     if (!current->memcg_may_oom)
1523         return;
1524     /*
1525      * We are in the middle of the charge context here, so we
1526      * don't want to block when potentially sitting on a callstack
1527      * that holds all kinds of filesystem and mm locks.
1528      *
1529      * Also, the caller may handle a failed allocation gracefully
1530      * (like optional page cache readahead) and so an OOM killer
1531      * invocation might not even be necessary.
1532      *
1533      * That's why we don't do anything here except remember the
1534      * OOM context and then deal with it at the end of the page
1535      * fault when the stack is unwound, the locks are released,
1536      * and when we know whether the fault was overall successful.
1537      */
1538     css_get(&memcg->css);
1539     current->memcg_in_oom = memcg;
1540     current->memcg_oom_gfp_mask = mask;
1541     current->memcg_oom_order = order;
1542 }
1543 
1544 /**
1545  * mem_cgroup_oom_synchronize - complete memcg OOM handling
1546  * @handle: actually kill/wait or just clean up the OOM state
1547  *
1548  * This has to be called at the end of a page fault if the memcg OOM
1549  * handler was enabled.
1550  *
1551  * Memcg supports userspace OOM handling where failed allocations must
1552  * sleep on a waitqueue until the userspace task resolves the
1553  * situation.  Sleeping directly in the charge context with all kinds
1554  * of locks held is not a good idea, instead we remember an OOM state
1555  * in the task and mem_cgroup_oom_synchronize() has to be called at
1556  * the end of the page fault to complete the OOM handling.
1557  *
1558  * Returns %true if an ongoing memcg OOM situation was detected and
1559  * completed, %false otherwise.
1560  */
1561 bool mem_cgroup_oom_synchronize(bool handle)
1562 {
1563     struct mem_cgroup *memcg = current->memcg_in_oom;
1564     struct oom_wait_info owait;
1565     bool locked;
1566 
1567     /* OOM is global, do not handle */
1568     if (!memcg)
1569         return false;
1570 
1571     if (!handle)
1572         goto cleanup;
1573 
1574     owait.memcg = memcg;
1575     owait.wait.flags = 0;
1576     owait.wait.func = memcg_oom_wake_function;
1577     owait.wait.private = current;
1578     INIT_LIST_HEAD(&owait.wait.task_list);
1579 
1580     prepare_to_wait(&memcg_oom_waitq, &owait.wait, TASK_KILLABLE);
1581     mem_cgroup_mark_under_oom(memcg);
1582 
1583     locked = mem_cgroup_oom_trylock(memcg);
1584 
1585     if (locked)
1586         mem_cgroup_oom_notify(memcg);
1587 
1588     if (locked && !memcg->oom_kill_disable) {
1589         mem_cgroup_unmark_under_oom(memcg);
1590         finish_wait(&memcg_oom_waitq, &owait.wait);
1591         mem_cgroup_out_of_memory(memcg, current->memcg_oom_gfp_mask,
1592                      current->memcg_oom_order);
1593     } else {
1594         schedule();
1595         mem_cgroup_unmark_under_oom(memcg);
1596         finish_wait(&memcg_oom_waitq, &owait.wait);
1597     }
1598 
1599     if (locked) {
1600         mem_cgroup_oom_unlock(memcg);
1601         /*
1602          * There is no guarantee that an OOM-lock contender
1603          * sees the wakeups triggered by the OOM kill
1604          * uncharges.  Wake any sleepers explicitely.
1605          */
1606         memcg_oom_recover(memcg);
1607     }
1608 cleanup:
1609     current->memcg_in_oom = NULL;
1610     css_put(&memcg->css);
1611     return true;
1612 }
1613 
1614 /**
1615  * lock_page_memcg - lock a page->mem_cgroup binding
1616  * @page: the page
1617  *
1618  * This function protects unlocked LRU pages from being moved to
1619  * another cgroup and stabilizes their page->mem_cgroup binding.
1620  */
1621 void lock_page_memcg(struct page *page)
1622 {
1623     struct mem_cgroup *memcg;
1624     unsigned long flags;
1625 
1626     /*
1627      * The RCU lock is held throughout the transaction.  The fast
1628      * path can get away without acquiring the memcg->move_lock
1629      * because page moving starts with an RCU grace period.
1630      */
1631     rcu_read_lock();
1632 
1633     if (mem_cgroup_disabled())
1634         return;
1635 again:
1636     memcg = page->mem_cgroup;
1637     if (unlikely(!memcg))
1638         return;
1639 
1640     if (atomic_read(&memcg->moving_account) <= 0)
1641         return;
1642 
1643     spin_lock_irqsave(&memcg->move_lock, flags);
1644     if (memcg != page->mem_cgroup) {
1645         spin_unlock_irqrestore(&memcg->move_lock, flags);
1646         goto again;
1647     }
1648 
1649     /*
1650      * When charge migration first begins, we can have locked and
1651      * unlocked page stat updates happening concurrently.  Track
1652      * the task who has the lock for unlock_page_memcg().
1653      */
1654     memcg->move_lock_task = current;
1655     memcg->move_lock_flags = flags;
1656 
1657     return;
1658 }
1659 EXPORT_SYMBOL(lock_page_memcg);
1660 
1661 /**
1662  * unlock_page_memcg - unlock a page->mem_cgroup binding
1663  * @page: the page
1664  */
1665 void unlock_page_memcg(struct page *page)
1666 {
1667     struct mem_cgroup *memcg = page->mem_cgroup;
1668 
1669     if (memcg && memcg->move_lock_task == current) {
1670         unsigned long flags = memcg->move_lock_flags;
1671 
1672         memcg->move_lock_task = NULL;
1673         memcg->move_lock_flags = 0;
1674 
1675         spin_unlock_irqrestore(&memcg->move_lock, flags);
1676     }
1677 
1678     rcu_read_unlock();
1679 }
1680 EXPORT_SYMBOL(unlock_page_memcg);
1681 
1682 /*
1683  * size of first charge trial. "32" comes from vmscan.c's magic value.
1684  * TODO: maybe necessary to use big numbers in big irons.
1685  */
1686 #define CHARGE_BATCH    32U
1687 struct memcg_stock_pcp {
1688     struct mem_cgroup *cached; /* this never be root cgroup */
1689     unsigned int nr_pages;
1690     struct work_struct work;
1691     unsigned long flags;
1692 #define FLUSHING_CACHED_CHARGE  0
1693 };
1694 static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_stock);
1695 static DEFINE_MUTEX(percpu_charge_mutex);
1696 
1697 /**
1698  * consume_stock: Try to consume stocked charge on this cpu.
1699  * @memcg: memcg to consume from.
1700  * @nr_pages: how many pages to charge.
1701  *
1702  * The charges will only happen if @memcg matches the current cpu's memcg
1703  * stock, and at least @nr_pages are available in that stock.  Failure to
1704  * service an allocation will refill the stock.
1705  *
1706  * returns true if successful, false otherwise.
1707  */
1708 static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
1709 {
1710     struct memcg_stock_pcp *stock;
1711     unsigned long flags;
1712     bool ret = false;
1713 
1714     if (nr_pages > CHARGE_BATCH)
1715         return ret;
1716 
1717     local_irq_save(flags);
1718 
1719     stock = this_cpu_ptr(&memcg_stock);
1720     if (memcg == stock->cached && stock->nr_pages >= nr_pages) {
1721         stock->nr_pages -= nr_pages;
1722         ret = true;
1723     }
1724 
1725     local_irq_restore(flags);
1726 
1727     return ret;
1728 }
1729 
1730 /*
1731  * Returns stocks cached in percpu and reset cached information.
1732  */
1733 static void drain_stock(struct memcg_stock_pcp *stock)
1734 {
1735     struct mem_cgroup *old = stock->cached;
1736 
1737     if (stock->nr_pages) {
1738         page_counter_uncharge(&old->memory, stock->nr_pages);
1739         if (do_memsw_account())
1740             page_counter_uncharge(&old->memsw, stock->nr_pages);
1741         css_put_many(&old->css, stock->nr_pages);
1742         stock->nr_pages = 0;
1743     }
1744     stock->cached = NULL;
1745 }
1746 
1747 static void drain_local_stock(struct work_struct *dummy)
1748 {
1749     struct memcg_stock_pcp *stock;
1750     unsigned long flags;
1751 
1752     local_irq_save(flags);
1753 
1754     stock = this_cpu_ptr(&memcg_stock);
1755     drain_stock(stock);
1756     clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags);
1757 
1758     local_irq_restore(flags);
1759 }
1760 
1761 /*
1762  * Cache charges(val) to local per_cpu area.
1763  * This will be consumed by consume_stock() function, later.
1764  */
1765 static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
1766 {
1767     struct memcg_stock_pcp *stock;
1768     unsigned long flags;
1769 
1770     local_irq_save(flags);
1771 
1772     stock = this_cpu_ptr(&memcg_stock);
1773     if (stock->cached != memcg) { /* reset if necessary */
1774         drain_stock(stock);
1775         stock->cached = memcg;
1776     }
1777     stock->nr_pages += nr_pages;
1778 
1779     local_irq_restore(flags);
1780 }
1781 
1782 /*
1783  * Drains all per-CPU charge caches for given root_memcg resp. subtree
1784  * of the hierarchy under it.
1785  */
1786 static void drain_all_stock(struct mem_cgroup *root_memcg)
1787 {
1788     int cpu, curcpu;
1789 
1790     /* If someone's already draining, avoid adding running more workers. */
1791     if (!mutex_trylock(&percpu_charge_mutex))
1792         return;
1793     /* Notify other cpus that system-wide "drain" is running */
1794     get_online_cpus();
1795     curcpu = get_cpu();
1796     for_each_online_cpu(cpu) {
1797         struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
1798         struct mem_cgroup *memcg;
1799 
1800         memcg = stock->cached;
1801         if (!memcg || !stock->nr_pages)
1802             continue;
1803         if (!mem_cgroup_is_descendant(memcg, root_memcg))
1804             continue;
1805         if (!test_and_set_bit(FLUSHING_CACHED_CHARGE, &stock->flags)) {
1806             if (cpu == curcpu)
1807                 drain_local_stock(&stock->work);
1808             else
1809                 schedule_work_on(cpu, &stock->work);
1810         }
1811     }
1812     put_cpu();
1813     put_online_cpus();
1814     mutex_unlock(&percpu_charge_mutex);
1815 }
1816 
1817 static int memcg_hotplug_cpu_dead(unsigned int cpu)
1818 {
1819     struct memcg_stock_pcp *stock;
1820 
1821     stock = &per_cpu(memcg_stock, cpu);
1822     drain_stock(stock);
1823     return 0;
1824 }
1825 
1826 static void reclaim_high(struct mem_cgroup *memcg,
1827              unsigned int nr_pages,
1828              gfp_t gfp_mask)
1829 {
1830     do {
1831         if (page_counter_read(&memcg->memory) <= memcg->high)
1832             continue;
1833         mem_cgroup_events(memcg, MEMCG_HIGH, 1);
1834         try_to_free_mem_cgroup_pages(memcg, nr_pages, gfp_mask, true);
1835     } while ((memcg = parent_mem_cgroup(memcg)));
1836 }
1837 
1838 static void high_work_func(struct work_struct *work)
1839 {
1840     struct mem_cgroup *memcg;
1841 
1842     memcg = container_of(work, struct mem_cgroup, high_work);
1843     reclaim_high(memcg, CHARGE_BATCH, GFP_KERNEL);
1844 }
1845 
1846 /*
1847  * Scheduled by try_charge() to be executed from the userland return path
1848  * and reclaims memory over the high limit.
1849  */
1850 void mem_cgroup_handle_over_high(void)
1851 {
1852     unsigned int nr_pages = current->memcg_nr_pages_over_high;
1853     struct mem_cgroup *memcg;
1854 
1855     if (likely(!nr_pages))
1856         return;
1857 
1858     memcg = get_mem_cgroup_from_mm(current->mm);
1859     reclaim_high(memcg, nr_pages, GFP_KERNEL);
1860     css_put(&memcg->css);
1861     current->memcg_nr_pages_over_high = 0;
1862 }
1863 
1864 static int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask,
1865               unsigned int nr_pages)
1866 {
1867     unsigned int batch = max(CHARGE_BATCH, nr_pages);
1868     int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
1869     struct mem_cgroup *mem_over_limit;
1870     struct page_counter *counter;
1871     unsigned long nr_reclaimed;
1872     bool may_swap = true;
1873     bool drained = false;
1874 
1875     if (mem_cgroup_is_root(memcg))
1876         return 0;
1877 retry:
1878     if (consume_stock(memcg, nr_pages))
1879         return 0;
1880 
1881     if (!do_memsw_account() ||
1882         page_counter_try_charge(&memcg->memsw, batch, &counter)) {
1883         if (page_counter_try_charge(&memcg->memory, batch, &counter))
1884             goto done_restock;
1885         if (do_memsw_account())
1886             page_counter_uncharge(&memcg->memsw, batch);
1887         mem_over_limit = mem_cgroup_from_counter(counter, memory);
1888     } else {
1889         mem_over_limit = mem_cgroup_from_counter(counter, memsw);
1890         may_swap = false;
1891     }
1892 
1893     if (batch > nr_pages) {
1894         batch = nr_pages;
1895         goto retry;
1896     }
1897 
1898     /*
1899      * Unlike in global OOM situations, memcg is not in a physical
1900      * memory shortage.  Allow dying and OOM-killed tasks to
1901      * bypass the last charges so that they can exit quickly and
1902      * free their memory.
1903      */
1904     if (unlikely(test_thread_flag(TIF_MEMDIE) ||
1905              fatal_signal_pending(current) ||
1906              current->flags & PF_EXITING))
1907         goto force;
1908 
1909     /*
1910      * Prevent unbounded recursion when reclaim operations need to
1911      * allocate memory. This might exceed the limits temporarily,
1912      * but we prefer facilitating memory reclaim and getting back
1913      * under the limit over triggering OOM kills in these cases.
1914      */
1915     if (unlikely(current->flags & PF_MEMALLOC))
1916         goto force;
1917 
1918     if (unlikely(task_in_memcg_oom(current)))
1919         goto nomem;
1920 
1921     if (!gfpflags_allow_blocking(gfp_mask))
1922         goto nomem;
1923 
1924     mem_cgroup_events(mem_over_limit, MEMCG_MAX, 1);
1925 
1926     nr_reclaimed = try_to_free_mem_cgroup_pages(mem_over_limit, nr_pages,
1927                             gfp_mask, may_swap);
1928 
1929     if (mem_cgroup_margin(mem_over_limit) >= nr_pages)
1930         goto retry;
1931 
1932     if (!drained) {
1933         drain_all_stock(mem_over_limit);
1934         drained = true;
1935         goto retry;
1936     }
1937 
1938     if (gfp_mask & __GFP_NORETRY)
1939         goto nomem;
1940     /*
1941      * Even though the limit is exceeded at this point, reclaim
1942      * may have been able to free some pages.  Retry the charge
1943      * before killing the task.
1944      *
1945      * Only for regular pages, though: huge pages are rather
1946      * unlikely to succeed so close to the limit, and we fall back
1947      * to regular pages anyway in case of failure.
1948      */
1949     if (nr_reclaimed && nr_pages <= (1 << PAGE_ALLOC_COSTLY_ORDER))
1950         goto retry;
1951     /*
1952      * At task move, charge accounts can be doubly counted. So, it's
1953      * better to wait until the end of task_move if something is going on.
1954      */
1955     if (mem_cgroup_wait_acct_move(mem_over_limit))
1956         goto retry;
1957 
1958     if (nr_retries--)
1959         goto retry;
1960 
1961     if (gfp_mask & __GFP_NOFAIL)
1962         goto force;
1963 
1964     if (fatal_signal_pending(current))
1965         goto force;
1966 
1967     mem_cgroup_events(mem_over_limit, MEMCG_OOM, 1);
1968 
1969     mem_cgroup_oom(mem_over_limit, gfp_mask,
1970                get_order(nr_pages * PAGE_SIZE));
1971 nomem:
1972     if (!(gfp_mask & __GFP_NOFAIL))
1973         return -ENOMEM;
1974 force:
1975     /*
1976      * The allocation either can't fail or will lead to more memory
1977      * being freed very soon.  Allow memory usage go over the limit
1978      * temporarily by force charging it.
1979      */
1980     page_counter_charge(&memcg->memory, nr_pages);
1981     if (do_memsw_account())
1982         page_counter_charge(&memcg->memsw, nr_pages);
1983     css_get_many(&memcg->css, nr_pages);
1984 
1985     return 0;
1986 
1987 done_restock:
1988     css_get_many(&memcg->css, batch);
1989     if (batch > nr_pages)
1990         refill_stock(memcg, batch - nr_pages);
1991 
1992     /*
1993      * If the hierarchy is above the normal consumption range, schedule
1994      * reclaim on returning to userland.  We can perform reclaim here
1995      * if __GFP_RECLAIM but let's always punt for simplicity and so that
1996      * GFP_KERNEL can consistently be used during reclaim.  @memcg is
1997      * not recorded as it most likely matches current's and won't
1998      * change in the meantime.  As high limit is checked again before
1999      * reclaim, the cost of mismatch is negligible.
2000      */
2001     do {
2002         if (page_counter_read(&memcg->memory) > memcg->high) {
2003             /* Don't bother a random interrupted task */
2004             if (in_interrupt()) {
2005                 schedule_work(&memcg->high_work);
2006                 break;
2007             }
2008             current->memcg_nr_pages_over_high += batch;
2009             set_notify_resume(current);
2010             break;
2011         }
2012     } while ((memcg = parent_mem_cgroup(memcg)));
2013 
2014     return 0;
2015 }
2016 
2017 static void cancel_charge(struct mem_cgroup *memcg, unsigned int nr_pages)
2018 {
2019     if (mem_cgroup_is_root(memcg))
2020         return;
2021 
2022     page_counter_uncharge(&memcg->memory, nr_pages);
2023     if (do_memsw_account())
2024         page_counter_uncharge(&memcg->memsw, nr_pages);
2025 
2026     css_put_many(&memcg->css, nr_pages);
2027 }
2028 
2029 static void lock_page_lru(struct page *page, int *isolated)
2030 {
2031     struct zone *zone = page_zone(page);
2032 
2033     spin_lock_irq(zone_lru_lock(zone));
2034     if (PageLRU(page)) {
2035         struct lruvec *lruvec;
2036 
2037         lruvec = mem_cgroup_page_lruvec(page, zone->zone_pgdat);
2038         ClearPageLRU(page);
2039         del_page_from_lru_list(page, lruvec, page_lru(page));
2040         *isolated = 1;
2041     } else
2042         *isolated = 0;
2043 }
2044 
2045 static void unlock_page_lru(struct page *page, int isolated)
2046 {
2047     struct zone *zone = page_zone(page);
2048 
2049     if (isolated) {
2050         struct lruvec *lruvec;
2051 
2052         lruvec = mem_cgroup_page_lruvec(page, zone->zone_pgdat);
2053         VM_BUG_ON_PAGE(PageLRU(page), page);
2054         SetPageLRU(page);
2055         add_page_to_lru_list(page, lruvec, page_lru(page));
2056     }
2057     spin_unlock_irq(zone_lru_lock(zone));
2058 }
2059 
2060 static void commit_charge(struct page *page, struct mem_cgroup *memcg,
2061               bool lrucare)
2062 {
2063     int isolated;
2064 
2065     VM_BUG_ON_PAGE(page->mem_cgroup, page);
2066 
2067     /*
2068      * In some cases, SwapCache and FUSE(splice_buf->radixtree), the page
2069      * may already be on some other mem_cgroup's LRU.  Take care of it.
2070      */
2071     if (lrucare)
2072         lock_page_lru(page, &isolated);
2073 
2074     /*
2075      * Nobody should be changing or seriously looking at
2076      * page->mem_cgroup at this point:
2077      *
2078      * - the page is uncharged
2079      *
2080      * - the page is off-LRU
2081      *
2082      * - an anonymous fault has exclusive page access, except for
2083      *   a locked page table
2084      *
2085      * - a page cache insertion, a swapin fault, or a migration
2086      *   have the page locked
2087      */
2088     page->mem_cgroup = memcg;
2089 
2090     if (lrucare)
2091         unlock_page_lru(page, isolated);
2092 }
2093 
2094 #ifndef CONFIG_SLOB
2095 static int memcg_alloc_cache_id(void)
2096 {
2097     int id, size;
2098     int err;
2099 
2100     id = ida_simple_get(&memcg_cache_ida,
2101                 0, MEMCG_CACHES_MAX_SIZE, GFP_KERNEL);
2102     if (id < 0)
2103         return id;
2104 
2105     if (id < memcg_nr_cache_ids)
2106         return id;
2107 
2108     /*
2109      * There's no space for the new id in memcg_caches arrays,
2110      * so we have to grow them.
2111      */
2112     down_write(&memcg_cache_ids_sem);
2113 
2114     size = 2 * (id + 1);
2115     if (size < MEMCG_CACHES_MIN_SIZE)
2116         size = MEMCG_CACHES_MIN_SIZE;
2117     else if (size > MEMCG_CACHES_MAX_SIZE)
2118         size = MEMCG_CACHES_MAX_SIZE;
2119 
2120     err = memcg_update_all_caches(size);
2121     if (!err)
2122         err = memcg_update_all_list_lrus(size);
2123     if (!err)
2124         memcg_nr_cache_ids = size;
2125 
2126     up_write(&memcg_cache_ids_sem);
2127 
2128     if (err) {
2129         ida_simple_remove(&memcg_cache_ida, id);
2130         return err;
2131     }
2132     return id;
2133 }
2134 
2135 static void memcg_free_cache_id(int id)
2136 {
2137     ida_simple_remove(&memcg_cache_ida, id);
2138 }
2139 
2140 struct memcg_kmem_cache_create_work {
2141     struct mem_cgroup *memcg;
2142     struct kmem_cache *cachep;
2143     struct work_struct work;
2144 };
2145 
2146 static struct workqueue_struct *memcg_kmem_cache_create_wq;
2147 
2148 static void memcg_kmem_cache_create_func(struct work_struct *w)
2149 {
2150     struct memcg_kmem_cache_create_work *cw =
2151         container_of(w, struct memcg_kmem_cache_create_work, work);
2152     struct mem_cgroup *memcg = cw->memcg;
2153     struct kmem_cache *cachep = cw->cachep;
2154 
2155     memcg_create_kmem_cache(memcg, cachep);
2156 
2157     css_put(&memcg->css);
2158     kfree(cw);
2159 }
2160 
2161 /*
2162  * Enqueue the creation of a per-memcg kmem_cache.
2163  */
2164 static void __memcg_schedule_kmem_cache_create(struct mem_cgroup *memcg,
2165                            struct kmem_cache *cachep)
2166 {
2167     struct memcg_kmem_cache_create_work *cw;
2168 
2169     cw = kmalloc(sizeof(*cw), GFP_NOWAIT);
2170     if (!cw)
2171         return;
2172 
2173     css_get(&memcg->css);
2174 
2175     cw->memcg = memcg;
2176     cw->cachep = cachep;
2177     INIT_WORK(&cw->work, memcg_kmem_cache_create_func);
2178 
2179     queue_work(memcg_kmem_cache_create_wq, &cw->work);
2180 }
2181 
2182 static void memcg_schedule_kmem_cache_create(struct mem_cgroup *memcg,
2183                          struct kmem_cache *cachep)
2184 {
2185     /*
2186      * We need to stop accounting when we kmalloc, because if the
2187      * corresponding kmalloc cache is not yet created, the first allocation
2188      * in __memcg_schedule_kmem_cache_create will recurse.
2189      *
2190      * However, it is better to enclose the whole function. Depending on
2191      * the debugging options enabled, INIT_WORK(), for instance, can
2192      * trigger an allocation. This too, will make us recurse. Because at
2193      * this point we can't allow ourselves back into memcg_kmem_get_cache,
2194      * the safest choice is to do it like this, wrapping the whole function.
2195      */
2196     current->memcg_kmem_skip_account = 1;
2197     __memcg_schedule_kmem_cache_create(memcg, cachep);
2198     current->memcg_kmem_skip_account = 0;
2199 }
2200 
2201 static inline bool memcg_kmem_bypass(void)
2202 {
2203     if (in_interrupt() || !current->mm || (current->flags & PF_KTHREAD))
2204         return true;
2205     return false;
2206 }
2207 
2208 /**
2209  * memcg_kmem_get_cache: select the correct per-memcg cache for allocation
2210  * @cachep: the original global kmem cache
2211  *
2212  * Return the kmem_cache we're supposed to use for a slab allocation.
2213  * We try to use the current memcg's version of the cache.
2214  *
2215  * If the cache does not exist yet, if we are the first user of it, we
2216  * create it asynchronously in a workqueue and let the current allocation
2217  * go through with the original cache.
2218  *
2219  * This function takes a reference to the cache it returns to assure it
2220  * won't get destroyed while we are working with it. Once the caller is
2221  * done with it, memcg_kmem_put_cache() must be called to release the
2222  * reference.
2223  */
2224 struct kmem_cache *memcg_kmem_get_cache(struct kmem_cache *cachep)
2225 {
2226     struct mem_cgroup *memcg;
2227     struct kmem_cache *memcg_cachep;
2228     int kmemcg_id;
2229 
2230     VM_BUG_ON(!is_root_cache(cachep));
2231 
2232     if (memcg_kmem_bypass())
2233         return cachep;
2234 
2235     if (current->memcg_kmem_skip_account)
2236         return cachep;
2237 
2238     memcg = get_mem_cgroup_from_mm(current->mm);
2239     kmemcg_id = READ_ONCE(memcg->kmemcg_id);
2240     if (kmemcg_id < 0)
2241         goto out;
2242 
2243     memcg_cachep = cache_from_memcg_idx(cachep, kmemcg_id);
2244     if (likely(memcg_cachep))
2245         return memcg_cachep;
2246 
2247     /*
2248      * If we are in a safe context (can wait, and not in interrupt
2249      * context), we could be be predictable and return right away.
2250      * This would guarantee that the allocation being performed
2251      * already belongs in the new cache.
2252      *
2253      * However, there are some clashes that can arrive from locking.
2254      * For instance, because we acquire the slab_mutex while doing
2255      * memcg_create_kmem_cache, this means no further allocation
2256      * could happen with the slab_mutex held. So it's better to
2257      * defer everything.
2258      */
2259     memcg_schedule_kmem_cache_create(memcg, cachep);
2260 out:
2261     css_put(&memcg->css);
2262     return cachep;
2263 }
2264 
2265 /**
2266  * memcg_kmem_put_cache: drop reference taken by memcg_kmem_get_cache
2267  * @cachep: the cache returned by memcg_kmem_get_cache
2268  */
2269 void memcg_kmem_put_cache(struct kmem_cache *cachep)
2270 {
2271     if (!is_root_cache(cachep))
2272         css_put(&cachep->memcg_params.memcg->css);
2273 }
2274 
2275 /**
2276  * memcg_kmem_charge: charge a kmem page
2277  * @page: page to charge
2278  * @gfp: reclaim mode
2279  * @order: allocation order
2280  * @memcg: memory cgroup to charge
2281  *
2282  * Returns 0 on success, an error code on failure.
2283  */
2284 int memcg_kmem_charge_memcg(struct page *page, gfp_t gfp, int order,
2285                 struct mem_cgroup *memcg)
2286 {
2287     unsigned int nr_pages = 1 << order;
2288     struct page_counter *counter;
2289     int ret;
2290 
2291     ret = try_charge(memcg, gfp, nr_pages);
2292     if (ret)
2293         return ret;
2294 
2295     if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) &&
2296         !page_counter_try_charge(&memcg->kmem, nr_pages, &counter)) {
2297         cancel_charge(memcg, nr_pages);
2298         return -ENOMEM;
2299     }
2300 
2301     page->mem_cgroup = memcg;
2302 
2303     return 0;
2304 }
2305 
2306 /**
2307  * memcg_kmem_charge: charge a kmem page to the current memory cgroup
2308  * @page: page to charge
2309  * @gfp: reclaim mode
2310  * @order: allocation order
2311  *
2312  * Returns 0 on success, an error code on failure.
2313  */
2314 int memcg_kmem_charge(struct page *page, gfp_t gfp, int order)
2315 {
2316     struct mem_cgroup *memcg;
2317     int ret = 0;
2318 
2319     if (memcg_kmem_bypass())
2320         return 0;
2321 
2322     memcg = get_mem_cgroup_from_mm(current->mm);
2323     if (!mem_cgroup_is_root(memcg)) {
2324         ret = memcg_kmem_charge_memcg(page, gfp, order, memcg);
2325         if (!ret)
2326             __SetPageKmemcg(page);
2327     }
2328     css_put(&memcg->css);
2329     return ret;
2330 }
2331 /**
2332  * memcg_kmem_uncharge: uncharge a kmem page
2333  * @page: page to uncharge
2334  * @order: allocation order
2335  */
2336 void memcg_kmem_uncharge(struct page *page, int order)
2337 {
2338     struct mem_cgroup *memcg = page->mem_cgroup;
2339     unsigned int nr_pages = 1 << order;
2340 
2341     if (!memcg)
2342         return;
2343 
2344     VM_BUG_ON_PAGE(mem_cgroup_is_root(memcg), page);
2345 
2346     if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
2347         page_counter_uncharge(&memcg->kmem, nr_pages);
2348 
2349     page_counter_uncharge(&memcg->memory, nr_pages);
2350     if (do_memsw_account())
2351         page_counter_uncharge(&memcg->memsw, nr_pages);
2352 
2353     page->mem_cgroup = NULL;
2354 
2355     /* slab pages do not have PageKmemcg flag set */
2356     if (PageKmemcg(page))
2357         __ClearPageKmemcg(page);
2358 
2359     css_put_many(&memcg->css, nr_pages);
2360 }
2361 #endif /* !CONFIG_SLOB */
2362 
2363 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
2364 
2365 /*
2366  * Because tail pages are not marked as "used", set it. We're under
2367  * zone_lru_lock and migration entries setup in all page mappings.
2368  */
2369 void mem_cgroup_split_huge_fixup(struct page *head)
2370 {
2371     int i;
2372 
2373     if (mem_cgroup_disabled())
2374         return;
2375 
2376     for (i = 1; i < HPAGE_PMD_NR; i++)
2377         head[i].mem_cgroup = head->mem_cgroup;
2378 
2379     __this_cpu_sub(head->mem_cgroup->stat->count[MEM_CGROUP_STAT_RSS_HUGE],
2380                HPAGE_PMD_NR);
2381 }
2382 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
2383 
2384 #ifdef CONFIG_MEMCG_SWAP
2385 static void mem_cgroup_swap_statistics(struct mem_cgroup *memcg,
2386                      bool charge)
2387 {
2388     int val = (charge) ? 1 : -1;
2389     this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_SWAP], val);
2390 }
2391 
2392 /**
2393  * mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record.
2394  * @entry: swap entry to be moved
2395  * @from:  mem_cgroup which the entry is moved from
2396  * @to:  mem_cgroup which the entry is moved to
2397  *
2398  * It succeeds only when the swap_cgroup's record for this entry is the same
2399  * as the mem_cgroup's id of @from.
2400  *
2401  * Returns 0 on success, -EINVAL on failure.
2402  *
2403  * The caller must have charged to @to, IOW, called page_counter_charge() about
2404  * both res and memsw, and called css_get().
2405  */
2406 static int mem_cgroup_move_swap_account(swp_entry_t entry,
2407                 struct mem_cgroup *from, struct mem_cgroup *to)
2408 {
2409     unsigned short old_id, new_id;
2410 
2411     old_id = mem_cgroup_id(from);
2412     new_id = mem_cgroup_id(to);
2413 
2414     if (swap_cgroup_cmpxchg(entry, old_id, new_id) == old_id) {
2415         mem_cgroup_swap_statistics(from, false);
2416         mem_cgroup_swap_statistics(to, true);
2417         return 0;
2418     }
2419     return -EINVAL;
2420 }
2421 #else
2422 static inline int mem_cgroup_move_swap_account(swp_entry_t entry,
2423                 struct mem_cgroup *from, struct mem_cgroup *to)
2424 {
2425     return -EINVAL;
2426 }
2427 #endif
2428 
2429 static DEFINE_MUTEX(memcg_limit_mutex);
2430 
2431 static int mem_cgroup_resize_limit(struct mem_cgroup *memcg,
2432                    unsigned long limit)
2433 {
2434     unsigned long curusage;
2435     unsigned long oldusage;
2436     bool enlarge = false;
2437     int retry_count;
2438     int ret;
2439 
2440     /*
2441      * For keeping hierarchical_reclaim simple, how long we should retry
2442      * is depends on callers. We set our retry-count to be function
2443      * of # of children which we should visit in this loop.
2444      */
2445     retry_count = MEM_CGROUP_RECLAIM_RETRIES *
2446               mem_cgroup_count_children(memcg);
2447 
2448     oldusage = page_counter_read(&memcg->memory);
2449 
2450     do {
2451         if (signal_pending(current)) {
2452             ret = -EINTR;
2453             break;
2454         }
2455 
2456         mutex_lock(&memcg_limit_mutex);
2457         if (limit > memcg->memsw.limit) {
2458             mutex_unlock(&memcg_limit_mutex);
2459             ret = -EINVAL;
2460             break;
2461         }
2462         if (limit > memcg->memory.limit)
2463             enlarge = true;
2464         ret = page_counter_limit(&memcg->memory, limit);
2465         mutex_unlock(&memcg_limit_mutex);
2466 
2467         if (!ret)
2468             break;
2469 
2470         try_to_free_mem_cgroup_pages(memcg, 1, GFP_KERNEL, true);
2471 
2472         curusage = page_counter_read(&memcg->memory);
2473         /* Usage is reduced ? */
2474         if (curusage >= oldusage)
2475             retry_count--;
2476         else
2477             oldusage = curusage;
2478     } while (retry_count);
2479 
2480     if (!ret && enlarge)
2481         memcg_oom_recover(memcg);
2482 
2483     return ret;
2484 }
2485 
2486 static int mem_cgroup_resize_memsw_limit(struct mem_cgroup *memcg,
2487                      unsigned long limit)
2488 {
2489     unsigned long curusage;
2490     unsigned long oldusage;
2491     bool enlarge = false;
2492     int retry_count;
2493     int ret;
2494 
2495     /* see mem_cgroup_resize_res_limit */
2496     retry_count = MEM_CGROUP_RECLAIM_RETRIES *
2497               mem_cgroup_count_children(memcg);
2498 
2499     oldusage = page_counter_read(&memcg->memsw);
2500 
2501     do {
2502         if (signal_pending(current)) {
2503             ret = -EINTR;
2504             break;
2505         }
2506 
2507         mutex_lock(&memcg_limit_mutex);
2508         if (limit < memcg->memory.limit) {
2509             mutex_unlock(&memcg_limit_mutex);
2510             ret = -EINVAL;
2511             break;
2512         }
2513         if (limit > memcg->memsw.limit)
2514             enlarge = true;
2515         ret = page_counter_limit(&memcg->memsw, limit);
2516         mutex_unlock(&memcg_limit_mutex);
2517 
2518         if (!ret)
2519             break;
2520 
2521         try_to_free_mem_cgroup_pages(memcg, 1, GFP_KERNEL, false);
2522 
2523         curusage = page_counter_read(&memcg->memsw);
2524         /* Usage is reduced ? */
2525         if (curusage >= oldusage)
2526             retry_count--;
2527         else
2528             oldusage = curusage;
2529     } while (retry_count);
2530 
2531     if (!ret && enlarge)
2532         memcg_oom_recover(memcg);
2533 
2534     return ret;
2535 }
2536 
2537 unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order,
2538                         gfp_t gfp_mask,
2539                         unsigned long *total_scanned)
2540 {
2541     unsigned long nr_reclaimed = 0;
2542     struct mem_cgroup_per_node *mz, *next_mz = NULL;
2543     unsigned long reclaimed;
2544     int loop = 0;
2545     struct mem_cgroup_tree_per_node *mctz;
2546     unsigned long excess;
2547     unsigned long nr_scanned;
2548 
2549     if (order > 0)
2550         return 0;
2551 
2552     mctz = soft_limit_tree_node(pgdat->node_id);
2553 
2554     /*
2555      * Do not even bother to check the largest node if the root
2556      * is empty. Do it lockless to prevent lock bouncing. Races
2557      * are acceptable as soft limit is best effort anyway.
2558      */
2559     if (RB_EMPTY_ROOT(&mctz->rb_root))
2560         return 0;
2561 
2562     /*
2563      * This loop can run a while, specially if mem_cgroup's continuously
2564      * keep exceeding their soft limit and putting the system under
2565      * pressure
2566      */
2567     do {
2568         if (next_mz)
2569             mz = next_mz;
2570         else
2571             mz = mem_cgroup_largest_soft_limit_node(mctz);
2572         if (!mz)
2573             break;
2574 
2575         nr_scanned = 0;
2576         reclaimed = mem_cgroup_soft_reclaim(mz->memcg, pgdat,
2577                             gfp_mask, &nr_scanned);
2578         nr_reclaimed += reclaimed;
2579         *total_scanned += nr_scanned;
2580         spin_lock_irq(&mctz->lock);
2581         __mem_cgroup_remove_exceeded(mz, mctz);
2582 
2583         /*
2584          * If we failed to reclaim anything from this memory cgroup
2585          * it is time to move on to the next cgroup
2586          */
2587         next_mz = NULL;
2588         if (!reclaimed)
2589             next_mz = __mem_cgroup_largest_soft_limit_node(mctz);
2590 
2591         excess = soft_limit_excess(mz->memcg);
2592         /*
2593          * One school of thought says that we should not add
2594          * back the node to the tree if reclaim returns 0.
2595          * But our reclaim could return 0, simply because due
2596          * to priority we are exposing a smaller subset of
2597          * memory to reclaim from. Consider this as a longer
2598          * term TODO.
2599          */
2600         /* If excess == 0, no tree ops */
2601         __mem_cgroup_insert_exceeded(mz, mctz, excess);
2602         spin_unlock_irq(&mctz->lock);
2603         css_put(&mz->memcg->css);
2604         loop++;
2605         /*
2606          * Could not reclaim anything and there are no more
2607          * mem cgroups to try or we seem to be looping without
2608          * reclaiming anything.
2609          */
2610         if (!nr_reclaimed &&
2611             (next_mz == NULL ||
2612             loop > MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS))
2613             break;
2614     } while (!nr_reclaimed);
2615     if (next_mz)
2616         css_put(&next_mz->memcg->css);
2617     return nr_reclaimed;
2618 }
2619 
2620 /*
2621  * Test whether @memcg has children, dead or alive.  Note that this
2622  * function doesn't care whether @memcg has use_hierarchy enabled and
2623  * returns %true if there are child csses according to the cgroup
2624  * hierarchy.  Testing use_hierarchy is the caller's responsiblity.
2625  */
2626 static inline bool memcg_has_children(struct mem_cgroup *memcg)
2627 {
2628     bool ret;
2629 
2630     rcu_read_lock();
2631     ret = css_next_child(NULL, &memcg->css);
2632     rcu_read_unlock();
2633     return ret;
2634 }
2635 
2636 /*
2637  * Reclaims as many pages from the given memcg as possible.
2638  *
2639  * Caller is responsible for holding css reference for memcg.
2640  */
2641 static int mem_cgroup_force_empty(struct mem_cgroup *memcg)
2642 {
2643     int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
2644 
2645     /* we call try-to-free pages for make this cgroup empty */
2646     lru_add_drain_all();
2647     /* try to free all pages in this cgroup */
2648     while (nr_retries && page_counter_read(&memcg->memory)) {
2649         int progress;
2650 
2651         if (signal_pending(current))
2652             return -EINTR;
2653 
2654         progress = try_to_free_mem_cgroup_pages(memcg, 1,
2655                             GFP_KERNEL, true);
2656         if (!progress) {
2657             nr_retries--;
2658             /* maybe some writeback is necessary */
2659             congestion_wait(BLK_RW_ASYNC, HZ/10);
2660         }
2661 
2662     }
2663 
2664     return 0;
2665 }
2666 
2667 static ssize_t mem_cgroup_force_empty_write(struct kernfs_open_file *of,
2668                         char *buf, size_t nbytes,
2669                         loff_t off)
2670 {
2671     struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
2672 
2673     if (mem_cgroup_is_root(memcg))
2674         return -EINVAL;
2675     return mem_cgroup_force_empty(memcg) ?: nbytes;
2676 }
2677 
2678 static u64 mem_cgroup_hierarchy_read(struct cgroup_subsys_state *css,
2679                      struct cftype *cft)
2680 {
2681     return mem_cgroup_from_css(css)->use_hierarchy;
2682 }
2683 
2684 static int mem_cgroup_hierarchy_write(struct cgroup_subsys_state *css,
2685                       struct cftype *cft, u64 val)
2686 {
2687     int retval = 0;
2688     struct mem_cgroup *memcg = mem_cgroup_from_css(css);
2689     struct mem_cgroup *parent_memcg = mem_cgroup_from_css(memcg->css.parent);
2690 
2691     if (memcg->use_hierarchy == val)
2692         return 0;
2693 
2694     /*
2695      * If parent's use_hierarchy is set, we can't make any modifications
2696      * in the child subtrees. If it is unset, then the change can
2697      * occur, provided the current cgroup has no children.
2698      *
2699      * For the root cgroup, parent_mem is NULL, we allow value to be
2700      * set if there are no children.
2701      */
2702     if ((!parent_memcg || !parent_memcg->use_hierarchy) &&
2703                 (val == 1 || val == 0)) {
2704         if (!memcg_has_children(memcg))
2705             memcg->use_hierarchy = val;
2706         else
2707             retval = -EBUSY;
2708     } else
2709         retval = -EINVAL;
2710 
2711     return retval;
2712 }
2713 
2714 static void tree_stat(struct mem_cgroup *memcg, unsigned long *stat)
2715 {
2716     struct mem_cgroup *iter;
2717     int i;
2718 
2719     memset(stat, 0, sizeof(*stat) * MEMCG_NR_STAT);
2720 
2721     for_each_mem_cgroup_tree(iter, memcg) {
2722         for (i = 0; i < MEMCG_NR_STAT; i++)
2723             stat[i] += mem_cgroup_read_stat(iter, i);
2724     }
2725 }
2726 
2727 static void tree_events(struct mem_cgroup *memcg, unsigned long *events)
2728 {
2729     struct mem_cgroup *iter;
2730     int i;
2731 
2732     memset(events, 0, sizeof(*events) * MEMCG_NR_EVENTS);
2733 
2734     for_each_mem_cgroup_tree(iter, memcg) {
2735         for (i = 0; i < MEMCG_NR_EVENTS; i++)
2736             events[i] += mem_cgroup_read_events(iter, i);
2737     }
2738 }
2739 
2740 static unsigned long mem_cgroup_usage(struct mem_cgroup *memcg, bool swap)
2741 {
2742     unsigned long val = 0;
2743 
2744     if (mem_cgroup_is_root(memcg)) {
2745         struct mem_cgroup *iter;
2746 
2747         for_each_mem_cgroup_tree(iter, memcg) {
2748             val += mem_cgroup_read_stat(iter,
2749                     MEM_CGROUP_STAT_CACHE);
2750             val += mem_cgroup_read_stat(iter,
2751                     MEM_CGROUP_STAT_RSS);
2752             if (swap)
2753                 val += mem_cgroup_read_stat(iter,
2754                         MEM_CGROUP_STAT_SWAP);
2755         }
2756     } else {
2757         if (!swap)
2758             val = page_counter_read(&memcg->memory);
2759         else
2760             val = page_counter_read(&memcg->memsw);
2761     }
2762     return val;
2763 }
2764 
2765 enum {
2766     RES_USAGE,
2767     RES_LIMIT,
2768     RES_MAX_USAGE,
2769     RES_FAILCNT,
2770     RES_SOFT_LIMIT,
2771 };
2772 
2773 static u64 mem_cgroup_read_u64(struct cgroup_subsys_state *css,
2774                    struct cftype *cft)
2775 {
2776     struct mem_cgroup *memcg = mem_cgroup_from_css(css);
2777     struct page_counter *counter;
2778 
2779     switch (MEMFILE_TYPE(cft->private)) {
2780     case _MEM:
2781         counter = &memcg->memory;
2782         break;
2783     case _MEMSWAP:
2784         counter = &memcg->memsw;
2785         break;
2786     case _KMEM:
2787         counter = &memcg->kmem;
2788         break;
2789     case _TCP:
2790         counter = &memcg->tcpmem;
2791         break;
2792     default:
2793         BUG();
2794     }
2795 
2796     switch (MEMFILE_ATTR(cft->private)) {
2797     case RES_USAGE:
2798         if (counter == &memcg->memory)
2799             return (u64)mem_cgroup_usage(memcg, false) * PAGE_SIZE;
2800         if (counter == &memcg->memsw)
2801             return (u64)mem_cgroup_usage(memcg, true) * PAGE_SIZE;
2802         return (u64)page_counter_read(counter) * PAGE_SIZE;
2803     case RES_LIMIT:
2804         return (u64)counter->limit * PAGE_SIZE;
2805     case RES_MAX_USAGE:
2806         return (u64)counter->watermark * PAGE_SIZE;
2807     case RES_FAILCNT:
2808         return counter->failcnt;
2809     case RES_SOFT_LIMIT:
2810         return (u64)memcg->soft_limit * PAGE_SIZE;
2811     default:
2812         BUG();
2813     }
2814 }
2815 
2816 #ifndef CONFIG_SLOB
2817 static int memcg_online_kmem(struct mem_cgroup *memcg)
2818 {
2819     int memcg_id;
2820 
2821     if (cgroup_memory_nokmem)
2822         return 0;
2823 
2824     BUG_ON(memcg->kmemcg_id >= 0);
2825     BUG_ON(memcg->kmem_state);
2826 
2827     memcg_id = memcg_alloc_cache_id();
2828     if (memcg_id < 0)
2829         return memcg_id;
2830 
2831     static_branch_inc(&memcg_kmem_enabled_key);
2832     /*
2833      * A memory cgroup is considered kmem-online as soon as it gets
2834      * kmemcg_id. Setting the id after enabling static branching will
2835      * guarantee no one starts accounting before all call sites are
2836      * patched.
2837      */
2838     memcg->kmemcg_id = memcg_id;
2839     memcg->kmem_state = KMEM_ONLINE;
2840 
2841     return 0;
2842 }
2843 
2844 static void memcg_offline_kmem(struct mem_cgroup *memcg)
2845 {
2846     struct cgroup_subsys_state *css;
2847     struct mem_cgroup *parent, *child;
2848     int kmemcg_id;
2849 
2850     if (memcg->kmem_state != KMEM_ONLINE)
2851         return;
2852     /*
2853      * Clear the online state before clearing memcg_caches array
2854      * entries. The slab_mutex in memcg_deactivate_kmem_caches()
2855      * guarantees that no cache will be created for this cgroup
2856      * after we are done (see memcg_create_kmem_cache()).
2857      */
2858     memcg->kmem_state = KMEM_ALLOCATED;
2859 
2860     memcg_deactivate_kmem_caches(memcg);
2861 
2862     kmemcg_id = memcg->kmemcg_id;
2863     BUG_ON(kmemcg_id < 0);
2864 
2865     parent = parent_mem_cgroup(memcg);
2866     if (!parent)
2867         parent = root_mem_cgroup;
2868 
2869     /*
2870      * Change kmemcg_id of this cgroup and all its descendants to the
2871      * parent's id, and then move all entries from this cgroup's list_lrus
2872      * to ones of the parent. After we have finished, all list_lrus
2873      * corresponding to this cgroup are guaranteed to remain empty. The
2874      * ordering is imposed by list_lru_node->lock taken by
2875      * memcg_drain_all_list_lrus().
2876      */
2877     rcu_read_lock(); /* can be called from css_free w/o cgroup_mutex */
2878     css_for_each_descendant_pre(css, &memcg->css) {
2879         child = mem_cgroup_from_css(css);
2880         BUG_ON(child->kmemcg_id != kmemcg_id);
2881         child->kmemcg_id = parent->kmemcg_id;
2882         if (!memcg->use_hierarchy)
2883             break;
2884     }
2885     rcu_read_unlock();
2886 
2887     memcg_drain_all_list_lrus(kmemcg_id, parent->kmemcg_id);
2888 
2889     memcg_free_cache_id(kmemcg_id);
2890 }
2891 
2892 static void memcg_free_kmem(struct mem_cgroup *memcg)
2893 {
2894     /* css_alloc() failed, offlining didn't happen */
2895     if (unlikely(memcg->kmem_state == KMEM_ONLINE))
2896         memcg_offline_kmem(memcg);
2897 
2898     if (memcg->kmem_state == KMEM_ALLOCATED) {
2899         memcg_destroy_kmem_caches(memcg);
2900         static_branch_dec(&memcg_kmem_enabled_key);
2901         WARN_ON(page_counter_read(&memcg->kmem));
2902     }
2903 }
2904 #else
2905 static int memcg_online_kmem(struct mem_cgroup *memcg)
2906 {
2907     return 0;
2908 }
2909 static void memcg_offline_kmem(struct mem_cgroup *memcg)
2910 {
2911 }
2912 static void memcg_free_kmem(struct mem_cgroup *memcg)
2913 {
2914 }
2915 #endif /* !CONFIG_SLOB */
2916 
2917 static int memcg_update_kmem_limit(struct mem_cgroup *memcg,
2918                    unsigned long limit)
2919 {
2920     int ret;
2921 
2922     mutex_lock(&memcg_limit_mutex);
2923     ret = page_counter_limit(&memcg->kmem, limit);
2924     mutex_unlock(&memcg_limit_mutex);
2925     return ret;
2926 }
2927 
2928 static int memcg_update_tcp_limit(struct mem_cgroup *memcg, unsigned long limit)
2929 {
2930     int ret;
2931 
2932     mutex_lock(&memcg_limit_mutex);
2933 
2934     ret = page_counter_limit(&memcg->tcpmem, limit);
2935     if (ret)
2936         goto out;
2937 
2938     if (!memcg->tcpmem_active) {
2939         /*
2940          * The active flag needs to be written after the static_key
2941          * update. This is what guarantees that the socket activation
2942          * function is the last one to run. See mem_cgroup_sk_alloc()
2943          * for details, and note that we don't mark any socket as
2944          * belonging to this memcg until that flag is up.
2945          *
2946          * We need to do this, because static_keys will span multiple
2947          * sites, but we can't control their order. If we mark a socket
2948          * as accounted, but the accounting functions are not patched in
2949          * yet, we'll lose accounting.
2950          *
2951          * We never race with the readers in mem_cgroup_sk_alloc(),
2952          * because when this value change, the code to process it is not
2953          * patched in yet.
2954          */
2955         static_branch_inc(&memcg_sockets_enabled_key);
2956         memcg->tcpmem_active = true;
2957     }
2958 out:
2959     mutex_unlock(&memcg_limit_mutex);
2960     return ret;
2961 }
2962 
2963 /*
2964  * The user of this function is...
2965  * RES_LIMIT.
2966  */
2967 static ssize_t mem_cgroup_write(struct kernfs_open_file *of,
2968                 char *buf, size_t nbytes, loff_t off)
2969 {
2970     struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
2971     unsigned long nr_pages;
2972     int ret;
2973 
2974     buf = strstrip(buf);
2975     ret = page_counter_memparse(buf, "-1", &nr_pages);
2976     if (ret)
2977         return ret;
2978 
2979     switch (MEMFILE_ATTR(of_cft(of)->private)) {
2980     case RES_LIMIT:
2981         if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */
2982             ret = -EINVAL;
2983             break;
2984         }
2985         switch (MEMFILE_TYPE(of_cft(of)->private)) {
2986         case _MEM:
2987             ret = mem_cgroup_resize_limit(memcg, nr_pages);
2988             break;
2989         case _MEMSWAP:
2990             ret = mem_cgroup_resize_memsw_limit(memcg, nr_pages);
2991             break;
2992         case _KMEM:
2993             ret = memcg_update_kmem_limit(memcg, nr_pages);
2994             break;
2995         case _TCP:
2996             ret = memcg_update_tcp_limit(memcg, nr_pages);
2997             break;
2998         }
2999         break;
3000     case RES_SOFT_LIMIT:
3001         memcg->soft_limit = nr_pages;
3002         ret = 0;
3003         break;
3004     }
3005     return ret ?: nbytes;
3006 }
3007 
3008 static ssize_t mem_cgroup_reset(struct kernfs_open_file *of, char *buf,
3009                 size_t nbytes, loff_t off)
3010 {
3011     struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
3012     struct page_counter *counter;
3013 
3014     switch (MEMFILE_TYPE(of_cft(of)->private)) {
3015     case _MEM:
3016         counter = &memcg->memory;
3017         break;
3018     case _MEMSWAP:
3019         counter = &memcg->memsw;
3020         break;
3021     case _KMEM:
3022         counter = &memcg->kmem;
3023         break;
3024     case _TCP:
3025         counter = &memcg->tcpmem;
3026         break;
3027     default:
3028         BUG();
3029     }
3030 
3031     switch (MEMFILE_ATTR(of_cft(of)->private)) {
3032     case RES_MAX_USAGE:
3033         page_counter_reset_watermark(counter);
3034         break;
3035     case RES_FAILCNT:
3036         counter->failcnt = 0;
3037         break;
3038     default:
3039         BUG();
3040     }
3041 
3042     return nbytes;
3043 }
3044 
3045 static u64 mem_cgroup_move_charge_read(struct cgroup_subsys_state *css,
3046                     struct cftype *cft)
3047 {
3048     return mem_cgroup_from_css(css)->move_charge_at_immigrate;
3049 }
3050 
3051 #ifdef CONFIG_MMU
3052 static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
3053                     struct cftype *cft, u64 val)
3054 {
3055     struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3056 
3057     if (val & ~MOVE_MASK)
3058         return -EINVAL;
3059 
3060     /*
3061      * No kind of locking is needed in here, because ->can_attach() will
3062      * check this value once in the beginning of the process, and then carry
3063      * on with stale data. This means that changes to this value will only
3064      * affect task migrations starting after the change.
3065      */
3066     memcg->move_charge_at_immigrate = val;
3067     return 0;
3068 }
3069 #else
3070 static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
3071                     struct cftype *cft, u64 val)
3072 {
3073     return -ENOSYS;
3074 }
3075 #endif
3076 
3077 #ifdef CONFIG_NUMA
3078 static int memcg_numa_stat_show(struct seq_file *m, void *v)
3079 {
3080     struct numa_stat {
3081         const char *name;
3082         unsigned int lru_mask;
3083     };
3084 
3085     static const struct numa_stat stats[] = {
3086         { "total", LRU_ALL },
3087         { "file", LRU_ALL_FILE },
3088         { "anon", LRU_ALL_ANON },
3089         { "unevictable", BIT(LRU_UNEVICTABLE) },
3090     };
3091     const struct numa_stat *stat;
3092     int nid;
3093     unsigned long nr;
3094     struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
3095 
3096     for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) {
3097         nr = mem_cgroup_nr_lru_pages(memcg, stat->lru_mask);
3098         seq_printf(m, "%s=%lu", stat->name, nr);
3099         for_each_node_state(nid, N_MEMORY) {
3100             nr = mem_cgroup_node_nr_lru_pages(memcg, nid,
3101                               stat->lru_mask);
3102             seq_printf(m, " N%d=%lu", nid, nr);
3103         }
3104         seq_putc(m, '\n');
3105     }
3106 
3107     for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) {
3108         struct mem_cgroup *iter;
3109 
3110         nr = 0;
3111         for_each_mem_cgroup_tree(iter, memcg)
3112             nr += mem_cgroup_nr_lru_pages(iter, stat->lru_mask);
3113         seq_printf(m, "hierarchical_%s=%lu", stat->name, nr);
3114         for_each_node_state(nid, N_MEMORY) {
3115             nr = 0;
3116             for_each_mem_cgroup_tree(iter, memcg)
3117                 nr += mem_cgroup_node_nr_lru_pages(
3118                     iter, nid, stat->lru_mask);
3119             seq_printf(m, " N%d=%lu", nid, nr);
3120         }
3121         seq_putc(m, '\n');
3122     }
3123 
3124     return 0;
3125 }
3126 #endif /* CONFIG_NUMA */
3127 
3128 static int memcg_stat_show(struct seq_file *m, void *v)
3129 {
3130     struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
3131     unsigned long memory, memsw;
3132     struct mem_cgroup *mi;
3133     unsigned int i;
3134 
3135     BUILD_BUG_ON(ARRAY_SIZE(mem_cgroup_stat_names) !=
3136              MEM_CGROUP_STAT_NSTATS);
3137     BUILD_BUG_ON(ARRAY_SIZE(mem_cgroup_events_names) !=
3138              MEM_CGROUP_EVENTS_NSTATS);
3139     BUILD_BUG_ON(ARRAY_SIZE(mem_cgroup_lru_names) != NR_LRU_LISTS);
3140 
3141     for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) {
3142         if (i == MEM_CGROUP_STAT_SWAP && !do_memsw_account())
3143             continue;
3144         seq_printf(m, "%s %lu\n", mem_cgroup_stat_names[i],
3145                mem_cgroup_read_stat(memcg, i) * PAGE_SIZE);
3146     }
3147 
3148     for (i = 0; i < MEM_CGROUP_EVENTS_NSTATS; i++)
3149         seq_printf(m, "%s %lu\n", mem_cgroup_events_names[i],
3150                mem_cgroup_read_events(memcg, i));
3151 
3152     for (i = 0; i < NR_LRU_LISTS; i++)
3153         seq_printf(m, "%s %lu\n", mem_cgroup_lru_names[i],
3154                mem_cgroup_nr_lru_pages(memcg, BIT(i)) * PAGE_SIZE);
3155 
3156     /* Hierarchical information */
3157     memory = memsw = PAGE_COUNTER_MAX;
3158     for (mi = memcg; mi; mi = parent_mem_cgroup(mi)) {
3159         memory = min(memory, mi->memory.limit);
3160         memsw = min(memsw, mi->memsw.limit);
3161     }
3162     seq_printf(m, "hierarchical_memory_limit %llu\n",
3163            (u64)memory * PAGE_SIZE);
3164     if (do_memsw_account())
3165         seq_printf(m, "hierarchical_memsw_limit %llu\n",
3166                (u64)memsw * PAGE_SIZE);
3167 
3168     for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) {
3169         unsigned long long val = 0;
3170 
3171         if (i == MEM_CGROUP_STAT_SWAP && !do_memsw_account())
3172             continue;
3173         for_each_mem_cgroup_tree(mi, memcg)
3174             val += mem_cgroup_read_stat(mi, i) * PAGE_SIZE;
3175         seq_printf(m, "total_%s %llu\n", mem_cgroup_stat_names[i], val);
3176     }
3177 
3178     for (i = 0; i < MEM_CGROUP_EVENTS_NSTATS; i++) {
3179         unsigned long long val = 0;
3180 
3181         for_each_mem_cgroup_tree(mi, memcg)
3182             val += mem_cgroup_read_events(mi, i);
3183         seq_printf(m, "total_%s %llu\n",
3184                mem_cgroup_events_names[i], val);
3185     }
3186 
3187     for (i = 0; i < NR_LRU_LISTS; i++) {
3188         unsigned long long val = 0;
3189 
3190         for_each_mem_cgroup_tree(mi, memcg)
3191             val += mem_cgroup_nr_lru_pages(mi, BIT(i)) * PAGE_SIZE;
3192         seq_printf(m, "total_%s %llu\n", mem_cgroup_lru_names[i], val);
3193     }
3194 
3195 #ifdef CONFIG_DEBUG_VM
3196     {
3197         pg_data_t *pgdat;
3198         struct mem_cgroup_per_node *mz;
3199         struct zone_reclaim_stat *rstat;
3200         unsigned long recent_rotated[2] = {0, 0};
3201         unsigned long recent_scanned[2] = {0, 0};
3202 
3203         for_each_online_pgdat(pgdat) {
3204             mz = mem_cgroup_nodeinfo(memcg, pgdat->node_id);
3205             rstat = &mz->lruvec.reclaim_stat;
3206 
3207             recent_rotated[0] += rstat->recent_rotated[0];
3208             recent_rotated[1] += rstat->recent_rotated[1];
3209             recent_scanned[0] += rstat->recent_scanned[0];
3210             recent_scanned[1] += rstat->recent_scanned[1];
3211         }
3212         seq_printf(m, "recent_rotated_anon %lu\n", recent_rotated[0]);
3213         seq_printf(m, "recent_rotated_file %lu\n", recent_rotated[1]);
3214         seq_printf(m, "recent_scanned_anon %lu\n", recent_scanned[0]);
3215         seq_printf(m, "recent_scanned_file %lu\n", recent_scanned[1]);
3216     }
3217 #endif
3218 
3219     return 0;
3220 }
3221 
3222 static u64 mem_cgroup_swappiness_read(struct cgroup_subsys_state *css,
3223                       struct cftype *cft)
3224 {
3225     struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3226 
3227     return mem_cgroup_swappiness(memcg);
3228 }
3229 
3230 static int mem_cgroup_swappiness_write(struct cgroup_subsys_state *css,
3231                        struct cftype *cft, u64 val)
3232 {
3233     struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3234 
3235     if (val > 100)
3236         return -EINVAL;
3237 
3238     if (css->parent)
3239         memcg->swappiness = val;
3240     else
3241         vm_swappiness = val;
3242 
3243     return 0;
3244 }
3245 
3246 static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap)
3247 {
3248     struct mem_cgroup_threshold_ary *t;
3249     unsigned long usage;
3250     int i;
3251 
3252     rcu_read_lock();
3253     if (!swap)
3254         t = rcu_dereference(memcg->thresholds.primary);
3255     else
3256         t = rcu_dereference(memcg->memsw_thresholds.primary);
3257 
3258     if (!t)
3259         goto unlock;
3260 
3261     usage = mem_cgroup_usage(memcg, swap);
3262 
3263     /*
3264      * current_threshold points to threshold just below or equal to usage.
3265      * If it's not true, a threshold was crossed after last
3266      * call of __mem_cgroup_threshold().
3267      */
3268     i = t->current_threshold;
3269 
3270     /*
3271      * Iterate backward over array of thresholds starting from
3272      * current_threshold and check if a threshold is crossed.
3273      * If none of thresholds below usage is crossed, we read
3274      * only one element of the array here.
3275      */
3276     for (; i >= 0 && unlikely(t->entries[i].threshold > usage); i--)
3277         eventfd_signal(t->entries[i].eventfd, 1);
3278 
3279     /* i = current_threshold + 1 */
3280     i++;
3281 
3282     /*
3283      * Iterate forward over array of thresholds starting from
3284      * current_threshold+1 and check if a threshold is crossed.
3285      * If none of thresholds above usage is crossed, we read
3286      * only one element of the array here.
3287      */
3288     for (; i < t->size && unlikely(t->entries[i].threshold <= usage); i++)
3289         eventfd_signal(t->entries[i].eventfd, 1);
3290 
3291     /* Update current_threshold */
3292     t->current_threshold = i - 1;
3293 unlock:
3294     rcu_read_unlock();
3295 }
3296 
3297 static void mem_cgroup_threshold(struct mem_cgroup *memcg)
3298 {
3299     while (memcg) {
3300         __mem_cgroup_threshold(memcg, false);
3301         if (do_memsw_account())
3302             __mem_cgroup_threshold(memcg, true);
3303 
3304         memcg = parent_mem_cgroup(memcg);
3305     }
3306 }
3307 
3308 static int compare_thresholds(const void *a, const void *b)
3309 {
3310     const struct mem_cgroup_threshold *_a = a;
3311     const struct mem_cgroup_threshold *_b = b;
3312 
3313     if (_a->threshold > _b->threshold)
3314         return 1;
3315 
3316     if (_a->threshold < _b->threshold)
3317         return -1;
3318 
3319     return 0;
3320 }
3321 
3322 static int mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg)
3323 {
3324     struct mem_cgroup_eventfd_list *ev;
3325 
3326     spin_lock(&memcg_oom_lock);
3327 
3328     list_for_each_entry(ev, &memcg->oom_notify, list)
3329         eventfd_signal(ev->eventfd, 1);
3330 
3331     spin_unlock(&memcg_oom_lock);
3332     return 0;
3333 }
3334 
3335 static void mem_cgroup_oom_notify(struct mem_cgroup *memcg)
3336 {
3337     struct mem_cgroup *iter;
3338 
3339     for_each_mem_cgroup_tree(iter, memcg)
3340         mem_cgroup_oom_notify_cb(iter);
3341 }
3342 
3343 static int __mem_cgroup_usage_register_event(struct mem_cgroup *memcg,
3344     struct eventfd_ctx *eventfd, const char *args, enum res_type type)
3345 {
3346     struct mem_cgroup_thresholds *thresholds;
3347     struct mem_cgroup_threshold_ary *new;
3348     unsigned long threshold;
3349     unsigned long usage;
3350     int i, size, ret;
3351 
3352     ret = page_counter_memparse(args, "-1", &threshold);
3353     if (ret)
3354         return ret;
3355 
3356     mutex_lock(&memcg->thresholds_lock);
3357 
3358     if (type == _MEM) {
3359         thresholds = &memcg->thresholds;
3360         usage = mem_cgroup_usage(memcg, false);
3361     } else if (type == _MEMSWAP) {
3362         thresholds = &memcg->memsw_thresholds;
3363         usage = mem_cgroup_usage(memcg, true);
3364     } else
3365         BUG();
3366 
3367     /* Check if a threshold crossed before adding a new one */
3368     if (thresholds->primary)
3369         __mem_cgroup_threshold(memcg, type == _MEMSWAP);
3370 
3371     size = thresholds->primary ? thresholds->primary->size + 1 : 1;
3372 
3373     /* Allocate memory for new array of thresholds */
3374     new = kmalloc(sizeof(*new) + size * sizeof(struct mem_cgroup_threshold),
3375             GFP_KERNEL);
3376     if (!new) {
3377         ret = -ENOMEM;
3378         goto unlock;
3379     }
3380     new->size = size;
3381 
3382     /* Copy thresholds (if any) to new array */
3383     if (thresholds->primary) {
3384         memcpy(new->entries, thresholds->primary->entries, (size - 1) *
3385                 sizeof(struct mem_cgroup_threshold));
3386     }
3387 
3388     /* Add new threshold */
3389     new->entries[size - 1].eventfd = eventfd;
3390     new->entries[size - 1].threshold = threshold;
3391 
3392     /* Sort thresholds. Registering of new threshold isn't time-critical */
3393     sort(new->entries, size, sizeof(struct mem_cgroup_threshold),
3394             compare_thresholds, NULL);
3395 
3396     /* Find current threshold */
3397     new->current_threshold = -1;
3398     for (i = 0; i < size; i++) {
3399         if (new->entries[i].threshold <= usage) {
3400             /*
3401              * new->current_threshold will not be used until
3402              * rcu_assign_pointer(), so it's safe to increment
3403              * it here.
3404              */
3405             ++new->current_threshold;
3406         } else
3407             break;
3408     }
3409 
3410     /* Free old spare buffer and save old primary buffer as spare */
3411     kfree(thresholds->spare);
3412     thresholds->spare = thresholds->primary;
3413 
3414     rcu_assign_pointer(thresholds->primary, new);
3415 
3416     /* To be sure that nobody uses thresholds */
3417     synchronize_rcu();
3418 
3419 unlock:
3420     mutex_unlock(&memcg->thresholds_lock);
3421 
3422     return ret;
3423 }
3424 
3425 static int mem_cgroup_usage_register_event(struct mem_cgroup *memcg,
3426     struct eventfd_ctx *eventfd, const char *args)
3427 {
3428     return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEM);
3429 }
3430 
3431 static int memsw_cgroup_usage_register_event(struct mem_cgroup *memcg,
3432     struct eventfd_ctx *eventfd, const char *args)
3433 {
3434     return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEMSWAP);
3435 }
3436 
3437 static void __mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
3438     struct eventfd_ctx *eventfd, enum res_type type)
3439 {
3440     struct mem_cgroup_thresholds *thresholds;
3441     struct mem_cgroup_threshold_ary *new;
3442     unsigned long usage;
3443     int i, j, size;
3444 
3445     mutex_lock(&memcg->thresholds_lock);
3446 
3447     if (type == _MEM) {
3448         thresholds = &memcg->thresholds;
3449         usage = mem_cgroup_usage(memcg, false);
3450     } else if (type == _MEMSWAP) {
3451         thresholds = &memcg->memsw_thresholds;
3452         usage = mem_cgroup_usage(memcg, true);
3453     } else
3454         BUG();
3455 
3456     if (!thresholds->primary)
3457         goto unlock;
3458 
3459     /* Check if a threshold crossed before removing */
3460     __mem_cgroup_threshold(memcg, type == _MEMSWAP);
3461 
3462     /* Calculate new number of threshold */
3463     size = 0;
3464     for (i = 0; i < thresholds->primary->size; i++) {
3465         if (thresholds->primary->entries[i].eventfd != eventfd)
3466             size++;
3467     }
3468 
3469     new = thresholds->spare;
3470 
3471     /* Set thresholds array to NULL if we don't have thresholds */
3472     if (!size) {
3473         kfree(new);
3474         new = NULL;
3475         goto swap_buffers;
3476     }
3477 
3478     new->size = size;
3479 
3480     /* Copy thresholds and find current threshold */
3481     new->current_threshold = -1;
3482     for (i = 0, j = 0; i < thresholds->primary->size; i++) {
3483         if (thresholds->primary->entries[i].eventfd == eventfd)
3484             continue;
3485 
3486         new->entries[j] = thresholds->primary->entries[i];
3487         if (new->entries[j].threshold <= usage) {
3488             /*
3489              * new->current_threshold will not be used
3490              * until rcu_assign_pointer(), so it's safe to increment
3491              * it here.
3492              */
3493             ++new->current_threshold;
3494         }
3495         j++;
3496     }
3497 
3498 swap_buffers:
3499     /* Swap primary and spare array */
3500     thresholds->spare = thresholds->primary;
3501 
3502     rcu_assign_pointer(thresholds->primary, new);
3503 
3504     /* To be sure that nobody uses thresholds */
3505     synchronize_rcu();
3506 
3507     /* If all events are unregistered, free the spare array */
3508     if (!new) {
3509         kfree(thresholds->spare);
3510         thresholds->spare = NULL;
3511     }
3512 unlock:
3513     mutex_unlock(&memcg->thresholds_lock);
3514 }
3515 
3516 static void mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
3517     struct eventfd_ctx *eventfd)
3518 {
3519     return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEM);
3520 }
3521 
3522 static void memsw_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
3523     struct eventfd_ctx *eventfd)
3524 {
3525     return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEMSWAP);
3526 }
3527 
3528 static int mem_cgroup_oom_register_event(struct mem_cgroup *memcg,
3529     struct eventfd_ctx *eventfd, const char *args)
3530 {
3531     struct mem_cgroup_eventfd_list *event;
3532 
3533     event = kmalloc(sizeof(*event), GFP_KERNEL);
3534     if (!event)
3535         return -ENOMEM;
3536 
3537     spin_lock(&memcg_oom_lock);
3538 
3539     event->eventfd = eventfd;
3540     list_add(&event->list, &memcg->oom_notify);
3541 
3542     /* already in OOM ? */
3543     if (memcg->under_oom)
3544         eventfd_signal(eventfd, 1);
3545     spin_unlock(&memcg_oom_lock);
3546 
3547     return 0;
3548 }
3549 
3550 static void mem_cgroup_oom_unregister_event(struct mem_cgroup *memcg,
3551     struct eventfd_ctx *eventfd)
3552 {
3553     struct mem_cgroup_eventfd_list *ev, *tmp;
3554 
3555     spin_lock(&memcg_oom_lock);
3556 
3557     list_for_each_entry_safe(ev, tmp, &memcg->oom_notify, list) {
3558         if (ev->eventfd == eventfd) {
3559             list_del(&ev->list);
3560             kfree(ev);
3561         }
3562     }
3563 
3564     spin_unlock(&memcg_oom_lock);
3565 }
3566 
3567 static int mem_cgroup_oom_control_read(struct seq_file *sf, void *v)
3568 {
3569     struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(sf));
3570 
3571     seq_printf(sf, "oom_kill_disable %d\n", memcg->oom_kill_disable);
3572     seq_printf(sf, "under_oom %d\n", (bool)memcg->under_oom);
3573     return 0;
3574 }
3575 
3576 static int mem_cgroup_oom_control_write(struct cgroup_subsys_state *css,
3577     struct cftype *cft, u64 val)
3578 {
3579     struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3580 
3581     /* cannot set to root cgroup and only 0 and 1 are allowed */
3582     if (!css->parent || !((val == 0) || (val == 1)))
3583         return -EINVAL;
3584 
3585     memcg->oom_kill_disable = val;
3586     if (!val)
3587         memcg_oom_recover(memcg);
3588 
3589     return 0;
3590 }
3591 
3592 #ifdef CONFIG_CGROUP_WRITEBACK
3593 
3594 struct list_head *mem_cgroup_cgwb_list(struct mem_cgroup *memcg)
3595 {
3596     return &memcg->cgwb_list;
3597 }
3598 
3599 static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp)
3600 {
3601     return wb_domain_init(&memcg->cgwb_domain, gfp);
3602 }
3603 
3604 static void memcg_wb_domain_exit(struct mem_cgroup *memcg)
3605 {
3606     wb_domain_exit(&memcg->cgwb_domain);
3607 }
3608 
3609 static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg)
3610 {
3611     wb_domain_size_changed(&memcg->cgwb_domain);
3612 }
3613 
3614 struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb)
3615 {
3616     struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
3617 
3618     if (!memcg->css.parent)
3619         return NULL;
3620 
3621     return &memcg->cgwb_domain;
3622 }
3623 
3624 /**
3625  * mem_cgroup_wb_stats - retrieve writeback related stats from its memcg
3626  * @wb: bdi_writeback in question
3627  * @pfilepages: out parameter for number of file pages
3628  * @pheadroom: out parameter for number of allocatable pages according to memcg
3629  * @pdirty: out parameter for number of dirty pages
3630  * @pwriteback: out parameter for number of pages under writeback
3631  *
3632  * Determine the numbers of file, headroom, dirty, and writeback pages in
3633  * @wb's memcg.  File, dirty and writeback are self-explanatory.  Headroom
3634  * is a bit more involved.
3635  *
3636  * A memcg's headroom is "min(max, high) - used".  In the hierarchy, the
3637  * headroom is calculated as the lowest headroom of itself and the
3638  * ancestors.  Note that this doesn't consider the actual amount of
3639  * available memory in the system.  The caller should further cap
3640  * *@pheadroom accordingly.
3641  */
3642 void mem_cgroup_wb_stats(struct bdi_writeback *wb, unsigned long *pfilepages,
3643              unsigned long *pheadroom, unsigned long *pdirty,
3644              unsigned long *pwriteback)
3645 {
3646     struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
3647     struct mem_cgroup *parent;
3648 
3649     *pdirty = mem_cgroup_read_stat(memcg, MEM_CGROUP_STAT_DIRTY);
3650 
3651     /* this should eventually include NR_UNSTABLE_NFS */
3652     *pwriteback = mem_cgroup_read_stat(memcg, MEM_CGROUP_STAT_WRITEBACK);
3653     *pfilepages = mem_cgroup_nr_lru_pages(memcg, (1 << LRU_INACTIVE_FILE) |
3654                              (1 << LRU_ACTIVE_FILE));
3655     *pheadroom = PAGE_COUNTER_MAX;
3656 
3657     while ((parent = parent_mem_cgroup(memcg))) {
3658         unsigned long ceiling = min(memcg->memory.limit, memcg->high);
3659         unsigned long used = page_counter_read(&memcg->memory);
3660 
3661         *pheadroom = min(*pheadroom, ceiling - min(ceiling, used));
3662         memcg = parent;
3663     }
3664 }
3665 
3666 #else   /* CONFIG_CGROUP_WRITEBACK */
3667 
3668 static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp)
3669 {
3670     return 0;
3671 }
3672 
3673 static void memcg_wb_domain_exit(struct mem_cgroup *memcg)
3674 {
3675 }
3676 
3677 static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg)
3678 {
3679 }
3680 
3681 #endif  /* CONFIG_CGROUP_WRITEBACK */
3682 
3683 /*
3684  * DO NOT USE IN NEW FILES.
3685  *
3686  * "cgroup.event_control" implementation.
3687  *
3688  * This is way over-engineered.  It tries to support fully configurable
3689  * events for each user.  Such level of flexibility is completely
3690  * unnecessary especially in the light of the planned unified hierarchy.
3691  *
3692  * Please deprecate this and replace with something simpler if at all
3693  * possible.
3694  */
3695 
3696 /*
3697  * Unregister event and free resources.
3698  *
3699  * Gets called from workqueue.
3700  */
3701 static void memcg_event_remove(struct work_struct *work)
3702 {
3703     struct mem_cgroup_event *event =
3704         container_of(work, struct mem_cgroup_event, remove);
3705     struct mem_cgroup *memcg = event->memcg;
3706 
3707     remove_wait_queue(event->wqh, &event->wait);
3708 
3709     event->unregister_event(memcg, event->eventfd);
3710 
3711     /* Notify userspace the event is going away. */
3712     eventfd_signal(event->eventfd, 1);
3713 
3714     eventfd_ctx_put(event->eventfd);
3715     kfree(event);
3716     css_put(&memcg->css);
3717 }
3718 
3719 /*
3720  * Gets called on POLLHUP on eventfd when user closes it.
3721  *
3722  * Called with wqh->lock held and interrupts disabled.
3723  */
3724 static int memcg_event_wake(wait_queue_t *wait, unsigned mode,
3725                 int sync, void *key)
3726 {
3727     struct mem_cgroup_event *event =
3728         container_of(wait, struct mem_cgroup_event, wait);
3729     struct mem_cgroup *memcg = event->memcg;
3730     unsigned long flags = (unsigned long)key;
3731 
3732     if (flags & POLLHUP) {
3733         /*
3734          * If the event has been detached at cgroup removal, we
3735          * can simply return knowing the other side will cleanup
3736          * for us.
3737          *
3738          * We can't race against event freeing since the other
3739          * side will require wqh->lock via remove_wait_queue(),
3740          * which we hold.
3741          */
3742         spin_lock(&memcg->event_list_lock);
3743         if (!list_empty(&event->list)) {
3744             list_del_init(&event->list);
3745             /*
3746              * We are in atomic context, but cgroup_event_remove()
3747              * may sleep, so we have to call it in workqueue.
3748              */
3749             schedule_work(&event->remove);
3750         }
3751         spin_unlock(&memcg->event_list_lock);
3752     }
3753 
3754     return 0;
3755 }
3756 
3757 static void memcg_event_ptable_queue_proc(struct file *file,
3758         wait_queue_head_t *wqh, poll_table *pt)
3759 {
3760     struct mem_cgroup_event *event =
3761         container_of(pt, struct mem_cgroup_event, pt);
3762 
3763     event->wqh = wqh;
3764     add_wait_queue(wqh, &event->wait);
3765 }
3766 
3767 /*
3768  * DO NOT USE IN NEW FILES.
3769  *
3770  * Parse input and register new cgroup event handler.
3771  *
3772  * Input must be in format '<event_fd> <control_fd> <args>'.
3773  * Interpretation of args is defined by control file implementation.
3774  */
3775 static ssize_t memcg_write_event_control(struct kernfs_open_file *of,
3776                      char *buf, size_t nbytes, loff_t off)
3777 {
3778     struct cgroup_subsys_state *css = of_css(of);
3779     struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3780     struct mem_cgroup_event *event;
3781     struct cgroup_subsys_state *cfile_css;
3782     unsigned int efd, cfd;
3783     struct fd efile;
3784     struct fd cfile;
3785     const char *name;
3786     char *endp;
3787     int ret;
3788 
3789     buf = strstrip(buf);
3790 
3791     efd = simple_strtoul(buf, &endp, 10);
3792     if (*endp != ' ')
3793         return -EINVAL;
3794     buf = endp + 1;
3795 
3796     cfd = simple_strtoul(buf, &endp, 10);
3797     if ((*endp != ' ') && (*endp != '\0'))
3798         return -EINVAL;
3799     buf = endp + 1;
3800 
3801     event = kzalloc(sizeof(*event), GFP_KERNEL);
3802     if (!event)
3803         return -ENOMEM;
3804 
3805     event->memcg = memcg;
3806     INIT_LIST_HEAD(&event->list);
3807     init_poll_funcptr(&event->pt, memcg_event_ptable_queue_proc);
3808     init_waitqueue_func_entry(&event->wait, memcg_event_wake);
3809     INIT_WORK(&event->remove, memcg_event_remove);
3810 
3811     efile = fdget(efd);
3812     if (!efile.file) {
3813         ret = -EBADF;
3814         goto out_kfree;
3815     }
3816 
3817     event->eventfd = eventfd_ctx_fileget(efile.file);
3818     if (IS_ERR(event->eventfd)) {
3819         ret = PTR_ERR(event->eventfd);
3820         goto out_put_efile;
3821     }
3822 
3823     cfile = fdget(cfd);
3824     if (!cfile.file) {
3825         ret = -EBADF;
3826         goto out_put_eventfd;
3827     }
3828 
3829     /* the process need read permission on control file */
3830     /* AV: shouldn't we check that it's been opened for read instead? */
3831     ret = inode_permission(file_inode(cfile.file), MAY_READ);
3832     if (ret < 0)
3833         goto out_put_cfile;
3834 
3835     /*
3836      * Determine the event callbacks and set them in @event.  This used
3837      * to be done via struct cftype but cgroup core no longer knows
3838      * about these events.  The following is crude but the whole thing
3839      * is for compatibility anyway.
3840      *
3841      * DO NOT ADD NEW FILES.
3842      */
3843     name = cfile.file->f_path.dentry->d_name.name;
3844 
3845     if (!strcmp(name, "memory.usage_in_bytes")) {
3846         event->register_event = mem_cgroup_usage_register_event;
3847         event->unregister_event = mem_cgroup_usage_unregister_event;
3848     } else if (!strcmp(name, "memory.oom_control")) {
3849         event->register_event = mem_cgroup_oom_register_event;
3850         event->unregister_event = mem_cgroup_oom_unregister_event;
3851     } else if (!strcmp(name, "memory.pressure_level")) {
3852         event->register_event = vmpressure_register_event;
3853         event->unregister_event = vmpressure_unregister_event;
3854     } else if (!strcmp(name, "memory.memsw.usage_in_bytes")) {
3855         event->register_event = memsw_cgroup_usage_register_event;
3856         event->unregister_event = memsw_cgroup_usage_unregister_event;
3857     } else {
3858         ret = -EINVAL;
3859         goto out_put_cfile;
3860     }
3861 
3862     /*
3863      * Verify @cfile should belong to @css.  Also, remaining events are
3864      * automatically removed on cgroup destruction but the removal is
3865      * asynchronous, so take an extra ref on @css.
3866      */
3867     cfile_css = css_tryget_online_from_dir(cfile.file->f_path.dentry->d_parent,
3868                            &memory_cgrp_subsys);
3869     ret = -EINVAL;
3870     if (IS_ERR(cfile_css))
3871         goto out_put_cfile;
3872     if (cfile_css != css) {
3873         css_put(cfile_css);
3874         goto out_put_cfile;
3875     }
3876 
3877     ret = event->register_event(memcg, event->eventfd, buf);
3878     if (ret)
3879         goto out_put_css;
3880 
3881     efile.file->f_op->poll(efile.file, &event->pt);
3882 
3883     spin_lock(&memcg->event_list_lock);
3884     list_add(&event->list, &memcg->event_list);
3885     spin_unlock(&memcg->event_list_lock);
3886 
3887     fdput(cfile);
3888     fdput(efile);
3889 
3890     return nbytes;
3891 
3892 out_put_css:
3893     css_put(css);
3894 out_put_cfile:
3895     fdput(cfile);
3896 out_put_eventfd:
3897     eventfd_ctx_put(event->eventfd);
3898 out_put_efile:
3899     fdput(efile);
3900 out_kfree:
3901     kfree(event);
3902 
3903     return ret;
3904 }
3905 
3906 static struct cftype mem_cgroup_legacy_files[] = {
3907     {
3908         .name = "usage_in_bytes",
3909         .private = MEMFILE_PRIVATE(_MEM, RES_USAGE),
3910         .read_u64 = mem_cgroup_read_u64,
3911     },
3912     {
3913         .name = "max_usage_in_bytes",
3914         .private = MEMFILE_PRIVATE(_MEM, RES_MAX_USAGE),
3915         .write = mem_cgroup_reset,
3916         .read_u64 = mem_cgroup_read_u64,
3917     },
3918     {
3919         .name = "limit_in_bytes",
3920         .private = MEMFILE_PRIVATE(_MEM, RES_LIMIT),
3921         .write = mem_cgroup_write,
3922         .read_u64 = mem_cgroup_read_u64,
3923     },
3924     {
3925         .name = "soft_limit_in_bytes",
3926         .private = MEMFILE_PRIVATE(_MEM, RES_SOFT_LIMIT),
3927         .write = mem_cgroup_write,
3928         .read_u64 = mem_cgroup_read_u64,
3929     },
3930     {
3931         .name = "failcnt",
3932         .private = MEMFILE_PRIVATE(_MEM, RES_FAILCNT),
3933         .write = mem_cgroup_reset,
3934         .read_u64 = mem_cgroup_read_u64,
3935     },
3936     {
3937         .name = "stat",
3938         .seq_show = memcg_stat_show,
3939     },
3940     {
3941         .name = "force_empty",
3942         .write = mem_cgroup_force_empty_write,
3943     },
3944     {
3945         .name = "use_hierarchy",
3946         .write_u64 = mem_cgroup_hierarchy_write,
3947         .read_u64 = mem_cgroup_hierarchy_read,
3948     },
3949     {
3950         .name = "cgroup.event_control",     /* XXX: for compat */
3951         .write = memcg_write_event_control,
3952         .flags = CFTYPE_NO_PREFIX | CFTYPE_WORLD_WRITABLE,
3953     },
3954     {
3955         .name = "swappiness",
3956         .read_u64 = mem_cgroup_swappiness_read,
3957         .write_u64 = mem_cgroup_swappiness_write,
3958     },
3959     {
3960         .name = "move_charge_at_immigrate",
3961         .read_u64 = mem_cgroup_move_charge_read,
3962         .write_u64 = mem_cgroup_move_charge_write,
3963     },
3964     {
3965         .name = "oom_control",
3966         .seq_show = mem_cgroup_oom_control_read,
3967         .write_u64 = mem_cgroup_oom_control_write,
3968         .private = MEMFILE_PRIVATE(_OOM_TYPE, OOM_CONTROL),
3969     },
3970     {
3971         .name = "pressure_level",
3972     },
3973 #ifdef CONFIG_NUMA
3974     {
3975         .name = "numa_stat",
3976         .seq_show = memcg_numa_stat_show,
3977     },
3978 #endif
3979     {
3980         .name = "kmem.limit_in_bytes",
3981         .private = MEMFILE_PRIVATE(_KMEM, RES_LIMIT),
3982         .write = mem_cgroup_write,
3983         .read_u64 = mem_cgroup_read_u64,
3984     },
3985     {
3986         .name = "kmem.usage_in_bytes",
3987         .private = MEMFILE_PRIVATE(_KMEM, RES_USAGE),
3988         .read_u64 = mem_cgroup_read_u64,
3989     },
3990     {
3991         .name = "kmem.failcnt",
3992         .private = MEMFILE_PRIVATE(_KMEM, RES_FAILCNT),
3993         .write = mem_cgroup_reset,
3994         .read_u64 = mem_cgroup_read_u64,
3995     },
3996     {
3997         .name = "kmem.max_usage_in_bytes",
3998         .private = MEMFILE_PRIVATE(_KMEM, RES_MAX_USAGE),
3999         .write = mem_cgroup_reset,
4000         .read_u64 = mem_cgroup_read_u64,
4001     },
4002 #ifdef CONFIG_SLABINFO
4003     {
4004         .name = "kmem.slabinfo",
4005         .seq_start = slab_start,
4006         .seq_next = slab_next,
4007         .seq_stop = slab_stop,
4008         .seq_show = memcg_slab_show,
4009     },
4010 #endif
4011     {
4012         .name = "kmem.tcp.limit_in_bytes",
4013         .private = MEMFILE_PRIVATE(_TCP, RES_LIMIT),
4014         .write = mem_cgroup_write,
4015         .read_u64 = mem_cgroup_read_u64,
4016     },
4017     {
4018         .name = "kmem.tcp.usage_in_bytes",
4019         .private = MEMFILE_PRIVATE(_TCP, RES_USAGE),
4020         .read_u64 = mem_cgroup_read_u64,
4021     },
4022     {
4023         .name = "kmem.tcp.failcnt",
4024         .private = MEMFILE_PRIVATE(_TCP, RES_FAILCNT),
4025         .write = mem_cgroup_reset,
4026         .read_u64 = mem_cgroup_read_u64,
4027     },
4028     {
4029         .name = "kmem.tcp.max_usage_in_bytes",
4030         .private = MEMFILE_PRIVATE(_TCP, RES_MAX_USAGE),
4031         .write = mem_cgroup_reset,
4032         .read_u64 = mem_cgroup_read_u64,
4033     },
4034     { },    /* terminate */
4035 };
4036 
4037 /*
4038  * Private memory cgroup IDR
4039  *
4040  * Swap-out records and page cache shadow entries need to store memcg
4041  * references in constrained space, so we maintain an ID space that is
4042  * limited to 16 bit (MEM_CGROUP_ID_MAX), limiting the total number of
4043  * memory-controlled cgroups to 64k.
4044  *
4045  * However, there usually are many references to the oflline CSS after
4046  * the cgroup has been destroyed, such as page cache or reclaimable
4047  * slab objects, that don't need to hang on to the ID. We want to keep
4048  * those dead CSS from occupying IDs, or we might quickly exhaust the
4049  * relatively small ID space and prevent the creation of new cgroups
4050  * even when there are much fewer than 64k cgroups - possibly none.
4051  *
4052  * Maintain a private 16-bit ID space for memcg, and allow the ID to
4053  * be freed and recycled when it's no longer needed, which is usually
4054  * when the CSS is offlined.
4055  *
4056  * The only exception to that are records of swapped out tmpfs/shmem
4057  * pages that need to be attributed to live ancestors on swapin. But
4058  * those references are manageable from userspace.
4059  */
4060 
4061 static DEFINE_IDR(mem_cgroup_idr);
4062 
4063 static void mem_cgroup_id_get_many(struct mem_cgroup *memcg, unsigned int n)
4064 {
4065     VM_BUG_ON(atomic_read(&memcg->id.ref) <= 0);
4066     atomic_add(n, &memcg->id.ref);
4067 }
4068 
4069 static void mem_cgroup_id_put_many(struct mem_cgroup *memcg, unsigned int n)
4070 {
4071     VM_BUG_ON(atomic_read(&memcg->id.ref) < n);
4072     if (atomic_sub_and_test(n, &memcg->id.ref)) {
4073         idr_remove(&mem_cgroup_idr, memcg->id.id);
4074         memcg->id.id = 0;
4075 
4076         /* Memcg ID pins CSS */
4077         css_put(&memcg->css);
4078     }
4079 }
4080 
4081 static inline void mem_cgroup_id_get(struct mem_cgroup *memcg)
4082 {
4083     mem_cgroup_id_get_many(memcg, 1);
4084 }
4085 
4086 static inline void mem_cgroup_id_put(struct mem_cgroup *memcg)
4087 {
4088     mem_cgroup_id_put_many(memcg, 1);
4089 }
4090 
4091 /**
4092  * mem_cgroup_from_id - look up a memcg from a memcg id
4093  * @id: the memcg id to look up
4094  *
4095  * Caller must hold rcu_read_lock().
4096  */
4097 struct mem_cgroup *mem_cgroup_from_id(unsigned short id)
4098 {
4099     WARN_ON_ONCE(!rcu_read_lock_held());
4100     return idr_find(&mem_cgroup_idr, id);
4101 }
4102 
4103 static int alloc_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node)
4104 {
4105     struct mem_cgroup_per_node *pn;
4106     int tmp = node;
4107     /*
4108      * This routine is called against possible nodes.
4109      * But it's BUG to call kmalloc() against offline node.
4110      *
4111      * TODO: this routine can waste much memory for nodes which will
4112      *       never be onlined. It's better to use memory hotplug callback
4113      *       function.
4114      */
4115     if (!node_state(node, N_NORMAL_MEMORY))
4116         tmp = -1;
4117     pn = kzalloc_node(sizeof(*pn), GFP_KERNEL, tmp);
4118     if (!pn)
4119         return 1;
4120 
4121     lruvec_init(&pn->lruvec);
4122     pn->usage_in_excess = 0;
4123     pn->on_tree = false;
4124     pn->memcg = memcg;
4125 
4126     memcg->nodeinfo[node] = pn;
4127     return 0;
4128 }
4129 
4130 static void free_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node)
4131 {
4132     kfree(memcg->nodeinfo[node]);
4133 }
4134 
4135 static void mem_cgroup_free(struct mem_cgroup *memcg)
4136 {
4137     int node;
4138 
4139     memcg_wb_domain_exit(memcg);
4140     for_each_node(node)
4141         free_mem_cgroup_per_node_info(memcg, node);
4142     free_percpu(memcg->stat);
4143     kfree(memcg);
4144 }
4145 
4146 static struct mem_cgroup *mem_cgroup_alloc(void)
4147 {
4148     struct mem_cgroup *memcg;
4149     size_t size;
4150     int node;
4151 
4152     size = sizeof(struct mem_cgroup);
4153     size += nr_node_ids * sizeof(struct mem_cgroup_per_node *);
4154 
4155     memcg = kzalloc(size, GFP_KERNEL);
4156     if (!memcg)
4157         return NULL;
4158 
4159     memcg->id.id = idr_alloc(&mem_cgroup_idr, NULL,
4160                  1, MEM_CGROUP_ID_MAX,
4161                  GFP_KERNEL);
4162     if (memcg->id.id < 0)
4163         goto fail;
4164 
4165     memcg->stat = alloc_percpu(struct mem_cgroup_stat_cpu);
4166     if (!memcg->stat)
4167         goto fail;
4168 
4169     for_each_node(node)
4170         if (alloc_mem_cgroup_per_node_info(memcg, node))
4171             goto fail;
4172 
4173     if (memcg_wb_domain_init(memcg, GFP_KERNEL))
4174         goto fail;
4175 
4176     INIT_WORK(&memcg->high_work, high_work_func);
4177     memcg->last_scanned_node = MAX_NUMNODES;
4178     INIT_LIST_HEAD(&memcg->oom_notify);
4179     mutex_init(&memcg->thresholds_lock);
4180     spin_lock_init(&memcg->move_lock);
4181     vmpressure_init(&memcg->vmpressure);
4182     INIT_LIST_HEAD(&memcg->event_list);
4183     spin_lock_init(&memcg->event_list_lock);
4184     memcg->socket_pressure = jiffies;
4185 #ifndef CONFIG_SLOB
4186     memcg->kmemcg_id = -1;
4187 #endif
4188 #ifdef CONFIG_CGROUP_WRITEBACK
4189     INIT_LIST_HEAD(&memcg->cgwb_list);
4190 #endif
4191     idr_replace(&mem_cgroup_idr, memcg, memcg->id.id);
4192     return memcg;
4193 fail:
4194     if (memcg->id.id > 0)
4195         idr_remove(&mem_cgroup_idr, memcg->id.id);
4196     mem_cgroup_free(memcg);
4197     return NULL;
4198 }
4199 
4200 static struct cgroup_subsys_state * __ref
4201 mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
4202 {
4203     struct mem_cgroup *parent = mem_cgroup_from_css(parent_css);
4204     struct mem_cgroup *memcg;
4205     long error = -ENOMEM;
4206 
4207     memcg = mem_cgroup_alloc();
4208     if (!memcg)
4209         return ERR_PTR(error);
4210 
4211     memcg->high = PAGE_COUNTER_MAX;
4212     memcg->soft_limit = PAGE_COUNTER_MAX;
4213     if (parent) {
4214         memcg->swappiness = mem_cgroup_swappiness(parent);
4215         memcg->oom_kill_disable = parent->oom_kill_disable;
4216     }
4217     if (parent && parent->use_hierarchy) {
4218         memcg->use_hierarchy = true;
4219         page_counter_init(&memcg->memory, &parent->memory);
4220         page_counter_init(&memcg->swap, &parent->swap);
4221         page_counter_init(&memcg->memsw, &parent->memsw);
4222         page_counter_init(&memcg->kmem, &parent->kmem);
4223         page_counter_init(&memcg->tcpmem, &parent->tcpmem);
4224     } else {
4225         page_counter_init(&memcg->memory, NULL);
4226         page_counter_init(&memcg->swap, NULL);
4227         page_counter_init(&memcg->memsw, NULL);
4228         page_counter_init(&memcg->kmem, NULL);
4229         page_counter_init(&memcg->tcpmem, NULL);
4230         /*
4231          * Deeper hierachy with use_hierarchy == false doesn't make
4232          * much sense so let cgroup subsystem know about this
4233          * unfortunate state in our controller.
4234          */
4235         if (parent != root_mem_cgroup)
4236             memory_cgrp_subsys.broken_hierarchy = true;
4237     }
4238 
4239     /* The following stuff does not apply to the root */
4240     if (!parent) {
4241         root_mem_cgroup = memcg;
4242         return &memcg->css;
4243     }
4244 
4245     error = memcg_online_kmem(memcg);
4246     if (error)
4247         goto fail;
4248 
4249     if (cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_nosocket)
4250         static_branch_inc(&memcg_sockets_enabled_key);
4251 
4252     return &memcg->css;
4253 fail:
4254     mem_cgroup_free(memcg);
4255     return ERR_PTR(-ENOMEM);
4256 }
4257 
4258 static int mem_cgroup_css_online(struct cgroup_subsys_state *css)
4259 {
4260     struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4261 
4262     /* Online state pins memcg ID, memcg ID pins CSS */
4263     atomic_set(&memcg->id.ref, 1);
4264     css_get(css);
4265     return 0;
4266 }
4267 
4268 static void mem_cgroup_css_offline(struct cgroup_subsys_state *css)
4269 {
4270     struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4271     struct mem_cgroup_event *event, *tmp;
4272 
4273     /*
4274      * Unregister events and notify userspace.
4275      * Notify userspace about cgroup removing only after rmdir of cgroup
4276      * directory to avoid race between userspace and kernelspace.
4277      */
4278     spin_lock(&memcg->event_list_lock);
4279     list_for_each_entry_safe(event, tmp, &memcg->event_list, list) {
4280         list_del_init(&event->list);
4281         schedule_work(&event->remove);
4282     }
4283     spin_unlock(&memcg->event_list_lock);
4284 
4285     memcg_offline_kmem(memcg);
4286     wb_memcg_offline(memcg);
4287 
4288     mem_cgroup_id_put(memcg);
4289 }
4290 
4291 static void mem_cgroup_css_released(struct cgroup_subsys_state *css)
4292 {
4293     struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4294 
4295     invalidate_reclaim_iterators(memcg);
4296 }
4297 
4298 static void mem_cgroup_css_free(struct cgroup_subsys_state *css)
4299 {
4300     struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4301 
4302     if (cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_nosocket)
4303         static_branch_dec(&memcg_sockets_enabled_key);
4304 
4305     if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && memcg->tcpmem_active)
4306         static_branch_dec(&memcg_sockets_enabled_key);
4307 
4308     vmpressure_cleanup(&memcg->vmpressure);
4309     cancel_work_sync(&memcg->high_work);
4310     mem_cgroup_remove_from_trees(memcg);
4311     memcg_free_kmem(memcg);
4312     mem_cgroup_free(memcg);
4313 }
4314 
4315 /**
4316  * mem_cgroup_css_reset - reset the states of a mem_cgroup
4317  * @css: the target css
4318  *
4319  * Reset the states of the mem_cgroup associated with @css.  This is
4320  * invoked when the userland requests disabling on the default hierarchy
4321  * but the memcg is pinned through dependency.  The memcg should stop
4322  * applying policies and should revert to the vanilla state as it may be
4323  * made visible again.
4324  *
4325  * The current implementation only resets the essential configurations.
4326  * This needs to be expanded to cover all the visible parts.
4327  */
4328 static void mem_cgroup_css_reset(struct cgroup_subsys_state *css)
4329 {
4330     struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4331 
4332     page_counter_limit(&memcg->memory, PAGE_COUNTER_MAX);
4333     page_counter_limit(&memcg->swap, PAGE_COUNTER_MAX);
4334     page_counter_limit(&memcg->memsw, PAGE_COUNTER_MAX);
4335     page_counter_limit(&memcg->kmem, PAGE_COUNTER_MAX);
4336     page_counter_limit(&memcg->tcpmem, PAGE_COUNTER_MAX);
4337     memcg->low = 0;
4338     memcg->high = PAGE_COUNTER_MAX;
4339     memcg->soft_limit = PAGE_COUNTER_MAX;
4340     memcg_wb_domain_size_changed(memcg);
4341 }
4342 
4343 #ifdef CONFIG_MMU
4344 /* Handlers for move charge at task migration. */
4345 static int mem_cgroup_do_precharge(unsigned long count)
4346 {
4347     int ret;
4348 
4349     /* Try a single bulk charge without reclaim first, kswapd may wake */
4350     ret = try_charge(mc.to, GFP_KERNEL & ~__GFP_DIRECT_RECLAIM, count);
4351     if (!ret) {
4352         mc.precharge += count;
4353         return ret;
4354     }
4355 
4356     /* Try charges one by one with reclaim, but do not retry */
4357     while (count--) {
4358         ret = try_charge(mc.to, GFP_KERNEL | __GFP_NORETRY, 1);
4359         if (ret)
4360             return ret;
4361         mc.precharge++;
4362         cond_resched();
4363     }
4364     return 0;
4365 }
4366 
4367 union mc_target {
4368     struct page *page;
4369     swp_entry_t ent;
4370 };
4371 
4372 enum mc_target_type {
4373     MC_TARGET_NONE = 0,
4374     MC_TARGET_PAGE,
4375     MC_TARGET_SWAP,
4376 };
4377 
4378 static struct page *mc_handle_present_pte(struct vm_area_struct *vma,
4379                         unsigned long addr, pte_t ptent)
4380 {
4381     struct page *page = vm_normal_page(vma, addr, ptent);
4382 
4383     if (!page || !page_mapped(page))
4384         return NULL;
4385     if (PageAnon(page)) {
4386         if (!(mc.flags & MOVE_ANON))
4387             return NULL;
4388     } else {
4389         if (!(mc.flags & MOVE_FILE))
4390             return NULL;
4391     }
4392     if (!get_page_unless_zero(page))
4393         return NULL;
4394 
4395     return page;
4396 }
4397 
4398 #ifdef CONFIG_SWAP
4399 static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
4400             pte_t ptent, swp_entry_t *entry)
4401 {
4402     struct page *page = NULL;
4403     swp_entry_t ent = pte_to_swp_entry(ptent);
4404 
4405     if (!(mc.flags & MOVE_ANON) || non_swap_entry(ent))
4406         return NULL;
4407     /*
4408      * Because lookup_swap_cache() updates some statistics counter,
4409      * we call find_get_page() with swapper_space directly.
4410      */
4411     page = find_get_page(swap_address_space(ent), swp_offset(ent));
4412     if (do_memsw_account())
4413         entry->val = ent.val;
4414 
4415     return page;
4416 }
4417 #else
4418 static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
4419             pte_t ptent, swp_entry_t *entry)
4420 {
4421     return NULL;
4422 }
4423 #endif
4424 
4425 static struct page *mc_handle_file_pte(struct vm_area_struct *vma,
4426             unsigned long addr, pte_t ptent, swp_entry_t *entry)
4427 {
4428     struct page *page = NULL;
4429     struct address_space *mapping;
4430     pgoff_t pgoff;
4431 
4432     if (!vma->vm_file) /* anonymous vma */
4433         return NULL;
4434     if (!(mc.flags & MOVE_FILE))
4435         return NULL;
4436 
4437     mapping = vma->vm_file->f_mapping;
4438     pgoff = linear_page_index(vma, addr);
4439 
4440     /* page is moved even if it's not RSS of this task(page-faulted). */
4441 #ifdef CONFIG_SWAP
4442     /* shmem/tmpfs may report page out on swap: account for that too. */
4443     if (shmem_mapping(mapping)) {
4444         page = find_get_entry(mapping, pgoff);
4445         if (radix_tree_exceptional_entry(page)) {
4446             swp_entry_t swp = radix_to_swp_entry(page);
4447             if (do_memsw_account())
4448                 *entry = swp;
4449             page = find_get_page(swap_address_space(swp),
4450                          swp_offset(swp));
4451         }
4452     } else
4453         page = find_get_page(mapping, pgoff);
4454 #else
4455     page = find_get_page(mapping, pgoff);
4456 #endif
4457     return page;
4458 }
4459 
4460 /**
4461  * mem_cgroup_move_account - move account of the page
4462  * @page: the page
4463  * @compound: charge the page as compound or small page
4464  * @from: mem_cgroup which the page is moved from.
4465  * @to: mem_cgroup which the page is moved to. @from != @to.
4466  *
4467  * The caller must make sure the page is not on LRU (isolate_page() is useful.)
4468  *
4469  * This function doesn't do "charge" to new cgroup and doesn't do "uncharge"
4470  * from old cgroup.
4471  */
4472 static int mem_cgroup_move_account(struct page *page,
4473                    bool compound,
4474                    struct mem_cgroup *from,
4475                    struct mem_cgroup *to)
4476 {
4477     unsigned long flags;
4478     unsigned int nr_pages = compound ? hpage_nr_pages(page) : 1;
4479     int ret;
4480     bool anon;
4481 
4482     VM_BUG_ON(from == to);
4483     VM_BUG_ON_PAGE(PageLRU(page), page);
4484     VM_BUG_ON(compound && !PageTransHuge(page));
4485 
4486     /*
4487      * Prevent mem_cgroup_migrate() from looking at
4488      * page->mem_cgroup of its source page while we change it.
4489      */
4490     ret = -EBUSY;
4491     if (!trylock_page(page))
4492         goto out;
4493 
4494     ret = -EINVAL;
4495     if (page->mem_cgroup != from)
4496         goto out_unlock;
4497 
4498     anon = PageAnon(page);
4499 
4500     spin_lock_irqsave(&from->move_lock, flags);
4501 
4502     if (!anon && page_mapped(page)) {
4503         __this_cpu_sub(from->stat->count[MEM_CGROUP_STAT_FILE_MAPPED],
4504                    nr_pages);
4505         __this_cpu_add(to->stat->count[MEM_CGROUP_STAT_FILE_MAPPED],
4506                    nr_pages);
4507     }
4508 
4509     /*
4510      * move_lock grabbed above and caller set from->moving_account, so
4511      * mem_cgroup_update_page_stat() will serialize updates to PageDirty.
4512      * So mapping should be stable for dirty pages.
4513      */
4514     if (!anon && PageDirty(page)) {
4515         struct address_space *mapping = page_mapping(page);
4516 
4517         if (mapping_cap_account_dirty(mapping)) {
4518             __this_cpu_sub(from->stat->count[MEM_CGROUP_STAT_DIRTY],
4519                        nr_pages);
4520             __this_cpu_add(to->stat->count[MEM_CGROUP_STAT_DIRTY],
4521                        nr_pages);
4522         }
4523     }
4524 
4525     if (PageWriteback(page)) {
4526         __this_cpu_sub(from->stat->count[MEM_CGROUP_STAT_WRITEBACK],
4527                    nr_pages);
4528         __this_cpu_add(to->stat->count[MEM_CGROUP_STAT_WRITEBACK],
4529                    nr_pages);
4530     }
4531 
4532     /*
4533      * It is safe to change page->mem_cgroup here because the page
4534      * is referenced, charged, and isolated - we can't race with
4535      * uncharging, charging, migration, or LRU putback.
4536      */
4537 
4538     /* caller should have done css_get */
4539     page->mem_cgroup = to;
4540     spin_unlock_irqrestore(&from->move_lock, flags);
4541 
4542     ret = 0;
4543 
4544     local_irq_disable();
4545     mem_cgroup_charge_statistics(to, page, compound, nr_pages);
4546     memcg_check_events(to, page);
4547     mem_cgroup_charge_statistics(from, page, compound, -nr_pages);
4548     memcg_check_events(from, page);
4549     local_irq_enable();
4550 out_unlock:
4551     unlock_page(page);
4552 out:
4553     return ret;
4554 }
4555 
4556 /**
4557  * get_mctgt_type - get target type of moving charge
4558  * @vma: the vma the pte to be checked belongs
4559  * @addr: the address corresponding to the pte to be checked
4560  * @ptent: the pte to be checked
4561  * @target: the pointer the target page or swap ent will be stored(can be NULL)
4562  *
4563  * Returns
4564  *   0(MC_TARGET_NONE): if the pte is not a target for move charge.
4565  *   1(MC_TARGET_PAGE): if the page corresponding to this pte is a target for
4566  *     move charge. if @target is not NULL, the page is stored in target->page
4567  *     with extra refcnt got(Callers should handle it).
4568  *   2(MC_TARGET_SWAP): if the swap entry corresponding to this pte is a
4569  *     target for charge migration. if @target is not NULL, the entry is stored
4570  *     in target->ent.
4571  *
4572  * Called with pte lock held.
4573  */
4574 
4575 static enum mc_target_type get_mctgt_type(struct vm_area_struct *vma,
4576         unsigned long addr, pte_t ptent, union mc_target *target)
4577 {
4578     struct page *page = NULL;
4579     enum mc_target_type ret = MC_TARGET_NONE;
4580     swp_entry_t ent = { .val = 0 };
4581 
4582     if (pte_present(ptent))
4583         page = mc_handle_present_pte(vma, addr, ptent);
4584     else if (is_swap_pte(ptent))
4585         page = mc_handle_swap_pte(vma, ptent, &ent);
4586     else if (pte_none(ptent))
4587         page = mc_handle_file_pte(vma, addr, ptent, &ent);
4588 
4589     if (!page && !ent.val)
4590         return ret;
4591     if (page) {
4592         /*
4593          * Do only loose check w/o serialization.
4594          * mem_cgroup_move_account() checks the page is valid or
4595          * not under LRU exclusion.
4596          */
4597         if (page->mem_cgroup == mc.from) {
4598             ret = MC_TARGET_PAGE;
4599             if (target)
4600                 target->page = page;
4601         }
4602         if (!ret || !target)
4603             put_page(page);
4604     }
4605     /* There is a swap entry and a page doesn't exist or isn't charged */
4606     if (ent.val && !ret &&
4607         mem_cgroup_id(mc.from) == lookup_swap_cgroup_id(ent)) {
4608         ret = MC_TARGET_SWAP;
4609         if (target)
4610             target->ent = ent;
4611     }
4612     return ret;
4613 }
4614 
4615 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
4616 /*
4617  * We don't consider swapping or file mapped pages because THP does not
4618  * support them for now.
4619  * Caller should make sure that pmd_trans_huge(pmd) is true.
4620  */
4621 static enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
4622         unsigned long addr, pmd_t pmd, union mc_target *target)
4623 {
4624     struct page *page = NULL;
4625     enum mc_target_type ret = MC_TARGET_NONE;
4626 
4627     page = pmd_page(pmd);
4628     VM_BUG_ON_PAGE(!page || !PageHead(page), page);
4629     if (!(mc.flags & MOVE_ANON))
4630         return ret;
4631     if (page->mem_cgroup == mc.from) {
4632         ret = MC_TARGET_PAGE;
4633         if (target) {
4634             get_page(page);
4635             target->page = page;
4636         }
4637     }
4638     return ret;
4639 }
4640 #else
4641 static inline enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
4642         unsigned long addr, pmd_t pmd, union mc_target *target)
4643 {
4644     return MC_TARGET_NONE;
4645 }
4646 #endif
4647 
4648 static int mem_cgroup_count_precharge_pte_range(pmd_t *pmd,
4649                     unsigned long addr, unsigned long end,
4650                     struct mm_walk *walk)
4651 {
4652     struct vm_area_struct *vma = walk->vma;
4653     pte_t *pte;
4654     spinlock_t *ptl;
4655 
4656     ptl = pmd_trans_huge_lock(pmd, vma);
4657     if (ptl) {
4658         if (get_mctgt_type_thp(vma, addr, *pmd, NULL) == MC_TARGET_PAGE)
4659             mc.precharge += HPAGE_PMD_NR;
4660         spin_unlock(ptl);
4661         return 0;
4662     }
4663 
4664     if (pmd_trans_unstable(pmd))
4665         return 0;
4666     pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
4667     for (; addr != end; pte++, addr += PAGE_SIZE)
4668         if (get_mctgt_type(vma, addr, *pte, NULL))
4669             mc.precharge++; /* increment precharge temporarily */
4670     pte_unmap_unlock(pte - 1, ptl);
4671     cond_resched();
4672 
4673     return 0;
4674 }
4675 
4676 static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm)
4677 {
4678     unsigned long precharge;
4679 
4680     struct mm_walk mem_cgroup_count_precharge_walk = {
4681         .pmd_entry = mem_cgroup_count_precharge_pte_range,
4682         .mm = mm,
4683     };
4684     down_read(&mm->mmap_sem);
4685     walk_page_range(0, mm->highest_vm_end,
4686             &mem_cgroup_count_precharge_walk);
4687     up_read(&mm->mmap_sem);
4688 
4689     precharge = mc.precharge;
4690     mc.precharge = 0;
4691 
4692     return precharge;
4693 }
4694 
4695 static int mem_cgroup_precharge_mc(struct mm_struct *mm)
4696 {
4697     unsigned long precharge = mem_cgroup_count_precharge(mm);
4698 
4699     VM_BUG_ON(mc.moving_task);
4700     mc.moving_task = current;
4701     return mem_cgroup_do_precharge(precharge);
4702 }
4703 
4704 /* cancels all extra charges on mc.from and mc.to, and wakes up all waiters. */
4705 static void __mem_cgroup_clear_mc(void)
4706 {
4707     struct mem_cgroup *from = mc.from;
4708     struct mem_cgroup *to = mc.to;
4709 
4710     /* we must uncharge all the leftover precharges from mc.to */
4711     if (mc.precharge) {
4712         cancel_charge(mc.to, mc.precharge);
4713         mc.precharge = 0;
4714     }
4715     /*
4716      * we didn't uncharge from mc.from at mem_cgroup_move_account(), so
4717      * we must uncharge here.
4718      */
4719     if (mc.moved_charge) {
4720         cancel_charge(mc.from, mc.moved_charge);
4721         mc.moved_charge = 0;
4722     }
4723     /* we must fixup refcnts and charges */
4724     if (mc.moved_swap) {
4725         /* uncharge swap account from the old cgroup */
4726         if (!mem_cgroup_is_root(mc.from))
4727             page_counter_uncharge(&mc.from->memsw, mc.moved_swap);
4728 
4729         mem_cgroup_id_put_many(mc.from, mc.moved_swap);
4730 
4731         /*
4732          * we charged both to->memory and to->memsw, so we
4733          * should uncharge to->memory.
4734          */
4735         if (!mem_cgroup_is_root(mc.to))
4736             page_counter_uncharge(&mc.to->memory, mc.moved_swap);
4737 
4738         mem_cgroup_id_get_many(mc.to, mc.moved_swap);
4739         css_put_many(&mc.to->css, mc.moved_swap);
4740 
4741         mc.moved_swap = 0;
4742     }
4743     memcg_oom_recover(from);
4744     memcg_oom_recover(to);
4745     wake_up_all(&mc.waitq);
4746 }
4747 
4748 static void mem_cgroup_clear_mc(void)
4749 {
4750     struct mm_struct *mm = mc.mm;
4751 
4752     /*
4753      * we must clear moving_task before waking up waiters at the end of
4754      * task migration.
4755      */
4756     mc.moving_task = NULL;
4757     __mem_cgroup_clear_mc();
4758     spin_lock(&mc.lock);
4759     mc.from = NULL;
4760     mc.to = NULL;
4761     mc.mm = NULL;
4762     spin_unlock(&mc.lock);
4763 
4764     mmput(mm);
4765 }
4766 
4767 static int mem_cgroup_can_attach(struct cgroup_taskset *tset)
4768 {
4769     struct cgroup_subsys_state *css;
4770     struct mem_cgroup *memcg = NULL; /* unneeded init to make gcc happy */
4771     struct mem_cgroup *from;
4772     struct task_struct *leader, *p;
4773     struct mm_struct *mm;
4774     unsigned long move_flags;
4775     int ret = 0;
4776 
4777     /* charge immigration isn't supported on the default hierarchy */
4778     if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
4779         return 0;
4780 
4781     /*
4782      * Multi-process migrations only happen on the default hierarchy
4783      * where charge immigration is not used.  Perform charge
4784      * immigration if @tset contains a leader and whine if there are
4785      * multiple.
4786      */
4787     p = NULL;
4788     cgroup_taskset_for_each_leader(leader, css, tset) {
4789         WARN_ON_ONCE(p);
4790         p = leader;
4791         memcg = mem_cgroup_from_css(css);
4792     }
4793     if (!p)
4794         return 0;
4795 
4796     /*
4797      * We are now commited to this value whatever it is. Changes in this
4798      * tunable will only affect upcoming migrations, not the current one.
4799      * So we need to save it, and keep it going.
4800      */
4801     move_flags = READ_ONCE(memcg->move_charge_at_immigrate);
4802     if (!move_flags)
4803         return 0;
4804 
4805     from = mem_cgroup_from_task(p);
4806 
4807     VM_BUG_ON(from == memcg);
4808 
4809     mm = get_task_mm(p);
4810     if (!mm)
4811         return 0;
4812     /* We move charges only when we move a owner of the mm */
4813     if (mm->owner == p) {
4814         VM_BUG_ON(mc.from);
4815         VM_BUG_ON(mc.to);
4816         VM_BUG_ON(mc.precharge);
4817         VM_BUG_ON(mc.moved_charge);
4818         VM_BUG_ON(mc.moved_swap);
4819 
4820         spin_lock(&mc.lock);
4821         mc.mm = mm;
4822         mc.from = from;
4823         mc.to = memcg;
4824         mc.flags = move_flags;
4825         spin_unlock(&mc.lock);
4826         /* We set mc.moving_task later */
4827 
4828         ret = mem_cgroup_precharge_mc(mm);
4829         if (ret)
4830             mem_cgroup_clear_mc();
4831     } else {
4832         mmput(mm);
4833     }
4834     return ret;
4835 }
4836 
4837 static void mem_cgroup_cancel_attach(struct cgroup_taskset *tset)
4838 {
4839     if (mc.to)
4840         mem_cgroup_clear_mc();
4841 }
4842 
4843 static int mem_cgroup_move_charge_pte_range(pmd_t *pmd,
4844                 unsigned long addr, unsigned long end,
4845                 struct mm_walk *walk)
4846 {
4847     int ret = 0;
4848     struct vm_area_struct *vma = walk->vma;
4849     pte_t *pte;
4850     spinlock_t *ptl;
4851     enum mc_target_type target_type;
4852     union mc_target target;
4853     struct page *page;
4854 
4855     ptl = pmd_trans_huge_lock(pmd, vma);
4856     if (ptl) {
4857         if (mc.precharge < HPAGE_PMD_NR) {
4858             spin_unlock(ptl);
4859             return 0;
4860         }
4861         target_type = get_mctgt_type_thp(vma, addr, *pmd, &target);
4862         if (target_type == MC_TARGET_PAGE) {
4863             page = target.page;
4864             if (!isolate_lru_page(page)) {
4865                 if (!mem_cgroup_move_account(page, true,
4866                                  mc.from, mc.to)) {
4867                     mc.precharge -= HPAGE_PMD_NR;
4868                     mc.moved_charge += HPAGE_PMD_NR;
4869                 }
4870                 putback_lru_page(page);
4871             }
4872             put_page(page);
4873         }
4874         spin_unlock(ptl);
4875         return 0;
4876     }
4877 
4878     if (pmd_trans_unstable(pmd))
4879         return 0;
4880 retry:
4881     pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
4882     for (; addr != end; addr += PAGE_SIZE) {
4883         pte_t ptent = *(pte++);
4884         swp_entry_t ent;
4885 
4886         if (!mc.precharge)
4887             break;
4888 
4889         switch (get_mctgt_type(vma, addr, ptent, &target)) {
4890         case MC_TARGET_PAGE:
4891             page = target.page;
4892             /*
4893              * We can have a part of the split pmd here. Moving it
4894              * can be done but it would be too convoluted so simply
4895              * ignore such a partial THP and keep it in original
4896              * memcg. There should be somebody mapping the head.
4897              */
4898             if (PageTransCompound(page))
4899                 goto put;
4900             if (isolate_lru_page(page))
4901                 goto put;
4902             if (!mem_cgroup_move_account(page, false,
4903                         mc.from, mc.to)) {
4904                 mc.precharge--;
4905                 /* we uncharge from mc.from later. */
4906                 mc.moved_charge++;
4907             }
4908             putback_lru_page(page);
4909 put:            /* get_mctgt_type() gets the page */
4910             put_page(page);
4911             break;
4912         case MC_TARGET_SWAP:
4913             ent = target.ent;
4914             if (!mem_cgroup_move_swap_account(ent, mc.from, mc.to)) {
4915                 mc.precharge--;
4916                 /* we fixup refcnts and charges later. */
4917                 mc.moved_swap++;
4918             }
4919             break;
4920         default:
4921             break;
4922         }
4923     }
4924     pte_unmap_unlock(pte - 1, ptl);
4925     cond_resched();
4926 
4927     if (addr != end) {
4928         /*
4929          * We have consumed all precharges we got in can_attach().
4930          * We try charge one by one, but don't do any additional
4931          * charges to mc.to if we have failed in charge once in attach()
4932          * phase.
4933          */
4934         ret = mem_cgroup_do_precharge(1);
4935         if (!ret)
4936             goto retry;
4937     }
4938 
4939     return ret;
4940 }
4941 
4942 static void mem_cgroup_move_charge(void)
4943 {
4944     struct mm_walk mem_cgroup_move_charge_walk = {
4945         .pmd_entry = mem_cgroup_move_charge_pte_range,
4946         .mm = mc.mm,
4947     };
4948 
4949     lru_add_drain_all();
4950     /*
4951      * Signal lock_page_memcg() to take the memcg's move_lock
4952      * while we're moving its pages to another memcg. Then wait
4953      * for already started RCU-only updates to finish.
4954      */
4955     atomic_inc(&mc.from->moving_account);
4956     synchronize_rcu();
4957 retry:
4958     if (unlikely(!down_read_trylock(&mc.mm->mmap_sem))) {
4959         /*
4960          * Someone who are holding the mmap_sem might be waiting in
4961          * waitq. So we cancel all extra charges, wake up all waiters,
4962          * and retry. Because we cancel precharges, we might not be able
4963          * to move enough charges, but moving charge is a best-effort
4964          * feature anyway, so it wouldn't be a big problem.
4965          */
4966         __mem_cgroup_clear_mc();
4967         cond_resched();
4968         goto retry;
4969     }
4970     /*
4971      * When we have consumed all precharges and failed in doing
4972      * additional charge, the page walk just aborts.
4973      */
4974     walk_page_range(0, mc.mm->highest_vm_end, &mem_cgroup_move_charge_walk);
4975 
4976     up_read(&mc.mm->mmap_sem);
4977     atomic_dec(&mc.from->moving_account);
4978 }
4979 
4980 static void mem_cgroup_move_task(void)
4981 {
4982     if (mc.to) {
4983         mem_cgroup_move_charge();
4984         mem_cgroup_clear_mc();
4985     }
4986 }
4987 #else   /* !CONFIG_MMU */
4988 static int mem_cgroup_can_attach(struct cgroup_taskset *tset)
4989 {
4990     return 0;
4991 }
4992 static void mem_cgroup_cancel_attach(struct cgroup_taskset *tset)
4993 {
4994 }
4995 static void mem_cgroup_move_task(void)
4996 {
4997 }
4998 #endif
4999 
5000 /*
5001  * Cgroup retains root cgroups across [un]mount cycles making it necessary
5002  * to verify whether we're attached to the default hierarchy on each mount
5003  * attempt.
5004  */
5005 static void mem_cgroup_bind(struct cgroup_subsys_state *root_css)
5006 {
5007     /*
5008      * use_hierarchy is forced on the default hierarchy.  cgroup core
5009      * guarantees that @root doesn't have any children, so turning it
5010      * on for the root memcg is enough.
5011      */
5012     if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
5013         root_mem_cgroup->use_hierarchy = true;
5014     else
5015         root_mem_cgroup->use_hierarchy = false;
5016 }
5017 
5018 static u64 memory_current_read(struct cgroup_subsys_state *css,
5019                    struct cftype *cft)
5020 {
5021     struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5022 
5023     return (u64)page_counter_read(&memcg->memory) * PAGE_SIZE;
5024 }
5025 
5026 static int memory_low_show(struct seq_file *m, void *v)
5027 {
5028     struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
5029     unsigned long low = READ_ONCE(memcg->low);
5030 
5031     if (low == PAGE_COUNTER_MAX)
5032         seq_puts(m, "max\n");
5033     else
5034         seq_printf(m, "%llu\n", (u64)low * PAGE_SIZE);
5035 
5036     return 0;
5037 }
5038 
5039 static ssize_t memory_low_write(struct kernfs_open_file *of,
5040                 char *buf, size_t nbytes, loff_t off)
5041 {
5042     struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
5043     unsigned long low;
5044     int err;
5045 
5046     buf = strstrip(buf);
5047     err = page_counter_memparse(buf, "max", &low);
5048     if (err)
5049         return err;
5050 
5051     memcg->low = low;
5052 
5053     return nbytes;
5054 }
5055 
5056 static int memory_high_show(struct seq_file *m, void *v)
5057 {
5058     struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
5059     unsigned long high = READ_ONCE(memcg->high);
5060 
5061     if (high == PAGE_COUNTER_MAX)
5062         seq_puts(m, "max\n");
5063     else
5064         seq_printf(m, "%llu\n", (u64)high * PAGE_SIZE);
5065 
5066     return 0;
5067 }
5068 
5069 static ssize_t memory_high_write(struct kernfs_open_file *of,
5070                  char *buf, size_t nbytes, loff_t off)
5071 {
5072     struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
5073     unsigned long nr_pages;
5074     unsigned long high;
5075     int err;
5076 
5077     buf = strstrip(buf);
5078     err = page_counter_memparse(buf, "max", &high);
5079     if (err)
5080         return err;
5081 
5082     memcg->high = high;
5083 
5084     nr_pages = page_counter_read(&memcg->memory);
5085     if (nr_pages > high)
5086         try_to_free_mem_cgroup_pages(memcg, nr_pages - high,
5087                          GFP_KERNEL, true);
5088 
5089     memcg_wb_domain_size_changed(memcg);
5090     return nbytes;
5091 }
5092 
5093 static int memory_max_show(struct seq_file *m, void *v)
5094 {
5095     struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
5096     unsigned long max = READ_ONCE(memcg->memory.limit);
5097 
5098     if (max == PAGE_COUNTER_MAX)
5099         seq_puts(m, "max\n");
5100     else
5101         seq_printf(m, "%llu\n", (u64)max * PAGE_SIZE);
5102 
5103     return 0;
5104 }
5105 
5106 static ssize_t memory_max_write(struct kernfs_open_file *of,
5107                 char *buf, size_t nbytes, loff_t off)
5108 {
5109     struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
5110     unsigned int nr_reclaims = MEM_CGROUP_RECLAIM_RETRIES;
5111     bool drained = false;
5112     unsigned long max;
5113     int err;
5114 
5115     buf = strstrip(buf);
5116     err = page_counter_memparse(buf, "max", &max);
5117     if (err)
5118         return err;
5119 
5120     xchg(&memcg->memory.limit, max);
5121 
5122     for (;;) {
5123         unsigned long nr_pages = page_counter_read(&memcg->memory);
5124 
5125         if (nr_pages <= max)
5126             break;
5127 
5128         if (signal_pending(current)) {
5129             err = -EINTR;
5130             break;
5131         }
5132 
5133         if (!drained) {
5134             drain_all_stock(memcg);
5135             drained = true;
5136             continue;
5137         }
5138 
5139         if (nr_reclaims) {
5140             if (!try_to_free_mem_cgroup_pages(memcg, nr_pages - max,
5141                               GFP_KERNEL, true))
5142                 nr_reclaims--;
5143             continue;
5144         }
5145 
5146         mem_cgroup_events(memcg, MEMCG_OOM, 1);
5147         if (!mem_cgroup_out_of_memory(memcg, GFP_KERNEL, 0))
5148             break;
5149     }
5150 
5151     memcg_wb_domain_size_changed(memcg);
5152     return nbytes;
5153 }
5154 
5155 static int memory_events_show(struct seq_file *m, void *v)
5156 {
5157     struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
5158 
5159     seq_printf(m, "low %lu\n", mem_cgroup_read_events(memcg, MEMCG_LOW));
5160     seq_printf(m, "high %lu\n", mem_cgroup_read_events(memcg, MEMCG_HIGH));
5161     seq_printf(m, "max %lu\n", mem_cgroup_read_events(memcg, MEMCG_MAX));
5162     seq_printf(m, "oom %lu\n", mem_cgroup_read_events(memcg, MEMCG_OOM));
5163 
5164     return 0;
5165 }
5166 
5167 static int memory_stat_show(struct seq_file *m, void *v)
5168 {
5169     struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
5170     unsigned long stat[MEMCG_NR_STAT];
5171     unsigned long events[MEMCG_NR_EVENTS];
5172     int i;
5173 
5174     /*
5175      * Provide statistics on the state of the memory subsystem as
5176      * well as cumulative event counters that show past behavior.
5177      *
5178      * This list is ordered following a combination of these gradients:
5179      * 1) generic big picture -> specifics and details
5180      * 2) reflecting userspace activity -> reflecting kernel heuristics
5181      *
5182      * Current memory state:
5183      */
5184 
5185     tree_stat(memcg, stat);
5186     tree_events(memcg, events);
5187 
5188     seq_printf(m, "anon %llu\n",
5189            (u64)stat[MEM_CGROUP_STAT_RSS] * PAGE_SIZE);
5190     seq_printf(m, "file %llu\n",
5191            (u64)stat[MEM_CGROUP_STAT_CACHE] * PAGE_SIZE);
5192     seq_printf(m, "kernel_stack %llu\n",
5193            (u64)stat[MEMCG_KERNEL_STACK_KB] * 1024);
5194     seq_printf(m, "slab %llu\n",
5195            (u64)(stat[MEMCG_SLAB_RECLAIMABLE] +
5196              stat[MEMCG_SLAB_UNRECLAIMABLE]) * PAGE_SIZE);
5197     seq_printf(m, "sock %llu\n",
5198            (u64)stat[MEMCG_SOCK] * PAGE_SIZE);
5199 
5200     seq_printf(m, "file_mapped %llu\n",
5201            (u64)stat[MEM_CGROUP_STAT_FILE_MAPPED] * PAGE_SIZE);
5202     seq_printf(m, "file_dirty %llu\n",
5203            (u64)stat[MEM_CGROUP_STAT_DIRTY] * PAGE_SIZE);
5204     seq_printf(m, "file_writeback %llu\n",
5205            (u64)stat[MEM_CGROUP_STAT_WRITEBACK] * PAGE_SIZE);
5206 
5207     for (i = 0; i < NR_LRU_LISTS; i++) {
5208         struct mem_cgroup *mi;
5209         unsigned long val = 0;
5210 
5211         for_each_mem_cgroup_tree(mi, memcg)
5212             val += mem_cgroup_nr_lru_pages(mi, BIT(i));
5213         seq_printf(m, "%s %llu\n",
5214                mem_cgroup_lru_names[i], (u64)val * PAGE_SIZE);
5215     }
5216 
5217     seq_printf(m, "slab_reclaimable %llu\n",
5218            (u64)stat[MEMCG_SLAB_RECLAIMABLE] * PAGE_SIZE);
5219     seq_printf(m, "slab_unreclaimable %llu\n",
5220            (u64)stat[MEMCG_SLAB_UNRECLAIMABLE] * PAGE_SIZE);
5221 
5222     /* Accumulated memory events */
5223 
5224     seq_printf(m, "pgfault %lu\n",
5225            events[MEM_CGROUP_EVENTS_PGFAULT]);
5226     seq_printf(m, "pgmajfault %lu\n",
5227            events[MEM_CGROUP_EVENTS_PGMAJFAULT]);
5228 
5229     return 0;
5230 }
5231 
5232 static struct cftype memory_files[] = {
5233     {
5234         .name = "current",
5235         .flags = CFTYPE_NOT_ON_ROOT,
5236         .read_u64 = memory_current_read,
5237     },
5238     {
5239         .name = "low",
5240         .flags = CFTYPE_NOT_ON_ROOT,
5241         .seq_show = memory_low_show,
5242         .write = memory_low_write,
5243     },
5244     {
5245         .name = "high",
5246         .flags = CFTYPE_NOT_ON_ROOT,
5247         .seq_show = memory_high_show,
5248         .write = memory_high_write,
5249     },
5250     {
5251         .name = "max",
5252         .flags = CFTYPE_NOT_ON_ROOT,
5253         .seq_show = memory_max_show,
5254         .write = memory_max_write,
5255     },
5256     {
5257         .name = "events",
5258         .flags = CFTYPE_NOT_ON_ROOT,
5259         .file_offset = offsetof(struct mem_cgroup, events_file),
5260         .seq_show = memory_events_show,
5261     },
5262     {
5263         .name = "stat",
5264         .flags = CFTYPE_NOT_ON_ROOT,
5265         .seq_show = memory_stat_show,
5266     },
5267     { } /* terminate */
5268 };
5269 
5270 struct cgroup_subsys memory_cgrp_subsys = {
5271     .css_alloc = mem_cgroup_css_alloc,
5272     .css_online = mem_cgroup_css_online,
5273     .css_offline = mem_cgroup_css_offline,
5274     .css_released = mem_cgroup_css_released,
5275     .css_free = mem_cgroup_css_free,
5276     .css_reset = mem_cgroup_css_reset,
5277     .can_attach = mem_cgroup_can_attach,
5278     .cancel_attach = mem_cgroup_cancel_attach,
5279     .post_attach = mem_cgroup_move_task,
5280     .bind = mem_cgroup_bind,
5281     .dfl_cftypes = memory_files,
5282     .legacy_cftypes = mem_cgroup_legacy_files,
5283     .early_init = 0,
5284 };
5285 
5286 /**
5287  * mem_cgroup_low - check if memory consumption is below the normal range
5288  * @root: the highest ancestor to consider
5289  * @memcg: the memory cgroup to check
5290  *
5291  * Returns %true if memory consumption of @memcg, and that of all
5292  * configurable ancestors up to @root, is below the normal range.
5293  */
5294 bool mem_cgroup_low(struct mem_cgroup *root, struct mem_cgroup *memcg)
5295 {
5296     if (mem_cgroup_disabled())
5297         return false;
5298 
5299     /*
5300      * The toplevel group doesn't have a configurable range, so
5301      * it's never low when looked at directly, and it is not
5302      * considered an ancestor when assessing the hierarchy.
5303      */
5304 
5305     if (memcg == root_mem_cgroup)
5306         return false;
5307 
5308     if (page_counter_read(&memcg->memory) >= memcg->low)
5309         return false;
5310 
5311     while (memcg != root) {
5312         memcg = parent_mem_cgroup(memcg);
5313 
5314         if (memcg == root_mem_cgroup)
5315             break;
5316 
5317         if (page_counter_read(&memcg->memory) >= memcg->low)
5318             return false;
5319     }
5320     return true;
5321 }
5322 
5323 /**
5324  * mem_cgroup_try_charge - try charging a page
5325  * @page: page to charge
5326  * @mm: mm context of the victim
5327  * @gfp_mask: reclaim mode
5328  * @memcgp: charged memcg return
5329  * @compound: charge the page as compound or small page
5330  *
5331  * Try to charge @page to the memcg that @mm belongs to, reclaiming
5332  * pages according to @gfp_mask if necessary.
5333  *
5334  * Returns 0 on success, with *@memcgp pointing to the charged memcg.
5335  * Otherwise, an error code is returned.
5336  *
5337  * After page->mapping has been set up, the caller must finalize the
5338  * charge with mem_cgroup_commit_charge().  Or abort the transaction
5339  * with mem_cgroup_cancel_charge() in case page instantiation fails.
5340  */
5341 int mem_cgroup_try_charge(struct page *page, struct mm_struct *mm,
5342               gfp_t gfp_mask, struct mem_cgroup **memcgp,
5343               bool compound)
5344 {
5345     struct mem_cgroup *memcg = NULL;
5346     unsigned int nr_pages = compound ? hpage_nr_pages(page) : 1;
5347     int ret = 0;
5348 
5349     if (mem_cgroup_disabled())
5350         goto out;
5351 
5352     if (PageSwapCache(page)) {
5353         /*
5354          * Every swap fault against a single page tries to charge the
5355          * page, bail as early as possible.  shmem_unuse() encounters
5356          * already charged pages, too.  The USED bit is protected by
5357          * the page lock, which serializes swap cache removal, which
5358          * in turn serializes uncharging.
5359          */
5360         VM_BUG_ON_PAGE(!PageLocked(page), page);
5361         if (page->mem_cgroup)
5362             goto out;
5363 
5364         if (do_swap_account) {
5365             swp_entry_t ent = { .val = page_private(page), };
5366             unsigned short id = lookup_swap_cgroup_id(ent);
5367 
5368             rcu_read_lock();
5369             memcg = mem_cgroup_from_id(id);
5370             if (memcg && !css_tryget_online(&memcg->css))
5371                 memcg = NULL;
5372             rcu_read_unlock();
5373         }
5374     }
5375 
5376     if (!memcg)
5377         memcg = get_mem_cgroup_from_mm(mm);
5378 
5379     ret = try_charge(memcg, gfp_mask, nr_pages);
5380 
5381     css_put(&memcg->css);
5382 out:
5383     *memcgp = memcg;
5384     return ret;
5385 }
5386 
5387 /**
5388  * mem_cgroup_commit_charge - commit a page charge
5389  * @page: page to charge
5390  * @memcg: memcg to charge the page to
5391  * @lrucare: page might be on LRU already
5392  * @compound: charge the page as compound or small page
5393  *
5394  * Finalize a charge transaction started by mem_cgroup_try_charge(),
5395  * after page->mapping has been set up.  This must happen atomically
5396  * as part of the page instantiation, i.e. under the page table lock
5397  * for anonymous pages, under the page lock for page and swap cache.
5398  *
5399  * In addition, the page must not be on the LRU during the commit, to
5400  * prevent racing with task migration.  If it might be, use @lrucare.
5401  *
5402  * Use mem_cgroup_cancel_charge() to cancel the transaction instead.
5403  */
5404 void mem_cgroup_commit_charge(struct page *page, struct mem_cgroup *memcg,
5405                   bool lrucare, bool compound)
5406 {
5407     unsigned int nr_pages = compound ? hpage_nr_pages(page) : 1;
5408 
5409     VM_BUG_ON_PAGE(!page->mapping, page);
5410     VM_BUG_ON_PAGE(PageLRU(page) && !lrucare, page);
5411 
5412     if (mem_cgroup_disabled())
5413         return;
5414     /*
5415      * Swap faults will attempt to charge the same page multiple
5416      * times.  But reuse_swap_page() might have removed the page
5417      * from swapcache already, so we can't check PageSwapCache().
5418      */
5419     if (!memcg)
5420         return;
5421 
5422     commit_charge(page, memcg, lrucare);
5423 
5424     local_irq_disable();
5425     mem_cgroup_charge_statistics(memcg, page, compound, nr_pages);
5426     memcg_check_events(memcg, page);
5427     local_irq_enable();
5428 
5429     if (do_memsw_account() && PageSwapCache(page)) {
5430         swp_entry_t entry = { .val = page_private(page) };
5431         /*
5432          * The swap entry might not get freed for a long time,
5433          * let's not wait for it.  The page already received a
5434          * memory+swap charge, drop the swap entry duplicate.
5435          */
5436         mem_cgroup_uncharge_swap(entry);
5437     }
5438 }
5439 
5440 /**
5441  * mem_cgroup_cancel_charge - cancel a page charge
5442  * @page: page to charge
5443  * @memcg: memcg to charge the page to
5444  * @compound: charge the page as compound or small page
5445  *
5446  * Cancel a charge transaction started by mem_cgroup_try_charge().
5447  */
5448 void mem_cgroup_cancel_charge(struct page *page, struct mem_cgroup *memcg,
5449         bool compound)
5450 {
5451     unsigned int nr_pages = compound ? hpage_nr_pages(page) : 1;
5452 
5453     if (mem_cgroup_disabled())
5454         return;
5455     /*
5456      * Swap faults will attempt to charge the same page multiple
5457      * times.  But reuse_swap_page() might have removed the page
5458      * from swapcache already, so we can't check PageSwapCache().
5459      */
5460     if (!memcg)
5461         return;
5462 
5463     cancel_charge(memcg, nr_pages);
5464 }
5465 
5466 static void uncharge_batch(struct mem_cgroup *memcg, unsigned long pgpgout,
5467                unsigned long nr_anon, unsigned long nr_file,
5468                unsigned long nr_huge, unsigned long nr_kmem,
5469                struct page *dummy_page)
5470 {
5471     unsigned long nr_pages = nr_anon + nr_file + nr_kmem;
5472     unsigned long flags;
5473 
5474     if (!mem_cgroup_is_root(memcg)) {
5475         page_counter_uncharge(&memcg->memory, nr_pages);
5476         if (do_memsw_account())
5477             page_counter_uncharge(&memcg->memsw, nr_pages);
5478         if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && nr_kmem)
5479             page_counter_uncharge(&memcg->kmem, nr_kmem);
5480         memcg_oom_recover(memcg);
5481     }
5482 
5483     local_irq_save(flags);
5484     __this_cpu_sub(memcg->stat->count[MEM_CGROUP_STAT_RSS], nr_anon);
5485     __this_cpu_sub(memcg->stat->count[MEM_CGROUP_STAT_CACHE], nr_file);
5486     __this_cpu_sub(memcg->stat->count[MEM_CGROUP_STAT_RSS_HUGE], nr_huge);
5487     __this_cpu_add(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGOUT], pgpgout);
5488     __this_cpu_add(memcg->stat->nr_page_events, nr_pages);
5489     memcg_check_events(memcg, dummy_page);
5490     local_irq_restore(flags);
5491 
5492     if (!mem_cgroup_is_root(memcg))
5493         css_put_many(&memcg->css, nr_pages);
5494 }
5495 
5496 static void uncharge_list(struct list_head *page_list)
5497 {
5498     struct mem_cgroup *memcg = NULL;
5499     unsigned long nr_anon = 0;
5500     unsigned long nr_file = 0;
5501     unsigned long nr_huge = 0;
5502     unsigned long nr_kmem = 0;
5503     unsigned long pgpgout = 0;
5504     struct list_head *next;
5505     struct page *page;
5506 
5507     /*
5508      * Note that the list can be a single page->lru; hence the
5509      * do-while loop instead of a simple list_for_each_entry().
5510      */
5511     next = page_list->next;
5512     do {
5513         page = list_entry(next, struct page, lru);
5514         next = page->lru.next;
5515 
5516         VM_BUG_ON_PAGE(PageLRU(page), page);
5517         VM_BUG_ON_PAGE(page_count(page), page);
5518 
5519         if (!page->mem_cgroup)
5520             continue;
5521 
5522         /*
5523          * Nobody should be changing or seriously looking at
5524          * page->mem_cgroup at this point, we have fully
5525          * exclusive access to the page.
5526          */
5527 
5528         if (memcg != page->mem_cgroup) {
5529             if (memcg) {
5530                 uncharge_batch(memcg, pgpgout, nr_anon, nr_file,
5531                            nr_huge, nr_kmem, page);
5532                 pgpgout = nr_anon = nr_file =
5533                     nr_huge = nr_kmem = 0;
5534             }
5535             memcg = page->mem_cgroup;
5536         }
5537 
5538         if (!PageKmemcg(page)) {
5539             unsigned int nr_pages = 1;
5540 
5541             if (PageTransHuge(page)) {
5542                 nr_pages <<= compound_order(page);
5543                 nr_huge += nr_pages;
5544             }
5545             if (PageAnon(page))
5546                 nr_anon += nr_pages;
5547             else
5548                 nr_file += nr_pages;
5549             pgpgout++;
5550         } else {
5551             nr_kmem += 1 << compound_order(page);
5552             __ClearPageKmemcg(page);
5553         }
5554 
5555         page->mem_cgroup = NULL;
5556     } while (next != page_list);
5557 
5558     if (memcg)
5559         uncharge_batch(memcg, pgpgout, nr_anon, nr_file,
5560                    nr_huge, nr_kmem, page);
5561 }
5562 
5563 /**
5564  * mem_cgroup_uncharge - uncharge a page
5565  * @page: page to uncharge
5566  *
5567  * Uncharge a page previously charged with mem_cgroup_try_charge() and
5568  * mem_cgroup_commit_charge().
5569  */
5570 void mem_cgroup_uncharge(struct page *page)
5571 {
5572     if (mem_cgroup_disabled())
5573         return;
5574 
5575     /* Don't touch page->lru of any random page, pre-check: */
5576     if (!page->mem_cgroup)
5577         return;
5578 
5579     INIT_LIST_HEAD(&page->lru);
5580     uncharge_list(&page->lru);
5581 }
5582 
5583 /**
5584  * mem_cgroup_uncharge_list - uncharge a list of page
5585  * @page_list: list of pages to uncharge
5586  *
5587  * Uncharge a list of pages previously charged with
5588  * mem_cgroup_try_charge() and mem_cgroup_commit_charge().
5589  */
5590 void mem_cgroup_uncharge_list(struct list_head *page_list)
5591 {
5592     if (mem_cgroup_disabled())
5593         return;
5594 
5595     if (!list_empty(page_list))
5596         uncharge_list(page_list);
5597 }
5598 
5599 /**
5600  * mem_cgroup_migrate - charge a page's replacement
5601  * @oldpage: currently circulating page
5602  * @newpage: replacement page
5603  *
5604  * Charge @newpage as a replacement page for @oldpage. @oldpage will
5605  * be uncharged upon free.
5606  *
5607  * Both pages must be locked, @newpage->mapping must be set up.
5608  */
5609 void mem_cgroup_migrate(struct page *oldpage, struct page *newpage)
5610 {
5611     struct mem_cgroup *memcg;
5612     unsigned int nr_pages;
5613     bool compound;
5614     unsigned long flags;
5615 
5616     VM_BUG_ON_PAGE(!PageLocked(oldpage), oldpage);
5617     VM_BUG_ON_PAGE(!PageLocked(newpage), newpage);
5618     VM_BUG_ON_PAGE(PageAnon(oldpage) != PageAnon(newpage), newpage);
5619     VM_BUG_ON_PAGE(PageTransHuge(oldpage) != PageTransHuge(newpage),
5620                newpage);
5621 
5622     if (mem_cgroup_disabled())
5623         return;
5624 
5625     /* Page cache replacement: new page already charged? */
5626     if (newpage->mem_cgroup)
5627         return;
5628 
5629     /* Swapcache readahead pages can get replaced before being charged */
5630     memcg = oldpage->mem_cgroup;
5631     if (!memcg)
5632         return;
5633 
5634     /* Force-charge the new page. The old one will be freed soon */
5635     compound = PageTransHuge(newpage);
5636     nr_pages = compound ? hpage_nr_pages(newpage) : 1;
5637 
5638     page_counter_charge(&memcg->memory, nr_pages);
5639     if (do_memsw_account())
5640         page_counter_charge(&memcg->memsw, nr_pages);
5641     css_get_many(&memcg->css, nr_pages);
5642 
5643     commit_charge(newpage, memcg, false);
5644 
5645     local_irq_save(flags);
5646     mem_cgroup_charge_statistics(memcg, newpage, compound, nr_pages);
5647     memcg_check_events(memcg, newpage);
5648     local_irq_restore(flags);
5649 }
5650 
5651 DEFINE_STATIC_KEY_FALSE(memcg_sockets_enabled_key);
5652 EXPORT_SYMBOL(memcg_sockets_enabled_key);
5653 
5654 void mem_cgroup_sk_alloc(struct sock *sk)
5655 {
5656     struct mem_cgroup *memcg;
5657 
5658     if (!mem_cgroup_sockets_enabled)
5659         return;
5660 
5661     /*
5662      * Socket cloning can throw us here with sk_memcg already
5663      * filled. It won't however, necessarily happen from
5664      * process context. So the test for root memcg given
5665      * the current task's memcg won't help us in this case.
5666      *
5667      * Respecting the original socket's memcg is a better
5668      * decision in this case.
5669      */
5670     if (sk->sk_memcg) {
5671         BUG_ON(mem_cgroup_is_root(sk->sk_memcg));
5672         css_get(&sk->sk_memcg->css);
5673         return;
5674     }
5675 
5676     rcu_read_lock();
5677     memcg = mem_cgroup_from_task(current);
5678     if (memcg == root_mem_cgroup)
5679         goto out;
5680     if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && !memcg->tcpmem_active)
5681         goto out;
5682     if (css_tryget_online(&memcg->css))
5683         sk->sk_memcg = memcg;
5684 out:
5685     rcu_read_unlock();
5686 }
5687 
5688 void mem_cgroup_sk_free(struct sock *sk)
5689 {
5690     if (sk->sk_memcg)
5691         css_put(&sk->sk_memcg->css);
5692 }
5693 
5694 /**
5695  * mem_cgroup_charge_skmem - charge socket memory
5696  * @memcg: memcg to charge
5697  * @nr_pages: number of pages to charge
5698  *
5699  * Charges @nr_pages to @memcg. Returns %true if the charge fit within
5700  * @memcg's configured limit, %false if the charge had to be forced.
5701  */
5702 bool mem_cgroup_charge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages)
5703 {
5704     gfp_t gfp_mask = GFP_KERNEL;
5705 
5706     if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) {
5707         struct page_counter *fail;
5708 
5709         if (page_counter_try_charge(&memcg->tcpmem, nr_pages, &fail)) {
5710             memcg->tcpmem_pressure = 0;
5711             return true;
5712         }
5713         page_counter_charge(&memcg->tcpmem, nr_pages);
5714         memcg->tcpmem_pressure = 1;
5715         return false;
5716     }
5717 
5718     /* Don't block in the packet receive path */
5719     if (in_softirq())
5720         gfp_mask = GFP_NOWAIT;
5721 
5722     this_cpu_add(memcg->stat->count[MEMCG_SOCK], nr_pages);
5723 
5724     if (try_charge(memcg, gfp_mask, nr_pages) == 0)
5725         return true;
5726 
5727     try_charge(memcg, gfp_mask|__GFP_NOFAIL, nr_pages);
5728     return false;
5729 }
5730 
5731 /**
5732  * mem_cgroup_uncharge_skmem - uncharge socket memory
5733  * @memcg - memcg to uncharge
5734  * @nr_pages - number of pages to uncharge
5735  */
5736 void mem_cgroup_uncharge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages)
5737 {
5738     if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) {
5739         page_counter_uncharge(&memcg->tcpmem, nr_pages);
5740         return;
5741     }
5742 
5743     this_cpu_sub(memcg->stat->count[MEMCG_SOCK], nr_pages);
5744 
5745     page_counter_uncharge(&memcg->memory, nr_pages);
5746     css_put_many(&memcg->css, nr_pages);
5747 }
5748 
5749 static int __init cgroup_memory(char *s)
5750 {
5751     char *token;
5752 
5753     while ((token = strsep(&s, ",")) != NULL) {
5754         if (!*token)
5755             continue;
5756         if (!strcmp(token, "nosocket"))
5757             cgroup_memory_nosocket = true;
5758         if (!strcmp(token, "nokmem"))
5759             cgroup_memory_nokmem = true;
5760     }
5761     return 0;
5762 }
5763 __setup("cgroup.memory=", cgroup_memory);
5764 
5765 /*
5766  * subsys_initcall() for memory controller.
5767  *
5768  * Some parts like memcg_hotplug_cpu_dead() have to be initialized from this
5769  * context because of lock dependencies (cgroup_lock -> cpu hotplug) but
5770  * basically everything that doesn't depend on a specific mem_cgroup structure
5771  * should be initialized from here.
5772  */
5773 static int __init mem_cgroup_init(void)
5774 {
5775     int cpu, node;
5776 
5777 #ifndef CONFIG_SLOB
5778     /*
5779      * Kmem cache creation is mostly done with the slab_mutex held,
5780      * so use a special workqueue to avoid stalling all worker
5781      * threads in case lots of cgroups are created simultaneously.
5782      */
5783     memcg_kmem_cache_create_wq =
5784         alloc_ordered_workqueue("memcg_kmem_cache_create", 0);
5785     BUG_ON(!memcg_kmem_cache_create_wq);
5786 #endif
5787 
5788     cpuhp_setup_state_nocalls(CPUHP_MM_MEMCQ_DEAD, "mm/memctrl:dead", NULL,
5789                   memcg_hotplug_cpu_dead);
5790 
5791     for_each_possible_cpu(cpu)
5792         INIT_WORK(&per_cpu_ptr(&memcg_stock, cpu)->work,
5793               drain_local_stock);
5794 
5795     for_each_node(node) {
5796         struct mem_cgroup_tree_per_node *rtpn;
5797 
5798         rtpn = kzalloc_node(sizeof(*rtpn), GFP_KERNEL,
5799                     node_online(node) ? node : NUMA_NO_NODE);
5800 
5801         rtpn->rb_root = RB_ROOT;
5802         spin_lock_init(&rtpn->lock);
5803         soft_limit_tree.rb_tree_per_node[node] = rtpn;
5804     }
5805 
5806     return 0;
5807 }
5808 subsys_initcall(mem_cgroup_init);
5809 
5810 #ifdef CONFIG_MEMCG_SWAP
5811 static struct mem_cgroup *mem_cgroup_id_get_online(struct mem_cgroup *memcg)
5812 {
5813     while (!atomic_inc_not_zero(&memcg->id.ref)) {
5814         /*
5815          * The root cgroup cannot be destroyed, so it's refcount must
5816          * always be >= 1.
5817          */
5818         if (WARN_ON_ONCE(memcg == root_mem_cgroup)) {
5819             VM_BUG_ON(1);
5820             break;
5821         }
5822         memcg = parent_mem_cgroup(memcg);
5823         if (!memcg)
5824             memcg = root_mem_cgroup;
5825     }
5826     return memcg;
5827 }
5828 
5829 /**
5830  * mem_cgroup_swapout - transfer a memsw charge to swap
5831  * @page: page whose memsw charge to transfer
5832  * @entry: swap entry to move the charge to
5833  *
5834  * Transfer the memsw charge of @page to @entry.
5835  */
5836 void mem_cgroup_swapout(struct page *page, swp_entry_t entry)
5837 {
5838     struct mem_cgroup *memcg, *swap_memcg;
5839     unsigned short oldid;
5840 
5841     VM_BUG_ON_PAGE(PageLRU(page), page);
5842     VM_BUG_ON_PAGE(page_count(page), page);
5843 
5844     if (!do_memsw_account())
5845         return;
5846 
5847     memcg = page->mem_cgroup;
5848 
5849     /* Readahead page, never charged */
5850     if (!memcg)
5851         return;
5852 
5853     /*
5854      * In case the memcg owning these pages has been offlined and doesn't
5855      * have an ID allocated to it anymore, charge the closest online
5856      * ancestor for the swap instead and transfer the memory+swap charge.
5857      */
5858     swap_memcg = mem_cgroup_id_get_online(memcg);
5859     oldid = swap_cgroup_record(entry, mem_cgroup_id(swap_memcg));
5860     VM_BUG_ON_PAGE(oldid, page);
5861     mem_cgroup_swap_statistics(swap_memcg, true);
5862 
5863     page->mem_cgroup = NULL;
5864 
5865     if (!mem_cgroup_is_root(memcg))
5866         page_counter_uncharge(&memcg->memory, 1);
5867 
5868     if (memcg != swap_memcg) {
5869         if (!mem_cgroup_is_root(swap_memcg))
5870             page_counter_charge(&swap_memcg->memsw, 1);
5871         page_counter_uncharge(&memcg->memsw, 1);
5872     }
5873 
5874     /*
5875      * Interrupts should be disabled here because the caller holds the
5876      * mapping->tree_lock lock which is taken with interrupts-off. It is
5877      * important here to have the interrupts disabled because it is the
5878      * only synchronisation we have for udpating the per-CPU variables.
5879      */
5880     VM_BUG_ON(!irqs_disabled());
5881     mem_cgroup_charge_statistics(memcg, page, false, -1);
5882     memcg_check_events(memcg, page);
5883 
5884     if (!mem_cgroup_is_root(memcg))
5885         css_put(&memcg->css);
5886 }
5887 
5888 /*
5889  * mem_cgroup_try_charge_swap - try charging a swap entry
5890  * @page: page being added to swap
5891  * @entry: swap entry to charge
5892  *
5893  * Try to charge @entry to the memcg that @page belongs to.
5894  *
5895  * Returns 0 on success, -ENOMEM on failure.
5896  */
5897 int mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry)
5898 {
5899     struct mem_cgroup *memcg;
5900     struct page_counter *counter;
5901     unsigned short oldid;
5902 
5903     if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) || !do_swap_account)
5904         return 0;
5905 
5906     memcg = page->mem_cgroup;
5907 
5908     /* Readahead page, never charged */
5909     if (!memcg)
5910         return 0;
5911 
5912     memcg = mem_cgroup_id_get_online(memcg);
5913 
5914     if (!mem_cgroup_is_root(memcg) &&
5915         !page_counter_try_charge(&memcg->swap, 1, &counter)) {
5916         mem_cgroup_id_put(memcg);
5917         return -ENOMEM;
5918     }
5919 
5920     oldid = swap_cgroup_record(entry, mem_cgroup_id(memcg));
5921     VM_BUG_ON_PAGE(oldid, page);
5922     mem_cgroup_swap_statistics(memcg, true);
5923 
5924     return 0;
5925 }
5926 
5927 /**
5928  * mem_cgroup_uncharge_swap - uncharge a swap entry
5929  * @entry: swap entry to uncharge
5930  *
5931  * Drop the swap charge associated with @entry.
5932  */
5933 void mem_cgroup_uncharge_swap(swp_entry_t entry)
5934 {
5935     struct mem_cgroup *memcg;
5936     unsigned short id;
5937 
5938     if (!do_swap_account)
5939         return;
5940 
5941     id = swap_cgroup_record(entry, 0);
5942     rcu_read_lock();
5943     memcg = mem_cgroup_from_id(id);
5944     if (memcg) {
5945         if (!mem_cgroup_is_root(memcg)) {
5946             if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
5947                 page_counter_uncharge(&memcg->swap, 1);
5948             else
5949                 page_counter_uncharge(&memcg->memsw, 1);
5950         }
5951         mem_cgroup_swap_statistics(memcg, false);
5952         mem_cgroup_id_put(memcg);
5953     }
5954     rcu_read_unlock();
5955 }
5956 
5957 long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg)
5958 {
5959     long nr_swap_pages = get_nr_swap_pages();
5960 
5961     if (!do_swap_account || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
5962         return nr_swap_pages;
5963     for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg))
5964         nr_swap_pages = min_t(long, nr_swap_pages,
5965                       READ_ONCE(memcg->swap.limit) -
5966                       page_counter_read(&memcg->swap));
5967     return nr_swap_pages;
5968 }
5969 
5970 bool mem_cgroup_swap_full(struct page *page)
5971 {
5972     struct mem_cgroup *memcg;
5973 
5974     VM_BUG_ON_PAGE(!PageLocked(page), page);
5975 
5976     if (vm_swap_full())
5977         return true;
5978     if (!do_swap_account || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
5979         return false;
5980 
5981     memcg = page->mem_cgroup;
5982     if (!memcg)
5983         return false;
5984 
5985     for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg))
5986         if (page_counter_read(&memcg->swap) * 2 >= memcg->swap.limit)
5987             return true;
5988 
5989     return false;
5990 }
5991 
5992 /* for remember boot option*/
5993 #ifdef CONFIG_MEMCG_SWAP_ENABLED
5994 static int really_do_swap_account __initdata = 1;
5995 #else
5996 static int really_do_swap_account __initdata;
5997 #endif
5998 
5999 static int __init enable_swap_account(char *s)
6000 {
6001     if (!strcmp(s, "1"))
6002         really_do_swap_account = 1;
6003     else if (!strcmp(s, "0"))
6004         really_do_swap_account = 0;
6005     return 1;
6006 }
6007 __setup("swapaccount=", enable_swap_account);
6008 
6009 static u64 swap_current_read(struct cgroup_subsys_state *css,
6010                  struct cftype *cft)
6011 {
6012     struct mem_cgroup *memcg = mem_cgroup_from_css(css);
6013 
6014     return (u64)page_counter_read(&memcg->swap) * PAGE_SIZE;
6015 }
6016 
6017 static int swap_max_show(struct seq_file *m, void *v)
6018 {
6019     struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
6020     unsigned long max = READ_ONCE(memcg->swap.limit);
6021 
6022     if (max == PAGE_COUNTER_MAX)
6023         seq_puts(m, "max\n");
6024     else
6025         seq_printf(m, "%llu\n", (u64)max * PAGE_SIZE);
6026 
6027     return 0;
6028 }
6029 
6030 static ssize_t swap_max_write(struct kernfs_open_file *of,
6031                   char *buf, size_t nbytes, loff_t off)
6032 {
6033     struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
6034     unsigned long max;
6035     int err;
6036 
6037     buf = strstrip(buf);
6038     err = page_counter_memparse(buf, "max", &max);
6039     if (err)
6040         return err;
6041 
6042     mutex_lock(&memcg_limit_mutex);
6043     err = page_counter_limit(&memcg->swap, max);
6044     mutex_unlock(&memcg_limit_mutex);
6045     if (err)
6046         return err;
6047 
6048     return nbytes;
6049 }
6050 
6051 static struct cftype swap_files[] = {
6052     {
6053         .name = "swap.current",
6054         .flags = CFTYPE_NOT_ON_ROOT,
6055         .read_u64 = swap_current_read,
6056     },
6057     {
6058         .name = "swap.max",
6059         .flags = CFTYPE_NOT_ON_ROOT,
6060         .seq_show = swap_max_show,
6061         .write = swap_max_write,
6062     },
6063     { } /* terminate */
6064 };
6065 
6066 static struct cftype memsw_cgroup_files[] = {
6067     {
6068         .name = "memsw.usage_in_bytes",
6069         .private = MEMFILE_PRIVATE(_MEMSWAP, RES_USAGE),
6070         .read_u64 = mem_cgroup_read_u64,
6071     },
6072     {
6073         .name = "memsw.max_usage_in_bytes",
6074         .private = MEMFILE_PRIVATE(_MEMSWAP, RES_MAX_USAGE),
6075         .write = mem_cgroup_reset,
6076         .read_u64 = mem_cgroup_read_u64,
6077     },
6078     {
6079         .name = "memsw.limit_in_bytes",
6080         .private = MEMFILE_PRIVATE(_MEMSWAP, RES_LIMIT),
6081         .write = mem_cgroup_write,
6082         .read_u64 = mem_cgroup_read_u64,
6083     },
6084     {
6085         .name = "memsw.failcnt",
6086         .private = MEMFILE_PRIVATE(_MEMSWAP, RES_FAILCNT),
6087         .write = mem_cgroup_reset,
6088         .read_u64 = mem_cgroup_read_u64,
6089     },
6090     { },    /* terminate */
6091 };
6092 
6093 static int __init mem_cgroup_swap_init(void)
6094 {
6095     if (!mem_cgroup_disabled() && really_do_swap_account) {
6096         do_swap_account = 1;
6097         WARN_ON(cgroup_add_dfl_cftypes(&memory_cgrp_subsys,
6098                            swap_files));
6099         WARN_ON(cgroup_add_legacy_cftypes(&memory_cgrp_subsys,
6100                           memsw_cgroup_files));
6101     }
6102     return 0;
6103 }
6104 subsys_initcall(mem_cgroup_swap_init);
6105 
6106 #endif /* CONFIG_MEMCG_SWAP */