mm/damon/ops-common.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * Common Primitives for Data Access Monitoring
0004  *
0005  * Author: SeongJae Park <sj@kernel.org>
0006  */
0007
0008 #include <linux/mmu_notifier.h>
0009 #include <linux/page_idle.h>
0010 #include <linux/pagemap.h>
0011 #include <linux/rmap.h>
0012
0013 #include "ops-common.h"
0014
0015 /*
0016  * Get an online page for a pfn if it's in the LRU list.  Otherwise, returns
0017  * NULL.
0018  *
0019  * The body of this function is stolen from the 'page_idle_get_page()'.  We
0020  * steal rather than reuse it because the code is quite simple.
0021  */
0022 struct page *damon_get_page(unsigned long pfn)
0023 {
0024     struct page *page = pfn_to_online_page(pfn);
0025
0026     if (!page || !PageLRU(page) || !get_page_unless_zero(page))
0027         return NULL;
0028
0029     if (unlikely(!PageLRU(page))) {
0030         put_page(page);
0031         page = NULL;
0032     }
0033     return page;
0034 }
0035
0036 void damon_ptep_mkold(pte_t *pte, struct mm_struct *mm, unsigned long addr)
0037 {
0038     bool referenced = false;
0039     struct page *page = damon_get_page(pte_pfn(*pte));
0040
0041     if (!page)
0042         return;
0043
0044     if (pte_young(*pte)) {
0045         referenced = true;
0046         *pte = pte_mkold(*pte);
0047     }
0048
0049 #ifdef CONFIG_MMU_NOTIFIER
0050     if (mmu_notifier_clear_young(mm, addr, addr + PAGE_SIZE))
0051         referenced = true;
0052 #endif /* CONFIG_MMU_NOTIFIER */
0053
0054     if (referenced)
0055         set_page_young(page);
0056
0057     set_page_idle(page);
0058     put_page(page);
0059 }
0060
0061 void damon_pmdp_mkold(pmd_t *pmd, struct mm_struct *mm, unsigned long addr)
0062 {
0063 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
0064     bool referenced = false;
0065     struct page *page = damon_get_page(pmd_pfn(*pmd));
0066
0067     if (!page)
0068         return;
0069
0070     if (pmd_young(*pmd)) {
0071         referenced = true;
0072         *pmd = pmd_mkold(*pmd);
0073     }
0074
0075 #ifdef CONFIG_MMU_NOTIFIER
0076     if (mmu_notifier_clear_young(mm, addr, addr + HPAGE_PMD_SIZE))
0077         referenced = true;
0078 #endif /* CONFIG_MMU_NOTIFIER */
0079
0080     if (referenced)
0081         set_page_young(page);
0082
0083     set_page_idle(page);
0084     put_page(page);
0085 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
0086 }
0087
0088 #define DAMON_MAX_SUBSCORE  (100)
0089 #define DAMON_MAX_AGE_IN_LOG    (32)
0090
0091 int damon_pageout_score(struct damon_ctx *c, struct damon_region *r,
0092             struct damos *s)
0093 {
0094     unsigned int max_nr_accesses;
0095     int freq_subscore;
0096     unsigned int age_in_sec;
0097     int age_in_log, age_subscore;
0098     unsigned int freq_weight = s->quota.weight_nr_accesses;
0099     unsigned int age_weight = s->quota.weight_age;
0100     int hotness;
0101
0102     max_nr_accesses = c->aggr_interval / c->sample_interval;
0103     freq_subscore = r->nr_accesses * DAMON_MAX_SUBSCORE / max_nr_accesses;
0104
0105     age_in_sec = (unsigned long)r->age * c->aggr_interval / 1000000;
0106     for (age_in_log = 0; age_in_log < DAMON_MAX_AGE_IN_LOG && age_in_sec;
0107             age_in_log++, age_in_sec >>= 1)
0108         ;
0109
0110     /* If frequency is 0, higher age means it's colder */
0111     if (freq_subscore == 0)
0112         age_in_log *= -1;
0113
0114     /*
0115      * Now age_in_log is in [-DAMON_MAX_AGE_IN_LOG, DAMON_MAX_AGE_IN_LOG].
0116      * Scale it to be in [0, 100] and set it as age subscore.
0117      */
0118     age_in_log += DAMON_MAX_AGE_IN_LOG;
0119     age_subscore = age_in_log * DAMON_MAX_SUBSCORE /
0120         DAMON_MAX_AGE_IN_LOG / 2;
0121
0122     hotness = (freq_weight * freq_subscore + age_weight * age_subscore);
0123     if (freq_weight + age_weight)
0124         hotness /= freq_weight + age_weight;
0125     /*
0126      * Transform it to fit in [0, DAMOS_MAX_SCORE]
0127      */
0128     hotness = hotness * DAMOS_MAX_SCORE / DAMON_MAX_SUBSCORE;
0129
0130     /* Return coldness of the region */
0131     return DAMOS_MAX_SCORE - hotness;
0132 }
0133
0134 int damon_hot_score(struct damon_ctx *c, struct damon_region *r,
0135             struct damos *s)
0136 {
0137     unsigned int max_nr_accesses;
0138     int freq_subscore;
0139     unsigned int age_in_sec;
0140     int age_in_log, age_subscore;
0141     unsigned int freq_weight = s->quota.weight_nr_accesses;
0142     unsigned int age_weight = s->quota.weight_age;
0143     int hotness;
0144
0145     max_nr_accesses = c->aggr_interval / c->sample_interval;
0146     freq_subscore = r->nr_accesses * DAMON_MAX_SUBSCORE / max_nr_accesses;
0147
0148     age_in_sec = (unsigned long)r->age * c->aggr_interval / 1000000;
0149     for (age_in_log = 0; age_in_log < DAMON_MAX_AGE_IN_LOG && age_in_sec;
0150             age_in_log++, age_in_sec >>= 1)
0151         ;
0152
0153     /* If frequency is 0, higher age means it's colder */
0154     if (freq_subscore == 0)
0155         age_in_log *= -1;
0156
0157     /*
0158      * Now age_in_log is in [-DAMON_MAX_AGE_IN_LOG, DAMON_MAX_AGE_IN_LOG].
0159      * Scale it to be in [0, 100] and set it as age subscore.
0160      */
0161     age_in_log += DAMON_MAX_AGE_IN_LOG;
0162     age_subscore = age_in_log * DAMON_MAX_SUBSCORE /
0163         DAMON_MAX_AGE_IN_LOG / 2;
0164
0165     hotness = (freq_weight * freq_subscore + age_weight * age_subscore);
0166     if (freq_weight + age_weight)
0167         hotness /= freq_weight + age_weight;
0168     /*
0169      * Transform it to fit in [0, DAMOS_MAX_SCORE]
0170      */
0171     hotness = hotness * DAMOS_MAX_SCORE / DAMON_MAX_SUBSCORE;
0172
0173     return hotness;
0174 }