kernel/sched/isolation.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  *  Housekeeping management. Manage the targets for routine code that can run on
0004  *  any CPU: unbound workqueues, timers, kthreads and any offloadable work.
0005  *
0006  * Copyright (C) 2017 Red Hat, Inc., Frederic Weisbecker
0007  * Copyright (C) 2017-2018 SUSE, Frederic Weisbecker
0008  *
0009  */
0010
0011 enum hk_flags {
0012     HK_FLAG_TIMER       = BIT(HK_TYPE_TIMER),
0013     HK_FLAG_RCU     = BIT(HK_TYPE_RCU),
0014     HK_FLAG_MISC        = BIT(HK_TYPE_MISC),
0015     HK_FLAG_SCHED       = BIT(HK_TYPE_SCHED),
0016     HK_FLAG_TICK        = BIT(HK_TYPE_TICK),
0017     HK_FLAG_DOMAIN      = BIT(HK_TYPE_DOMAIN),
0018     HK_FLAG_WQ      = BIT(HK_TYPE_WQ),
0019     HK_FLAG_MANAGED_IRQ = BIT(HK_TYPE_MANAGED_IRQ),
0020     HK_FLAG_KTHREAD     = BIT(HK_TYPE_KTHREAD),
0021 };
0022
0023 DEFINE_STATIC_KEY_FALSE(housekeeping_overridden);
0024 EXPORT_SYMBOL_GPL(housekeeping_overridden);
0025
0026 struct housekeeping {
0027     cpumask_var_t cpumasks[HK_TYPE_MAX];
0028     unsigned long flags;
0029 };
0030
0031 static struct housekeeping housekeeping;
0032
0033 bool housekeeping_enabled(enum hk_type type)
0034 {
0035     return !!(housekeeping.flags & BIT(type));
0036 }
0037 EXPORT_SYMBOL_GPL(housekeeping_enabled);
0038
0039 int housekeeping_any_cpu(enum hk_type type)
0040 {
0041     int cpu;
0042
0043     if (static_branch_unlikely(&housekeeping_overridden)) {
0044         if (housekeeping.flags & BIT(type)) {
0045             cpu = sched_numa_find_closest(housekeeping.cpumasks[type], smp_processor_id());
0046             if (cpu < nr_cpu_ids)
0047                 return cpu;
0048
0049             return cpumask_any_and(housekeeping.cpumasks[type], cpu_online_mask);
0050         }
0051     }
0052     return smp_processor_id();
0053 }
0054 EXPORT_SYMBOL_GPL(housekeeping_any_cpu);
0055
0056 const struct cpumask *housekeeping_cpumask(enum hk_type type)
0057 {
0058     if (static_branch_unlikely(&housekeeping_overridden))
0059         if (housekeeping.flags & BIT(type))
0060             return housekeeping.cpumasks[type];
0061     return cpu_possible_mask;
0062 }
0063 EXPORT_SYMBOL_GPL(housekeeping_cpumask);
0064
0065 void housekeeping_affine(struct task_struct *t, enum hk_type type)
0066 {
0067     if (static_branch_unlikely(&housekeeping_overridden))
0068         if (housekeeping.flags & BIT(type))
0069             set_cpus_allowed_ptr(t, housekeeping.cpumasks[type]);
0070 }
0071 EXPORT_SYMBOL_GPL(housekeeping_affine);
0072
0073 bool housekeeping_test_cpu(int cpu, enum hk_type type)
0074 {
0075     if (static_branch_unlikely(&housekeeping_overridden))
0076         if (housekeeping.flags & BIT(type))
0077             return cpumask_test_cpu(cpu, housekeeping.cpumasks[type]);
0078     return true;
0079 }
0080 EXPORT_SYMBOL_GPL(housekeeping_test_cpu);
0081
0082 void __init housekeeping_init(void)
0083 {
0084     enum hk_type type;
0085
0086     if (!housekeeping.flags)
0087         return;
0088
0089     static_branch_enable(&housekeeping_overridden);
0090
0091     if (housekeeping.flags & HK_FLAG_TICK)
0092         sched_tick_offload_init();
0093
0094     for_each_set_bit(type, &housekeeping.flags, HK_TYPE_MAX) {
0095         /* We need at least one CPU to handle housekeeping work */
0096         WARN_ON_ONCE(cpumask_empty(housekeeping.cpumasks[type]));
0097     }
0098 }
0099
0100 static void __init housekeeping_setup_type(enum hk_type type,
0101                        cpumask_var_t housekeeping_staging)
0102 {
0103
0104     alloc_bootmem_cpumask_var(&housekeeping.cpumasks[type]);
0105     cpumask_copy(housekeeping.cpumasks[type],
0106              housekeeping_staging);
0107 }
0108
0109 static int __init housekeeping_setup(char *str, unsigned long flags)
0110 {
0111     cpumask_var_t non_housekeeping_mask, housekeeping_staging;
0112     int err = 0;
0113
0114     if ((flags & HK_FLAG_TICK) && !(housekeeping.flags & HK_FLAG_TICK)) {
0115         if (!IS_ENABLED(CONFIG_NO_HZ_FULL)) {
0116             pr_warn("Housekeeping: nohz unsupported."
0117                 " Build with CONFIG_NO_HZ_FULL\n");
0118             return 0;
0119         }
0120     }
0121
0122     alloc_bootmem_cpumask_var(&non_housekeeping_mask);
0123     if (cpulist_parse(str, non_housekeeping_mask) < 0) {
0124         pr_warn("Housekeeping: nohz_full= or isolcpus= incorrect CPU range\n");
0125         goto free_non_housekeeping_mask;
0126     }
0127
0128     alloc_bootmem_cpumask_var(&housekeeping_staging);
0129     cpumask_andnot(housekeeping_staging,
0130                cpu_possible_mask, non_housekeeping_mask);
0131
0132     if (!cpumask_intersects(cpu_present_mask, housekeeping_staging)) {
0133         __cpumask_set_cpu(smp_processor_id(), housekeeping_staging);
0134         __cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask);
0135         if (!housekeeping.flags) {
0136             pr_warn("Housekeeping: must include one present CPU, "
0137                 "using boot CPU:%d\n", smp_processor_id());
0138         }
0139     }
0140
0141     if (!housekeeping.flags) {
0142         /* First setup call ("nohz_full=" or "isolcpus=") */
0143         enum hk_type type;
0144
0145         for_each_set_bit(type, &flags, HK_TYPE_MAX)
0146             housekeeping_setup_type(type, housekeeping_staging);
0147     } else {
0148         /* Second setup call ("nohz_full=" after "isolcpus=" or the reverse) */
0149         enum hk_type type;
0150         unsigned long iter_flags = flags & housekeeping.flags;
0151
0152         for_each_set_bit(type, &iter_flags, HK_TYPE_MAX) {
0153             if (!cpumask_equal(housekeeping_staging,
0154                        housekeeping.cpumasks[type])) {
0155                 pr_warn("Housekeeping: nohz_full= must match isolcpus=\n");
0156                 goto free_housekeeping_staging;
0157             }
0158         }
0159
0160         iter_flags = flags & ~housekeeping.flags;
0161
0162         for_each_set_bit(type, &iter_flags, HK_TYPE_MAX)
0163             housekeeping_setup_type(type, housekeeping_staging);
0164     }
0165
0166     if ((flags & HK_FLAG_TICK) && !(housekeeping.flags & HK_FLAG_TICK))
0167         tick_nohz_full_setup(non_housekeeping_mask);
0168
0169     housekeeping.flags |= flags;
0170     err = 1;
0171
0172 free_housekeeping_staging:
0173     free_bootmem_cpumask_var(housekeeping_staging);
0174 free_non_housekeeping_mask:
0175     free_bootmem_cpumask_var(non_housekeeping_mask);
0176
0177     return err;
0178 }
0179
0180 static int __init housekeeping_nohz_full_setup(char *str)
0181 {
0182     unsigned long flags;
0183
0184     flags = HK_FLAG_TICK | HK_FLAG_WQ | HK_FLAG_TIMER | HK_FLAG_RCU |
0185         HK_FLAG_MISC | HK_FLAG_KTHREAD;
0186
0187     return housekeeping_setup(str, flags);
0188 }
0189 __setup("nohz_full=", housekeeping_nohz_full_setup);
0190
0191 static int __init housekeeping_isolcpus_setup(char *str)
0192 {
0193     unsigned long flags = 0;
0194     bool illegal = false;
0195     char *par;
0196     int len;
0197
0198     while (isalpha(*str)) {
0199         if (!strncmp(str, "nohz,", 5)) {
0200             str += 5;
0201             flags |= HK_FLAG_TICK;
0202             continue;
0203         }
0204
0205         if (!strncmp(str, "domain,", 7)) {
0206             str += 7;
0207             flags |= HK_FLAG_DOMAIN;
0208             continue;
0209         }
0210
0211         if (!strncmp(str, "managed_irq,", 12)) {
0212             str += 12;
0213             flags |= HK_FLAG_MANAGED_IRQ;
0214             continue;
0215         }
0216
0217         /*
0218          * Skip unknown sub-parameter and validate that it is not
0219          * containing an invalid character.
0220          */
0221         for (par = str, len = 0; *str && *str != ','; str++, len++) {
0222             if (!isalpha(*str) && *str != '_')
0223                 illegal = true;
0224         }
0225
0226         if (illegal) {
0227             pr_warn("isolcpus: Invalid flag %.*s\n", len, par);
0228             return 0;
0229         }
0230
0231         pr_info("isolcpus: Skipped unknown flag %.*s\n", len, par);
0232         str++;
0233     }
0234
0235     /* Default behaviour for isolcpus without flags */
0236     if (!flags)
0237         flags |= HK_FLAG_DOMAIN;
0238
0239     return housekeeping_setup(str, flags);
0240 }
0241 __setup("isolcpus=", housekeeping_isolcpus_setup);