Back to home page

OSCL-LXR

 
 

    


0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * Based on arch/arm/kernel/process.c
0004  *
0005  * Original Copyright (C) 1995  Linus Torvalds
0006  * Copyright (C) 1996-2000 Russell King - Converted to ARM.
0007  * Copyright (C) 2012 ARM Ltd.
0008  */
0009 #include <linux/compat.h>
0010 #include <linux/efi.h>
0011 #include <linux/elf.h>
0012 #include <linux/export.h>
0013 #include <linux/sched.h>
0014 #include <linux/sched/debug.h>
0015 #include <linux/sched/task.h>
0016 #include <linux/sched/task_stack.h>
0017 #include <linux/kernel.h>
0018 #include <linux/mman.h>
0019 #include <linux/mm.h>
0020 #include <linux/nospec.h>
0021 #include <linux/stddef.h>
0022 #include <linux/sysctl.h>
0023 #include <linux/unistd.h>
0024 #include <linux/user.h>
0025 #include <linux/delay.h>
0026 #include <linux/reboot.h>
0027 #include <linux/interrupt.h>
0028 #include <linux/init.h>
0029 #include <linux/cpu.h>
0030 #include <linux/elfcore.h>
0031 #include <linux/pm.h>
0032 #include <linux/tick.h>
0033 #include <linux/utsname.h>
0034 #include <linux/uaccess.h>
0035 #include <linux/random.h>
0036 #include <linux/hw_breakpoint.h>
0037 #include <linux/personality.h>
0038 #include <linux/notifier.h>
0039 #include <trace/events/power.h>
0040 #include <linux/percpu.h>
0041 #include <linux/thread_info.h>
0042 #include <linux/prctl.h>
0043 #include <linux/stacktrace.h>
0044 
0045 #include <asm/alternative.h>
0046 #include <asm/compat.h>
0047 #include <asm/cpufeature.h>
0048 #include <asm/cacheflush.h>
0049 #include <asm/exec.h>
0050 #include <asm/fpsimd.h>
0051 #include <asm/mmu_context.h>
0052 #include <asm/mte.h>
0053 #include <asm/processor.h>
0054 #include <asm/pointer_auth.h>
0055 #include <asm/stacktrace.h>
0056 #include <asm/switch_to.h>
0057 #include <asm/system_misc.h>
0058 
0059 #if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_STACKPROTECTOR_PER_TASK)
0060 #include <linux/stackprotector.h>
0061 unsigned long __stack_chk_guard __ro_after_init;
0062 EXPORT_SYMBOL(__stack_chk_guard);
0063 #endif
0064 
0065 /*
0066  * Function pointers to optional machine specific functions
0067  */
0068 void (*pm_power_off)(void);
0069 EXPORT_SYMBOL_GPL(pm_power_off);
0070 
0071 #ifdef CONFIG_HOTPLUG_CPU
0072 void arch_cpu_idle_dead(void)
0073 {
0074        cpu_die();
0075 }
0076 #endif
0077 
0078 /*
0079  * Called by kexec, immediately prior to machine_kexec().
0080  *
0081  * This must completely disable all secondary CPUs; simply causing those CPUs
0082  * to execute e.g. a RAM-based pin loop is not sufficient. This allows the
0083  * kexec'd kernel to use any and all RAM as it sees fit, without having to
0084  * avoid any code or data used by any SW CPU pin loop. The CPU hotplug
0085  * functionality embodied in smpt_shutdown_nonboot_cpus() to achieve this.
0086  */
0087 void machine_shutdown(void)
0088 {
0089     smp_shutdown_nonboot_cpus(reboot_cpu);
0090 }
0091 
0092 /*
0093  * Halting simply requires that the secondary CPUs stop performing any
0094  * activity (executing tasks, handling interrupts). smp_send_stop()
0095  * achieves this.
0096  */
0097 void machine_halt(void)
0098 {
0099     local_irq_disable();
0100     smp_send_stop();
0101     while (1);
0102 }
0103 
0104 /*
0105  * Power-off simply requires that the secondary CPUs stop performing any
0106  * activity (executing tasks, handling interrupts). smp_send_stop()
0107  * achieves this. When the system power is turned off, it will take all CPUs
0108  * with it.
0109  */
0110 void machine_power_off(void)
0111 {
0112     local_irq_disable();
0113     smp_send_stop();
0114     do_kernel_power_off();
0115 }
0116 
0117 /*
0118  * Restart requires that the secondary CPUs stop performing any activity
0119  * while the primary CPU resets the system. Systems with multiple CPUs must
0120  * provide a HW restart implementation, to ensure that all CPUs reset at once.
0121  * This is required so that any code running after reset on the primary CPU
0122  * doesn't have to co-ordinate with other CPUs to ensure they aren't still
0123  * executing pre-reset code, and using RAM that the primary CPU's code wishes
0124  * to use. Implementing such co-ordination would be essentially impossible.
0125  */
0126 void machine_restart(char *cmd)
0127 {
0128     /* Disable interrupts first */
0129     local_irq_disable();
0130     smp_send_stop();
0131 
0132     /*
0133      * UpdateCapsule() depends on the system being reset via
0134      * ResetSystem().
0135      */
0136     if (efi_enabled(EFI_RUNTIME_SERVICES))
0137         efi_reboot(reboot_mode, NULL);
0138 
0139     /* Now call the architecture specific reboot code. */
0140     do_kernel_restart(cmd);
0141 
0142     /*
0143      * Whoops - the architecture was unable to reboot.
0144      */
0145     printk("Reboot failed -- System halted\n");
0146     while (1);
0147 }
0148 
0149 #define bstr(suffix, str) [PSR_BTYPE_ ## suffix >> PSR_BTYPE_SHIFT] = str
0150 static const char *const btypes[] = {
0151     bstr(NONE, "--"),
0152     bstr(  JC, "jc"),
0153     bstr(   C, "-c"),
0154     bstr(  J , "j-")
0155 };
0156 #undef bstr
0157 
0158 static void print_pstate(struct pt_regs *regs)
0159 {
0160     u64 pstate = regs->pstate;
0161 
0162     if (compat_user_mode(regs)) {
0163         printk("pstate: %08llx (%c%c%c%c %c %s %s %c%c%c %cDIT %cSSBS)\n",
0164             pstate,
0165             pstate & PSR_AA32_N_BIT ? 'N' : 'n',
0166             pstate & PSR_AA32_Z_BIT ? 'Z' : 'z',
0167             pstate & PSR_AA32_C_BIT ? 'C' : 'c',
0168             pstate & PSR_AA32_V_BIT ? 'V' : 'v',
0169             pstate & PSR_AA32_Q_BIT ? 'Q' : 'q',
0170             pstate & PSR_AA32_T_BIT ? "T32" : "A32",
0171             pstate & PSR_AA32_E_BIT ? "BE" : "LE",
0172             pstate & PSR_AA32_A_BIT ? 'A' : 'a',
0173             pstate & PSR_AA32_I_BIT ? 'I' : 'i',
0174             pstate & PSR_AA32_F_BIT ? 'F' : 'f',
0175             pstate & PSR_AA32_DIT_BIT ? '+' : '-',
0176             pstate & PSR_AA32_SSBS_BIT ? '+' : '-');
0177     } else {
0178         const char *btype_str = btypes[(pstate & PSR_BTYPE_MASK) >>
0179                            PSR_BTYPE_SHIFT];
0180 
0181         printk("pstate: %08llx (%c%c%c%c %c%c%c%c %cPAN %cUAO %cTCO %cDIT %cSSBS BTYPE=%s)\n",
0182             pstate,
0183             pstate & PSR_N_BIT ? 'N' : 'n',
0184             pstate & PSR_Z_BIT ? 'Z' : 'z',
0185             pstate & PSR_C_BIT ? 'C' : 'c',
0186             pstate & PSR_V_BIT ? 'V' : 'v',
0187             pstate & PSR_D_BIT ? 'D' : 'd',
0188             pstate & PSR_A_BIT ? 'A' : 'a',
0189             pstate & PSR_I_BIT ? 'I' : 'i',
0190             pstate & PSR_F_BIT ? 'F' : 'f',
0191             pstate & PSR_PAN_BIT ? '+' : '-',
0192             pstate & PSR_UAO_BIT ? '+' : '-',
0193             pstate & PSR_TCO_BIT ? '+' : '-',
0194             pstate & PSR_DIT_BIT ? '+' : '-',
0195             pstate & PSR_SSBS_BIT ? '+' : '-',
0196             btype_str);
0197     }
0198 }
0199 
0200 void __show_regs(struct pt_regs *regs)
0201 {
0202     int i, top_reg;
0203     u64 lr, sp;
0204 
0205     if (compat_user_mode(regs)) {
0206         lr = regs->compat_lr;
0207         sp = regs->compat_sp;
0208         top_reg = 12;
0209     } else {
0210         lr = regs->regs[30];
0211         sp = regs->sp;
0212         top_reg = 29;
0213     }
0214 
0215     show_regs_print_info(KERN_DEFAULT);
0216     print_pstate(regs);
0217 
0218     if (!user_mode(regs)) {
0219         printk("pc : %pS\n", (void *)regs->pc);
0220         printk("lr : %pS\n", (void *)ptrauth_strip_insn_pac(lr));
0221     } else {
0222         printk("pc : %016llx\n", regs->pc);
0223         printk("lr : %016llx\n", lr);
0224     }
0225 
0226     printk("sp : %016llx\n", sp);
0227 
0228     if (system_uses_irq_prio_masking())
0229         printk("pmr_save: %08llx\n", regs->pmr_save);
0230 
0231     i = top_reg;
0232 
0233     while (i >= 0) {
0234         printk("x%-2d: %016llx", i, regs->regs[i]);
0235 
0236         while (i-- % 3)
0237             pr_cont(" x%-2d: %016llx", i, regs->regs[i]);
0238 
0239         pr_cont("\n");
0240     }
0241 }
0242 
0243 void show_regs(struct pt_regs *regs)
0244 {
0245     __show_regs(regs);
0246     dump_backtrace(regs, NULL, KERN_DEFAULT);
0247 }
0248 
0249 static void tls_thread_flush(void)
0250 {
0251     write_sysreg(0, tpidr_el0);
0252     if (system_supports_tpidr2())
0253         write_sysreg_s(0, SYS_TPIDR2_EL0);
0254 
0255     if (is_compat_task()) {
0256         current->thread.uw.tp_value = 0;
0257 
0258         /*
0259          * We need to ensure ordering between the shadow state and the
0260          * hardware state, so that we don't corrupt the hardware state
0261          * with a stale shadow state during context switch.
0262          */
0263         barrier();
0264         write_sysreg(0, tpidrro_el0);
0265     }
0266 }
0267 
0268 static void flush_tagged_addr_state(void)
0269 {
0270     if (IS_ENABLED(CONFIG_ARM64_TAGGED_ADDR_ABI))
0271         clear_thread_flag(TIF_TAGGED_ADDR);
0272 }
0273 
0274 void flush_thread(void)
0275 {
0276     fpsimd_flush_thread();
0277     tls_thread_flush();
0278     flush_ptrace_hw_breakpoint(current);
0279     flush_tagged_addr_state();
0280 }
0281 
0282 void release_thread(struct task_struct *dead_task)
0283 {
0284 }
0285 
0286 void arch_release_task_struct(struct task_struct *tsk)
0287 {
0288     fpsimd_release_task(tsk);
0289 }
0290 
0291 int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src)
0292 {
0293     if (current->mm)
0294         fpsimd_preserve_current_state();
0295     *dst = *src;
0296 
0297     /* We rely on the above assignment to initialize dst's thread_flags: */
0298     BUILD_BUG_ON(!IS_ENABLED(CONFIG_THREAD_INFO_IN_TASK));
0299 
0300     /*
0301      * Detach src's sve_state (if any) from dst so that it does not
0302      * get erroneously used or freed prematurely.  dst's copies
0303      * will be allocated on demand later on if dst uses SVE.
0304      * For consistency, also clear TIF_SVE here: this could be done
0305      * later in copy_process(), but to avoid tripping up future
0306      * maintainers it is best not to leave TIF flags and buffers in
0307      * an inconsistent state, even temporarily.
0308      */
0309     dst->thread.sve_state = NULL;
0310     clear_tsk_thread_flag(dst, TIF_SVE);
0311 
0312     /*
0313      * In the unlikely event that we create a new thread with ZA
0314      * enabled we should retain the ZA state so duplicate it here.
0315      * This may be shortly freed if we exec() or if CLONE_SETTLS
0316      * but it's simpler to do it here. To avoid confusing the rest
0317      * of the code ensure that we have a sve_state allocated
0318      * whenever za_state is allocated.
0319      */
0320     if (thread_za_enabled(&src->thread)) {
0321         dst->thread.sve_state = kzalloc(sve_state_size(src),
0322                         GFP_KERNEL);
0323         if (!dst->thread.sve_state)
0324             return -ENOMEM;
0325         dst->thread.za_state = kmemdup(src->thread.za_state,
0326                            za_state_size(src),
0327                            GFP_KERNEL);
0328         if (!dst->thread.za_state) {
0329             kfree(dst->thread.sve_state);
0330             dst->thread.sve_state = NULL;
0331             return -ENOMEM;
0332         }
0333     } else {
0334         dst->thread.za_state = NULL;
0335         clear_tsk_thread_flag(dst, TIF_SME);
0336     }
0337 
0338     /* clear any pending asynchronous tag fault raised by the parent */
0339     clear_tsk_thread_flag(dst, TIF_MTE_ASYNC_FAULT);
0340 
0341     return 0;
0342 }
0343 
0344 asmlinkage void ret_from_fork(void) asm("ret_from_fork");
0345 
0346 int copy_thread(struct task_struct *p, const struct kernel_clone_args *args)
0347 {
0348     unsigned long clone_flags = args->flags;
0349     unsigned long stack_start = args->stack;
0350     unsigned long tls = args->tls;
0351     struct pt_regs *childregs = task_pt_regs(p);
0352 
0353     memset(&p->thread.cpu_context, 0, sizeof(struct cpu_context));
0354 
0355     /*
0356      * In case p was allocated the same task_struct pointer as some
0357      * other recently-exited task, make sure p is disassociated from
0358      * any cpu that may have run that now-exited task recently.
0359      * Otherwise we could erroneously skip reloading the FPSIMD
0360      * registers for p.
0361      */
0362     fpsimd_flush_task_state(p);
0363 
0364     ptrauth_thread_init_kernel(p);
0365 
0366     if (likely(!args->fn)) {
0367         *childregs = *current_pt_regs();
0368         childregs->regs[0] = 0;
0369 
0370         /*
0371          * Read the current TLS pointer from tpidr_el0 as it may be
0372          * out-of-sync with the saved value.
0373          */
0374         *task_user_tls(p) = read_sysreg(tpidr_el0);
0375         if (system_supports_tpidr2())
0376             p->thread.tpidr2_el0 = read_sysreg_s(SYS_TPIDR2_EL0);
0377 
0378         if (stack_start) {
0379             if (is_compat_thread(task_thread_info(p)))
0380                 childregs->compat_sp = stack_start;
0381             else
0382                 childregs->sp = stack_start;
0383         }
0384 
0385         /*
0386          * If a TLS pointer was passed to clone, use it for the new
0387          * thread.  We also reset TPIDR2 if it's in use.
0388          */
0389         if (clone_flags & CLONE_SETTLS) {
0390             p->thread.uw.tp_value = tls;
0391             p->thread.tpidr2_el0 = 0;
0392         }
0393     } else {
0394         /*
0395          * A kthread has no context to ERET to, so ensure any buggy
0396          * ERET is treated as an illegal exception return.
0397          *
0398          * When a user task is created from a kthread, childregs will
0399          * be initialized by start_thread() or start_compat_thread().
0400          */
0401         memset(childregs, 0, sizeof(struct pt_regs));
0402         childregs->pstate = PSR_MODE_EL1h | PSR_IL_BIT;
0403 
0404         p->thread.cpu_context.x19 = (unsigned long)args->fn;
0405         p->thread.cpu_context.x20 = (unsigned long)args->fn_arg;
0406     }
0407     p->thread.cpu_context.pc = (unsigned long)ret_from_fork;
0408     p->thread.cpu_context.sp = (unsigned long)childregs;
0409     /*
0410      * For the benefit of the unwinder, set up childregs->stackframe
0411      * as the final frame for the new task.
0412      */
0413     p->thread.cpu_context.fp = (unsigned long)childregs->stackframe;
0414 
0415     ptrace_hw_copy_thread(p);
0416 
0417     return 0;
0418 }
0419 
0420 void tls_preserve_current_state(void)
0421 {
0422     *task_user_tls(current) = read_sysreg(tpidr_el0);
0423     if (system_supports_tpidr2() && !is_compat_task())
0424         current->thread.tpidr2_el0 = read_sysreg_s(SYS_TPIDR2_EL0);
0425 }
0426 
0427 static void tls_thread_switch(struct task_struct *next)
0428 {
0429     tls_preserve_current_state();
0430 
0431     if (is_compat_thread(task_thread_info(next)))
0432         write_sysreg(next->thread.uw.tp_value, tpidrro_el0);
0433     else if (!arm64_kernel_unmapped_at_el0())
0434         write_sysreg(0, tpidrro_el0);
0435 
0436     write_sysreg(*task_user_tls(next), tpidr_el0);
0437     if (system_supports_tpidr2())
0438         write_sysreg_s(next->thread.tpidr2_el0, SYS_TPIDR2_EL0);
0439 }
0440 
0441 /*
0442  * Force SSBS state on context-switch, since it may be lost after migrating
0443  * from a CPU which treats the bit as RES0 in a heterogeneous system.
0444  */
0445 static void ssbs_thread_switch(struct task_struct *next)
0446 {
0447     /*
0448      * Nothing to do for kernel threads, but 'regs' may be junk
0449      * (e.g. idle task) so check the flags and bail early.
0450      */
0451     if (unlikely(next->flags & PF_KTHREAD))
0452         return;
0453 
0454     /*
0455      * If all CPUs implement the SSBS extension, then we just need to
0456      * context-switch the PSTATE field.
0457      */
0458     if (cpus_have_const_cap(ARM64_SSBS))
0459         return;
0460 
0461     spectre_v4_enable_task_mitigation(next);
0462 }
0463 
0464 /*
0465  * We store our current task in sp_el0, which is clobbered by userspace. Keep a
0466  * shadow copy so that we can restore this upon entry from userspace.
0467  *
0468  * This is *only* for exception entry from EL0, and is not valid until we
0469  * __switch_to() a user task.
0470  */
0471 DEFINE_PER_CPU(struct task_struct *, __entry_task);
0472 
0473 static void entry_task_switch(struct task_struct *next)
0474 {
0475     __this_cpu_write(__entry_task, next);
0476 }
0477 
0478 /*
0479  * ARM erratum 1418040 handling, affecting the 32bit view of CNTVCT.
0480  * Ensure access is disabled when switching to a 32bit task, ensure
0481  * access is enabled when switching to a 64bit task.
0482  */
0483 static void erratum_1418040_thread_switch(struct task_struct *next)
0484 {
0485     if (!IS_ENABLED(CONFIG_ARM64_ERRATUM_1418040) ||
0486         !this_cpu_has_cap(ARM64_WORKAROUND_1418040))
0487         return;
0488 
0489     if (is_compat_thread(task_thread_info(next)))
0490         sysreg_clear_set(cntkctl_el1, ARCH_TIMER_USR_VCT_ACCESS_EN, 0);
0491     else
0492         sysreg_clear_set(cntkctl_el1, 0, ARCH_TIMER_USR_VCT_ACCESS_EN);
0493 }
0494 
0495 static void erratum_1418040_new_exec(void)
0496 {
0497     preempt_disable();
0498     erratum_1418040_thread_switch(current);
0499     preempt_enable();
0500 }
0501 
0502 /*
0503  * __switch_to() checks current->thread.sctlr_user as an optimisation. Therefore
0504  * this function must be called with preemption disabled and the update to
0505  * sctlr_user must be made in the same preemption disabled block so that
0506  * __switch_to() does not see the variable update before the SCTLR_EL1 one.
0507  */
0508 void update_sctlr_el1(u64 sctlr)
0509 {
0510     /*
0511      * EnIA must not be cleared while in the kernel as this is necessary for
0512      * in-kernel PAC. It will be cleared on kernel exit if needed.
0513      */
0514     sysreg_clear_set(sctlr_el1, SCTLR_USER_MASK & ~SCTLR_ELx_ENIA, sctlr);
0515 
0516     /* ISB required for the kernel uaccess routines when setting TCF0. */
0517     isb();
0518 }
0519 
0520 /*
0521  * Thread switching.
0522  */
0523 __notrace_funcgraph __sched
0524 struct task_struct *__switch_to(struct task_struct *prev,
0525                 struct task_struct *next)
0526 {
0527     struct task_struct *last;
0528 
0529     fpsimd_thread_switch(next);
0530     tls_thread_switch(next);
0531     hw_breakpoint_thread_switch(next);
0532     contextidr_thread_switch(next);
0533     entry_task_switch(next);
0534     ssbs_thread_switch(next);
0535     erratum_1418040_thread_switch(next);
0536     ptrauth_thread_switch_user(next);
0537 
0538     /*
0539      * Complete any pending TLB or cache maintenance on this CPU in case
0540      * the thread migrates to a different CPU.
0541      * This full barrier is also required by the membarrier system
0542      * call.
0543      */
0544     dsb(ish);
0545 
0546     /*
0547      * MTE thread switching must happen after the DSB above to ensure that
0548      * any asynchronous tag check faults have been logged in the TFSR*_EL1
0549      * registers.
0550      */
0551     mte_thread_switch(next);
0552     /* avoid expensive SCTLR_EL1 accesses if no change */
0553     if (prev->thread.sctlr_user != next->thread.sctlr_user)
0554         update_sctlr_el1(next->thread.sctlr_user);
0555 
0556     /* the actual thread switch */
0557     last = cpu_switch_to(prev, next);
0558 
0559     return last;
0560 }
0561 
0562 struct wchan_info {
0563     unsigned long   pc;
0564     int     count;
0565 };
0566 
0567 static bool get_wchan_cb(void *arg, unsigned long pc)
0568 {
0569     struct wchan_info *wchan_info = arg;
0570 
0571     if (!in_sched_functions(pc)) {
0572         wchan_info->pc = pc;
0573         return false;
0574     }
0575     return wchan_info->count++ < 16;
0576 }
0577 
0578 unsigned long __get_wchan(struct task_struct *p)
0579 {
0580     struct wchan_info wchan_info = {
0581         .pc = 0,
0582         .count = 0,
0583     };
0584 
0585     if (!try_get_task_stack(p))
0586         return 0;
0587 
0588     arch_stack_walk(get_wchan_cb, &wchan_info, p, NULL);
0589 
0590     put_task_stack(p);
0591 
0592     return wchan_info.pc;
0593 }
0594 
0595 unsigned long arch_align_stack(unsigned long sp)
0596 {
0597     if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space)
0598         sp -= get_random_int() & ~PAGE_MASK;
0599     return sp & ~0xf;
0600 }
0601 
0602 #ifdef CONFIG_COMPAT
0603 int compat_elf_check_arch(const struct elf32_hdr *hdr)
0604 {
0605     if (!system_supports_32bit_el0())
0606         return false;
0607 
0608     if ((hdr)->e_machine != EM_ARM)
0609         return false;
0610 
0611     if (!((hdr)->e_flags & EF_ARM_EABI_MASK))
0612         return false;
0613 
0614     /*
0615      * Prevent execve() of a 32-bit program from a deadline task
0616      * if the restricted affinity mask would be inadmissible on an
0617      * asymmetric system.
0618      */
0619     return !static_branch_unlikely(&arm64_mismatched_32bit_el0) ||
0620            !dl_task_check_affinity(current, system_32bit_el0_cpumask());
0621 }
0622 #endif
0623 
0624 /*
0625  * Called from setup_new_exec() after (COMPAT_)SET_PERSONALITY.
0626  */
0627 void arch_setup_new_exec(void)
0628 {
0629     unsigned long mmflags = 0;
0630 
0631     if (is_compat_task()) {
0632         mmflags = MMCF_AARCH32;
0633 
0634         /*
0635          * Restrict the CPU affinity mask for a 32-bit task so that
0636          * it contains only 32-bit-capable CPUs.
0637          *
0638          * From the perspective of the task, this looks similar to
0639          * what would happen if the 64-bit-only CPUs were hot-unplugged
0640          * at the point of execve(), although we try a bit harder to
0641          * honour the cpuset hierarchy.
0642          */
0643         if (static_branch_unlikely(&arm64_mismatched_32bit_el0))
0644             force_compatible_cpus_allowed_ptr(current);
0645     } else if (static_branch_unlikely(&arm64_mismatched_32bit_el0)) {
0646         relax_compatible_cpus_allowed_ptr(current);
0647     }
0648 
0649     current->mm->context.flags = mmflags;
0650     ptrauth_thread_init_user();
0651     mte_thread_init_user();
0652     erratum_1418040_new_exec();
0653 
0654     if (task_spec_ssb_noexec(current)) {
0655         arch_prctl_spec_ctrl_set(current, PR_SPEC_STORE_BYPASS,
0656                      PR_SPEC_ENABLE);
0657     }
0658 }
0659 
0660 #ifdef CONFIG_ARM64_TAGGED_ADDR_ABI
0661 /*
0662  * Control the relaxed ABI allowing tagged user addresses into the kernel.
0663  */
0664 static unsigned int tagged_addr_disabled;
0665 
0666 long set_tagged_addr_ctrl(struct task_struct *task, unsigned long arg)
0667 {
0668     unsigned long valid_mask = PR_TAGGED_ADDR_ENABLE;
0669     struct thread_info *ti = task_thread_info(task);
0670 
0671     if (is_compat_thread(ti))
0672         return -EINVAL;
0673 
0674     if (system_supports_mte())
0675         valid_mask |= PR_MTE_TCF_SYNC | PR_MTE_TCF_ASYNC \
0676             | PR_MTE_TAG_MASK;
0677 
0678     if (arg & ~valid_mask)
0679         return -EINVAL;
0680 
0681     /*
0682      * Do not allow the enabling of the tagged address ABI if globally
0683      * disabled via sysctl abi.tagged_addr_disabled.
0684      */
0685     if (arg & PR_TAGGED_ADDR_ENABLE && tagged_addr_disabled)
0686         return -EINVAL;
0687 
0688     if (set_mte_ctrl(task, arg) != 0)
0689         return -EINVAL;
0690 
0691     update_ti_thread_flag(ti, TIF_TAGGED_ADDR, arg & PR_TAGGED_ADDR_ENABLE);
0692 
0693     return 0;
0694 }
0695 
0696 long get_tagged_addr_ctrl(struct task_struct *task)
0697 {
0698     long ret = 0;
0699     struct thread_info *ti = task_thread_info(task);
0700 
0701     if (is_compat_thread(ti))
0702         return -EINVAL;
0703 
0704     if (test_ti_thread_flag(ti, TIF_TAGGED_ADDR))
0705         ret = PR_TAGGED_ADDR_ENABLE;
0706 
0707     ret |= get_mte_ctrl(task);
0708 
0709     return ret;
0710 }
0711 
0712 /*
0713  * Global sysctl to disable the tagged user addresses support. This control
0714  * only prevents the tagged address ABI enabling via prctl() and does not
0715  * disable it for tasks that already opted in to the relaxed ABI.
0716  */
0717 
0718 static struct ctl_table tagged_addr_sysctl_table[] = {
0719     {
0720         .procname   = "tagged_addr_disabled",
0721         .mode       = 0644,
0722         .data       = &tagged_addr_disabled,
0723         .maxlen     = sizeof(int),
0724         .proc_handler   = proc_dointvec_minmax,
0725         .extra1     = SYSCTL_ZERO,
0726         .extra2     = SYSCTL_ONE,
0727     },
0728     { }
0729 };
0730 
0731 static int __init tagged_addr_init(void)
0732 {
0733     if (!register_sysctl("abi", tagged_addr_sysctl_table))
0734         return -EINVAL;
0735     return 0;
0736 }
0737 
0738 core_initcall(tagged_addr_init);
0739 #endif  /* CONFIG_ARM64_TAGGED_ADDR_ABI */
0740 
0741 #ifdef CONFIG_BINFMT_ELF
0742 int arch_elf_adjust_prot(int prot, const struct arch_elf_state *state,
0743              bool has_interp, bool is_interp)
0744 {
0745     /*
0746      * For dynamically linked executables the interpreter is
0747      * responsible for setting PROT_BTI on everything except
0748      * itself.
0749      */
0750     if (is_interp != has_interp)
0751         return prot;
0752 
0753     if (!(state->flags & ARM64_ELF_BTI))
0754         return prot;
0755 
0756     if (prot & PROT_EXEC)
0757         prot |= PROT_BTI;
0758 
0759     return prot;
0760 }
0761 #endif