kernel/cpu/tsx.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * Intel Transactional Synchronization Extensions (TSX) control.
0004  *
0005  * Copyright (C) 2019-2021 Intel Corporation
0006  *
0007  * Author:
0008  *  Pawan Gupta <pawan.kumar.gupta@linux.intel.com>
0009  */
0010
0011 #include <linux/cpufeature.h>
0012
0013 #include <asm/cmdline.h>
0014
0015 #include "cpu.h"
0016
0017 #undef pr_fmt
0018 #define pr_fmt(fmt) "tsx: " fmt
0019
0020 enum tsx_ctrl_states tsx_ctrl_state __ro_after_init = TSX_CTRL_NOT_SUPPORTED;
0021
0022 static void tsx_disable(void)
0023 {
0024     u64 tsx;
0025
0026     rdmsrl(MSR_IA32_TSX_CTRL, tsx);
0027
0028     /* Force all transactions to immediately abort */
0029     tsx |= TSX_CTRL_RTM_DISABLE;
0030
0031     /*
0032      * Ensure TSX support is not enumerated in CPUID.
0033      * This is visible to userspace and will ensure they
0034      * do not waste resources trying TSX transactions that
0035      * will always abort.
0036      */
0037     tsx |= TSX_CTRL_CPUID_CLEAR;
0038
0039     wrmsrl(MSR_IA32_TSX_CTRL, tsx);
0040 }
0041
0042 static void tsx_enable(void)
0043 {
0044     u64 tsx;
0045
0046     rdmsrl(MSR_IA32_TSX_CTRL, tsx);
0047
0048     /* Enable the RTM feature in the cpu */
0049     tsx &= ~TSX_CTRL_RTM_DISABLE;
0050
0051     /*
0052      * Ensure TSX support is enumerated in CPUID.
0053      * This is visible to userspace and will ensure they
0054      * can enumerate and use the TSX feature.
0055      */
0056     tsx &= ~TSX_CTRL_CPUID_CLEAR;
0057
0058     wrmsrl(MSR_IA32_TSX_CTRL, tsx);
0059 }
0060
0061 static bool tsx_ctrl_is_supported(void)
0062 {
0063     u64 ia32_cap = x86_read_arch_cap_msr();
0064
0065     /*
0066      * TSX is controlled via MSR_IA32_TSX_CTRL.  However, support for this
0067      * MSR is enumerated by ARCH_CAP_TSX_MSR bit in MSR_IA32_ARCH_CAPABILITIES.
0068      *
0069      * TSX control (aka MSR_IA32_TSX_CTRL) is only available after a
0070      * microcode update on CPUs that have their MSR_IA32_ARCH_CAPABILITIES
0071      * bit MDS_NO=1. CPUs with MDS_NO=0 are not planned to get
0072      * MSR_IA32_TSX_CTRL support even after a microcode update. Thus,
0073      * tsx= cmdline requests will do nothing on CPUs without
0074      * MSR_IA32_TSX_CTRL support.
0075      */
0076     return !!(ia32_cap & ARCH_CAP_TSX_CTRL_MSR);
0077 }
0078
0079 static enum tsx_ctrl_states x86_get_tsx_auto_mode(void)
0080 {
0081     if (boot_cpu_has_bug(X86_BUG_TAA))
0082         return TSX_CTRL_DISABLE;
0083
0084     return TSX_CTRL_ENABLE;
0085 }
0086
0087 /*
0088  * Disabling TSX is not a trivial business.
0089  *
0090  * First of all, there's a CPUID bit: X86_FEATURE_RTM_ALWAYS_ABORT
0091  * which says that TSX is practically disabled (all transactions are
0092  * aborted by default). When that bit is set, the kernel unconditionally
0093  * disables TSX.
0094  *
0095  * In order to do that, however, it needs to dance a bit:
0096  *
0097  * 1. The first method to disable it is through MSR_TSX_FORCE_ABORT and
0098  * the MSR is present only when *two* CPUID bits are set:
0099  *
0100  * - X86_FEATURE_RTM_ALWAYS_ABORT
0101  * - X86_FEATURE_TSX_FORCE_ABORT
0102  *
0103  * 2. The second method is for CPUs which do not have the above-mentioned
0104  * MSR: those use a different MSR - MSR_IA32_TSX_CTRL and disable TSX
0105  * through that one. Those CPUs can also have the initially mentioned
0106  * CPUID bit X86_FEATURE_RTM_ALWAYS_ABORT set and for those the same strategy
0107  * applies: TSX gets disabled unconditionally.
0108  *
0109  * When either of the two methods are present, the kernel disables TSX and
0110  * clears the respective RTM and HLE feature flags.
0111  *
0112  * An additional twist in the whole thing presents late microcode loading
0113  * which, when done, may cause for the X86_FEATURE_RTM_ALWAYS_ABORT CPUID
0114  * bit to be set after the update.
0115  *
0116  * A subsequent hotplug operation on any logical CPU except the BSP will
0117  * cause for the supported CPUID feature bits to get re-detected and, if
0118  * RTM and HLE get cleared all of a sudden, but, userspace did consult
0119  * them before the update, then funny explosions will happen. Long story
0120  * short: the kernel doesn't modify CPUID feature bits after booting.
0121  *
0122  * That's why, this function's call in init_intel() doesn't clear the
0123  * feature flags.
0124  */
0125 static void tsx_clear_cpuid(void)
0126 {
0127     u64 msr;
0128
0129     /*
0130      * MSR_TFA_TSX_CPUID_CLEAR bit is only present when both CPUID
0131      * bits RTM_ALWAYS_ABORT and TSX_FORCE_ABORT are present.
0132      */
0133     if (boot_cpu_has(X86_FEATURE_RTM_ALWAYS_ABORT) &&
0134         boot_cpu_has(X86_FEATURE_TSX_FORCE_ABORT)) {
0135         rdmsrl(MSR_TSX_FORCE_ABORT, msr);
0136         msr |= MSR_TFA_TSX_CPUID_CLEAR;
0137         wrmsrl(MSR_TSX_FORCE_ABORT, msr);
0138     } else if (tsx_ctrl_is_supported()) {
0139         rdmsrl(MSR_IA32_TSX_CTRL, msr);
0140         msr |= TSX_CTRL_CPUID_CLEAR;
0141         wrmsrl(MSR_IA32_TSX_CTRL, msr);
0142     }
0143 }
0144
0145 /*
0146  * Disable TSX development mode
0147  *
0148  * When the microcode released in Feb 2022 is applied, TSX will be disabled by
0149  * default on some processors. MSR 0x122 (TSX_CTRL) and MSR 0x123
0150  * (IA32_MCU_OPT_CTRL) can be used to re-enable TSX for development, doing so is
0151  * not recommended for production deployments. In particular, applying MD_CLEAR
0152  * flows for mitigation of the Intel TSX Asynchronous Abort (TAA) transient
0153  * execution attack may not be effective on these processors when Intel TSX is
0154  * enabled with updated microcode.
0155  */
0156 static void tsx_dev_mode_disable(void)
0157 {
0158     u64 mcu_opt_ctrl;
0159
0160     /* Check if RTM_ALLOW exists */
0161     if (!boot_cpu_has_bug(X86_BUG_TAA) || !tsx_ctrl_is_supported() ||
0162         !cpu_feature_enabled(X86_FEATURE_SRBDS_CTRL))
0163         return;
0164
0165     rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_opt_ctrl);
0166
0167     if (mcu_opt_ctrl & RTM_ALLOW) {
0168         mcu_opt_ctrl &= ~RTM_ALLOW;
0169         wrmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_opt_ctrl);
0170         setup_force_cpu_cap(X86_FEATURE_RTM_ALWAYS_ABORT);
0171     }
0172 }
0173
0174 void __init tsx_init(void)
0175 {
0176     char arg[5] = {};
0177     int ret;
0178
0179     tsx_dev_mode_disable();
0180
0181     /*
0182      * Hardware will always abort a TSX transaction when the CPUID bit
0183      * RTM_ALWAYS_ABORT is set. In this case, it is better not to enumerate
0184      * CPUID.RTM and CPUID.HLE bits. Clear them here.
0185      */
0186     if (boot_cpu_has(X86_FEATURE_RTM_ALWAYS_ABORT)) {
0187         tsx_ctrl_state = TSX_CTRL_RTM_ALWAYS_ABORT;
0188         tsx_clear_cpuid();
0189         setup_clear_cpu_cap(X86_FEATURE_RTM);
0190         setup_clear_cpu_cap(X86_FEATURE_HLE);
0191         return;
0192     }
0193
0194     if (!tsx_ctrl_is_supported()) {
0195         tsx_ctrl_state = TSX_CTRL_NOT_SUPPORTED;
0196         return;
0197     }
0198
0199     ret = cmdline_find_option(boot_command_line, "tsx", arg, sizeof(arg));
0200     if (ret >= 0) {
0201         if (!strcmp(arg, "on")) {
0202             tsx_ctrl_state = TSX_CTRL_ENABLE;
0203         } else if (!strcmp(arg, "off")) {
0204             tsx_ctrl_state = TSX_CTRL_DISABLE;
0205         } else if (!strcmp(arg, "auto")) {
0206             tsx_ctrl_state = x86_get_tsx_auto_mode();
0207         } else {
0208             tsx_ctrl_state = TSX_CTRL_DISABLE;
0209             pr_err("invalid option, defaulting to off\n");
0210         }
0211     } else {
0212         /* tsx= not provided */
0213         if (IS_ENABLED(CONFIG_X86_INTEL_TSX_MODE_AUTO))
0214             tsx_ctrl_state = x86_get_tsx_auto_mode();
0215         else if (IS_ENABLED(CONFIG_X86_INTEL_TSX_MODE_OFF))
0216             tsx_ctrl_state = TSX_CTRL_DISABLE;
0217         else
0218             tsx_ctrl_state = TSX_CTRL_ENABLE;
0219     }
0220
0221     if (tsx_ctrl_state == TSX_CTRL_DISABLE) {
0222         tsx_disable();
0223
0224         /*
0225          * tsx_disable() will change the state of the RTM and HLE CPUID
0226          * bits. Clear them here since they are now expected to be not
0227          * set.
0228          */
0229         setup_clear_cpu_cap(X86_FEATURE_RTM);
0230         setup_clear_cpu_cap(X86_FEATURE_HLE);
0231     } else if (tsx_ctrl_state == TSX_CTRL_ENABLE) {
0232
0233         /*
0234          * HW defaults TSX to be enabled at bootup.
0235          * We may still need the TSX enable support
0236          * during init for special cases like
0237          * kexec after TSX is disabled.
0238          */
0239         tsx_enable();
0240
0241         /*
0242          * tsx_enable() will change the state of the RTM and HLE CPUID
0243          * bits. Force them here since they are now expected to be set.
0244          */
0245         setup_force_cpu_cap(X86_FEATURE_RTM);
0246         setup_force_cpu_cap(X86_FEATURE_HLE);
0247     }
0248 }
0249
0250 void tsx_ap_init(void)
0251 {
0252     tsx_dev_mode_disable();
0253
0254     if (tsx_ctrl_state == TSX_CTRL_ENABLE)
0255         tsx_enable();
0256     else if (tsx_ctrl_state == TSX_CTRL_DISABLE)
0257         tsx_disable();
0258     else if (tsx_ctrl_state == TSX_CTRL_RTM_ALWAYS_ABORT)
0259         /* See comment over that function for more details. */
0260         tsx_clear_cpuid();
0261 }