include/asm/segment.h

0001 /* SPDX-License-Identifier: GPL-2.0 */
0002 #ifndef _ASM_X86_SEGMENT_H
0003 #define _ASM_X86_SEGMENT_H
0004
0005 #include <linux/const.h>
0006 #include <asm/alternative.h>
0007 #include <asm/ibt.h>
0008
0009 /*
0010  * Constructor for a conventional segment GDT (or LDT) entry.
0011  * This is a macro so it can be used in initializers.
0012  */
0013 #define GDT_ENTRY(flags, base, limit)           \
0014     ((((base)  & _AC(0xff000000,ULL)) << (56-24)) | \
0015      (((flags) & _AC(0x0000f0ff,ULL)) << 40) |  \
0016      (((limit) & _AC(0x000f0000,ULL)) << (48-16)) | \
0017      (((base)  & _AC(0x00ffffff,ULL)) << 16) |  \
0018      (((limit) & _AC(0x0000ffff,ULL))))
0019
0020 /* Simple and small GDT entries for booting only: */
0021
0022 #define GDT_ENTRY_BOOT_CS   2
0023 #define GDT_ENTRY_BOOT_DS   3
0024 #define GDT_ENTRY_BOOT_TSS  4
0025 #define __BOOT_CS       (GDT_ENTRY_BOOT_CS*8)
0026 #define __BOOT_DS       (GDT_ENTRY_BOOT_DS*8)
0027 #define __BOOT_TSS      (GDT_ENTRY_BOOT_TSS*8)
0028
0029 /*
0030  * Bottom two bits of selector give the ring
0031  * privilege level
0032  */
0033 #define SEGMENT_RPL_MASK    0x3
0034
0035 /*
0036  * When running on Xen PV, the actual privilege level of the kernel is 1,
0037  * not 0. Testing the Requested Privilege Level in a segment selector to
0038  * determine whether the context is user mode or kernel mode with
0039  * SEGMENT_RPL_MASK is wrong because the PV kernel's privilege level
0040  * matches the 0x3 mask.
0041  *
0042  * Testing with USER_SEGMENT_RPL_MASK is valid for both native and Xen PV
0043  * kernels because privilege level 2 is never used.
0044  */
0045 #define USER_SEGMENT_RPL_MASK   0x2
0046
0047 /* User mode is privilege level 3: */
0048 #define USER_RPL        0x3
0049
0050 /* Bit 2 is Table Indicator (TI): selects between LDT or GDT */
0051 #define SEGMENT_TI_MASK     0x4
0052 /* LDT segment has TI set ... */
0053 #define SEGMENT_LDT     0x4
0054 /* ... GDT has it cleared */
0055 #define SEGMENT_GDT     0x0
0056
0057 #define GDT_ENTRY_INVALID_SEG   0
0058
0059 #ifdef CONFIG_X86_32
0060 /*
0061  * The layout of the per-CPU GDT under Linux:
0062  *
0063  *   0 - null                               <=== cacheline #1
0064  *   1 - reserved
0065  *   2 - reserved
0066  *   3 - reserved
0067  *
0068  *   4 - unused                             <=== cacheline #2
0069  *   5 - unused
0070  *
0071  *  ------- start of TLS (Thread-Local Storage) segments:
0072  *
0073  *   6 - TLS segment #1         [ glibc's TLS segment ]
0074  *   7 - TLS segment #2         [ Wine's %fs Win32 segment ]
0075  *   8 - TLS segment #3                         <=== cacheline #3
0076  *   9 - reserved
0077  *  10 - reserved
0078  *  11 - reserved
0079  *
0080  *  ------- start of kernel segments:
0081  *
0082  *  12 - kernel code segment                        <=== cacheline #4
0083  *  13 - kernel data segment
0084  *  14 - default user CS
0085  *  15 - default user DS
0086  *  16 - TSS                                <=== cacheline #5
0087  *  17 - LDT
0088  *  18 - PNPBIOS support (16->32 gate)
0089  *  19 - PNPBIOS support
0090  *  20 - PNPBIOS support                        <=== cacheline #6
0091  *  21 - PNPBIOS support
0092  *  22 - PNPBIOS support
0093  *  23 - APM BIOS support
0094  *  24 - APM BIOS support                       <=== cacheline #7
0095  *  25 - APM BIOS support
0096  *
0097  *  26 - ESPFIX small SS
0098  *  27 - per-cpu            [ offset to per-cpu data area ]
0099  *  28 - unused
0100  *  29 - unused
0101  *  30 - unused
0102  *  31 - TSS for double fault handler
0103  */
0104 #define GDT_ENTRY_TLS_MIN       6
0105 #define GDT_ENTRY_TLS_MAX       (GDT_ENTRY_TLS_MIN + GDT_ENTRY_TLS_ENTRIES - 1)
0106
0107 #define GDT_ENTRY_KERNEL_CS     12
0108 #define GDT_ENTRY_KERNEL_DS     13
0109 #define GDT_ENTRY_DEFAULT_USER_CS   14
0110 #define GDT_ENTRY_DEFAULT_USER_DS   15
0111 #define GDT_ENTRY_TSS           16
0112 #define GDT_ENTRY_LDT           17
0113 #define GDT_ENTRY_PNPBIOS_CS32      18
0114 #define GDT_ENTRY_PNPBIOS_CS16      19
0115 #define GDT_ENTRY_PNPBIOS_DS        20
0116 #define GDT_ENTRY_PNPBIOS_TS1       21
0117 #define GDT_ENTRY_PNPBIOS_TS2       22
0118 #define GDT_ENTRY_APMBIOS_BASE      23
0119
0120 #define GDT_ENTRY_ESPFIX_SS     26
0121 #define GDT_ENTRY_PERCPU        27
0122
0123 #define GDT_ENTRY_DOUBLEFAULT_TSS   31
0124
0125 /*
0126  * Number of entries in the GDT table:
0127  */
0128 #define GDT_ENTRIES         32
0129
0130 /*
0131  * Segment selector values corresponding to the above entries:
0132  */
0133
0134 #define __KERNEL_CS         (GDT_ENTRY_KERNEL_CS*8)
0135 #define __KERNEL_DS         (GDT_ENTRY_KERNEL_DS*8)
0136 #define __USER_DS           (GDT_ENTRY_DEFAULT_USER_DS*8 + 3)
0137 #define __USER_CS           (GDT_ENTRY_DEFAULT_USER_CS*8 + 3)
0138 #define __ESPFIX_SS         (GDT_ENTRY_ESPFIX_SS*8)
0139
0140 /* segment for calling fn: */
0141 #define PNP_CS32            (GDT_ENTRY_PNPBIOS_CS32*8)
0142 /* code segment for BIOS: */
0143 #define PNP_CS16            (GDT_ENTRY_PNPBIOS_CS16*8)
0144
0145 /* "Is this PNP code selector (PNP_CS32 or PNP_CS16)?" */
0146 #define SEGMENT_IS_PNP_CODE(x)      (((x) & 0xf4) == PNP_CS32)
0147
0148 /* data segment for BIOS: */
0149 #define PNP_DS              (GDT_ENTRY_PNPBIOS_DS*8)
0150 /* transfer data segment: */
0151 #define PNP_TS1             (GDT_ENTRY_PNPBIOS_TS1*8)
0152 /* another data segment: */
0153 #define PNP_TS2             (GDT_ENTRY_PNPBIOS_TS2*8)
0154
0155 #ifdef CONFIG_SMP
0156 # define __KERNEL_PERCPU        (GDT_ENTRY_PERCPU*8)
0157 #else
0158 # define __KERNEL_PERCPU        0
0159 #endif
0160
0161 #else /* 64-bit: */
0162
0163 #include <asm/cache.h>
0164
0165 #define GDT_ENTRY_KERNEL32_CS       1
0166 #define GDT_ENTRY_KERNEL_CS     2
0167 #define GDT_ENTRY_KERNEL_DS     3
0168
0169 /*
0170  * We cannot use the same code segment descriptor for user and kernel mode,
0171  * not even in long flat mode, because of different DPL.
0172  *
0173  * GDT layout to get 64-bit SYSCALL/SYSRET support right. SYSRET hardcodes
0174  * selectors:
0175  *
0176  *   if returning to 32-bit userspace: cs = STAR.SYSRET_CS,
0177  *   if returning to 64-bit userspace: cs = STAR.SYSRET_CS+16,
0178  *
0179  * ss = STAR.SYSRET_CS+8 (in either case)
0180  *
0181  * thus USER_DS should be between 32-bit and 64-bit code selectors:
0182  */
0183 #define GDT_ENTRY_DEFAULT_USER32_CS 4
0184 #define GDT_ENTRY_DEFAULT_USER_DS   5
0185 #define GDT_ENTRY_DEFAULT_USER_CS   6
0186
0187 /* Needs two entries */
0188 #define GDT_ENTRY_TSS           8
0189 /* Needs two entries */
0190 #define GDT_ENTRY_LDT           10
0191
0192 #define GDT_ENTRY_TLS_MIN       12
0193 #define GDT_ENTRY_TLS_MAX       14
0194
0195 #define GDT_ENTRY_CPUNODE       15
0196
0197 /*
0198  * Number of entries in the GDT table:
0199  */
0200 #define GDT_ENTRIES         16
0201
0202 /*
0203  * Segment selector values corresponding to the above entries:
0204  *
0205  * Note, selectors also need to have a correct RPL,
0206  * expressed with the +3 value for user-space selectors:
0207  */
0208 #define __KERNEL32_CS           (GDT_ENTRY_KERNEL32_CS*8)
0209 #define __KERNEL_CS         (GDT_ENTRY_KERNEL_CS*8)
0210 #define __KERNEL_DS         (GDT_ENTRY_KERNEL_DS*8)
0211 #define __USER32_CS         (GDT_ENTRY_DEFAULT_USER32_CS*8 + 3)
0212 #define __USER_DS           (GDT_ENTRY_DEFAULT_USER_DS*8 + 3)
0213 #define __USER32_DS         __USER_DS
0214 #define __USER_CS           (GDT_ENTRY_DEFAULT_USER_CS*8 + 3)
0215 #define __CPUNODE_SEG           (GDT_ENTRY_CPUNODE*8 + 3)
0216
0217 #endif
0218
0219 #define IDT_ENTRIES         256
0220 #define NUM_EXCEPTION_VECTORS       32
0221
0222 /* Bitmask of exception vectors which push an error code on the stack: */
0223 #define EXCEPTION_ERRCODE_MASK      0x20027d00
0224
0225 #define GDT_SIZE            (GDT_ENTRIES*8)
0226 #define GDT_ENTRY_TLS_ENTRIES       3
0227 #define TLS_SIZE            (GDT_ENTRY_TLS_ENTRIES* 8)
0228
0229 #ifdef CONFIG_X86_64
0230
0231 /* Bit size and mask of CPU number stored in the per CPU data (and TSC_AUX) */
0232 #define VDSO_CPUNODE_BITS       12
0233 #define VDSO_CPUNODE_MASK       0xfff
0234
0235 #ifndef __ASSEMBLY__
0236
0237 /* Helper functions to store/load CPU and node numbers */
0238
0239 static inline unsigned long vdso_encode_cpunode(int cpu, unsigned long node)
0240 {
0241     return (node << VDSO_CPUNODE_BITS) | cpu;
0242 }
0243
0244 static inline void vdso_read_cpunode(unsigned *cpu, unsigned *node)
0245 {
0246     unsigned int p;
0247
0248     /*
0249      * Load CPU and node number from the GDT.  LSL is faster than RDTSCP
0250      * and works on all CPUs.  This is volatile so that it orders
0251      * correctly with respect to barrier() and to keep GCC from cleverly
0252      * hoisting it out of the calling function.
0253      *
0254      * If RDPID is available, use it.
0255      */
0256     alternative_io ("lsl %[seg],%[p]",
0257             ".byte 0xf3,0x0f,0xc7,0xf8", /* RDPID %eax/rax */
0258             X86_FEATURE_RDPID,
0259             [p] "=a" (p), [seg] "r" (__CPUNODE_SEG));
0260
0261     if (cpu)
0262         *cpu = (p & VDSO_CPUNODE_MASK);
0263     if (node)
0264         *node = (p >> VDSO_CPUNODE_BITS);
0265 }
0266
0267 #endif /* !__ASSEMBLY__ */
0268 #endif /* CONFIG_X86_64 */
0269
0270 #ifdef __KERNEL__
0271
0272 /*
0273  * early_idt_handler_array is an array of entry points referenced in the
0274  * early IDT.  For simplicity, it's a real array with one entry point
0275  * every nine bytes.  That leaves room for an optional 'push $0' if the
0276  * vector has no error code (two bytes), a 'push $vector_number' (two
0277  * bytes), and a jump to the common entry code (up to five bytes).
0278  */
0279 #define EARLY_IDT_HANDLER_SIZE (9 + ENDBR_INSN_SIZE)
0280
0281 /*
0282  * xen_early_idt_handler_array is for Xen pv guests: for each entry in
0283  * early_idt_handler_array it contains a prequel in the form of
0284  * pop %rcx; pop %r11; jmp early_idt_handler_array[i]; summing up to
0285  * max 8 bytes.
0286  */
0287 #define XEN_EARLY_IDT_HANDLER_SIZE (8 + ENDBR_INSN_SIZE)
0288
0289 #ifndef __ASSEMBLY__
0290
0291 extern const char early_idt_handler_array[NUM_EXCEPTION_VECTORS][EARLY_IDT_HANDLER_SIZE];
0292 extern void early_ignore_irq(void);
0293
0294 #ifdef CONFIG_XEN_PV
0295 extern const char xen_early_idt_handler_array[NUM_EXCEPTION_VECTORS][XEN_EARLY_IDT_HANDLER_SIZE];
0296 #endif
0297
0298 /*
0299  * Load a segment. Fall back on loading the zero segment if something goes
0300  * wrong.  This variant assumes that loading zero fully clears the segment.
0301  * This is always the case on Intel CPUs and, even on 64-bit AMD CPUs, any
0302  * failure to fully clear the cached descriptor is only observable for
0303  * FS and GS.
0304  */
0305 #define __loadsegment_simple(seg, value)                \
0306 do {                                    \
0307     unsigned short __val = (value);                 \
0308                                     \
0309     asm volatile("                      \n" \
0310              "1:    movl %k0,%%" #seg "     \n" \
0311              _ASM_EXTABLE_TYPE_REG(1b, 1b, EX_TYPE_ZERO_REG, %k0)\
0312              : "+r" (__val) : : "memory");          \
0313 } while (0)
0314
0315 #define __loadsegment_ss(value) __loadsegment_simple(ss, (value))
0316 #define __loadsegment_ds(value) __loadsegment_simple(ds, (value))
0317 #define __loadsegment_es(value) __loadsegment_simple(es, (value))
0318
0319 #ifdef CONFIG_X86_32
0320
0321 /*
0322  * On 32-bit systems, the hidden parts of FS and GS are unobservable if
0323  * the selector is NULL, so there's no funny business here.
0324  */
0325 #define __loadsegment_fs(value) __loadsegment_simple(fs, (value))
0326 #define __loadsegment_gs(value) __loadsegment_simple(gs, (value))
0327
0328 #else
0329
0330 static inline void __loadsegment_fs(unsigned short value)
0331 {
0332     asm volatile("                      \n"
0333              "1:    movw %0, %%fs           \n"
0334              "2:                    \n"
0335
0336              _ASM_EXTABLE_TYPE(1b, 2b, EX_TYPE_CLEAR_FS)
0337
0338              : : "rm" (value) : "memory");
0339 }
0340
0341 /* __loadsegment_gs is intentionally undefined.  Use load_gs_index instead. */
0342
0343 #endif
0344
0345 #define loadsegment(seg, value) __loadsegment_ ## seg (value)
0346
0347 /*
0348  * Save a segment register away:
0349  */
0350 #define savesegment(seg, value)             \
0351     asm("mov %%" #seg ",%0":"=r" (value) : : "memory")
0352
0353 #endif /* !__ASSEMBLY__ */
0354 #endif /* __KERNEL__ */
0355
0356 #endif /* _ASM_X86_SEGMENT_H */