Back to home page

OSCL-LXR

 
 

    


0001 /* SPDX-License-Identifier: GPL-2.0 */
0002 /*
0003  * Asm versions of Xen pv-ops, suitable for direct use.
0004  *
0005  * We only bother with direct forms (ie, vcpu in percpu data) of the
0006  * operations here; the indirect forms are better handled in C.
0007  */
0008 
0009 #include <asm/errno.h>
0010 #include <asm/asm-offsets.h>
0011 #include <asm/percpu.h>
0012 #include <asm/processor-flags.h>
0013 #include <asm/segment.h>
0014 #include <asm/thread_info.h>
0015 #include <asm/asm.h>
0016 #include <asm/frame.h>
0017 #include <asm/unwind_hints.h>
0018 
0019 #include <xen/interface/xen.h>
0020 
0021 #include <linux/init.h>
0022 #include <linux/linkage.h>
0023 #include <../entry/calling.h>
0024 
0025 .pushsection .noinstr.text, "ax"
0026 /*
0027  * Disabling events is simply a matter of making the event mask
0028  * non-zero.
0029  */
0030 SYM_FUNC_START(xen_irq_disable_direct)
0031     movb $1, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
0032     RET
0033 SYM_FUNC_END(xen_irq_disable_direct)
0034 
0035 /*
0036  * Force an event check by making a hypercall, but preserve regs
0037  * before making the call.
0038  */
0039 SYM_FUNC_START(check_events)
0040     FRAME_BEGIN
0041     push %rax
0042     push %rcx
0043     push %rdx
0044     push %rsi
0045     push %rdi
0046     push %r8
0047     push %r9
0048     push %r10
0049     push %r11
0050     call xen_force_evtchn_callback
0051     pop %r11
0052     pop %r10
0053     pop %r9
0054     pop %r8
0055     pop %rdi
0056     pop %rsi
0057     pop %rdx
0058     pop %rcx
0059     pop %rax
0060     FRAME_END
0061     RET
0062 SYM_FUNC_END(check_events)
0063 
0064 /*
0065  * Enable events.  This clears the event mask and tests the pending
0066  * event status with one and operation.  If there are pending events,
0067  * then enter the hypervisor to get them handled.
0068  */
0069 SYM_FUNC_START(xen_irq_enable_direct)
0070     FRAME_BEGIN
0071     /* Unmask events */
0072     movb $0, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
0073 
0074     /*
0075      * Preempt here doesn't matter because that will deal with any
0076      * pending interrupts.  The pending check may end up being run
0077      * on the wrong CPU, but that doesn't hurt.
0078      */
0079 
0080     /* Test for pending */
0081     testb $0xff, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_pending
0082     jz 1f
0083 
0084     call check_events
0085 1:
0086     FRAME_END
0087     RET
0088 SYM_FUNC_END(xen_irq_enable_direct)
0089 
0090 /*
0091  * (xen_)save_fl is used to get the current interrupt enable status.
0092  * Callers expect the status to be in X86_EFLAGS_IF, and other bits
0093  * may be set in the return value.  We take advantage of this by
0094  * making sure that X86_EFLAGS_IF has the right value (and other bits
0095  * in that byte are 0), but other bits in the return value are
0096  * undefined.  We need to toggle the state of the bit, because Xen and
0097  * x86 use opposite senses (mask vs enable).
0098  */
0099 SYM_FUNC_START(xen_save_fl_direct)
0100     testb $0xff, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
0101     setz %ah
0102     addb %ah, %ah
0103     RET
0104 SYM_FUNC_END(xen_save_fl_direct)
0105 
0106 SYM_FUNC_START(xen_read_cr2)
0107     FRAME_BEGIN
0108     _ASM_MOV PER_CPU_VAR(xen_vcpu), %_ASM_AX
0109     _ASM_MOV XEN_vcpu_info_arch_cr2(%_ASM_AX), %_ASM_AX
0110     FRAME_END
0111     RET
0112 SYM_FUNC_END(xen_read_cr2);
0113 
0114 SYM_FUNC_START(xen_read_cr2_direct)
0115     FRAME_BEGIN
0116     _ASM_MOV PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_arch_cr2, %_ASM_AX
0117     FRAME_END
0118     RET
0119 SYM_FUNC_END(xen_read_cr2_direct);
0120 .popsection
0121 
0122 .macro xen_pv_trap name
0123 SYM_CODE_START(xen_\name)
0124     UNWIND_HINT_ENTRY
0125     ENDBR
0126     pop %rcx
0127     pop %r11
0128     jmp  \name
0129 SYM_CODE_END(xen_\name)
0130 _ASM_NOKPROBE(xen_\name)
0131 .endm
0132 
0133 xen_pv_trap asm_exc_divide_error
0134 xen_pv_trap asm_xenpv_exc_debug
0135 xen_pv_trap asm_exc_int3
0136 xen_pv_trap asm_xenpv_exc_nmi
0137 xen_pv_trap asm_exc_overflow
0138 xen_pv_trap asm_exc_bounds
0139 xen_pv_trap asm_exc_invalid_op
0140 xen_pv_trap asm_exc_device_not_available
0141 xen_pv_trap asm_xenpv_exc_double_fault
0142 xen_pv_trap asm_exc_coproc_segment_overrun
0143 xen_pv_trap asm_exc_invalid_tss
0144 xen_pv_trap asm_exc_segment_not_present
0145 xen_pv_trap asm_exc_stack_segment
0146 xen_pv_trap asm_exc_general_protection
0147 xen_pv_trap asm_exc_page_fault
0148 xen_pv_trap asm_exc_spurious_interrupt_bug
0149 xen_pv_trap asm_exc_coprocessor_error
0150 xen_pv_trap asm_exc_alignment_check
0151 #ifdef CONFIG_X86_KERNEL_IBT
0152 xen_pv_trap asm_exc_control_protection
0153 #endif
0154 #ifdef CONFIG_X86_MCE
0155 xen_pv_trap asm_xenpv_exc_machine_check
0156 #endif /* CONFIG_X86_MCE */
0157 xen_pv_trap asm_exc_simd_coprocessor_error
0158 #ifdef CONFIG_IA32_EMULATION
0159 xen_pv_trap entry_INT80_compat
0160 #endif
0161 xen_pv_trap asm_exc_xen_unknown_trap
0162 xen_pv_trap asm_exc_xen_hypervisor_callback
0163 
0164     __INIT
0165 SYM_CODE_START(xen_early_idt_handler_array)
0166     i = 0
0167     .rept NUM_EXCEPTION_VECTORS
0168     UNWIND_HINT_EMPTY
0169     ENDBR
0170     pop %rcx
0171     pop %r11
0172     jmp early_idt_handler_array + i*EARLY_IDT_HANDLER_SIZE
0173     i = i + 1
0174     .fill xen_early_idt_handler_array + i*XEN_EARLY_IDT_HANDLER_SIZE - ., 1, 0xcc
0175     .endr
0176 SYM_CODE_END(xen_early_idt_handler_array)
0177     __FINIT
0178 
0179 hypercall_iret = hypercall_page + __HYPERVISOR_iret * 32
0180 /*
0181  * Xen64 iret frame:
0182  *
0183  *  ss
0184  *  rsp
0185  *  rflags
0186  *  cs
0187  *  rip     <-- standard iret frame
0188  *
0189  *  flags
0190  *
0191  *  rcx     }
0192  *  r11     }<-- pushed by hypercall page
0193  * rsp->rax     }
0194  */
0195 SYM_CODE_START(xen_iret)
0196     UNWIND_HINT_EMPTY
0197     ANNOTATE_NOENDBR
0198     pushq $0
0199     jmp hypercall_iret
0200 SYM_CODE_END(xen_iret)
0201 
0202 /*
0203  * XEN pv doesn't use trampoline stack, PER_CPU_VAR(cpu_tss_rw + TSS_sp0) is
0204  * also the kernel stack.  Reusing swapgs_restore_regs_and_return_to_usermode()
0205  * in XEN pv would cause %rsp to move up to the top of the kernel stack and
0206  * leave the IRET frame below %rsp, which is dangerous to be corrupted if #NMI
0207  * interrupts. And swapgs_restore_regs_and_return_to_usermode() pushing the IRET
0208  * frame at the same address is useless.
0209  */
0210 SYM_CODE_START(xenpv_restore_regs_and_return_to_usermode)
0211     UNWIND_HINT_REGS
0212     POP_REGS
0213 
0214     /* stackleak_erase() can work safely on the kernel stack. */
0215     STACKLEAK_ERASE_NOCLOBBER
0216 
0217     addq    $8, %rsp    /* skip regs->orig_ax */
0218     jmp xen_iret
0219 SYM_CODE_END(xenpv_restore_regs_and_return_to_usermode)
0220 
0221 /*
0222  * Xen handles syscall callbacks much like ordinary exceptions, which
0223  * means we have:
0224  * - kernel gs
0225  * - kernel rsp
0226  * - an iret-like stack frame on the stack (including rcx and r11):
0227  *  ss
0228  *  rsp
0229  *  rflags
0230  *  cs
0231  *  rip
0232  *  r11
0233  * rsp->rcx
0234  */
0235 
0236 /* Normal 64-bit system call target */
0237 SYM_CODE_START(xen_entry_SYSCALL_64)
0238     UNWIND_HINT_ENTRY
0239     ENDBR
0240     popq %rcx
0241     popq %r11
0242 
0243     /*
0244      * Neither Xen nor the kernel really knows what the old SS and
0245      * CS were.  The kernel expects __USER_DS and __USER_CS, so
0246      * report those values even though Xen will guess its own values.
0247      */
0248     movq $__USER_DS, 4*8(%rsp)
0249     movq $__USER_CS, 1*8(%rsp)
0250 
0251     jmp entry_SYSCALL_64_after_hwframe
0252 SYM_CODE_END(xen_entry_SYSCALL_64)
0253 
0254 #ifdef CONFIG_IA32_EMULATION
0255 
0256 /* 32-bit compat syscall target */
0257 SYM_CODE_START(xen_entry_SYSCALL_compat)
0258     UNWIND_HINT_ENTRY
0259     ENDBR
0260     popq %rcx
0261     popq %r11
0262 
0263     /*
0264      * Neither Xen nor the kernel really knows what the old SS and
0265      * CS were.  The kernel expects __USER32_DS and __USER32_CS, so
0266      * report those values even though Xen will guess its own values.
0267      */
0268     movq $__USER32_DS, 4*8(%rsp)
0269     movq $__USER32_CS, 1*8(%rsp)
0270 
0271     jmp entry_SYSCALL_compat_after_hwframe
0272 SYM_CODE_END(xen_entry_SYSCALL_compat)
0273 
0274 /* 32-bit compat sysenter target */
0275 SYM_CODE_START(xen_entry_SYSENTER_compat)
0276     UNWIND_HINT_ENTRY
0277     ENDBR
0278     /*
0279      * NB: Xen is polite and clears TF from EFLAGS for us.  This means
0280      * that we don't need to guard against single step exceptions here.
0281      */
0282     popq %rcx
0283     popq %r11
0284 
0285     /*
0286      * Neither Xen nor the kernel really knows what the old SS and
0287      * CS were.  The kernel expects __USER32_DS and __USER32_CS, so
0288      * report those values even though Xen will guess its own values.
0289      */
0290     movq $__USER32_DS, 4*8(%rsp)
0291     movq $__USER32_CS, 1*8(%rsp)
0292 
0293     jmp entry_SYSENTER_compat_after_hwframe
0294 SYM_CODE_END(xen_entry_SYSENTER_compat)
0295 
0296 #else /* !CONFIG_IA32_EMULATION */
0297 
0298 SYM_CODE_START(xen_entry_SYSCALL_compat)
0299 SYM_CODE_START(xen_entry_SYSENTER_compat)
0300     UNWIND_HINT_ENTRY
0301     ENDBR
0302     lea 16(%rsp), %rsp  /* strip %rcx, %r11 */
0303     mov $-ENOSYS, %rax
0304     pushq $0
0305     jmp hypercall_iret
0306 SYM_CODE_END(xen_entry_SYSENTER_compat)
0307 SYM_CODE_END(xen_entry_SYSCALL_compat)
0308 
0309 #endif  /* CONFIG_IA32_EMULATION */