0001
0002
0003
0004
0005
0006
0007
0008
0009 #include <asm/errno.h>
0010 #include <asm/asm-offsets.h>
0011 #include <asm/percpu.h>
0012 #include <asm/processor-flags.h>
0013 #include <asm/segment.h>
0014 #include <asm/thread_info.h>
0015 #include <asm/asm.h>
0016 #include <asm/frame.h>
0017 #include <asm/unwind_hints.h>
0018
0019 #include <xen/interface/xen.h>
0020
0021 #include <linux/init.h>
0022 #include <linux/linkage.h>
0023 #include <../entry/calling.h>
0024
0025 .pushsection .noinstr.text, "ax"
0026
0027
0028
0029
0030 SYM_FUNC_START(xen_irq_disable_direct)
0031 movb $1, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
0032 RET
0033 SYM_FUNC_END(xen_irq_disable_direct)
0034
0035
0036
0037
0038
0039 SYM_FUNC_START(check_events)
0040 FRAME_BEGIN
0041 push %rax
0042 push %rcx
0043 push %rdx
0044 push %rsi
0045 push %rdi
0046 push %r8
0047 push %r9
0048 push %r10
0049 push %r11
0050 call xen_force_evtchn_callback
0051 pop %r11
0052 pop %r10
0053 pop %r9
0054 pop %r8
0055 pop %rdi
0056 pop %rsi
0057 pop %rdx
0058 pop %rcx
0059 pop %rax
0060 FRAME_END
0061 RET
0062 SYM_FUNC_END(check_events)
0063
0064
0065
0066
0067
0068
0069 SYM_FUNC_START(xen_irq_enable_direct)
0070 FRAME_BEGIN
0071
0072 movb $0, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
0073
0074
0075
0076
0077
0078
0079
0080
0081 testb $0xff, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_pending
0082 jz 1f
0083
0084 call check_events
0085 1:
0086 FRAME_END
0087 RET
0088 SYM_FUNC_END(xen_irq_enable_direct)
0089
0090
0091
0092
0093
0094
0095
0096
0097
0098
0099 SYM_FUNC_START(xen_save_fl_direct)
0100 testb $0xff, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
0101 setz %ah
0102 addb %ah, %ah
0103 RET
0104 SYM_FUNC_END(xen_save_fl_direct)
0105
0106 SYM_FUNC_START(xen_read_cr2)
0107 FRAME_BEGIN
0108 _ASM_MOV PER_CPU_VAR(xen_vcpu), %_ASM_AX
0109 _ASM_MOV XEN_vcpu_info_arch_cr2(%_ASM_AX), %_ASM_AX
0110 FRAME_END
0111 RET
0112 SYM_FUNC_END(xen_read_cr2);
0113
0114 SYM_FUNC_START(xen_read_cr2_direct)
0115 FRAME_BEGIN
0116 _ASM_MOV PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_arch_cr2, %_ASM_AX
0117 FRAME_END
0118 RET
0119 SYM_FUNC_END(xen_read_cr2_direct);
0120 .popsection
0121
0122 .macro xen_pv_trap name
0123 SYM_CODE_START(xen_\name)
0124 UNWIND_HINT_ENTRY
0125 ENDBR
0126 pop %rcx
0127 pop %r11
0128 jmp \name
0129 SYM_CODE_END(xen_\name)
0130 _ASM_NOKPROBE(xen_\name)
0131 .endm
0132
0133 xen_pv_trap asm_exc_divide_error
0134 xen_pv_trap asm_xenpv_exc_debug
0135 xen_pv_trap asm_exc_int3
0136 xen_pv_trap asm_xenpv_exc_nmi
0137 xen_pv_trap asm_exc_overflow
0138 xen_pv_trap asm_exc_bounds
0139 xen_pv_trap asm_exc_invalid_op
0140 xen_pv_trap asm_exc_device_not_available
0141 xen_pv_trap asm_xenpv_exc_double_fault
0142 xen_pv_trap asm_exc_coproc_segment_overrun
0143 xen_pv_trap asm_exc_invalid_tss
0144 xen_pv_trap asm_exc_segment_not_present
0145 xen_pv_trap asm_exc_stack_segment
0146 xen_pv_trap asm_exc_general_protection
0147 xen_pv_trap asm_exc_page_fault
0148 xen_pv_trap asm_exc_spurious_interrupt_bug
0149 xen_pv_trap asm_exc_coprocessor_error
0150 xen_pv_trap asm_exc_alignment_check
0151 #ifdef CONFIG_X86_KERNEL_IBT
0152 xen_pv_trap asm_exc_control_protection
0153 #endif
0154 #ifdef CONFIG_X86_MCE
0155 xen_pv_trap asm_xenpv_exc_machine_check
0156 #endif
0157 xen_pv_trap asm_exc_simd_coprocessor_error
0158 #ifdef CONFIG_IA32_EMULATION
0159 xen_pv_trap entry_INT80_compat
0160 #endif
0161 xen_pv_trap asm_exc_xen_unknown_trap
0162 xen_pv_trap asm_exc_xen_hypervisor_callback
0163
0164 __INIT
0165 SYM_CODE_START(xen_early_idt_handler_array)
0166 i = 0
0167 .rept NUM_EXCEPTION_VECTORS
0168 UNWIND_HINT_EMPTY
0169 ENDBR
0170 pop %rcx
0171 pop %r11
0172 jmp early_idt_handler_array + i*EARLY_IDT_HANDLER_SIZE
0173 i = i + 1
0174 .fill xen_early_idt_handler_array + i*XEN_EARLY_IDT_HANDLER_SIZE - ., 1, 0xcc
0175 .endr
0176 SYM_CODE_END(xen_early_idt_handler_array)
0177 __FINIT
0178
0179 hypercall_iret = hypercall_page + __HYPERVISOR_iret * 32
0180
0181
0182
0183
0184
0185
0186
0187
0188
0189
0190
0191
0192
0193
0194
0195 SYM_CODE_START(xen_iret)
0196 UNWIND_HINT_EMPTY
0197 ANNOTATE_NOENDBR
0198 pushq $0
0199 jmp hypercall_iret
0200 SYM_CODE_END(xen_iret)
0201
0202
0203
0204
0205
0206
0207
0208
0209
0210 SYM_CODE_START(xenpv_restore_regs_and_return_to_usermode)
0211 UNWIND_HINT_REGS
0212 POP_REGS
0213
0214
0215 STACKLEAK_ERASE_NOCLOBBER
0216
0217 addq $8, %rsp
0218 jmp xen_iret
0219 SYM_CODE_END(xenpv_restore_regs_and_return_to_usermode)
0220
0221
0222
0223
0224
0225
0226
0227
0228
0229
0230
0231
0232
0233
0234
0235
0236
0237 SYM_CODE_START(xen_entry_SYSCALL_64)
0238 UNWIND_HINT_ENTRY
0239 ENDBR
0240 popq %rcx
0241 popq %r11
0242
0243
0244
0245
0246
0247
0248 movq $__USER_DS, 4*8(%rsp)
0249 movq $__USER_CS, 1*8(%rsp)
0250
0251 jmp entry_SYSCALL_64_after_hwframe
0252 SYM_CODE_END(xen_entry_SYSCALL_64)
0253
0254 #ifdef CONFIG_IA32_EMULATION
0255
0256
0257 SYM_CODE_START(xen_entry_SYSCALL_compat)
0258 UNWIND_HINT_ENTRY
0259 ENDBR
0260 popq %rcx
0261 popq %r11
0262
0263
0264
0265
0266
0267
0268 movq $__USER32_DS, 4*8(%rsp)
0269 movq $__USER32_CS, 1*8(%rsp)
0270
0271 jmp entry_SYSCALL_compat_after_hwframe
0272 SYM_CODE_END(xen_entry_SYSCALL_compat)
0273
0274
0275 SYM_CODE_START(xen_entry_SYSENTER_compat)
0276 UNWIND_HINT_ENTRY
0277 ENDBR
0278
0279
0280
0281
0282 popq %rcx
0283 popq %r11
0284
0285
0286
0287
0288
0289
0290 movq $__USER32_DS, 4*8(%rsp)
0291 movq $__USER32_CS, 1*8(%rsp)
0292
0293 jmp entry_SYSENTER_compat_after_hwframe
0294 SYM_CODE_END(xen_entry_SYSENTER_compat)
0295
0296 #else
0297
0298 SYM_CODE_START(xen_entry_SYSCALL_compat)
0299 SYM_CODE_START(xen_entry_SYSENTER_compat)
0300 UNWIND_HINT_ENTRY
0301 ENDBR
0302 lea 16(%rsp), %rsp
0303 mov $-ENOSYS, %rax
0304 pushq $0
0305 jmp hypercall_iret
0306 SYM_CODE_END(xen_entry_SYSENTER_compat)
0307 SYM_CODE_END(xen_entry_SYSCALL_compat)
0308
0309 #endif