Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 #include <linux/kernel.h>
0003 #include <linux/kvm_host.h>
0004 #include <asm/asm-prototypes.h>
0005 #include <asm/dbell.h>
0006 #include <asm/ppc-opcode.h>
0007 
0008 #include "book3s_hv.h"
0009 
0010 static void load_spr_state(struct kvm_vcpu *vcpu,
0011                 struct p9_host_os_sprs *host_os_sprs)
0012 {
0013     /* TAR is very fast */
0014     mtspr(SPRN_TAR, vcpu->arch.tar);
0015 
0016 #ifdef CONFIG_ALTIVEC
0017     if (cpu_has_feature(CPU_FTR_ALTIVEC) &&
0018         current->thread.vrsave != vcpu->arch.vrsave)
0019         mtspr(SPRN_VRSAVE, vcpu->arch.vrsave);
0020 #endif
0021 
0022     if (vcpu->arch.hfscr & HFSCR_EBB) {
0023         if (current->thread.ebbhr != vcpu->arch.ebbhr)
0024             mtspr(SPRN_EBBHR, vcpu->arch.ebbhr);
0025         if (current->thread.ebbrr != vcpu->arch.ebbrr)
0026             mtspr(SPRN_EBBRR, vcpu->arch.ebbrr);
0027         if (current->thread.bescr != vcpu->arch.bescr)
0028             mtspr(SPRN_BESCR, vcpu->arch.bescr);
0029     }
0030 
0031     if (cpu_has_feature(CPU_FTR_P9_TIDR) &&
0032             current->thread.tidr != vcpu->arch.tid)
0033         mtspr(SPRN_TIDR, vcpu->arch.tid);
0034     if (host_os_sprs->iamr != vcpu->arch.iamr)
0035         mtspr(SPRN_IAMR, vcpu->arch.iamr);
0036     if (host_os_sprs->amr != vcpu->arch.amr)
0037         mtspr(SPRN_AMR, vcpu->arch.amr);
0038     if (vcpu->arch.uamor != 0)
0039         mtspr(SPRN_UAMOR, vcpu->arch.uamor);
0040     if (current->thread.fscr != vcpu->arch.fscr)
0041         mtspr(SPRN_FSCR, vcpu->arch.fscr);
0042     if (current->thread.dscr != vcpu->arch.dscr)
0043         mtspr(SPRN_DSCR, vcpu->arch.dscr);
0044     if (vcpu->arch.pspb != 0)
0045         mtspr(SPRN_PSPB, vcpu->arch.pspb);
0046 
0047     /*
0048      * DAR, DSISR, and for nested HV, SPRGs must be set with MSR[RI]
0049      * clear (or hstate set appropriately to catch those registers
0050      * being clobbered if we take a MCE or SRESET), so those are done
0051      * later.
0052      */
0053 
0054     if (!(vcpu->arch.ctrl & 1))
0055         mtspr(SPRN_CTRLT, 0);
0056 }
0057 
0058 static void store_spr_state(struct kvm_vcpu *vcpu)
0059 {
0060     vcpu->arch.tar = mfspr(SPRN_TAR);
0061 
0062 #ifdef CONFIG_ALTIVEC
0063     if (cpu_has_feature(CPU_FTR_ALTIVEC))
0064         vcpu->arch.vrsave = mfspr(SPRN_VRSAVE);
0065 #endif
0066 
0067     if (vcpu->arch.hfscr & HFSCR_EBB) {
0068         vcpu->arch.ebbhr = mfspr(SPRN_EBBHR);
0069         vcpu->arch.ebbrr = mfspr(SPRN_EBBRR);
0070         vcpu->arch.bescr = mfspr(SPRN_BESCR);
0071     }
0072 
0073     if (cpu_has_feature(CPU_FTR_P9_TIDR))
0074         vcpu->arch.tid = mfspr(SPRN_TIDR);
0075     vcpu->arch.iamr = mfspr(SPRN_IAMR);
0076     vcpu->arch.amr = mfspr(SPRN_AMR);
0077     vcpu->arch.uamor = mfspr(SPRN_UAMOR);
0078     vcpu->arch.fscr = mfspr(SPRN_FSCR);
0079     vcpu->arch.dscr = mfspr(SPRN_DSCR);
0080     vcpu->arch.pspb = mfspr(SPRN_PSPB);
0081 
0082     vcpu->arch.ctrl = mfspr(SPRN_CTRLF);
0083 }
0084 
0085 /* Returns true if current MSR and/or guest MSR may have changed */
0086 bool load_vcpu_state(struct kvm_vcpu *vcpu,
0087              struct p9_host_os_sprs *host_os_sprs)
0088 {
0089     bool ret = false;
0090 
0091 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
0092     if (cpu_has_feature(CPU_FTR_TM) ||
0093         cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST)) {
0094         unsigned long guest_msr = vcpu->arch.shregs.msr;
0095         if (MSR_TM_ACTIVE(guest_msr)) {
0096             kvmppc_restore_tm_hv(vcpu, guest_msr, true);
0097             ret = true;
0098         } else if (vcpu->arch.hfscr & HFSCR_TM) {
0099             mtspr(SPRN_TEXASR, vcpu->arch.texasr);
0100             mtspr(SPRN_TFHAR, vcpu->arch.tfhar);
0101             mtspr(SPRN_TFIAR, vcpu->arch.tfiar);
0102         }
0103     }
0104 #endif
0105 
0106     load_spr_state(vcpu, host_os_sprs);
0107 
0108     load_fp_state(&vcpu->arch.fp);
0109 #ifdef CONFIG_ALTIVEC
0110     load_vr_state(&vcpu->arch.vr);
0111 #endif
0112 
0113     return ret;
0114 }
0115 EXPORT_SYMBOL_GPL(load_vcpu_state);
0116 
0117 void store_vcpu_state(struct kvm_vcpu *vcpu)
0118 {
0119     store_spr_state(vcpu);
0120 
0121     store_fp_state(&vcpu->arch.fp);
0122 #ifdef CONFIG_ALTIVEC
0123     store_vr_state(&vcpu->arch.vr);
0124 #endif
0125 
0126 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
0127     if (cpu_has_feature(CPU_FTR_TM) ||
0128         cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST)) {
0129         unsigned long guest_msr = vcpu->arch.shregs.msr;
0130         if (MSR_TM_ACTIVE(guest_msr)) {
0131             kvmppc_save_tm_hv(vcpu, guest_msr, true);
0132         } else if (vcpu->arch.hfscr & HFSCR_TM) {
0133             vcpu->arch.texasr = mfspr(SPRN_TEXASR);
0134             vcpu->arch.tfhar = mfspr(SPRN_TFHAR);
0135             vcpu->arch.tfiar = mfspr(SPRN_TFIAR);
0136 
0137             if (!vcpu->arch.nested) {
0138                 vcpu->arch.load_tm++; /* see load_ebb comment */
0139                 if (!vcpu->arch.load_tm)
0140                     vcpu->arch.hfscr &= ~HFSCR_TM;
0141             }
0142         }
0143     }
0144 #endif
0145 }
0146 EXPORT_SYMBOL_GPL(store_vcpu_state);
0147 
0148 void save_p9_host_os_sprs(struct p9_host_os_sprs *host_os_sprs)
0149 {
0150     host_os_sprs->iamr = mfspr(SPRN_IAMR);
0151     host_os_sprs->amr = mfspr(SPRN_AMR);
0152 }
0153 EXPORT_SYMBOL_GPL(save_p9_host_os_sprs);
0154 
0155 /* vcpu guest regs must already be saved */
0156 void restore_p9_host_os_sprs(struct kvm_vcpu *vcpu,
0157                  struct p9_host_os_sprs *host_os_sprs)
0158 {
0159     /*
0160      * current->thread.xxx registers must all be restored to host
0161      * values before a potential context switch, otherwise the context
0162      * switch itself will overwrite current->thread.xxx with the values
0163      * from the guest SPRs.
0164      */
0165 
0166     mtspr(SPRN_SPRG_VDSO_WRITE, local_paca->sprg_vdso);
0167 
0168     if (cpu_has_feature(CPU_FTR_P9_TIDR) &&
0169             current->thread.tidr != vcpu->arch.tid)
0170         mtspr(SPRN_TIDR, current->thread.tidr);
0171     if (host_os_sprs->iamr != vcpu->arch.iamr)
0172         mtspr(SPRN_IAMR, host_os_sprs->iamr);
0173     if (vcpu->arch.uamor != 0)
0174         mtspr(SPRN_UAMOR, 0);
0175     if (host_os_sprs->amr != vcpu->arch.amr)
0176         mtspr(SPRN_AMR, host_os_sprs->amr);
0177     if (current->thread.fscr != vcpu->arch.fscr)
0178         mtspr(SPRN_FSCR, current->thread.fscr);
0179     if (current->thread.dscr != vcpu->arch.dscr)
0180         mtspr(SPRN_DSCR, current->thread.dscr);
0181     if (vcpu->arch.pspb != 0)
0182         mtspr(SPRN_PSPB, 0);
0183 
0184     /* Save guest CTRL register, set runlatch to 1 */
0185     if (!(vcpu->arch.ctrl & 1))
0186         mtspr(SPRN_CTRLT, 1);
0187 
0188 #ifdef CONFIG_ALTIVEC
0189     if (cpu_has_feature(CPU_FTR_ALTIVEC) &&
0190         vcpu->arch.vrsave != current->thread.vrsave)
0191         mtspr(SPRN_VRSAVE, current->thread.vrsave);
0192 #endif
0193     if (vcpu->arch.hfscr & HFSCR_EBB) {
0194         if (vcpu->arch.bescr != current->thread.bescr)
0195             mtspr(SPRN_BESCR, current->thread.bescr);
0196         if (vcpu->arch.ebbhr != current->thread.ebbhr)
0197             mtspr(SPRN_EBBHR, current->thread.ebbhr);
0198         if (vcpu->arch.ebbrr != current->thread.ebbrr)
0199             mtspr(SPRN_EBBRR, current->thread.ebbrr);
0200 
0201         if (!vcpu->arch.nested) {
0202             /*
0203              * This is like load_fp in context switching, turn off
0204              * the facility after it wraps the u8 to try avoiding
0205              * saving and restoring the registers each partition
0206              * switch.
0207              */
0208             vcpu->arch.load_ebb++;
0209             if (!vcpu->arch.load_ebb)
0210                 vcpu->arch.hfscr &= ~HFSCR_EBB;
0211         }
0212     }
0213 
0214     if (vcpu->arch.tar != current->thread.tar)
0215         mtspr(SPRN_TAR, current->thread.tar);
0216 }
0217 EXPORT_SYMBOL_GPL(restore_p9_host_os_sprs);
0218 
0219 #ifdef CONFIG_KVM_BOOK3S_HV_P9_TIMING
0220 void accumulate_time(struct kvm_vcpu *vcpu, struct kvmhv_tb_accumulator *next)
0221 {
0222     struct kvmppc_vcore *vc = vcpu->arch.vcore;
0223     struct kvmhv_tb_accumulator *curr;
0224     u64 tb = mftb() - vc->tb_offset_applied;
0225     u64 prev_tb;
0226     u64 delta;
0227     u64 seq;
0228 
0229     curr = vcpu->arch.cur_activity;
0230     vcpu->arch.cur_activity = next;
0231     prev_tb = vcpu->arch.cur_tb_start;
0232     vcpu->arch.cur_tb_start = tb;
0233 
0234     if (!curr)
0235         return;
0236 
0237     delta = tb - prev_tb;
0238 
0239     seq = curr->seqcount;
0240     curr->seqcount = seq + 1;
0241     smp_wmb();
0242     curr->tb_total += delta;
0243     if (seq == 0 || delta < curr->tb_min)
0244         curr->tb_min = delta;
0245     if (delta > curr->tb_max)
0246         curr->tb_max = delta;
0247     smp_wmb();
0248     curr->seqcount = seq + 2;
0249 }
0250 EXPORT_SYMBOL_GPL(accumulate_time);
0251 #endif
0252 
0253 static inline u64 mfslbv(unsigned int idx)
0254 {
0255     u64 slbev;
0256 
0257     asm volatile("slbmfev  %0,%1" : "=r" (slbev) : "r" (idx));
0258 
0259     return slbev;
0260 }
0261 
0262 static inline u64 mfslbe(unsigned int idx)
0263 {
0264     u64 slbee;
0265 
0266     asm volatile("slbmfee  %0,%1" : "=r" (slbee) : "r" (idx));
0267 
0268     return slbee;
0269 }
0270 
0271 static inline void mtslb(u64 slbee, u64 slbev)
0272 {
0273     asm volatile("slbmte %0,%1" :: "r" (slbev), "r" (slbee));
0274 }
0275 
0276 static inline void clear_slb_entry(unsigned int idx)
0277 {
0278     mtslb(idx, 0);
0279 }
0280 
0281 static inline void slb_clear_invalidate_partition(void)
0282 {
0283     clear_slb_entry(0);
0284     asm volatile(PPC_SLBIA(6));
0285 }
0286 
0287 /*
0288  * Malicious or buggy radix guests may have inserted SLB entries
0289  * (only 0..3 because radix always runs with UPRT=1), so these must
0290  * be cleared here to avoid side-channels. slbmte is used rather
0291  * than slbia, as it won't clear cached translations.
0292  */
0293 static void radix_clear_slb(void)
0294 {
0295     int i;
0296 
0297     for (i = 0; i < 4; i++)
0298         clear_slb_entry(i);
0299 }
0300 
0301 static void switch_mmu_to_guest_radix(struct kvm *kvm, struct kvm_vcpu *vcpu, u64 lpcr)
0302 {
0303     struct kvm_nested_guest *nested = vcpu->arch.nested;
0304     u32 lpid;
0305     u32 pid;
0306 
0307     lpid = nested ? nested->shadow_lpid : kvm->arch.lpid;
0308     pid = vcpu->arch.pid;
0309 
0310     /*
0311      * Prior memory accesses to host PID Q3 must be completed before we
0312      * start switching, and stores must be drained to avoid not-my-LPAR
0313      * logic (see switch_mmu_to_host).
0314      */
0315     asm volatile("hwsync" ::: "memory");
0316     isync();
0317     mtspr(SPRN_LPID, lpid);
0318     mtspr(SPRN_LPCR, lpcr);
0319     mtspr(SPRN_PID, pid);
0320     /*
0321      * isync not required here because we are HRFID'ing to guest before
0322      * any guest context access, which is context synchronising.
0323      */
0324 }
0325 
0326 static void switch_mmu_to_guest_hpt(struct kvm *kvm, struct kvm_vcpu *vcpu, u64 lpcr)
0327 {
0328     u32 lpid;
0329     u32 pid;
0330     int i;
0331 
0332     lpid = kvm->arch.lpid;
0333     pid = vcpu->arch.pid;
0334 
0335     /*
0336      * See switch_mmu_to_guest_radix. ptesync should not be required here
0337      * even if the host is in HPT mode because speculative accesses would
0338      * not cause RC updates (we are in real mode).
0339      */
0340     asm volatile("hwsync" ::: "memory");
0341     isync();
0342     mtspr(SPRN_LPID, lpid);
0343     mtspr(SPRN_LPCR, lpcr);
0344     mtspr(SPRN_PID, pid);
0345 
0346     for (i = 0; i < vcpu->arch.slb_max; i++)
0347         mtslb(vcpu->arch.slb[i].orige, vcpu->arch.slb[i].origv);
0348     /*
0349      * isync not required here, see switch_mmu_to_guest_radix.
0350      */
0351 }
0352 
0353 static void switch_mmu_to_host(struct kvm *kvm, u32 pid)
0354 {
0355     u32 lpid = kvm->arch.host_lpid;
0356     u64 lpcr = kvm->arch.host_lpcr;
0357 
0358     /*
0359      * The guest has exited, so guest MMU context is no longer being
0360      * non-speculatively accessed, but a hwsync is needed before the
0361      * mtLPIDR / mtPIDR switch, in order to ensure all stores are drained,
0362      * so the not-my-LPAR tlbie logic does not overlook them.
0363      */
0364     asm volatile("hwsync" ::: "memory");
0365     isync();
0366     mtspr(SPRN_PID, pid);
0367     mtspr(SPRN_LPID, lpid);
0368     mtspr(SPRN_LPCR, lpcr);
0369     /*
0370      * isync is not required after the switch, because mtmsrd with L=0
0371      * is performed after this switch, which is context synchronising.
0372      */
0373 
0374     if (!radix_enabled())
0375         slb_restore_bolted_realmode();
0376 }
0377 
0378 static void save_clear_host_mmu(struct kvm *kvm)
0379 {
0380     if (!radix_enabled()) {
0381         /*
0382          * Hash host could save and restore host SLB entries to
0383          * reduce SLB fault overheads of VM exits, but for now the
0384          * existing code clears all entries and restores just the
0385          * bolted ones when switching back to host.
0386          */
0387         slb_clear_invalidate_partition();
0388     }
0389 }
0390 
0391 static void save_clear_guest_mmu(struct kvm *kvm, struct kvm_vcpu *vcpu)
0392 {
0393     if (kvm_is_radix(kvm)) {
0394         radix_clear_slb();
0395     } else {
0396         int i;
0397         int nr = 0;
0398 
0399         /*
0400          * This must run before switching to host (radix host can't
0401          * access all SLBs).
0402          */
0403         for (i = 0; i < vcpu->arch.slb_nr; i++) {
0404             u64 slbee, slbev;
0405 
0406             slbee = mfslbe(i);
0407             if (slbee & SLB_ESID_V) {
0408                 slbev = mfslbv(i);
0409                 vcpu->arch.slb[nr].orige = slbee | i;
0410                 vcpu->arch.slb[nr].origv = slbev;
0411                 nr++;
0412             }
0413         }
0414         vcpu->arch.slb_max = nr;
0415         slb_clear_invalidate_partition();
0416     }
0417 }
0418 
0419 static void flush_guest_tlb(struct kvm *kvm)
0420 {
0421     unsigned long rb, set;
0422 
0423     rb = PPC_BIT(52);   /* IS = 2 */
0424     if (kvm_is_radix(kvm)) {
0425         /* R=1 PRS=1 RIC=2 */
0426         asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1)
0427                  : : "r" (rb), "i" (1), "i" (1), "i" (2),
0428                    "r" (0) : "memory");
0429         for (set = 1; set < kvm->arch.tlb_sets; ++set) {
0430             rb += PPC_BIT(51);  /* increment set number */
0431             /* R=1 PRS=1 RIC=0 */
0432             asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1)
0433                      : : "r" (rb), "i" (1), "i" (1), "i" (0),
0434                        "r" (0) : "memory");
0435         }
0436         asm volatile("ptesync": : :"memory");
0437         // POWER9 congruence-class TLBIEL leaves ERAT. Flush it now.
0438         asm volatile(PPC_RADIX_INVALIDATE_ERAT_GUEST : : :"memory");
0439     } else {
0440         for (set = 0; set < kvm->arch.tlb_sets; ++set) {
0441             /* R=0 PRS=0 RIC=0 */
0442             asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1)
0443                      : : "r" (rb), "i" (0), "i" (0), "i" (0),
0444                        "r" (0) : "memory");
0445             rb += PPC_BIT(51);  /* increment set number */
0446         }
0447         asm volatile("ptesync": : :"memory");
0448         // POWER9 congruence-class TLBIEL leaves ERAT. Flush it now.
0449         asm volatile(PPC_ISA_3_0_INVALIDATE_ERAT : : :"memory");
0450     }
0451 }
0452 
0453 static void check_need_tlb_flush(struct kvm *kvm, int pcpu,
0454                  struct kvm_nested_guest *nested)
0455 {
0456     cpumask_t *need_tlb_flush;
0457     bool all_set = true;
0458     int i;
0459 
0460     if (nested)
0461         need_tlb_flush = &nested->need_tlb_flush;
0462     else
0463         need_tlb_flush = &kvm->arch.need_tlb_flush;
0464 
0465     if (likely(!cpumask_test_cpu(pcpu, need_tlb_flush)))
0466         return;
0467 
0468     /*
0469      * Individual threads can come in here, but the TLB is shared between
0470      * the 4 threads in a core, hence invalidating on one thread
0471      * invalidates for all, so only invalidate the first time (if all bits
0472      * were set.  The others must still execute a ptesync.
0473      *
0474      * If a race occurs and two threads do the TLB flush, that is not a
0475      * problem, just sub-optimal.
0476      */
0477     for (i = cpu_first_tlb_thread_sibling(pcpu);
0478             i <= cpu_last_tlb_thread_sibling(pcpu);
0479             i += cpu_tlb_thread_sibling_step()) {
0480         if (!cpumask_test_cpu(i, need_tlb_flush)) {
0481             all_set = false;
0482             break;
0483         }
0484     }
0485     if (all_set)
0486         flush_guest_tlb(kvm);
0487     else
0488         asm volatile("ptesync" ::: "memory");
0489 
0490     /* Clear the bit after the TLB flush */
0491     cpumask_clear_cpu(pcpu, need_tlb_flush);
0492 }
0493 
0494 unsigned long kvmppc_msr_hard_disable_set_facilities(struct kvm_vcpu *vcpu, unsigned long msr)
0495 {
0496     unsigned long msr_needed = 0;
0497 
0498     msr &= ~MSR_EE;
0499 
0500     /* MSR bits may have been cleared by context switch so must recheck */
0501     if (IS_ENABLED(CONFIG_PPC_FPU))
0502         msr_needed |= MSR_FP;
0503     if (cpu_has_feature(CPU_FTR_ALTIVEC))
0504         msr_needed |= MSR_VEC;
0505     if (cpu_has_feature(CPU_FTR_VSX))
0506         msr_needed |= MSR_VSX;
0507     if ((cpu_has_feature(CPU_FTR_TM) ||
0508         cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST)) &&
0509             (vcpu->arch.hfscr & HFSCR_TM))
0510         msr_needed |= MSR_TM;
0511 
0512     /*
0513      * This could be combined with MSR[RI] clearing, but that expands
0514      * the unrecoverable window. It would be better to cover unrecoverable
0515      * with KVM bad interrupt handling rather than use MSR[RI] at all.
0516      *
0517      * Much more difficult and less worthwhile to combine with IR/DR
0518      * disable.
0519      */
0520     if ((msr & msr_needed) != msr_needed) {
0521         msr |= msr_needed;
0522         __mtmsrd(msr, 0);
0523     } else {
0524         __hard_irq_disable();
0525     }
0526     local_paca->irq_happened |= PACA_IRQ_HARD_DIS;
0527 
0528     return msr;
0529 }
0530 EXPORT_SYMBOL_GPL(kvmppc_msr_hard_disable_set_facilities);
0531 
0532 int kvmhv_vcpu_entry_p9(struct kvm_vcpu *vcpu, u64 time_limit, unsigned long lpcr, u64 *tb)
0533 {
0534     struct p9_host_os_sprs host_os_sprs;
0535     struct kvm *kvm = vcpu->kvm;
0536     struct kvm_nested_guest *nested = vcpu->arch.nested;
0537     struct kvmppc_vcore *vc = vcpu->arch.vcore;
0538     s64 hdec, dec;
0539     u64 purr, spurr;
0540     u64 *exsave;
0541     int trap;
0542     unsigned long msr;
0543     unsigned long host_hfscr;
0544     unsigned long host_ciabr;
0545     unsigned long host_dawr0;
0546     unsigned long host_dawrx0;
0547     unsigned long host_psscr;
0548     unsigned long host_hpsscr;
0549     unsigned long host_pidr;
0550     unsigned long host_dawr1;
0551     unsigned long host_dawrx1;
0552     unsigned long dpdes;
0553 
0554     hdec = time_limit - *tb;
0555     if (hdec < 0)
0556         return BOOK3S_INTERRUPT_HV_DECREMENTER;
0557 
0558     WARN_ON_ONCE(vcpu->arch.shregs.msr & MSR_HV);
0559     WARN_ON_ONCE(!(vcpu->arch.shregs.msr & MSR_ME));
0560 
0561     vcpu->arch.ceded = 0;
0562 
0563     /* Save MSR for restore, with EE clear. */
0564     msr = mfmsr() & ~MSR_EE;
0565 
0566     host_hfscr = mfspr(SPRN_HFSCR);
0567     host_ciabr = mfspr(SPRN_CIABR);
0568     host_psscr = mfspr(SPRN_PSSCR_PR);
0569     if (cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST))
0570         host_hpsscr = mfspr(SPRN_PSSCR);
0571     host_pidr = mfspr(SPRN_PID);
0572 
0573     if (dawr_enabled()) {
0574         host_dawr0 = mfspr(SPRN_DAWR0);
0575         host_dawrx0 = mfspr(SPRN_DAWRX0);
0576         if (cpu_has_feature(CPU_FTR_DAWR1)) {
0577             host_dawr1 = mfspr(SPRN_DAWR1);
0578             host_dawrx1 = mfspr(SPRN_DAWRX1);
0579         }
0580     }
0581 
0582     local_paca->kvm_hstate.host_purr = mfspr(SPRN_PURR);
0583     local_paca->kvm_hstate.host_spurr = mfspr(SPRN_SPURR);
0584 
0585     save_p9_host_os_sprs(&host_os_sprs);
0586 
0587     msr = kvmppc_msr_hard_disable_set_facilities(vcpu, msr);
0588     if (lazy_irq_pending()) {
0589         trap = 0;
0590         goto out;
0591     }
0592 
0593     if (unlikely(load_vcpu_state(vcpu, &host_os_sprs)))
0594         msr = mfmsr(); /* MSR may have been updated */
0595 
0596     if (vc->tb_offset) {
0597         u64 new_tb = *tb + vc->tb_offset;
0598         mtspr(SPRN_TBU40, new_tb);
0599         if ((mftb() & 0xffffff) < (new_tb & 0xffffff)) {
0600             new_tb += 0x1000000;
0601             mtspr(SPRN_TBU40, new_tb);
0602         }
0603         *tb = new_tb;
0604         vc->tb_offset_applied = vc->tb_offset;
0605     }
0606 
0607     mtspr(SPRN_VTB, vc->vtb);
0608     mtspr(SPRN_PURR, vcpu->arch.purr);
0609     mtspr(SPRN_SPURR, vcpu->arch.spurr);
0610 
0611     if (vc->pcr)
0612         mtspr(SPRN_PCR, vc->pcr | PCR_MASK);
0613     if (vcpu->arch.doorbell_request) {
0614         vcpu->arch.doorbell_request = 0;
0615         mtspr(SPRN_DPDES, 1);
0616     }
0617 
0618     if (dawr_enabled()) {
0619         if (vcpu->arch.dawr0 != host_dawr0)
0620             mtspr(SPRN_DAWR0, vcpu->arch.dawr0);
0621         if (vcpu->arch.dawrx0 != host_dawrx0)
0622             mtspr(SPRN_DAWRX0, vcpu->arch.dawrx0);
0623         if (cpu_has_feature(CPU_FTR_DAWR1)) {
0624             if (vcpu->arch.dawr1 != host_dawr1)
0625                 mtspr(SPRN_DAWR1, vcpu->arch.dawr1);
0626             if (vcpu->arch.dawrx1 != host_dawrx1)
0627                 mtspr(SPRN_DAWRX1, vcpu->arch.dawrx1);
0628         }
0629     }
0630     if (vcpu->arch.ciabr != host_ciabr)
0631         mtspr(SPRN_CIABR, vcpu->arch.ciabr);
0632 
0633 
0634     if (cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST)) {
0635         mtspr(SPRN_PSSCR, vcpu->arch.psscr | PSSCR_EC |
0636               (local_paca->kvm_hstate.fake_suspend << PSSCR_FAKE_SUSPEND_LG));
0637     } else {
0638         if (vcpu->arch.psscr != host_psscr)
0639             mtspr(SPRN_PSSCR_PR, vcpu->arch.psscr);
0640     }
0641 
0642     mtspr(SPRN_HFSCR, vcpu->arch.hfscr);
0643 
0644     mtspr(SPRN_HSRR0, vcpu->arch.regs.nip);
0645     mtspr(SPRN_HSRR1, (vcpu->arch.shregs.msr & ~MSR_HV) | MSR_ME);
0646 
0647     /*
0648      * On POWER9 DD2.1 and below, sometimes on a Hypervisor Data Storage
0649      * Interrupt (HDSI) the HDSISR is not be updated at all.
0650      *
0651      * To work around this we put a canary value into the HDSISR before
0652      * returning to a guest and then check for this canary when we take a
0653      * HDSI. If we find the canary on a HDSI, we know the hardware didn't
0654      * update the HDSISR. In this case we return to the guest to retake the
0655      * HDSI which should correctly update the HDSISR the second time HDSI
0656      * entry.
0657      *
0658      * The "radix prefetch bug" test can be used to test for this bug, as
0659      * it also exists fo DD2.1 and below.
0660      */
0661     if (cpu_has_feature(CPU_FTR_P9_RADIX_PREFETCH_BUG))
0662         mtspr(SPRN_HDSISR, HDSISR_CANARY);
0663 
0664     mtspr(SPRN_SPRG0, vcpu->arch.shregs.sprg0);
0665     mtspr(SPRN_SPRG1, vcpu->arch.shregs.sprg1);
0666     mtspr(SPRN_SPRG2, vcpu->arch.shregs.sprg2);
0667     mtspr(SPRN_SPRG3, vcpu->arch.shregs.sprg3);
0668 
0669     /*
0670      * It might be preferable to load_vcpu_state here, in order to get the
0671      * GPR/FP register loads executing in parallel with the previous mtSPR
0672      * instructions, but for now that can't be done because the TM handling
0673      * in load_vcpu_state can change some SPRs and vcpu state (nip, msr).
0674      * But TM could be split out if this would be a significant benefit.
0675      */
0676 
0677     /*
0678      * MSR[RI] does not need to be cleared (and is not, for radix guests
0679      * with no prefetch bug), because in_guest is set. If we take a SRESET
0680      * or MCE with in_guest set but still in HV mode, then
0681      * kvmppc_p9_bad_interrupt handles the interrupt, which effectively
0682      * clears MSR[RI] and doesn't return.
0683      */
0684     WRITE_ONCE(local_paca->kvm_hstate.in_guest, KVM_GUEST_MODE_HV_P9);
0685     barrier(); /* Open in_guest critical section */
0686 
0687     /*
0688      * Hash host, hash guest, or radix guest with prefetch bug, all have
0689      * to disable the MMU before switching to guest MMU state.
0690      */
0691     if (!radix_enabled() || !kvm_is_radix(kvm) ||
0692             cpu_has_feature(CPU_FTR_P9_RADIX_PREFETCH_BUG))
0693         __mtmsrd(msr & ~(MSR_IR|MSR_DR|MSR_RI), 0);
0694 
0695     save_clear_host_mmu(kvm);
0696 
0697     if (kvm_is_radix(kvm))
0698         switch_mmu_to_guest_radix(kvm, vcpu, lpcr);
0699     else
0700         switch_mmu_to_guest_hpt(kvm, vcpu, lpcr);
0701 
0702     /* TLBIEL uses LPID=LPIDR, so run this after setting guest LPID */
0703     check_need_tlb_flush(kvm, vc->pcpu, nested);
0704 
0705     /*
0706      * P9 suppresses the HDEC exception when LPCR[HDICE] = 0,
0707      * so set guest LPCR (with HDICE) before writing HDEC.
0708      */
0709     mtspr(SPRN_HDEC, hdec);
0710 
0711     mtspr(SPRN_DEC, vcpu->arch.dec_expires - *tb);
0712 
0713 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
0714 tm_return_to_guest:
0715 #endif
0716     mtspr(SPRN_DAR, vcpu->arch.shregs.dar);
0717     mtspr(SPRN_DSISR, vcpu->arch.shregs.dsisr);
0718     mtspr(SPRN_SRR0, vcpu->arch.shregs.srr0);
0719     mtspr(SPRN_SRR1, vcpu->arch.shregs.srr1);
0720 
0721     switch_pmu_to_guest(vcpu, &host_os_sprs);
0722     accumulate_time(vcpu, &vcpu->arch.in_guest);
0723 
0724     kvmppc_p9_enter_guest(vcpu);
0725 
0726     accumulate_time(vcpu, &vcpu->arch.guest_exit);
0727     switch_pmu_to_host(vcpu, &host_os_sprs);
0728 
0729     /* XXX: Could get these from r11/12 and paca exsave instead */
0730     vcpu->arch.shregs.srr0 = mfspr(SPRN_SRR0);
0731     vcpu->arch.shregs.srr1 = mfspr(SPRN_SRR1);
0732     vcpu->arch.shregs.dar = mfspr(SPRN_DAR);
0733     vcpu->arch.shregs.dsisr = mfspr(SPRN_DSISR);
0734 
0735     /* 0x2 bit for HSRR is only used by PR and P7/8 HV paths, clear it */
0736     trap = local_paca->kvm_hstate.scratch0 & ~0x2;
0737 
0738     if (likely(trap > BOOK3S_INTERRUPT_MACHINE_CHECK))
0739         exsave = local_paca->exgen;
0740     else if (trap == BOOK3S_INTERRUPT_SYSTEM_RESET)
0741         exsave = local_paca->exnmi;
0742     else /* trap == 0x200 */
0743         exsave = local_paca->exmc;
0744 
0745     vcpu->arch.regs.gpr[1] = local_paca->kvm_hstate.scratch1;
0746     vcpu->arch.regs.gpr[3] = local_paca->kvm_hstate.scratch2;
0747 
0748     /*
0749      * After reading machine check regs (DAR, DSISR, SRR0/1) and hstate
0750      * scratch (which we need to move into exsave to make re-entrant vs
0751      * SRESET/MCE), register state is protected from reentrancy. However
0752      * timebase, MMU, among other state is still set to guest, so don't
0753      * enable MSR[RI] here. It gets enabled at the end, after in_guest
0754      * is cleared.
0755      *
0756      * It is possible an NMI could come in here, which is why it is
0757      * important to save the above state early so it can be debugged.
0758      */
0759 
0760     vcpu->arch.regs.gpr[9] = exsave[EX_R9/sizeof(u64)];
0761     vcpu->arch.regs.gpr[10] = exsave[EX_R10/sizeof(u64)];
0762     vcpu->arch.regs.gpr[11] = exsave[EX_R11/sizeof(u64)];
0763     vcpu->arch.regs.gpr[12] = exsave[EX_R12/sizeof(u64)];
0764     vcpu->arch.regs.gpr[13] = exsave[EX_R13/sizeof(u64)];
0765     vcpu->arch.ppr = exsave[EX_PPR/sizeof(u64)];
0766     vcpu->arch.cfar = exsave[EX_CFAR/sizeof(u64)];
0767     vcpu->arch.regs.ctr = exsave[EX_CTR/sizeof(u64)];
0768 
0769     vcpu->arch.last_inst = KVM_INST_FETCH_FAILED;
0770 
0771     if (unlikely(trap == BOOK3S_INTERRUPT_MACHINE_CHECK)) {
0772         vcpu->arch.fault_dar = exsave[EX_DAR/sizeof(u64)];
0773         vcpu->arch.fault_dsisr = exsave[EX_DSISR/sizeof(u64)];
0774         kvmppc_realmode_machine_check(vcpu);
0775 
0776     } else if (unlikely(trap == BOOK3S_INTERRUPT_HMI)) {
0777         kvmppc_p9_realmode_hmi_handler(vcpu);
0778 
0779     } else if (trap == BOOK3S_INTERRUPT_H_EMUL_ASSIST) {
0780         vcpu->arch.emul_inst = mfspr(SPRN_HEIR);
0781 
0782     } else if (trap == BOOK3S_INTERRUPT_H_DATA_STORAGE) {
0783         vcpu->arch.fault_dar = exsave[EX_DAR/sizeof(u64)];
0784         vcpu->arch.fault_dsisr = exsave[EX_DSISR/sizeof(u64)];
0785         vcpu->arch.fault_gpa = mfspr(SPRN_ASDR);
0786 
0787     } else if (trap == BOOK3S_INTERRUPT_H_INST_STORAGE) {
0788         vcpu->arch.fault_gpa = mfspr(SPRN_ASDR);
0789 
0790     } else if (trap == BOOK3S_INTERRUPT_H_FAC_UNAVAIL) {
0791         vcpu->arch.hfscr = mfspr(SPRN_HFSCR);
0792 
0793 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
0794     /*
0795      * Softpatch interrupt for transactional memory emulation cases
0796      * on POWER9 DD2.2.  This is early in the guest exit path - we
0797      * haven't saved registers or done a treclaim yet.
0798      */
0799     } else if (trap == BOOK3S_INTERRUPT_HV_SOFTPATCH) {
0800         vcpu->arch.emul_inst = mfspr(SPRN_HEIR);
0801 
0802         /*
0803          * The cases we want to handle here are those where the guest
0804          * is in real suspend mode and is trying to transition to
0805          * transactional mode.
0806          */
0807         if (!local_paca->kvm_hstate.fake_suspend &&
0808                 (vcpu->arch.shregs.msr & MSR_TS_S)) {
0809             if (kvmhv_p9_tm_emulation_early(vcpu)) {
0810                 /*
0811                  * Go straight back into the guest with the
0812                  * new NIP/MSR as set by TM emulation.
0813                  */
0814                 mtspr(SPRN_HSRR0, vcpu->arch.regs.nip);
0815                 mtspr(SPRN_HSRR1, vcpu->arch.shregs.msr);
0816                 goto tm_return_to_guest;
0817             }
0818         }
0819 #endif
0820     }
0821 
0822     /* Advance host PURR/SPURR by the amount used by guest */
0823     purr = mfspr(SPRN_PURR);
0824     spurr = mfspr(SPRN_SPURR);
0825     local_paca->kvm_hstate.host_purr += purr - vcpu->arch.purr;
0826     local_paca->kvm_hstate.host_spurr += spurr - vcpu->arch.spurr;
0827     vcpu->arch.purr = purr;
0828     vcpu->arch.spurr = spurr;
0829 
0830     vcpu->arch.ic = mfspr(SPRN_IC);
0831     vcpu->arch.pid = mfspr(SPRN_PID);
0832     vcpu->arch.psscr = mfspr(SPRN_PSSCR_PR);
0833 
0834     vcpu->arch.shregs.sprg0 = mfspr(SPRN_SPRG0);
0835     vcpu->arch.shregs.sprg1 = mfspr(SPRN_SPRG1);
0836     vcpu->arch.shregs.sprg2 = mfspr(SPRN_SPRG2);
0837     vcpu->arch.shregs.sprg3 = mfspr(SPRN_SPRG3);
0838 
0839     dpdes = mfspr(SPRN_DPDES);
0840     if (dpdes)
0841         vcpu->arch.doorbell_request = 1;
0842 
0843     vc->vtb = mfspr(SPRN_VTB);
0844 
0845     dec = mfspr(SPRN_DEC);
0846     if (!(lpcr & LPCR_LD)) /* Sign extend if not using large decrementer */
0847         dec = (s32) dec;
0848     *tb = mftb();
0849     vcpu->arch.dec_expires = dec + *tb;
0850 
0851     if (vc->tb_offset_applied) {
0852         u64 new_tb = *tb - vc->tb_offset_applied;
0853         mtspr(SPRN_TBU40, new_tb);
0854         if ((mftb() & 0xffffff) < (new_tb & 0xffffff)) {
0855             new_tb += 0x1000000;
0856             mtspr(SPRN_TBU40, new_tb);
0857         }
0858         *tb = new_tb;
0859         vc->tb_offset_applied = 0;
0860     }
0861 
0862     save_clear_guest_mmu(kvm, vcpu);
0863     switch_mmu_to_host(kvm, host_pidr);
0864 
0865     /*
0866      * Enable MSR here in order to have facilities enabled to save
0867      * guest registers. This enables MMU (if we were in realmode), so
0868      * only switch MMU on after the MMU is switched to host, to avoid
0869      * the P9_RADIX_PREFETCH_BUG or hash guest context.
0870      */
0871     if (IS_ENABLED(CONFIG_PPC_TRANSACTIONAL_MEM) &&
0872             vcpu->arch.shregs.msr & MSR_TS_MASK)
0873         msr |= MSR_TS_S;
0874     __mtmsrd(msr, 0);
0875 
0876     store_vcpu_state(vcpu);
0877 
0878     mtspr(SPRN_PURR, local_paca->kvm_hstate.host_purr);
0879     mtspr(SPRN_SPURR, local_paca->kvm_hstate.host_spurr);
0880 
0881     if (cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST)) {
0882         /* Preserve PSSCR[FAKE_SUSPEND] until we've called kvmppc_save_tm_hv */
0883         mtspr(SPRN_PSSCR, host_hpsscr |
0884               (local_paca->kvm_hstate.fake_suspend << PSSCR_FAKE_SUSPEND_LG));
0885     }
0886 
0887     mtspr(SPRN_HFSCR, host_hfscr);
0888     if (vcpu->arch.ciabr != host_ciabr)
0889         mtspr(SPRN_CIABR, host_ciabr);
0890 
0891     if (dawr_enabled()) {
0892         if (vcpu->arch.dawr0 != host_dawr0)
0893             mtspr(SPRN_DAWR0, host_dawr0);
0894         if (vcpu->arch.dawrx0 != host_dawrx0)
0895             mtspr(SPRN_DAWRX0, host_dawrx0);
0896         if (cpu_has_feature(CPU_FTR_DAWR1)) {
0897             if (vcpu->arch.dawr1 != host_dawr1)
0898                 mtspr(SPRN_DAWR1, host_dawr1);
0899             if (vcpu->arch.dawrx1 != host_dawrx1)
0900                 mtspr(SPRN_DAWRX1, host_dawrx1);
0901         }
0902     }
0903 
0904     if (dpdes)
0905         mtspr(SPRN_DPDES, 0);
0906     if (vc->pcr)
0907         mtspr(SPRN_PCR, PCR_MASK);
0908 
0909     /* HDEC must be at least as large as DEC, so decrementer_max fits */
0910     mtspr(SPRN_HDEC, decrementer_max);
0911 
0912     timer_rearm_host_dec(*tb);
0913 
0914     restore_p9_host_os_sprs(vcpu, &host_os_sprs);
0915 
0916     barrier(); /* Close in_guest critical section */
0917     WRITE_ONCE(local_paca->kvm_hstate.in_guest, KVM_GUEST_MODE_NONE);
0918     /* Interrupts are recoverable at this point */
0919 
0920     /*
0921      * cp_abort is required if the processor supports local copy-paste
0922      * to clear the copy buffer that was under control of the guest.
0923      */
0924     if (cpu_has_feature(CPU_FTR_ARCH_31))
0925         asm volatile(PPC_CP_ABORT);
0926 
0927 out:
0928     return trap;
0929 }
0930 EXPORT_SYMBOL_GPL(kvmhv_vcpu_entry_p9);
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!