misc/cxl/fault.c

0001 // SPDX-License-Identifier: GPL-2.0-or-later
0002 /*
0003  * Copyright 2014 IBM Corp.
0004  */
0005
0006 #include <linux/workqueue.h>
0007 #include <linux/sched/signal.h>
0008 #include <linux/sched/mm.h>
0009 #include <linux/pid.h>
0010 #include <linux/mm.h>
0011 #include <linux/moduleparam.h>
0012
0013 #undef MODULE_PARAM_PREFIX
0014 #define MODULE_PARAM_PREFIX "cxl" "."
0015 #include <asm/current.h>
0016 #include <asm/copro.h>
0017 #include <asm/mmu.h>
0018
0019 #include "cxl.h"
0020 #include "trace.h"
0021
0022 static bool sste_matches(struct cxl_sste *sste, struct copro_slb *slb)
0023 {
0024     return ((sste->vsid_data == cpu_to_be64(slb->vsid)) &&
0025         (sste->esid_data == cpu_to_be64(slb->esid)));
0026 }
0027
0028 /*
0029  * This finds a free SSTE for the given SLB, or returns NULL if it's already in
0030  * the segment table.
0031  */
0032 static struct cxl_sste *find_free_sste(struct cxl_context *ctx,
0033                        struct copro_slb *slb)
0034 {
0035     struct cxl_sste *primary, *sste, *ret = NULL;
0036     unsigned int mask = (ctx->sst_size >> 7) - 1; /* SSTP0[SegTableSize] */
0037     unsigned int entry;
0038     unsigned int hash;
0039
0040     if (slb->vsid & SLB_VSID_B_1T)
0041         hash = (slb->esid >> SID_SHIFT_1T) & mask;
0042     else /* 256M */
0043         hash = (slb->esid >> SID_SHIFT) & mask;
0044
0045     primary = ctx->sstp + (hash << 3);
0046
0047     for (entry = 0, sste = primary; entry < 8; entry++, sste++) {
0048         if (!ret && !(be64_to_cpu(sste->esid_data) & SLB_ESID_V))
0049             ret = sste;
0050         if (sste_matches(sste, slb))
0051             return NULL;
0052     }
0053     if (ret)
0054         return ret;
0055
0056     /* Nothing free, select an entry to cast out */
0057     ret = primary + ctx->sst_lru;
0058     ctx->sst_lru = (ctx->sst_lru + 1) & 0x7;
0059
0060     return ret;
0061 }
0062
0063 static void cxl_load_segment(struct cxl_context *ctx, struct copro_slb *slb)
0064 {
0065     /* mask is the group index, we search primary and secondary here. */
0066     struct cxl_sste *sste;
0067     unsigned long flags;
0068
0069     spin_lock_irqsave(&ctx->sste_lock, flags);
0070     sste = find_free_sste(ctx, slb);
0071     if (!sste)
0072         goto out_unlock;
0073
0074     pr_devel("CXL Populating SST[%li]: %#llx %#llx\n",
0075             sste - ctx->sstp, slb->vsid, slb->esid);
0076     trace_cxl_ste_write(ctx, sste - ctx->sstp, slb->esid, slb->vsid);
0077
0078     sste->vsid_data = cpu_to_be64(slb->vsid);
0079     sste->esid_data = cpu_to_be64(slb->esid);
0080 out_unlock:
0081     spin_unlock_irqrestore(&ctx->sste_lock, flags);
0082 }
0083
0084 static int cxl_fault_segment(struct cxl_context *ctx, struct mm_struct *mm,
0085                  u64 ea)
0086 {
0087     struct copro_slb slb = {0,0};
0088     int rc;
0089
0090     if (!(rc = copro_calculate_slb(mm, ea, &slb))) {
0091         cxl_load_segment(ctx, &slb);
0092     }
0093
0094     return rc;
0095 }
0096
0097 static void cxl_ack_ae(struct cxl_context *ctx)
0098 {
0099     unsigned long flags;
0100
0101     cxl_ops->ack_irq(ctx, CXL_PSL_TFC_An_AE, 0);
0102
0103     spin_lock_irqsave(&ctx->lock, flags);
0104     ctx->pending_fault = true;
0105     ctx->fault_addr = ctx->dar;
0106     ctx->fault_dsisr = ctx->dsisr;
0107     spin_unlock_irqrestore(&ctx->lock, flags);
0108
0109     wake_up_all(&ctx->wq);
0110 }
0111
0112 static int cxl_handle_segment_miss(struct cxl_context *ctx,
0113                    struct mm_struct *mm, u64 ea)
0114 {
0115     int rc;
0116
0117     pr_devel("CXL interrupt: Segment fault pe: %i ea: %#llx\n", ctx->pe, ea);
0118     trace_cxl_ste_miss(ctx, ea);
0119
0120     if ((rc = cxl_fault_segment(ctx, mm, ea)))
0121         cxl_ack_ae(ctx);
0122     else {
0123
0124         mb(); /* Order seg table write to TFC MMIO write */
0125         cxl_ops->ack_irq(ctx, CXL_PSL_TFC_An_R, 0);
0126     }
0127
0128     return IRQ_HANDLED;
0129 }
0130
0131 int cxl_handle_mm_fault(struct mm_struct *mm, u64 dsisr, u64 dar)
0132 {
0133     vm_fault_t flt = 0;
0134     int result;
0135     unsigned long access, flags, inv_flags = 0;
0136
0137     /*
0138      * Add the fault handling cpu to task mm cpumask so that we
0139      * can do a safe lockless page table walk when inserting the
0140      * hash page table entry. This function get called with a
0141      * valid mm for user space addresses. Hence using the if (mm)
0142      * check is sufficient here.
0143      */
0144     if (mm && !cpumask_test_cpu(smp_processor_id(), mm_cpumask(mm))) {
0145         cpumask_set_cpu(smp_processor_id(), mm_cpumask(mm));
0146         /*
0147          * We need to make sure we walk the table only after
0148          * we update the cpumask. The other side of the barrier
0149          * is explained in serialize_against_pte_lookup()
0150          */
0151         smp_mb();
0152     }
0153     if ((result = copro_handle_mm_fault(mm, dar, dsisr, &flt))) {
0154         pr_devel("copro_handle_mm_fault failed: %#x\n", result);
0155         return result;
0156     }
0157
0158     if (!radix_enabled()) {
0159         /*
0160          * update_mmu_cache() will not have loaded the hash since current->trap
0161          * is not a 0x400 or 0x300, so just call hash_page_mm() here.
0162          */
0163         access = _PAGE_PRESENT | _PAGE_READ;
0164         if (dsisr & CXL_PSL_DSISR_An_S)
0165             access |= _PAGE_WRITE;
0166
0167         if (!mm && (get_region_id(dar) != USER_REGION_ID))
0168             access |= _PAGE_PRIVILEGED;
0169
0170         if (dsisr & DSISR_NOHPTE)
0171             inv_flags |= HPTE_NOHPTE_UPDATE;
0172
0173         local_irq_save(flags);
0174         hash_page_mm(mm, dar, access, 0x300, inv_flags);
0175         local_irq_restore(flags);
0176     }
0177     return 0;
0178 }
0179
0180 static void cxl_handle_page_fault(struct cxl_context *ctx,
0181                   struct mm_struct *mm,
0182                   u64 dsisr, u64 dar)
0183 {
0184     trace_cxl_pte_miss(ctx, dsisr, dar);
0185
0186     if (cxl_handle_mm_fault(mm, dsisr, dar)) {
0187         cxl_ack_ae(ctx);
0188     } else {
0189         pr_devel("Page fault successfully handled for pe: %i!\n", ctx->pe);
0190         cxl_ops->ack_irq(ctx, CXL_PSL_TFC_An_R, 0);
0191     }
0192 }
0193
0194 /*
0195  * Returns the mm_struct corresponding to the context ctx.
0196  * mm_users == 0, the context may be in the process of being closed.
0197  */
0198 static struct mm_struct *get_mem_context(struct cxl_context *ctx)
0199 {
0200     if (ctx->mm == NULL)
0201         return NULL;
0202
0203     if (!mmget_not_zero(ctx->mm))
0204         return NULL;
0205
0206     return ctx->mm;
0207 }
0208
0209 static bool cxl_is_segment_miss(struct cxl_context *ctx, u64 dsisr)
0210 {
0211     if ((cxl_is_power8() && (dsisr & CXL_PSL_DSISR_An_DS)))
0212         return true;
0213
0214     return false;
0215 }
0216
0217 static bool cxl_is_page_fault(struct cxl_context *ctx, u64 dsisr)
0218 {
0219     if ((cxl_is_power8()) && (dsisr & CXL_PSL_DSISR_An_DM))
0220         return true;
0221
0222     if (cxl_is_power9())
0223         return true;
0224
0225     return false;
0226 }
0227
0228 void cxl_handle_fault(struct work_struct *fault_work)
0229 {
0230     struct cxl_context *ctx =
0231         container_of(fault_work, struct cxl_context, fault_work);
0232     u64 dsisr = ctx->dsisr;
0233     u64 dar = ctx->dar;
0234     struct mm_struct *mm = NULL;
0235
0236     if (cpu_has_feature(CPU_FTR_HVMODE)) {
0237         if (cxl_p2n_read(ctx->afu, CXL_PSL_DSISR_An) != dsisr ||
0238             cxl_p2n_read(ctx->afu, CXL_PSL_DAR_An) != dar ||
0239             cxl_p2n_read(ctx->afu, CXL_PSL_PEHandle_An) != ctx->pe) {
0240             /* Most likely explanation is harmless - a dedicated
0241              * process has detached and these were cleared by the
0242              * PSL purge, but warn about it just in case
0243              */
0244             dev_notice(&ctx->afu->dev, "cxl_handle_fault: Translation fault regs changed\n");
0245             return;
0246         }
0247     }
0248
0249     /* Early return if the context is being / has been detached */
0250     if (ctx->status == CLOSED) {
0251         cxl_ack_ae(ctx);
0252         return;
0253     }
0254
0255     pr_devel("CXL BOTTOM HALF handling fault for afu pe: %i. "
0256         "DSISR: %#llx DAR: %#llx\n", ctx->pe, dsisr, dar);
0257
0258     if (!ctx->kernel) {
0259
0260         mm = get_mem_context(ctx);
0261         if (mm == NULL) {
0262             pr_devel("%s: unable to get mm for pe=%d pid=%i\n",
0263                  __func__, ctx->pe, pid_nr(ctx->pid));
0264             cxl_ack_ae(ctx);
0265             return;
0266         } else {
0267             pr_devel("Handling page fault for pe=%d pid=%i\n",
0268                  ctx->pe, pid_nr(ctx->pid));
0269         }
0270     }
0271
0272     if (cxl_is_segment_miss(ctx, dsisr))
0273         cxl_handle_segment_miss(ctx, mm, dar);
0274     else if (cxl_is_page_fault(ctx, dsisr))
0275         cxl_handle_page_fault(ctx, mm, dsisr, dar);
0276     else
0277         WARN(1, "cxl_handle_fault has nothing to handle\n");
0278
0279     if (mm)
0280         mmput(mm);
0281 }
0282
0283 static void cxl_prefault_one(struct cxl_context *ctx, u64 ea)
0284 {
0285     struct mm_struct *mm;
0286
0287     mm = get_mem_context(ctx);
0288     if (mm == NULL) {
0289         pr_devel("cxl_prefault_one unable to get mm %i\n",
0290              pid_nr(ctx->pid));
0291         return;
0292     }
0293
0294     cxl_fault_segment(ctx, mm, ea);
0295
0296     mmput(mm);
0297 }
0298
0299 static u64 next_segment(u64 ea, u64 vsid)
0300 {
0301     if (vsid & SLB_VSID_B_1T)
0302         ea |= (1ULL << 40) - 1;
0303     else
0304         ea |= (1ULL << 28) - 1;
0305
0306     return ea + 1;
0307 }
0308
0309 static void cxl_prefault_vma(struct cxl_context *ctx)
0310 {
0311     u64 ea, last_esid = 0;
0312     struct copro_slb slb;
0313     struct vm_area_struct *vma;
0314     int rc;
0315     struct mm_struct *mm;
0316
0317     mm = get_mem_context(ctx);
0318     if (mm == NULL) {
0319         pr_devel("cxl_prefault_vm unable to get mm %i\n",
0320              pid_nr(ctx->pid));
0321         return;
0322     }
0323
0324     mmap_read_lock(mm);
0325     for (vma = mm->mmap; vma; vma = vma->vm_next) {
0326         for (ea = vma->vm_start; ea < vma->vm_end;
0327                 ea = next_segment(ea, slb.vsid)) {
0328             rc = copro_calculate_slb(mm, ea, &slb);
0329             if (rc)
0330                 continue;
0331
0332             if (last_esid == slb.esid)
0333                 continue;
0334
0335             cxl_load_segment(ctx, &slb);
0336             last_esid = slb.esid;
0337         }
0338     }
0339     mmap_read_unlock(mm);
0340
0341     mmput(mm);
0342 }
0343
0344 void cxl_prefault(struct cxl_context *ctx, u64 wed)
0345 {
0346     switch (ctx->afu->prefault_mode) {
0347     case CXL_PREFAULT_WED:
0348         cxl_prefault_one(ctx, wed);
0349         break;
0350     case CXL_PREFAULT_ALL:
0351         cxl_prefault_vma(ctx);
0352         break;
0353     default:
0354         break;
0355     }
0356 }