Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0-or-later
0002 /*
0003  * Copyright 2014 IBM Corp.
0004  */
0005 
0006 #include <linux/spinlock.h>
0007 #include <linux/sched.h>
0008 #include <linux/sched/clock.h>
0009 #include <linux/slab.h>
0010 #include <linux/mutex.h>
0011 #include <linux/mm.h>
0012 #include <linux/uaccess.h>
0013 #include <linux/delay.h>
0014 #include <linux/irqdomain.h>
0015 #include <asm/synch.h>
0016 #include <asm/switch_to.h>
0017 #include <misc/cxl-base.h>
0018 
0019 #include "cxl.h"
0020 #include "trace.h"
0021 
0022 static int afu_control(struct cxl_afu *afu, u64 command, u64 clear,
0023                u64 result, u64 mask, bool enabled)
0024 {
0025     u64 AFU_Cntl;
0026     unsigned long timeout = jiffies + (HZ * CXL_TIMEOUT);
0027     int rc = 0;
0028 
0029     spin_lock(&afu->afu_cntl_lock);
0030     pr_devel("AFU command starting: %llx\n", command);
0031 
0032     trace_cxl_afu_ctrl(afu, command);
0033 
0034     AFU_Cntl = cxl_p2n_read(afu, CXL_AFU_Cntl_An);
0035     cxl_p2n_write(afu, CXL_AFU_Cntl_An, (AFU_Cntl & ~clear) | command);
0036 
0037     AFU_Cntl = cxl_p2n_read(afu, CXL_AFU_Cntl_An);
0038     while ((AFU_Cntl & mask) != result) {
0039         if (time_after_eq(jiffies, timeout)) {
0040             dev_warn(&afu->dev, "WARNING: AFU control timed out!\n");
0041             rc = -EBUSY;
0042             goto out;
0043         }
0044 
0045         if (!cxl_ops->link_ok(afu->adapter, afu)) {
0046             afu->enabled = enabled;
0047             rc = -EIO;
0048             goto out;
0049         }
0050 
0051         pr_devel_ratelimited("AFU control... (0x%016llx)\n",
0052                      AFU_Cntl | command);
0053         cpu_relax();
0054         AFU_Cntl = cxl_p2n_read(afu, CXL_AFU_Cntl_An);
0055     }
0056 
0057     if (AFU_Cntl & CXL_AFU_Cntl_An_RA) {
0058         /*
0059          * Workaround for a bug in the XSL used in the Mellanox CX4
0060          * that fails to clear the RA bit after an AFU reset,
0061          * preventing subsequent AFU resets from working.
0062          */
0063         cxl_p2n_write(afu, CXL_AFU_Cntl_An, AFU_Cntl & ~CXL_AFU_Cntl_An_RA);
0064     }
0065 
0066     pr_devel("AFU command complete: %llx\n", command);
0067     afu->enabled = enabled;
0068 out:
0069     trace_cxl_afu_ctrl_done(afu, command, rc);
0070     spin_unlock(&afu->afu_cntl_lock);
0071 
0072     return rc;
0073 }
0074 
0075 static int afu_enable(struct cxl_afu *afu)
0076 {
0077     pr_devel("AFU enable request\n");
0078 
0079     return afu_control(afu, CXL_AFU_Cntl_An_E, 0,
0080                CXL_AFU_Cntl_An_ES_Enabled,
0081                CXL_AFU_Cntl_An_ES_MASK, true);
0082 }
0083 
0084 int cxl_afu_disable(struct cxl_afu *afu)
0085 {
0086     pr_devel("AFU disable request\n");
0087 
0088     return afu_control(afu, 0, CXL_AFU_Cntl_An_E,
0089                CXL_AFU_Cntl_An_ES_Disabled,
0090                CXL_AFU_Cntl_An_ES_MASK, false);
0091 }
0092 
0093 /* This will disable as well as reset */
0094 static int native_afu_reset(struct cxl_afu *afu)
0095 {
0096     int rc;
0097     u64 serr;
0098 
0099     pr_devel("AFU reset request\n");
0100 
0101     rc = afu_control(afu, CXL_AFU_Cntl_An_RA, 0,
0102                CXL_AFU_Cntl_An_RS_Complete | CXL_AFU_Cntl_An_ES_Disabled,
0103                CXL_AFU_Cntl_An_RS_MASK | CXL_AFU_Cntl_An_ES_MASK,
0104                false);
0105 
0106     /*
0107      * Re-enable any masked interrupts when the AFU is not
0108      * activated to avoid side effects after attaching a process
0109      * in dedicated mode.
0110      */
0111     if (afu->current_mode == 0) {
0112         serr = cxl_p1n_read(afu, CXL_PSL_SERR_An);
0113         serr &= ~CXL_PSL_SERR_An_IRQ_MASKS;
0114         cxl_p1n_write(afu, CXL_PSL_SERR_An, serr);
0115     }
0116 
0117     return rc;
0118 }
0119 
0120 static int native_afu_check_and_enable(struct cxl_afu *afu)
0121 {
0122     if (!cxl_ops->link_ok(afu->adapter, afu)) {
0123         WARN(1, "Refusing to enable afu while link down!\n");
0124         return -EIO;
0125     }
0126     if (afu->enabled)
0127         return 0;
0128     return afu_enable(afu);
0129 }
0130 
0131 int cxl_psl_purge(struct cxl_afu *afu)
0132 {
0133     u64 PSL_CNTL = cxl_p1n_read(afu, CXL_PSL_SCNTL_An);
0134     u64 AFU_Cntl = cxl_p2n_read(afu, CXL_AFU_Cntl_An);
0135     u64 dsisr, dar;
0136     u64 start, end;
0137     u64 trans_fault = 0x0ULL;
0138     unsigned long timeout = jiffies + (HZ * CXL_TIMEOUT);
0139     int rc = 0;
0140 
0141     trace_cxl_psl_ctrl(afu, CXL_PSL_SCNTL_An_Pc);
0142 
0143     pr_devel("PSL purge request\n");
0144 
0145     if (cxl_is_power8())
0146         trans_fault = CXL_PSL_DSISR_TRANS;
0147     if (cxl_is_power9())
0148         trans_fault = CXL_PSL9_DSISR_An_TF;
0149 
0150     if (!cxl_ops->link_ok(afu->adapter, afu)) {
0151         dev_warn(&afu->dev, "PSL Purge called with link down, ignoring\n");
0152         rc = -EIO;
0153         goto out;
0154     }
0155 
0156     if ((AFU_Cntl & CXL_AFU_Cntl_An_ES_MASK) != CXL_AFU_Cntl_An_ES_Disabled) {
0157         WARN(1, "psl_purge request while AFU not disabled!\n");
0158         cxl_afu_disable(afu);
0159     }
0160 
0161     cxl_p1n_write(afu, CXL_PSL_SCNTL_An,
0162                PSL_CNTL | CXL_PSL_SCNTL_An_Pc);
0163     start = local_clock();
0164     PSL_CNTL = cxl_p1n_read(afu, CXL_PSL_SCNTL_An);
0165     while ((PSL_CNTL &  CXL_PSL_SCNTL_An_Ps_MASK)
0166             == CXL_PSL_SCNTL_An_Ps_Pending) {
0167         if (time_after_eq(jiffies, timeout)) {
0168             dev_warn(&afu->dev, "WARNING: PSL Purge timed out!\n");
0169             rc = -EBUSY;
0170             goto out;
0171         }
0172         if (!cxl_ops->link_ok(afu->adapter, afu)) {
0173             rc = -EIO;
0174             goto out;
0175         }
0176 
0177         dsisr = cxl_p2n_read(afu, CXL_PSL_DSISR_An);
0178         pr_devel_ratelimited("PSL purging... PSL_CNTL: 0x%016llx  PSL_DSISR: 0x%016llx\n",
0179                      PSL_CNTL, dsisr);
0180 
0181         if (dsisr & trans_fault) {
0182             dar = cxl_p2n_read(afu, CXL_PSL_DAR_An);
0183             dev_notice(&afu->dev, "PSL purge terminating pending translation, DSISR: 0x%016llx, DAR: 0x%016llx\n",
0184                    dsisr, dar);
0185             cxl_p2n_write(afu, CXL_PSL_TFC_An, CXL_PSL_TFC_An_AE);
0186         } else if (dsisr) {
0187             dev_notice(&afu->dev, "PSL purge acknowledging pending non-translation fault, DSISR: 0x%016llx\n",
0188                    dsisr);
0189             cxl_p2n_write(afu, CXL_PSL_TFC_An, CXL_PSL_TFC_An_A);
0190         } else {
0191             cpu_relax();
0192         }
0193         PSL_CNTL = cxl_p1n_read(afu, CXL_PSL_SCNTL_An);
0194     }
0195     end = local_clock();
0196     pr_devel("PSL purged in %lld ns\n", end - start);
0197 
0198     cxl_p1n_write(afu, CXL_PSL_SCNTL_An,
0199                PSL_CNTL & ~CXL_PSL_SCNTL_An_Pc);
0200 out:
0201     trace_cxl_psl_ctrl_done(afu, CXL_PSL_SCNTL_An_Pc, rc);
0202     return rc;
0203 }
0204 
0205 static int spa_max_procs(int spa_size)
0206 {
0207     /*
0208      * From the CAIA:
0209      *    end_of_SPA_area = SPA_Base + ((n+4) * 128) + (( ((n*8) + 127) >> 7) * 128) + 255
0210      * Most of that junk is really just an overly-complicated way of saying
0211      * the last 256 bytes are __aligned(128), so it's really:
0212      *    end_of_SPA_area = end_of_PSL_queue_area + __aligned(128) 255
0213      * and
0214      *    end_of_PSL_queue_area = SPA_Base + ((n+4) * 128) + (n*8) - 1
0215      * so
0216      *    sizeof(SPA) = ((n+4) * 128) + (n*8) + __aligned(128) 256
0217      * Ignore the alignment (which is safe in this case as long as we are
0218      * careful with our rounding) and solve for n:
0219      */
0220     return ((spa_size / 8) - 96) / 17;
0221 }
0222 
0223 static int cxl_alloc_spa(struct cxl_afu *afu, int mode)
0224 {
0225     unsigned spa_size;
0226 
0227     /* Work out how many pages to allocate */
0228     afu->native->spa_order = -1;
0229     do {
0230         afu->native->spa_order++;
0231         spa_size = (1 << afu->native->spa_order) * PAGE_SIZE;
0232 
0233         if (spa_size > 0x100000) {
0234             dev_warn(&afu->dev, "num_of_processes too large for the SPA, limiting to %i (0x%x)\n",
0235                     afu->native->spa_max_procs, afu->native->spa_size);
0236             if (mode != CXL_MODE_DEDICATED)
0237                 afu->num_procs = afu->native->spa_max_procs;
0238             break;
0239         }
0240 
0241         afu->native->spa_size = spa_size;
0242         afu->native->spa_max_procs = spa_max_procs(afu->native->spa_size);
0243     } while (afu->native->spa_max_procs < afu->num_procs);
0244 
0245     if (!(afu->native->spa = (struct cxl_process_element *)
0246           __get_free_pages(GFP_KERNEL | __GFP_ZERO, afu->native->spa_order))) {
0247         pr_err("cxl_alloc_spa: Unable to allocate scheduled process area\n");
0248         return -ENOMEM;
0249     }
0250     pr_devel("spa pages: %i afu->spa_max_procs: %i   afu->num_procs: %i\n",
0251          1<<afu->native->spa_order, afu->native->spa_max_procs, afu->num_procs);
0252 
0253     return 0;
0254 }
0255 
0256 static void attach_spa(struct cxl_afu *afu)
0257 {
0258     u64 spap;
0259 
0260     afu->native->sw_command_status = (__be64 *)((char *)afu->native->spa +
0261                         ((afu->native->spa_max_procs + 3) * 128));
0262 
0263     spap = virt_to_phys(afu->native->spa) & CXL_PSL_SPAP_Addr;
0264     spap |= ((afu->native->spa_size >> (12 - CXL_PSL_SPAP_Size_Shift)) - 1) & CXL_PSL_SPAP_Size;
0265     spap |= CXL_PSL_SPAP_V;
0266     pr_devel("cxl: SPA allocated at 0x%p. Max processes: %i, sw_command_status: 0x%p CXL_PSL_SPAP_An=0x%016llx\n",
0267         afu->native->spa, afu->native->spa_max_procs,
0268         afu->native->sw_command_status, spap);
0269     cxl_p1n_write(afu, CXL_PSL_SPAP_An, spap);
0270 }
0271 
0272 static inline void detach_spa(struct cxl_afu *afu)
0273 {
0274     cxl_p1n_write(afu, CXL_PSL_SPAP_An, 0);
0275 }
0276 
0277 void cxl_release_spa(struct cxl_afu *afu)
0278 {
0279     if (afu->native->spa) {
0280         free_pages((unsigned long) afu->native->spa,
0281             afu->native->spa_order);
0282         afu->native->spa = NULL;
0283     }
0284 }
0285 
0286 /*
0287  * Invalidation of all ERAT entries is no longer required by CAIA2. Use
0288  * only for debug.
0289  */
0290 int cxl_invalidate_all_psl9(struct cxl *adapter)
0291 {
0292     unsigned long timeout = jiffies + (HZ * CXL_TIMEOUT);
0293     u64 ierat;
0294 
0295     pr_devel("CXL adapter - invalidation of all ERAT entries\n");
0296 
0297     /* Invalidates all ERAT entries for Radix or HPT */
0298     ierat = CXL_XSL9_IERAT_IALL;
0299     if (radix_enabled())
0300         ierat |= CXL_XSL9_IERAT_INVR;
0301     cxl_p1_write(adapter, CXL_XSL9_IERAT, ierat);
0302 
0303     while (cxl_p1_read(adapter, CXL_XSL9_IERAT) & CXL_XSL9_IERAT_IINPROG) {
0304         if (time_after_eq(jiffies, timeout)) {
0305             dev_warn(&adapter->dev,
0306             "WARNING: CXL adapter invalidation of all ERAT entries timed out!\n");
0307             return -EBUSY;
0308         }
0309         if (!cxl_ops->link_ok(adapter, NULL))
0310             return -EIO;
0311         cpu_relax();
0312     }
0313     return 0;
0314 }
0315 
0316 int cxl_invalidate_all_psl8(struct cxl *adapter)
0317 {
0318     unsigned long timeout = jiffies + (HZ * CXL_TIMEOUT);
0319 
0320     pr_devel("CXL adapter wide TLBIA & SLBIA\n");
0321 
0322     cxl_p1_write(adapter, CXL_PSL_AFUSEL, CXL_PSL_AFUSEL_A);
0323 
0324     cxl_p1_write(adapter, CXL_PSL_TLBIA, CXL_TLB_SLB_IQ_ALL);
0325     while (cxl_p1_read(adapter, CXL_PSL_TLBIA) & CXL_TLB_SLB_P) {
0326         if (time_after_eq(jiffies, timeout)) {
0327             dev_warn(&adapter->dev, "WARNING: CXL adapter wide TLBIA timed out!\n");
0328             return -EBUSY;
0329         }
0330         if (!cxl_ops->link_ok(adapter, NULL))
0331             return -EIO;
0332         cpu_relax();
0333     }
0334 
0335     cxl_p1_write(adapter, CXL_PSL_SLBIA, CXL_TLB_SLB_IQ_ALL);
0336     while (cxl_p1_read(adapter, CXL_PSL_SLBIA) & CXL_TLB_SLB_P) {
0337         if (time_after_eq(jiffies, timeout)) {
0338             dev_warn(&adapter->dev, "WARNING: CXL adapter wide SLBIA timed out!\n");
0339             return -EBUSY;
0340         }
0341         if (!cxl_ops->link_ok(adapter, NULL))
0342             return -EIO;
0343         cpu_relax();
0344     }
0345     return 0;
0346 }
0347 
0348 int cxl_data_cache_flush(struct cxl *adapter)
0349 {
0350     u64 reg;
0351     unsigned long timeout = jiffies + (HZ * CXL_TIMEOUT);
0352 
0353     /*
0354      * Do a datacache flush only if datacache is available.
0355      * In case of PSL9D datacache absent hence flush operation.
0356      * would timeout.
0357      */
0358     if (adapter->native->no_data_cache) {
0359         pr_devel("No PSL data cache. Ignoring cache flush req.\n");
0360         return 0;
0361     }
0362 
0363     pr_devel("Flushing data cache\n");
0364     reg = cxl_p1_read(adapter, CXL_PSL_Control);
0365     reg |= CXL_PSL_Control_Fr;
0366     cxl_p1_write(adapter, CXL_PSL_Control, reg);
0367 
0368     reg = cxl_p1_read(adapter, CXL_PSL_Control);
0369     while ((reg & CXL_PSL_Control_Fs_MASK) != CXL_PSL_Control_Fs_Complete) {
0370         if (time_after_eq(jiffies, timeout)) {
0371             dev_warn(&adapter->dev, "WARNING: cache flush timed out!\n");
0372             return -EBUSY;
0373         }
0374 
0375         if (!cxl_ops->link_ok(adapter, NULL)) {
0376             dev_warn(&adapter->dev, "WARNING: link down when flushing cache\n");
0377             return -EIO;
0378         }
0379         cpu_relax();
0380         reg = cxl_p1_read(adapter, CXL_PSL_Control);
0381     }
0382 
0383     reg &= ~CXL_PSL_Control_Fr;
0384     cxl_p1_write(adapter, CXL_PSL_Control, reg);
0385     return 0;
0386 }
0387 
0388 static int cxl_write_sstp(struct cxl_afu *afu, u64 sstp0, u64 sstp1)
0389 {
0390     int rc;
0391 
0392     /* 1. Disable SSTP by writing 0 to SSTP1[V] */
0393     cxl_p2n_write(afu, CXL_SSTP1_An, 0);
0394 
0395     /* 2. Invalidate all SLB entries */
0396     if ((rc = cxl_afu_slbia(afu)))
0397         return rc;
0398 
0399     /* 3. Set SSTP0_An */
0400     cxl_p2n_write(afu, CXL_SSTP0_An, sstp0);
0401 
0402     /* 4. Set SSTP1_An */
0403     cxl_p2n_write(afu, CXL_SSTP1_An, sstp1);
0404 
0405     return 0;
0406 }
0407 
0408 /* Using per slice version may improve performance here. (ie. SLBIA_An) */
0409 static void slb_invalid(struct cxl_context *ctx)
0410 {
0411     struct cxl *adapter = ctx->afu->adapter;
0412     u64 slbia;
0413 
0414     WARN_ON(!mutex_is_locked(&ctx->afu->native->spa_mutex));
0415 
0416     cxl_p1_write(adapter, CXL_PSL_LBISEL,
0417             ((u64)be32_to_cpu(ctx->elem->common.pid) << 32) |
0418             be32_to_cpu(ctx->elem->lpid));
0419     cxl_p1_write(adapter, CXL_PSL_SLBIA, CXL_TLB_SLB_IQ_LPIDPID);
0420 
0421     while (1) {
0422         if (!cxl_ops->link_ok(adapter, NULL))
0423             break;
0424         slbia = cxl_p1_read(adapter, CXL_PSL_SLBIA);
0425         if (!(slbia & CXL_TLB_SLB_P))
0426             break;
0427         cpu_relax();
0428     }
0429 }
0430 
0431 static int do_process_element_cmd(struct cxl_context *ctx,
0432                   u64 cmd, u64 pe_state)
0433 {
0434     u64 state;
0435     unsigned long timeout = jiffies + (HZ * CXL_TIMEOUT);
0436     int rc = 0;
0437 
0438     trace_cxl_llcmd(ctx, cmd);
0439 
0440     WARN_ON(!ctx->afu->enabled);
0441 
0442     ctx->elem->software_state = cpu_to_be32(pe_state);
0443     smp_wmb();
0444     *(ctx->afu->native->sw_command_status) = cpu_to_be64(cmd | 0 | ctx->pe);
0445     smp_mb();
0446     cxl_p1n_write(ctx->afu, CXL_PSL_LLCMD_An, cmd | ctx->pe);
0447     while (1) {
0448         if (time_after_eq(jiffies, timeout)) {
0449             dev_warn(&ctx->afu->dev, "WARNING: Process Element Command timed out!\n");
0450             rc = -EBUSY;
0451             goto out;
0452         }
0453         if (!cxl_ops->link_ok(ctx->afu->adapter, ctx->afu)) {
0454             dev_warn(&ctx->afu->dev, "WARNING: Device link down, aborting Process Element Command!\n");
0455             rc = -EIO;
0456             goto out;
0457         }
0458         state = be64_to_cpup(ctx->afu->native->sw_command_status);
0459         if (state == ~0ULL) {
0460             pr_err("cxl: Error adding process element to AFU\n");
0461             rc = -1;
0462             goto out;
0463         }
0464         if ((state & (CXL_SPA_SW_CMD_MASK | CXL_SPA_SW_STATE_MASK  | CXL_SPA_SW_LINK_MASK)) ==
0465             (cmd | (cmd >> 16) | ctx->pe))
0466             break;
0467         /*
0468          * The command won't finish in the PSL if there are
0469          * outstanding DSIs.  Hence we need to yield here in
0470          * case there are outstanding DSIs that we need to
0471          * service.  Tuning possiblity: we could wait for a
0472          * while before sched
0473          */
0474         schedule();
0475 
0476     }
0477 out:
0478     trace_cxl_llcmd_done(ctx, cmd, rc);
0479     return rc;
0480 }
0481 
0482 static int add_process_element(struct cxl_context *ctx)
0483 {
0484     int rc = 0;
0485 
0486     mutex_lock(&ctx->afu->native->spa_mutex);
0487     pr_devel("%s Adding pe: %i started\n", __func__, ctx->pe);
0488     if (!(rc = do_process_element_cmd(ctx, CXL_SPA_SW_CMD_ADD, CXL_PE_SOFTWARE_STATE_V)))
0489         ctx->pe_inserted = true;
0490     pr_devel("%s Adding pe: %i finished\n", __func__, ctx->pe);
0491     mutex_unlock(&ctx->afu->native->spa_mutex);
0492     return rc;
0493 }
0494 
0495 static int terminate_process_element(struct cxl_context *ctx)
0496 {
0497     int rc = 0;
0498 
0499     /* fast path terminate if it's already invalid */
0500     if (!(ctx->elem->software_state & cpu_to_be32(CXL_PE_SOFTWARE_STATE_V)))
0501         return rc;
0502 
0503     mutex_lock(&ctx->afu->native->spa_mutex);
0504     pr_devel("%s Terminate pe: %i started\n", __func__, ctx->pe);
0505     /* We could be asked to terminate when the hw is down. That
0506      * should always succeed: it's not running if the hw has gone
0507      * away and is being reset.
0508      */
0509     if (cxl_ops->link_ok(ctx->afu->adapter, ctx->afu))
0510         rc = do_process_element_cmd(ctx, CXL_SPA_SW_CMD_TERMINATE,
0511                         CXL_PE_SOFTWARE_STATE_V | CXL_PE_SOFTWARE_STATE_T);
0512     ctx->elem->software_state = 0;  /* Remove Valid bit */
0513     pr_devel("%s Terminate pe: %i finished\n", __func__, ctx->pe);
0514     mutex_unlock(&ctx->afu->native->spa_mutex);
0515     return rc;
0516 }
0517 
0518 static int remove_process_element(struct cxl_context *ctx)
0519 {
0520     int rc = 0;
0521 
0522     mutex_lock(&ctx->afu->native->spa_mutex);
0523     pr_devel("%s Remove pe: %i started\n", __func__, ctx->pe);
0524 
0525     /* We could be asked to remove when the hw is down. Again, if
0526      * the hw is down, the PE is gone, so we succeed.
0527      */
0528     if (cxl_ops->link_ok(ctx->afu->adapter, ctx->afu))
0529         rc = do_process_element_cmd(ctx, CXL_SPA_SW_CMD_REMOVE, 0);
0530 
0531     if (!rc)
0532         ctx->pe_inserted = false;
0533     if (cxl_is_power8())
0534         slb_invalid(ctx);
0535     pr_devel("%s Remove pe: %i finished\n", __func__, ctx->pe);
0536     mutex_unlock(&ctx->afu->native->spa_mutex);
0537 
0538     return rc;
0539 }
0540 
0541 void cxl_assign_psn_space(struct cxl_context *ctx)
0542 {
0543     if (!ctx->afu->pp_size || ctx->master) {
0544         ctx->psn_phys = ctx->afu->psn_phys;
0545         ctx->psn_size = ctx->afu->adapter->ps_size;
0546     } else {
0547         ctx->psn_phys = ctx->afu->psn_phys +
0548             (ctx->afu->native->pp_offset + ctx->afu->pp_size * ctx->pe);
0549         ctx->psn_size = ctx->afu->pp_size;
0550     }
0551 }
0552 
0553 static int activate_afu_directed(struct cxl_afu *afu)
0554 {
0555     int rc;
0556 
0557     dev_info(&afu->dev, "Activating AFU directed mode\n");
0558 
0559     afu->num_procs = afu->max_procs_virtualised;
0560     if (afu->native->spa == NULL) {
0561         if (cxl_alloc_spa(afu, CXL_MODE_DIRECTED))
0562             return -ENOMEM;
0563     }
0564     attach_spa(afu);
0565 
0566     cxl_p1n_write(afu, CXL_PSL_SCNTL_An, CXL_PSL_SCNTL_An_PM_AFU);
0567     if (cxl_is_power8())
0568         cxl_p1n_write(afu, CXL_PSL_AMOR_An, 0xFFFFFFFFFFFFFFFFULL);
0569     cxl_p1n_write(afu, CXL_PSL_ID_An, CXL_PSL_ID_An_F | CXL_PSL_ID_An_L);
0570 
0571     afu->current_mode = CXL_MODE_DIRECTED;
0572 
0573     if ((rc = cxl_chardev_m_afu_add(afu)))
0574         return rc;
0575 
0576     if ((rc = cxl_sysfs_afu_m_add(afu)))
0577         goto err;
0578 
0579     if ((rc = cxl_chardev_s_afu_add(afu)))
0580         goto err1;
0581 
0582     return 0;
0583 err1:
0584     cxl_sysfs_afu_m_remove(afu);
0585 err:
0586     cxl_chardev_afu_remove(afu);
0587     return rc;
0588 }
0589 
0590 #ifdef CONFIG_CPU_LITTLE_ENDIAN
0591 #define set_endian(sr) ((sr) |= CXL_PSL_SR_An_LE)
0592 #else
0593 #define set_endian(sr) ((sr) &= ~(CXL_PSL_SR_An_LE))
0594 #endif
0595 
0596 u64 cxl_calculate_sr(bool master, bool kernel, bool real_mode, bool p9)
0597 {
0598     u64 sr = 0;
0599 
0600     set_endian(sr);
0601     if (master)
0602         sr |= CXL_PSL_SR_An_MP;
0603     if (mfspr(SPRN_LPCR) & LPCR_TC)
0604         sr |= CXL_PSL_SR_An_TC;
0605 
0606     if (kernel) {
0607         if (!real_mode)
0608             sr |= CXL_PSL_SR_An_R;
0609         sr |= (mfmsr() & MSR_SF) | CXL_PSL_SR_An_HV;
0610     } else {
0611         sr |= CXL_PSL_SR_An_PR | CXL_PSL_SR_An_R;
0612         if (radix_enabled())
0613             sr |= CXL_PSL_SR_An_HV;
0614         else
0615             sr &= ~(CXL_PSL_SR_An_HV);
0616         if (!test_tsk_thread_flag(current, TIF_32BIT))
0617             sr |= CXL_PSL_SR_An_SF;
0618     }
0619     if (p9) {
0620         if (radix_enabled())
0621             sr |= CXL_PSL_SR_An_XLAT_ror;
0622         else
0623             sr |= CXL_PSL_SR_An_XLAT_hpt;
0624     }
0625     return sr;
0626 }
0627 
0628 static u64 calculate_sr(struct cxl_context *ctx)
0629 {
0630     return cxl_calculate_sr(ctx->master, ctx->kernel, false,
0631                 cxl_is_power9());
0632 }
0633 
0634 static void update_ivtes_directed(struct cxl_context *ctx)
0635 {
0636     bool need_update = (ctx->status == STARTED);
0637     int r;
0638 
0639     if (need_update) {
0640         WARN_ON(terminate_process_element(ctx));
0641         WARN_ON(remove_process_element(ctx));
0642     }
0643 
0644     for (r = 0; r < CXL_IRQ_RANGES; r++) {
0645         ctx->elem->ivte_offsets[r] = cpu_to_be16(ctx->irqs.offset[r]);
0646         ctx->elem->ivte_ranges[r] = cpu_to_be16(ctx->irqs.range[r]);
0647     }
0648 
0649     /*
0650      * Theoretically we could use the update llcmd, instead of a
0651      * terminate/remove/add (or if an atomic update was required we could
0652      * do a suspend/update/resume), however it seems there might be issues
0653      * with the update llcmd on some cards (including those using an XSL on
0654      * an ASIC) so for now it's safest to go with the commands that are
0655      * known to work. In the future if we come across a situation where the
0656      * card may be performing transactions using the same PE while we are
0657      * doing this update we might need to revisit this.
0658      */
0659     if (need_update)
0660         WARN_ON(add_process_element(ctx));
0661 }
0662 
0663 static int process_element_entry_psl9(struct cxl_context *ctx, u64 wed, u64 amr)
0664 {
0665     u32 pid;
0666     int rc;
0667 
0668     cxl_assign_psn_space(ctx);
0669 
0670     ctx->elem->ctxtime = 0; /* disable */
0671     ctx->elem->lpid = cpu_to_be32(mfspr(SPRN_LPID));
0672     ctx->elem->haurp = 0; /* disable */
0673 
0674     if (ctx->kernel)
0675         pid = 0;
0676     else {
0677         if (ctx->mm == NULL) {
0678             pr_devel("%s: unable to get mm for pe=%d pid=%i\n",
0679                 __func__, ctx->pe, pid_nr(ctx->pid));
0680             return -EINVAL;
0681         }
0682         pid = ctx->mm->context.id;
0683     }
0684 
0685     /* Assign a unique TIDR (thread id) for the current thread */
0686     if (!(ctx->tidr) && (ctx->assign_tidr)) {
0687         rc = set_thread_tidr(current);
0688         if (rc)
0689             return -ENODEV;
0690         ctx->tidr = current->thread.tidr;
0691         pr_devel("%s: current tidr: %d\n", __func__, ctx->tidr);
0692     }
0693 
0694     ctx->elem->common.tid = cpu_to_be32(ctx->tidr);
0695     ctx->elem->common.pid = cpu_to_be32(pid);
0696 
0697     ctx->elem->sr = cpu_to_be64(calculate_sr(ctx));
0698 
0699     ctx->elem->common.csrp = 0; /* disable */
0700 
0701     cxl_prefault(ctx, wed);
0702 
0703     /*
0704      * Ensure we have the multiplexed PSL interrupt set up to take faults
0705      * for kernel contexts that may not have allocated any AFU IRQs at all:
0706      */
0707     if (ctx->irqs.range[0] == 0) {
0708         ctx->irqs.offset[0] = ctx->afu->native->psl_hwirq;
0709         ctx->irqs.range[0] = 1;
0710     }
0711 
0712     ctx->elem->common.amr = cpu_to_be64(amr);
0713     ctx->elem->common.wed = cpu_to_be64(wed);
0714 
0715     return 0;
0716 }
0717 
0718 int cxl_attach_afu_directed_psl9(struct cxl_context *ctx, u64 wed, u64 amr)
0719 {
0720     int result;
0721 
0722     /* fill the process element entry */
0723     result = process_element_entry_psl9(ctx, wed, amr);
0724     if (result)
0725         return result;
0726 
0727     update_ivtes_directed(ctx);
0728 
0729     /* first guy needs to enable */
0730     result = cxl_ops->afu_check_and_enable(ctx->afu);
0731     if (result)
0732         return result;
0733 
0734     return add_process_element(ctx);
0735 }
0736 
0737 int cxl_attach_afu_directed_psl8(struct cxl_context *ctx, u64 wed, u64 amr)
0738 {
0739     u32 pid;
0740     int result;
0741 
0742     cxl_assign_psn_space(ctx);
0743 
0744     ctx->elem->ctxtime = 0; /* disable */
0745     ctx->elem->lpid = cpu_to_be32(mfspr(SPRN_LPID));
0746     ctx->elem->haurp = 0; /* disable */
0747     ctx->elem->u.sdr = cpu_to_be64(mfspr(SPRN_SDR1));
0748 
0749     pid = current->pid;
0750     if (ctx->kernel)
0751         pid = 0;
0752     ctx->elem->common.tid = 0;
0753     ctx->elem->common.pid = cpu_to_be32(pid);
0754 
0755     ctx->elem->sr = cpu_to_be64(calculate_sr(ctx));
0756 
0757     ctx->elem->common.csrp = 0; /* disable */
0758     ctx->elem->common.u.psl8.aurp0 = 0; /* disable */
0759     ctx->elem->common.u.psl8.aurp1 = 0; /* disable */
0760 
0761     cxl_prefault(ctx, wed);
0762 
0763     ctx->elem->common.u.psl8.sstp0 = cpu_to_be64(ctx->sstp0);
0764     ctx->elem->common.u.psl8.sstp1 = cpu_to_be64(ctx->sstp1);
0765 
0766     /*
0767      * Ensure we have the multiplexed PSL interrupt set up to take faults
0768      * for kernel contexts that may not have allocated any AFU IRQs at all:
0769      */
0770     if (ctx->irqs.range[0] == 0) {
0771         ctx->irqs.offset[0] = ctx->afu->native->psl_hwirq;
0772         ctx->irqs.range[0] = 1;
0773     }
0774 
0775     update_ivtes_directed(ctx);
0776 
0777     ctx->elem->common.amr = cpu_to_be64(amr);
0778     ctx->elem->common.wed = cpu_to_be64(wed);
0779 
0780     /* first guy needs to enable */
0781     if ((result = cxl_ops->afu_check_and_enable(ctx->afu)))
0782         return result;
0783 
0784     return add_process_element(ctx);
0785 }
0786 
0787 static int deactivate_afu_directed(struct cxl_afu *afu)
0788 {
0789     dev_info(&afu->dev, "Deactivating AFU directed mode\n");
0790 
0791     afu->current_mode = 0;
0792     afu->num_procs = 0;
0793 
0794     cxl_sysfs_afu_m_remove(afu);
0795     cxl_chardev_afu_remove(afu);
0796 
0797     /*
0798      * The CAIA section 2.2.1 indicates that the procedure for starting and
0799      * stopping an AFU in AFU directed mode is AFU specific, which is not
0800      * ideal since this code is generic and with one exception has no
0801      * knowledge of the AFU. This is in contrast to the procedure for
0802      * disabling a dedicated process AFU, which is documented to just
0803      * require a reset. The architecture does indicate that both an AFU
0804      * reset and an AFU disable should result in the AFU being disabled and
0805      * we do both followed by a PSL purge for safety.
0806      *
0807      * Notably we used to have some issues with the disable sequence on PSL
0808      * cards, which is why we ended up using this heavy weight procedure in
0809      * the first place, however a bug was discovered that had rendered the
0810      * disable operation ineffective, so it is conceivable that was the
0811      * sole explanation for those difficulties. Careful regression testing
0812      * is recommended if anyone attempts to remove or reorder these
0813      * operations.
0814      *
0815      * The XSL on the Mellanox CX4 behaves a little differently from the
0816      * PSL based cards and will time out an AFU reset if the AFU is still
0817      * enabled. That card is special in that we do have a means to identify
0818      * it from this code, so in that case we skip the reset and just use a
0819      * disable/purge to avoid the timeout and corresponding noise in the
0820      * kernel log.
0821      */
0822     if (afu->adapter->native->sl_ops->needs_reset_before_disable)
0823         cxl_ops->afu_reset(afu);
0824     cxl_afu_disable(afu);
0825     cxl_psl_purge(afu);
0826 
0827     return 0;
0828 }
0829 
0830 int cxl_activate_dedicated_process_psl9(struct cxl_afu *afu)
0831 {
0832     dev_info(&afu->dev, "Activating dedicated process mode\n");
0833 
0834     /*
0835      * If XSL is set to dedicated mode (Set in PSL_SCNTL reg), the
0836      * XSL and AFU are programmed to work with a single context.
0837      * The context information should be configured in the SPA area
0838      * index 0 (so PSL_SPAP must be configured before enabling the
0839      * AFU).
0840      */
0841     afu->num_procs = 1;
0842     if (afu->native->spa == NULL) {
0843         if (cxl_alloc_spa(afu, CXL_MODE_DEDICATED))
0844             return -ENOMEM;
0845     }
0846     attach_spa(afu);
0847 
0848     cxl_p1n_write(afu, CXL_PSL_SCNTL_An, CXL_PSL_SCNTL_An_PM_Process);
0849     cxl_p1n_write(afu, CXL_PSL_ID_An, CXL_PSL_ID_An_F | CXL_PSL_ID_An_L);
0850 
0851     afu->current_mode = CXL_MODE_DEDICATED;
0852 
0853     return cxl_chardev_d_afu_add(afu);
0854 }
0855 
0856 int cxl_activate_dedicated_process_psl8(struct cxl_afu *afu)
0857 {
0858     dev_info(&afu->dev, "Activating dedicated process mode\n");
0859 
0860     cxl_p1n_write(afu, CXL_PSL_SCNTL_An, CXL_PSL_SCNTL_An_PM_Process);
0861 
0862     cxl_p1n_write(afu, CXL_PSL_CtxTime_An, 0); /* disable */
0863     cxl_p1n_write(afu, CXL_PSL_SPAP_An, 0);    /* disable */
0864     cxl_p1n_write(afu, CXL_PSL_AMOR_An, 0xFFFFFFFFFFFFFFFFULL);
0865     cxl_p1n_write(afu, CXL_PSL_LPID_An, mfspr(SPRN_LPID));
0866     cxl_p1n_write(afu, CXL_HAURP_An, 0);       /* disable */
0867     cxl_p1n_write(afu, CXL_PSL_SDR_An, mfspr(SPRN_SDR1));
0868 
0869     cxl_p2n_write(afu, CXL_CSRP_An, 0);        /* disable */
0870     cxl_p2n_write(afu, CXL_AURP0_An, 0);       /* disable */
0871     cxl_p2n_write(afu, CXL_AURP1_An, 0);       /* disable */
0872 
0873     afu->current_mode = CXL_MODE_DEDICATED;
0874     afu->num_procs = 1;
0875 
0876     return cxl_chardev_d_afu_add(afu);
0877 }
0878 
0879 void cxl_update_dedicated_ivtes_psl9(struct cxl_context *ctx)
0880 {
0881     int r;
0882 
0883     for (r = 0; r < CXL_IRQ_RANGES; r++) {
0884         ctx->elem->ivte_offsets[r] = cpu_to_be16(ctx->irqs.offset[r]);
0885         ctx->elem->ivte_ranges[r] = cpu_to_be16(ctx->irqs.range[r]);
0886     }
0887 }
0888 
0889 void cxl_update_dedicated_ivtes_psl8(struct cxl_context *ctx)
0890 {
0891     struct cxl_afu *afu = ctx->afu;
0892 
0893     cxl_p1n_write(afu, CXL_PSL_IVTE_Offset_An,
0894                (((u64)ctx->irqs.offset[0] & 0xffff) << 48) |
0895                (((u64)ctx->irqs.offset[1] & 0xffff) << 32) |
0896                (((u64)ctx->irqs.offset[2] & 0xffff) << 16) |
0897             ((u64)ctx->irqs.offset[3] & 0xffff));
0898     cxl_p1n_write(afu, CXL_PSL_IVTE_Limit_An, (u64)
0899                (((u64)ctx->irqs.range[0] & 0xffff) << 48) |
0900                (((u64)ctx->irqs.range[1] & 0xffff) << 32) |
0901                (((u64)ctx->irqs.range[2] & 0xffff) << 16) |
0902             ((u64)ctx->irqs.range[3] & 0xffff));
0903 }
0904 
0905 int cxl_attach_dedicated_process_psl9(struct cxl_context *ctx, u64 wed, u64 amr)
0906 {
0907     struct cxl_afu *afu = ctx->afu;
0908     int result;
0909 
0910     /* fill the process element entry */
0911     result = process_element_entry_psl9(ctx, wed, amr);
0912     if (result)
0913         return result;
0914 
0915     if (ctx->afu->adapter->native->sl_ops->update_dedicated_ivtes)
0916         afu->adapter->native->sl_ops->update_dedicated_ivtes(ctx);
0917 
0918     ctx->elem->software_state = cpu_to_be32(CXL_PE_SOFTWARE_STATE_V);
0919     /*
0920      * Ideally we should do a wmb() here to make sure the changes to the
0921      * PE are visible to the card before we call afu_enable.
0922      * On ppc64 though all mmios are preceded by a 'sync' instruction hence
0923      * we dont dont need one here.
0924      */
0925 
0926     result = cxl_ops->afu_reset(afu);
0927     if (result)
0928         return result;
0929 
0930     return afu_enable(afu);
0931 }
0932 
0933 int cxl_attach_dedicated_process_psl8(struct cxl_context *ctx, u64 wed, u64 amr)
0934 {
0935     struct cxl_afu *afu = ctx->afu;
0936     u64 pid;
0937     int rc;
0938 
0939     pid = (u64)current->pid << 32;
0940     if (ctx->kernel)
0941         pid = 0;
0942     cxl_p2n_write(afu, CXL_PSL_PID_TID_An, pid);
0943 
0944     cxl_p1n_write(afu, CXL_PSL_SR_An, calculate_sr(ctx));
0945 
0946     if ((rc = cxl_write_sstp(afu, ctx->sstp0, ctx->sstp1)))
0947         return rc;
0948 
0949     cxl_prefault(ctx, wed);
0950 
0951     if (ctx->afu->adapter->native->sl_ops->update_dedicated_ivtes)
0952         afu->adapter->native->sl_ops->update_dedicated_ivtes(ctx);
0953 
0954     cxl_p2n_write(afu, CXL_PSL_AMR_An, amr);
0955 
0956     /* master only context for dedicated */
0957     cxl_assign_psn_space(ctx);
0958 
0959     if ((rc = cxl_ops->afu_reset(afu)))
0960         return rc;
0961 
0962     cxl_p2n_write(afu, CXL_PSL_WED_An, wed);
0963 
0964     return afu_enable(afu);
0965 }
0966 
0967 static int deactivate_dedicated_process(struct cxl_afu *afu)
0968 {
0969     dev_info(&afu->dev, "Deactivating dedicated process mode\n");
0970 
0971     afu->current_mode = 0;
0972     afu->num_procs = 0;
0973 
0974     cxl_chardev_afu_remove(afu);
0975 
0976     return 0;
0977 }
0978 
0979 static int native_afu_deactivate_mode(struct cxl_afu *afu, int mode)
0980 {
0981     if (mode == CXL_MODE_DIRECTED)
0982         return deactivate_afu_directed(afu);
0983     if (mode == CXL_MODE_DEDICATED)
0984         return deactivate_dedicated_process(afu);
0985     return 0;
0986 }
0987 
0988 static int native_afu_activate_mode(struct cxl_afu *afu, int mode)
0989 {
0990     if (!mode)
0991         return 0;
0992     if (!(mode & afu->modes_supported))
0993         return -EINVAL;
0994 
0995     if (!cxl_ops->link_ok(afu->adapter, afu)) {
0996         WARN(1, "Device link is down, refusing to activate!\n");
0997         return -EIO;
0998     }
0999 
1000     if (mode == CXL_MODE_DIRECTED)
1001         return activate_afu_directed(afu);
1002     if ((mode == CXL_MODE_DEDICATED) &&
1003         (afu->adapter->native->sl_ops->activate_dedicated_process))
1004         return afu->adapter->native->sl_ops->activate_dedicated_process(afu);
1005 
1006     return -EINVAL;
1007 }
1008 
1009 static int native_attach_process(struct cxl_context *ctx, bool kernel,
1010                 u64 wed, u64 amr)
1011 {
1012     if (!cxl_ops->link_ok(ctx->afu->adapter, ctx->afu)) {
1013         WARN(1, "Device link is down, refusing to attach process!\n");
1014         return -EIO;
1015     }
1016 
1017     ctx->kernel = kernel;
1018     if ((ctx->afu->current_mode == CXL_MODE_DIRECTED) &&
1019         (ctx->afu->adapter->native->sl_ops->attach_afu_directed))
1020         return ctx->afu->adapter->native->sl_ops->attach_afu_directed(ctx, wed, amr);
1021 
1022     if ((ctx->afu->current_mode == CXL_MODE_DEDICATED) &&
1023         (ctx->afu->adapter->native->sl_ops->attach_dedicated_process))
1024         return ctx->afu->adapter->native->sl_ops->attach_dedicated_process(ctx, wed, amr);
1025 
1026     return -EINVAL;
1027 }
1028 
1029 static inline int detach_process_native_dedicated(struct cxl_context *ctx)
1030 {
1031     /*
1032      * The CAIA section 2.1.1 indicates that we need to do an AFU reset to
1033      * stop the AFU in dedicated mode (we therefore do not make that
1034      * optional like we do in the afu directed path). It does not indicate
1035      * that we need to do an explicit disable (which should occur
1036      * implicitly as part of the reset) or purge, but we do these as well
1037      * to be on the safe side.
1038      *
1039      * Notably we used to have some issues with the disable sequence
1040      * (before the sequence was spelled out in the architecture) which is
1041      * why we were so heavy weight in the first place, however a bug was
1042      * discovered that had rendered the disable operation ineffective, so
1043      * it is conceivable that was the sole explanation for those
1044      * difficulties. Point is, we should be careful and do some regression
1045      * testing if we ever attempt to remove any part of this procedure.
1046      */
1047     cxl_ops->afu_reset(ctx->afu);
1048     cxl_afu_disable(ctx->afu);
1049     cxl_psl_purge(ctx->afu);
1050     return 0;
1051 }
1052 
1053 static void native_update_ivtes(struct cxl_context *ctx)
1054 {
1055     if (ctx->afu->current_mode == CXL_MODE_DIRECTED)
1056         return update_ivtes_directed(ctx);
1057     if ((ctx->afu->current_mode == CXL_MODE_DEDICATED) &&
1058         (ctx->afu->adapter->native->sl_ops->update_dedicated_ivtes))
1059         return ctx->afu->adapter->native->sl_ops->update_dedicated_ivtes(ctx);
1060     WARN(1, "native_update_ivtes: Bad mode\n");
1061 }
1062 
1063 static inline int detach_process_native_afu_directed(struct cxl_context *ctx)
1064 {
1065     if (!ctx->pe_inserted)
1066         return 0;
1067     if (terminate_process_element(ctx))
1068         return -1;
1069     if (remove_process_element(ctx))
1070         return -1;
1071 
1072     return 0;
1073 }
1074 
1075 static int native_detach_process(struct cxl_context *ctx)
1076 {
1077     trace_cxl_detach(ctx);
1078 
1079     if (ctx->afu->current_mode == CXL_MODE_DEDICATED)
1080         return detach_process_native_dedicated(ctx);
1081 
1082     return detach_process_native_afu_directed(ctx);
1083 }
1084 
1085 static int native_get_irq_info(struct cxl_afu *afu, struct cxl_irq_info *info)
1086 {
1087     /* If the adapter has gone away, we can't get any meaningful
1088      * information.
1089      */
1090     if (!cxl_ops->link_ok(afu->adapter, afu))
1091         return -EIO;
1092 
1093     info->dsisr = cxl_p2n_read(afu, CXL_PSL_DSISR_An);
1094     info->dar = cxl_p2n_read(afu, CXL_PSL_DAR_An);
1095     if (cxl_is_power8())
1096         info->dsr = cxl_p2n_read(afu, CXL_PSL_DSR_An);
1097     info->afu_err = cxl_p2n_read(afu, CXL_AFU_ERR_An);
1098     info->errstat = cxl_p2n_read(afu, CXL_PSL_ErrStat_An);
1099     info->proc_handle = 0;
1100 
1101     return 0;
1102 }
1103 
1104 void cxl_native_irq_dump_regs_psl9(struct cxl_context *ctx)
1105 {
1106     u64 fir1, serr;
1107 
1108     fir1 = cxl_p1_read(ctx->afu->adapter, CXL_PSL9_FIR1);
1109 
1110     dev_crit(&ctx->afu->dev, "PSL_FIR1: 0x%016llx\n", fir1);
1111     if (ctx->afu->adapter->native->sl_ops->register_serr_irq) {
1112         serr = cxl_p1n_read(ctx->afu, CXL_PSL_SERR_An);
1113         cxl_afu_decode_psl_serr(ctx->afu, serr);
1114     }
1115 }
1116 
1117 void cxl_native_irq_dump_regs_psl8(struct cxl_context *ctx)
1118 {
1119     u64 fir1, fir2, fir_slice, serr, afu_debug;
1120 
1121     fir1 = cxl_p1_read(ctx->afu->adapter, CXL_PSL_FIR1);
1122     fir2 = cxl_p1_read(ctx->afu->adapter, CXL_PSL_FIR2);
1123     fir_slice = cxl_p1n_read(ctx->afu, CXL_PSL_FIR_SLICE_An);
1124     afu_debug = cxl_p1n_read(ctx->afu, CXL_AFU_DEBUG_An);
1125 
1126     dev_crit(&ctx->afu->dev, "PSL_FIR1: 0x%016llx\n", fir1);
1127     dev_crit(&ctx->afu->dev, "PSL_FIR2: 0x%016llx\n", fir2);
1128     if (ctx->afu->adapter->native->sl_ops->register_serr_irq) {
1129         serr = cxl_p1n_read(ctx->afu, CXL_PSL_SERR_An);
1130         cxl_afu_decode_psl_serr(ctx->afu, serr);
1131     }
1132     dev_crit(&ctx->afu->dev, "PSL_FIR_SLICE_An: 0x%016llx\n", fir_slice);
1133     dev_crit(&ctx->afu->dev, "CXL_PSL_AFU_DEBUG_An: 0x%016llx\n", afu_debug);
1134 }
1135 
1136 static irqreturn_t native_handle_psl_slice_error(struct cxl_context *ctx,
1137                         u64 dsisr, u64 errstat)
1138 {
1139 
1140     dev_crit(&ctx->afu->dev, "PSL ERROR STATUS: 0x%016llx\n", errstat);
1141 
1142     if (ctx->afu->adapter->native->sl_ops->psl_irq_dump_registers)
1143         ctx->afu->adapter->native->sl_ops->psl_irq_dump_registers(ctx);
1144 
1145     if (ctx->afu->adapter->native->sl_ops->debugfs_stop_trace) {
1146         dev_crit(&ctx->afu->dev, "STOPPING CXL TRACE\n");
1147         ctx->afu->adapter->native->sl_ops->debugfs_stop_trace(ctx->afu->adapter);
1148     }
1149 
1150     return cxl_ops->ack_irq(ctx, 0, errstat);
1151 }
1152 
1153 static bool cxl_is_translation_fault(struct cxl_afu *afu, u64 dsisr)
1154 {
1155     if ((cxl_is_power8()) && (dsisr & CXL_PSL_DSISR_TRANS))
1156         return true;
1157 
1158     if ((cxl_is_power9()) && (dsisr & CXL_PSL9_DSISR_An_TF))
1159         return true;
1160 
1161     return false;
1162 }
1163 
1164 irqreturn_t cxl_fail_irq_psl(struct cxl_afu *afu, struct cxl_irq_info *irq_info)
1165 {
1166     if (cxl_is_translation_fault(afu, irq_info->dsisr))
1167         cxl_p2n_write(afu, CXL_PSL_TFC_An, CXL_PSL_TFC_An_AE);
1168     else
1169         cxl_p2n_write(afu, CXL_PSL_TFC_An, CXL_PSL_TFC_An_A);
1170 
1171     return IRQ_HANDLED;
1172 }
1173 
1174 static irqreturn_t native_irq_multiplexed(int irq, void *data)
1175 {
1176     struct cxl_afu *afu = data;
1177     struct cxl_context *ctx;
1178     struct cxl_irq_info irq_info;
1179     u64 phreg = cxl_p2n_read(afu, CXL_PSL_PEHandle_An);
1180     int ph, ret = IRQ_HANDLED, res;
1181 
1182     /* check if eeh kicked in while the interrupt was in flight */
1183     if (unlikely(phreg == ~0ULL)) {
1184         dev_warn(&afu->dev,
1185              "Ignoring slice interrupt(%d) due to fenced card",
1186              irq);
1187         return IRQ_HANDLED;
1188     }
1189     /* Mask the pe-handle from register value */
1190     ph = phreg & 0xffff;
1191     if ((res = native_get_irq_info(afu, &irq_info))) {
1192         WARN(1, "Unable to get CXL IRQ Info: %i\n", res);
1193         if (afu->adapter->native->sl_ops->fail_irq)
1194             return afu->adapter->native->sl_ops->fail_irq(afu, &irq_info);
1195         return ret;
1196     }
1197 
1198     rcu_read_lock();
1199     ctx = idr_find(&afu->contexts_idr, ph);
1200     if (ctx) {
1201         if (afu->adapter->native->sl_ops->handle_interrupt)
1202             ret = afu->adapter->native->sl_ops->handle_interrupt(irq, ctx, &irq_info);
1203         rcu_read_unlock();
1204         return ret;
1205     }
1206     rcu_read_unlock();
1207 
1208     WARN(1, "Unable to demultiplex CXL PSL IRQ for PE %i DSISR %016llx DAR"
1209         " %016llx\n(Possible AFU HW issue - was a term/remove acked"
1210         " with outstanding transactions?)\n", ph, irq_info.dsisr,
1211         irq_info.dar);
1212     if (afu->adapter->native->sl_ops->fail_irq)
1213         ret = afu->adapter->native->sl_ops->fail_irq(afu, &irq_info);
1214     return ret;
1215 }
1216 
1217 static void native_irq_wait(struct cxl_context *ctx)
1218 {
1219     u64 dsisr;
1220     int timeout = 1000;
1221     int ph;
1222 
1223     /*
1224      * Wait until no further interrupts are presented by the PSL
1225      * for this context.
1226      */
1227     while (timeout--) {
1228         ph = cxl_p2n_read(ctx->afu, CXL_PSL_PEHandle_An) & 0xffff;
1229         if (ph != ctx->pe)
1230             return;
1231         dsisr = cxl_p2n_read(ctx->afu, CXL_PSL_DSISR_An);
1232         if (cxl_is_power8() &&
1233            ((dsisr & CXL_PSL_DSISR_PENDING) == 0))
1234             return;
1235         if (cxl_is_power9() &&
1236            ((dsisr & CXL_PSL9_DSISR_PENDING) == 0))
1237             return;
1238         /*
1239          * We are waiting for the workqueue to process our
1240          * irq, so need to let that run here.
1241          */
1242         msleep(1);
1243     }
1244 
1245     dev_warn(&ctx->afu->dev, "WARNING: waiting on DSI for PE %i"
1246          " DSISR %016llx!\n", ph, dsisr);
1247     return;
1248 }
1249 
1250 static irqreturn_t native_slice_irq_err(int irq, void *data)
1251 {
1252     struct cxl_afu *afu = data;
1253     u64 errstat, serr, afu_error, dsisr;
1254     u64 fir_slice, afu_debug, irq_mask;
1255 
1256     /*
1257      * slice err interrupt is only used with full PSL (no XSL)
1258      */
1259     serr = cxl_p1n_read(afu, CXL_PSL_SERR_An);
1260     errstat = cxl_p2n_read(afu, CXL_PSL_ErrStat_An);
1261     afu_error = cxl_p2n_read(afu, CXL_AFU_ERR_An);
1262     dsisr = cxl_p2n_read(afu, CXL_PSL_DSISR_An);
1263     cxl_afu_decode_psl_serr(afu, serr);
1264 
1265     if (cxl_is_power8()) {
1266         fir_slice = cxl_p1n_read(afu, CXL_PSL_FIR_SLICE_An);
1267         afu_debug = cxl_p1n_read(afu, CXL_AFU_DEBUG_An);
1268         dev_crit(&afu->dev, "PSL_FIR_SLICE_An: 0x%016llx\n", fir_slice);
1269         dev_crit(&afu->dev, "CXL_PSL_AFU_DEBUG_An: 0x%016llx\n", afu_debug);
1270     }
1271     dev_crit(&afu->dev, "CXL_PSL_ErrStat_An: 0x%016llx\n", errstat);
1272     dev_crit(&afu->dev, "AFU_ERR_An: 0x%.16llx\n", afu_error);
1273     dev_crit(&afu->dev, "PSL_DSISR_An: 0x%.16llx\n", dsisr);
1274 
1275     /* mask off the IRQ so it won't retrigger until the AFU is reset */
1276     irq_mask = (serr & CXL_PSL_SERR_An_IRQS) >> 32;
1277     serr |= irq_mask;
1278     cxl_p1n_write(afu, CXL_PSL_SERR_An, serr);
1279     dev_info(&afu->dev, "Further such interrupts will be masked until the AFU is reset\n");
1280 
1281     return IRQ_HANDLED;
1282 }
1283 
1284 void cxl_native_err_irq_dump_regs_psl9(struct cxl *adapter)
1285 {
1286     u64 fir1;
1287 
1288     fir1 = cxl_p1_read(adapter, CXL_PSL9_FIR1);
1289     dev_crit(&adapter->dev, "PSL_FIR: 0x%016llx\n", fir1);
1290 }
1291 
1292 void cxl_native_err_irq_dump_regs_psl8(struct cxl *adapter)
1293 {
1294     u64 fir1, fir2;
1295 
1296     fir1 = cxl_p1_read(adapter, CXL_PSL_FIR1);
1297     fir2 = cxl_p1_read(adapter, CXL_PSL_FIR2);
1298     dev_crit(&adapter->dev,
1299          "PSL_FIR1: 0x%016llx\nPSL_FIR2: 0x%016llx\n",
1300          fir1, fir2);
1301 }
1302 
1303 static irqreturn_t native_irq_err(int irq, void *data)
1304 {
1305     struct cxl *adapter = data;
1306     u64 err_ivte;
1307 
1308     WARN(1, "CXL ERROR interrupt %i\n", irq);
1309 
1310     err_ivte = cxl_p1_read(adapter, CXL_PSL_ErrIVTE);
1311     dev_crit(&adapter->dev, "PSL_ErrIVTE: 0x%016llx\n", err_ivte);
1312 
1313     if (adapter->native->sl_ops->debugfs_stop_trace) {
1314         dev_crit(&adapter->dev, "STOPPING CXL TRACE\n");
1315         adapter->native->sl_ops->debugfs_stop_trace(adapter);
1316     }
1317 
1318     if (adapter->native->sl_ops->err_irq_dump_registers)
1319         adapter->native->sl_ops->err_irq_dump_registers(adapter);
1320 
1321     return IRQ_HANDLED;
1322 }
1323 
1324 int cxl_native_register_psl_err_irq(struct cxl *adapter)
1325 {
1326     int rc;
1327 
1328     adapter->irq_name = kasprintf(GFP_KERNEL, "cxl-%s-err",
1329                       dev_name(&adapter->dev));
1330     if (!adapter->irq_name)
1331         return -ENOMEM;
1332 
1333     if ((rc = cxl_register_one_irq(adapter, native_irq_err, adapter,
1334                        &adapter->native->err_hwirq,
1335                        &adapter->native->err_virq,
1336                        adapter->irq_name))) {
1337         kfree(adapter->irq_name);
1338         adapter->irq_name = NULL;
1339         return rc;
1340     }
1341 
1342     cxl_p1_write(adapter, CXL_PSL_ErrIVTE, adapter->native->err_hwirq & 0xffff);
1343 
1344     return 0;
1345 }
1346 
1347 void cxl_native_release_psl_err_irq(struct cxl *adapter)
1348 {
1349     if (adapter->native->err_virq == 0 ||
1350         adapter->native->err_virq !=
1351         irq_find_mapping(NULL, adapter->native->err_hwirq))
1352         return;
1353 
1354     cxl_p1_write(adapter, CXL_PSL_ErrIVTE, 0x0000000000000000);
1355     cxl_unmap_irq(adapter->native->err_virq, adapter);
1356     cxl_ops->release_one_irq(adapter, adapter->native->err_hwirq);
1357     kfree(adapter->irq_name);
1358     adapter->native->err_virq = 0;
1359 }
1360 
1361 int cxl_native_register_serr_irq(struct cxl_afu *afu)
1362 {
1363     u64 serr;
1364     int rc;
1365 
1366     afu->err_irq_name = kasprintf(GFP_KERNEL, "cxl-%s-err",
1367                       dev_name(&afu->dev));
1368     if (!afu->err_irq_name)
1369         return -ENOMEM;
1370 
1371     if ((rc = cxl_register_one_irq(afu->adapter, native_slice_irq_err, afu,
1372                        &afu->serr_hwirq,
1373                        &afu->serr_virq, afu->err_irq_name))) {
1374         kfree(afu->err_irq_name);
1375         afu->err_irq_name = NULL;
1376         return rc;
1377     }
1378 
1379     serr = cxl_p1n_read(afu, CXL_PSL_SERR_An);
1380     if (cxl_is_power8())
1381         serr = (serr & 0x00ffffffffff0000ULL) | (afu->serr_hwirq & 0xffff);
1382     if (cxl_is_power9()) {
1383         /*
1384          * By default, all errors are masked. So don't set all masks.
1385          * Slice errors will be transfered.
1386          */
1387         serr = (serr & ~0xff0000007fffffffULL) | (afu->serr_hwirq & 0xffff);
1388     }
1389     cxl_p1n_write(afu, CXL_PSL_SERR_An, serr);
1390 
1391     return 0;
1392 }
1393 
1394 void cxl_native_release_serr_irq(struct cxl_afu *afu)
1395 {
1396     if (afu->serr_virq == 0 ||
1397         afu->serr_virq != irq_find_mapping(NULL, afu->serr_hwirq))
1398         return;
1399 
1400     cxl_p1n_write(afu, CXL_PSL_SERR_An, 0x0000000000000000);
1401     cxl_unmap_irq(afu->serr_virq, afu);
1402     cxl_ops->release_one_irq(afu->adapter, afu->serr_hwirq);
1403     kfree(afu->err_irq_name);
1404     afu->serr_virq = 0;
1405 }
1406 
1407 int cxl_native_register_psl_irq(struct cxl_afu *afu)
1408 {
1409     int rc;
1410 
1411     afu->psl_irq_name = kasprintf(GFP_KERNEL, "cxl-%s",
1412                       dev_name(&afu->dev));
1413     if (!afu->psl_irq_name)
1414         return -ENOMEM;
1415 
1416     if ((rc = cxl_register_one_irq(afu->adapter, native_irq_multiplexed,
1417                     afu, &afu->native->psl_hwirq, &afu->native->psl_virq,
1418                     afu->psl_irq_name))) {
1419         kfree(afu->psl_irq_name);
1420         afu->psl_irq_name = NULL;
1421     }
1422     return rc;
1423 }
1424 
1425 void cxl_native_release_psl_irq(struct cxl_afu *afu)
1426 {
1427     if (afu->native->psl_virq == 0 ||
1428         afu->native->psl_virq !=
1429         irq_find_mapping(NULL, afu->native->psl_hwirq))
1430         return;
1431 
1432     cxl_unmap_irq(afu->native->psl_virq, afu);
1433     cxl_ops->release_one_irq(afu->adapter, afu->native->psl_hwirq);
1434     kfree(afu->psl_irq_name);
1435     afu->native->psl_virq = 0;
1436 }
1437 
1438 static void recover_psl_err(struct cxl_afu *afu, u64 errstat)
1439 {
1440     u64 dsisr;
1441 
1442     pr_devel("RECOVERING FROM PSL ERROR... (0x%016llx)\n", errstat);
1443 
1444     /* Clear PSL_DSISR[PE] */
1445     dsisr = cxl_p2n_read(afu, CXL_PSL_DSISR_An);
1446     cxl_p2n_write(afu, CXL_PSL_DSISR_An, dsisr & ~CXL_PSL_DSISR_An_PE);
1447 
1448     /* Write 1s to clear error status bits */
1449     cxl_p2n_write(afu, CXL_PSL_ErrStat_An, errstat);
1450 }
1451 
1452 static int native_ack_irq(struct cxl_context *ctx, u64 tfc, u64 psl_reset_mask)
1453 {
1454     trace_cxl_psl_irq_ack(ctx, tfc);
1455     if (tfc)
1456         cxl_p2n_write(ctx->afu, CXL_PSL_TFC_An, tfc);
1457     if (psl_reset_mask)
1458         recover_psl_err(ctx->afu, psl_reset_mask);
1459 
1460     return 0;
1461 }
1462 
1463 int cxl_check_error(struct cxl_afu *afu)
1464 {
1465     return (cxl_p1n_read(afu, CXL_PSL_SCNTL_An) == ~0ULL);
1466 }
1467 
1468 static bool native_support_attributes(const char *attr_name,
1469                       enum cxl_attrs type)
1470 {
1471     return true;
1472 }
1473 
1474 static int native_afu_cr_read64(struct cxl_afu *afu, int cr, u64 off, u64 *out)
1475 {
1476     if (unlikely(!cxl_ops->link_ok(afu->adapter, afu)))
1477         return -EIO;
1478     if (unlikely(off >= afu->crs_len))
1479         return -ERANGE;
1480     *out = in_le64(afu->native->afu_desc_mmio + afu->crs_offset +
1481         (cr * afu->crs_len) + off);
1482     return 0;
1483 }
1484 
1485 static int native_afu_cr_read32(struct cxl_afu *afu, int cr, u64 off, u32 *out)
1486 {
1487     if (unlikely(!cxl_ops->link_ok(afu->adapter, afu)))
1488         return -EIO;
1489     if (unlikely(off >= afu->crs_len))
1490         return -ERANGE;
1491     *out = in_le32(afu->native->afu_desc_mmio + afu->crs_offset +
1492         (cr * afu->crs_len) + off);
1493     return 0;
1494 }
1495 
1496 static int native_afu_cr_read16(struct cxl_afu *afu, int cr, u64 off, u16 *out)
1497 {
1498     u64 aligned_off = off & ~0x3L;
1499     u32 val;
1500     int rc;
1501 
1502     rc = native_afu_cr_read32(afu, cr, aligned_off, &val);
1503     if (!rc)
1504         *out = (val >> ((off & 0x3) * 8)) & 0xffff;
1505     return rc;
1506 }
1507 
1508 static int native_afu_cr_read8(struct cxl_afu *afu, int cr, u64 off, u8 *out)
1509 {
1510     u64 aligned_off = off & ~0x3L;
1511     u32 val;
1512     int rc;
1513 
1514     rc = native_afu_cr_read32(afu, cr, aligned_off, &val);
1515     if (!rc)
1516         *out = (val >> ((off & 0x3) * 8)) & 0xff;
1517     return rc;
1518 }
1519 
1520 static int native_afu_cr_write32(struct cxl_afu *afu, int cr, u64 off, u32 in)
1521 {
1522     if (unlikely(!cxl_ops->link_ok(afu->adapter, afu)))
1523         return -EIO;
1524     if (unlikely(off >= afu->crs_len))
1525         return -ERANGE;
1526     out_le32(afu->native->afu_desc_mmio + afu->crs_offset +
1527         (cr * afu->crs_len) + off, in);
1528     return 0;
1529 }
1530 
1531 static int native_afu_cr_write16(struct cxl_afu *afu, int cr, u64 off, u16 in)
1532 {
1533     u64 aligned_off = off & ~0x3L;
1534     u32 val32, mask, shift;
1535     int rc;
1536 
1537     rc = native_afu_cr_read32(afu, cr, aligned_off, &val32);
1538     if (rc)
1539         return rc;
1540     shift = (off & 0x3) * 8;
1541     WARN_ON(shift == 24);
1542     mask = 0xffff << shift;
1543     val32 = (val32 & ~mask) | (in << shift);
1544 
1545     rc = native_afu_cr_write32(afu, cr, aligned_off, val32);
1546     return rc;
1547 }
1548 
1549 static int native_afu_cr_write8(struct cxl_afu *afu, int cr, u64 off, u8 in)
1550 {
1551     u64 aligned_off = off & ~0x3L;
1552     u32 val32, mask, shift;
1553     int rc;
1554 
1555     rc = native_afu_cr_read32(afu, cr, aligned_off, &val32);
1556     if (rc)
1557         return rc;
1558     shift = (off & 0x3) * 8;
1559     mask = 0xff << shift;
1560     val32 = (val32 & ~mask) | (in << shift);
1561 
1562     rc = native_afu_cr_write32(afu, cr, aligned_off, val32);
1563     return rc;
1564 }
1565 
1566 const struct cxl_backend_ops cxl_native_ops = {
1567     .module = THIS_MODULE,
1568     .adapter_reset = cxl_pci_reset,
1569     .alloc_one_irq = cxl_pci_alloc_one_irq,
1570     .release_one_irq = cxl_pci_release_one_irq,
1571     .alloc_irq_ranges = cxl_pci_alloc_irq_ranges,
1572     .release_irq_ranges = cxl_pci_release_irq_ranges,
1573     .setup_irq = cxl_pci_setup_irq,
1574     .handle_psl_slice_error = native_handle_psl_slice_error,
1575     .psl_interrupt = NULL,
1576     .ack_irq = native_ack_irq,
1577     .irq_wait = native_irq_wait,
1578     .attach_process = native_attach_process,
1579     .detach_process = native_detach_process,
1580     .update_ivtes = native_update_ivtes,
1581     .support_attributes = native_support_attributes,
1582     .link_ok = cxl_adapter_link_ok,
1583     .release_afu = cxl_pci_release_afu,
1584     .afu_read_err_buffer = cxl_pci_afu_read_err_buffer,
1585     .afu_check_and_enable = native_afu_check_and_enable,
1586     .afu_activate_mode = native_afu_activate_mode,
1587     .afu_deactivate_mode = native_afu_deactivate_mode,
1588     .afu_reset = native_afu_reset,
1589     .afu_cr_read8 = native_afu_cr_read8,
1590     .afu_cr_read16 = native_afu_cr_read16,
1591     .afu_cr_read32 = native_afu_cr_read32,
1592     .afu_cr_read64 = native_afu_cr_read64,
1593     .afu_cr_write8 = native_afu_cr_write8,
1594     .afu_cr_write16 = native_afu_cr_write16,
1595     .afu_cr_write32 = native_afu_cr_write32,
1596     .read_adapter_vpd = cxl_pci_read_adapter_vpd,
1597 };
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!