Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * Copyright (C) 2017 Marvell
0004  *
0005  * Antoine Tenart <antoine.tenart@free-electrons.com>
0006  */
0007 
0008 #include <linux/clk.h>
0009 #include <linux/device.h>
0010 #include <linux/dma-mapping.h>
0011 #include <linux/dmapool.h>
0012 #include <linux/firmware.h>
0013 #include <linux/interrupt.h>
0014 #include <linux/module.h>
0015 #include <linux/of_platform.h>
0016 #include <linux/of_irq.h>
0017 #include <linux/pci.h>
0018 #include <linux/platform_device.h>
0019 #include <linux/workqueue.h>
0020 
0021 #include <crypto/internal/aead.h>
0022 #include <crypto/internal/hash.h>
0023 #include <crypto/internal/skcipher.h>
0024 
0025 #include "safexcel.h"
0026 
0027 static u32 max_rings = EIP197_MAX_RINGS;
0028 module_param(max_rings, uint, 0644);
0029 MODULE_PARM_DESC(max_rings, "Maximum number of rings to use.");
0030 
0031 static void eip197_trc_cache_setupvirt(struct safexcel_crypto_priv *priv)
0032 {
0033     int i;
0034 
0035     /*
0036      * Map all interfaces/rings to register index 0
0037      * so they can share contexts. Without this, the EIP197 will
0038      * assume each interface/ring to be in its own memory domain
0039      * i.e. have its own subset of UNIQUE memory addresses.
0040      * Which would cause records with the SAME memory address to
0041      * use DIFFERENT cache buffers, causing both poor cache utilization
0042      * AND serious coherence/invalidation issues.
0043      */
0044     for (i = 0; i < 4; i++)
0045         writel(0, priv->base + EIP197_FLUE_IFC_LUT(i));
0046 
0047     /*
0048      * Initialize other virtualization regs for cache
0049      * These may not be in their reset state ...
0050      */
0051     for (i = 0; i < priv->config.rings; i++) {
0052         writel(0, priv->base + EIP197_FLUE_CACHEBASE_LO(i));
0053         writel(0, priv->base + EIP197_FLUE_CACHEBASE_HI(i));
0054         writel(EIP197_FLUE_CONFIG_MAGIC,
0055                priv->base + EIP197_FLUE_CONFIG(i));
0056     }
0057     writel(0, priv->base + EIP197_FLUE_OFFSETS);
0058     writel(0, priv->base + EIP197_FLUE_ARC4_OFFSET);
0059 }
0060 
0061 static void eip197_trc_cache_banksel(struct safexcel_crypto_priv *priv,
0062                      u32 addrmid, int *actbank)
0063 {
0064     u32 val;
0065     int curbank;
0066 
0067     curbank = addrmid >> 16;
0068     if (curbank != *actbank) {
0069         val = readl(priv->base + EIP197_CS_RAM_CTRL);
0070         val = (val & ~EIP197_CS_BANKSEL_MASK) |
0071               (curbank << EIP197_CS_BANKSEL_OFS);
0072         writel(val, priv->base + EIP197_CS_RAM_CTRL);
0073         *actbank = curbank;
0074     }
0075 }
0076 
0077 static u32 eip197_trc_cache_probe(struct safexcel_crypto_priv *priv,
0078                   int maxbanks, u32 probemask, u32 stride)
0079 {
0080     u32 val, addrhi, addrlo, addrmid, addralias, delta, marker;
0081     int actbank;
0082 
0083     /*
0084      * And probe the actual size of the physically attached cache data RAM
0085      * Using a binary subdivision algorithm downto 32 byte cache lines.
0086      */
0087     addrhi = 1 << (16 + maxbanks);
0088     addrlo = 0;
0089     actbank = min(maxbanks - 1, 0);
0090     while ((addrhi - addrlo) > stride) {
0091         /* write marker to lowest address in top half */
0092         addrmid = (addrhi + addrlo) >> 1;
0093         marker = (addrmid ^ 0xabadbabe) & probemask; /* Unique */
0094         eip197_trc_cache_banksel(priv, addrmid, &actbank);
0095         writel(marker,
0096             priv->base + EIP197_CLASSIFICATION_RAMS +
0097             (addrmid & 0xffff));
0098 
0099         /* write invalid markers to possible aliases */
0100         delta = 1 << __fls(addrmid);
0101         while (delta >= stride) {
0102             addralias = addrmid - delta;
0103             eip197_trc_cache_banksel(priv, addralias, &actbank);
0104             writel(~marker,
0105                    priv->base + EIP197_CLASSIFICATION_RAMS +
0106                    (addralias & 0xffff));
0107             delta >>= 1;
0108         }
0109 
0110         /* read back marker from top half */
0111         eip197_trc_cache_banksel(priv, addrmid, &actbank);
0112         val = readl(priv->base + EIP197_CLASSIFICATION_RAMS +
0113                 (addrmid & 0xffff));
0114 
0115         if ((val & probemask) == marker)
0116             /* read back correct, continue with top half */
0117             addrlo = addrmid;
0118         else
0119             /* not read back correct, continue with bottom half */
0120             addrhi = addrmid;
0121     }
0122     return addrhi;
0123 }
0124 
0125 static void eip197_trc_cache_clear(struct safexcel_crypto_priv *priv,
0126                    int cs_rc_max, int cs_ht_wc)
0127 {
0128     int i;
0129     u32 htable_offset, val, offset;
0130 
0131     /* Clear all records in administration RAM */
0132     for (i = 0; i < cs_rc_max; i++) {
0133         offset = EIP197_CLASSIFICATION_RAMS + i * EIP197_CS_RC_SIZE;
0134 
0135         writel(EIP197_CS_RC_NEXT(EIP197_RC_NULL) |
0136                EIP197_CS_RC_PREV(EIP197_RC_NULL),
0137                priv->base + offset);
0138 
0139         val = EIP197_CS_RC_NEXT(i + 1) | EIP197_CS_RC_PREV(i - 1);
0140         if (i == 0)
0141             val |= EIP197_CS_RC_PREV(EIP197_RC_NULL);
0142         else if (i == cs_rc_max - 1)
0143             val |= EIP197_CS_RC_NEXT(EIP197_RC_NULL);
0144         writel(val, priv->base + offset + 4);
0145         /* must also initialize the address key due to ECC! */
0146         writel(0, priv->base + offset + 8);
0147         writel(0, priv->base + offset + 12);
0148     }
0149 
0150     /* Clear the hash table entries */
0151     htable_offset = cs_rc_max * EIP197_CS_RC_SIZE;
0152     for (i = 0; i < cs_ht_wc; i++)
0153         writel(GENMASK(29, 0),
0154                priv->base + EIP197_CLASSIFICATION_RAMS +
0155                htable_offset + i * sizeof(u32));
0156 }
0157 
0158 static int eip197_trc_cache_init(struct safexcel_crypto_priv *priv)
0159 {
0160     u32 val, dsize, asize;
0161     int cs_rc_max, cs_ht_wc, cs_trc_rec_wc, cs_trc_lg_rec_wc;
0162     int cs_rc_abs_max, cs_ht_sz;
0163     int maxbanks;
0164 
0165     /* Setup (dummy) virtualization for cache */
0166     eip197_trc_cache_setupvirt(priv);
0167 
0168     /*
0169      * Enable the record cache memory access and
0170      * probe the bank select width
0171      */
0172     val = readl(priv->base + EIP197_CS_RAM_CTRL);
0173     val &= ~EIP197_TRC_ENABLE_MASK;
0174     val |= EIP197_TRC_ENABLE_0 | EIP197_CS_BANKSEL_MASK;
0175     writel(val, priv->base + EIP197_CS_RAM_CTRL);
0176     val = readl(priv->base + EIP197_CS_RAM_CTRL);
0177     maxbanks = ((val&EIP197_CS_BANKSEL_MASK)>>EIP197_CS_BANKSEL_OFS) + 1;
0178 
0179     /* Clear all ECC errors */
0180     writel(0, priv->base + EIP197_TRC_ECCCTRL);
0181 
0182     /*
0183      * Make sure the cache memory is accessible by taking record cache into
0184      * reset. Need data memory access here, not admin access.
0185      */
0186     val = readl(priv->base + EIP197_TRC_PARAMS);
0187     val |= EIP197_TRC_PARAMS_SW_RESET | EIP197_TRC_PARAMS_DATA_ACCESS;
0188     writel(val, priv->base + EIP197_TRC_PARAMS);
0189 
0190     /* Probed data RAM size in bytes */
0191     dsize = eip197_trc_cache_probe(priv, maxbanks, 0xffffffff, 32);
0192 
0193     /*
0194      * Now probe the administration RAM size pretty much the same way
0195      * Except that only the lower 30 bits are writable and we don't need
0196      * bank selects
0197      */
0198     val = readl(priv->base + EIP197_TRC_PARAMS);
0199     /* admin access now */
0200     val &= ~(EIP197_TRC_PARAMS_DATA_ACCESS | EIP197_CS_BANKSEL_MASK);
0201     writel(val, priv->base + EIP197_TRC_PARAMS);
0202 
0203     /* Probed admin RAM size in admin words */
0204     asize = eip197_trc_cache_probe(priv, 0, 0x3fffffff, 16) >> 4;
0205 
0206     /* Clear any ECC errors detected while probing! */
0207     writel(0, priv->base + EIP197_TRC_ECCCTRL);
0208 
0209     /* Sanity check probing results */
0210     if (dsize < EIP197_MIN_DSIZE || asize < EIP197_MIN_ASIZE) {
0211         dev_err(priv->dev, "Record cache probing failed (%d,%d).",
0212             dsize, asize);
0213         return -ENODEV;
0214     }
0215 
0216     /*
0217      * Determine optimal configuration from RAM sizes
0218      * Note that we assume that the physical RAM configuration is sane
0219      * Therefore, we don't do any parameter error checking here ...
0220      */
0221 
0222     /* For now, just use a single record format covering everything */
0223     cs_trc_rec_wc = EIP197_CS_TRC_REC_WC;
0224     cs_trc_lg_rec_wc = EIP197_CS_TRC_REC_WC;
0225 
0226     /*
0227      * Step #1: How many records will physically fit?
0228      * Hard upper limit is 1023!
0229      */
0230     cs_rc_abs_max = min_t(uint, ((dsize >> 2) / cs_trc_lg_rec_wc), 1023);
0231     /* Step #2: Need at least 2 words in the admin RAM per record */
0232     cs_rc_max = min_t(uint, cs_rc_abs_max, (asize >> 1));
0233     /* Step #3: Determine log2 of hash table size */
0234     cs_ht_sz = __fls(asize - cs_rc_max) - 2;
0235     /* Step #4: determine current size of hash table in dwords */
0236     cs_ht_wc = 16 << cs_ht_sz; /* dwords, not admin words */
0237     /* Step #5: add back excess words and see if we can fit more records */
0238     cs_rc_max = min_t(uint, cs_rc_abs_max, asize - (cs_ht_wc >> 2));
0239 
0240     /* Clear the cache RAMs */
0241     eip197_trc_cache_clear(priv, cs_rc_max, cs_ht_wc);
0242 
0243     /* Disable the record cache memory access */
0244     val = readl(priv->base + EIP197_CS_RAM_CTRL);
0245     val &= ~EIP197_TRC_ENABLE_MASK;
0246     writel(val, priv->base + EIP197_CS_RAM_CTRL);
0247 
0248     /* Write head and tail pointers of the record free chain */
0249     val = EIP197_TRC_FREECHAIN_HEAD_PTR(0) |
0250           EIP197_TRC_FREECHAIN_TAIL_PTR(cs_rc_max - 1);
0251     writel(val, priv->base + EIP197_TRC_FREECHAIN);
0252 
0253     /* Configure the record cache #1 */
0254     val = EIP197_TRC_PARAMS2_RC_SZ_SMALL(cs_trc_rec_wc) |
0255           EIP197_TRC_PARAMS2_HTABLE_PTR(cs_rc_max);
0256     writel(val, priv->base + EIP197_TRC_PARAMS2);
0257 
0258     /* Configure the record cache #2 */
0259     val = EIP197_TRC_PARAMS_RC_SZ_LARGE(cs_trc_lg_rec_wc) |
0260           EIP197_TRC_PARAMS_BLK_TIMER_SPEED(1) |
0261           EIP197_TRC_PARAMS_HTABLE_SZ(cs_ht_sz);
0262     writel(val, priv->base + EIP197_TRC_PARAMS);
0263 
0264     dev_info(priv->dev, "TRC init: %dd,%da (%dr,%dh)\n",
0265          dsize, asize, cs_rc_max, cs_ht_wc + cs_ht_wc);
0266     return 0;
0267 }
0268 
0269 static void eip197_init_firmware(struct safexcel_crypto_priv *priv)
0270 {
0271     int pe, i;
0272     u32 val;
0273 
0274     for (pe = 0; pe < priv->config.pes; pe++) {
0275         /* Configure the token FIFO's */
0276         writel(3, EIP197_PE(priv) + EIP197_PE_ICE_PUTF_CTRL(pe));
0277         writel(0, EIP197_PE(priv) + EIP197_PE_ICE_PPTF_CTRL(pe));
0278 
0279         /* Clear the ICE scratchpad memory */
0280         val = readl(EIP197_PE(priv) + EIP197_PE_ICE_SCRATCH_CTRL(pe));
0281         val |= EIP197_PE_ICE_SCRATCH_CTRL_CHANGE_TIMER |
0282                EIP197_PE_ICE_SCRATCH_CTRL_TIMER_EN |
0283                EIP197_PE_ICE_SCRATCH_CTRL_SCRATCH_ACCESS |
0284                EIP197_PE_ICE_SCRATCH_CTRL_CHANGE_ACCESS;
0285         writel(val, EIP197_PE(priv) + EIP197_PE_ICE_SCRATCH_CTRL(pe));
0286 
0287         /* clear the scratchpad RAM using 32 bit writes only */
0288         for (i = 0; i < EIP197_NUM_OF_SCRATCH_BLOCKS; i++)
0289             writel(0, EIP197_PE(priv) +
0290                   EIP197_PE_ICE_SCRATCH_RAM(pe) + (i << 2));
0291 
0292         /* Reset the IFPP engine to make its program mem accessible */
0293         writel(EIP197_PE_ICE_x_CTRL_SW_RESET |
0294                EIP197_PE_ICE_x_CTRL_CLR_ECC_CORR |
0295                EIP197_PE_ICE_x_CTRL_CLR_ECC_NON_CORR,
0296                EIP197_PE(priv) + EIP197_PE_ICE_FPP_CTRL(pe));
0297 
0298         /* Reset the IPUE engine to make its program mem accessible */
0299         writel(EIP197_PE_ICE_x_CTRL_SW_RESET |
0300                EIP197_PE_ICE_x_CTRL_CLR_ECC_CORR |
0301                EIP197_PE_ICE_x_CTRL_CLR_ECC_NON_CORR,
0302                EIP197_PE(priv) + EIP197_PE_ICE_PUE_CTRL(pe));
0303 
0304         /* Enable access to all IFPP program memories */
0305         writel(EIP197_PE_ICE_RAM_CTRL_FPP_PROG_EN,
0306                EIP197_PE(priv) + EIP197_PE_ICE_RAM_CTRL(pe));
0307 
0308         /* bypass the OCE, if present */
0309         if (priv->flags & EIP197_OCE)
0310             writel(EIP197_DEBUG_OCE_BYPASS, EIP197_PE(priv) +
0311                             EIP197_PE_DEBUG(pe));
0312     }
0313 
0314 }
0315 
0316 static int eip197_write_firmware(struct safexcel_crypto_priv *priv,
0317                   const struct firmware *fw)
0318 {
0319     const __be32 *data = (const __be32 *)fw->data;
0320     int i;
0321 
0322     /* Write the firmware */
0323     for (i = 0; i < fw->size / sizeof(u32); i++)
0324         writel(be32_to_cpu(data[i]),
0325                priv->base + EIP197_CLASSIFICATION_RAMS +
0326                i * sizeof(__be32));
0327 
0328     /* Exclude final 2 NOPs from size */
0329     return i - EIP197_FW_TERMINAL_NOPS;
0330 }
0331 
0332 /*
0333  * If FW is actual production firmware, then poll for its initialization
0334  * to complete and check if it is good for the HW, otherwise just return OK.
0335  */
0336 static bool poll_fw_ready(struct safexcel_crypto_priv *priv, int fpp)
0337 {
0338     int pe, pollcnt;
0339     u32 base, pollofs;
0340 
0341     if (fpp)
0342         pollofs  = EIP197_FW_FPP_READY;
0343     else
0344         pollofs  = EIP197_FW_PUE_READY;
0345 
0346     for (pe = 0; pe < priv->config.pes; pe++) {
0347         base = EIP197_PE_ICE_SCRATCH_RAM(pe);
0348         pollcnt = EIP197_FW_START_POLLCNT;
0349         while (pollcnt &&
0350                (readl_relaxed(EIP197_PE(priv) + base +
0351                   pollofs) != 1)) {
0352             pollcnt--;
0353         }
0354         if (!pollcnt) {
0355             dev_err(priv->dev, "FW(%d) for PE %d failed to start\n",
0356                 fpp, pe);
0357             return false;
0358         }
0359     }
0360     return true;
0361 }
0362 
0363 static bool eip197_start_firmware(struct safexcel_crypto_priv *priv,
0364                   int ipuesz, int ifppsz, int minifw)
0365 {
0366     int pe;
0367     u32 val;
0368 
0369     for (pe = 0; pe < priv->config.pes; pe++) {
0370         /* Disable access to all program memory */
0371         writel(0, EIP197_PE(priv) + EIP197_PE_ICE_RAM_CTRL(pe));
0372 
0373         /* Start IFPP microengines */
0374         if (minifw)
0375             val = 0;
0376         else
0377             val = EIP197_PE_ICE_UENG_START_OFFSET((ifppsz - 1) &
0378                     EIP197_PE_ICE_UENG_INIT_ALIGN_MASK) |
0379                 EIP197_PE_ICE_UENG_DEBUG_RESET;
0380         writel(val, EIP197_PE(priv) + EIP197_PE_ICE_FPP_CTRL(pe));
0381 
0382         /* Start IPUE microengines */
0383         if (minifw)
0384             val = 0;
0385         else
0386             val = EIP197_PE_ICE_UENG_START_OFFSET((ipuesz - 1) &
0387                     EIP197_PE_ICE_UENG_INIT_ALIGN_MASK) |
0388                 EIP197_PE_ICE_UENG_DEBUG_RESET;
0389         writel(val, EIP197_PE(priv) + EIP197_PE_ICE_PUE_CTRL(pe));
0390     }
0391 
0392     /* For miniFW startup, there is no initialization, so always succeed */
0393     if (minifw)
0394         return true;
0395 
0396     /* Wait until all the firmwares have properly started up */
0397     if (!poll_fw_ready(priv, 1))
0398         return false;
0399     if (!poll_fw_ready(priv, 0))
0400         return false;
0401 
0402     return true;
0403 }
0404 
0405 static int eip197_load_firmwares(struct safexcel_crypto_priv *priv)
0406 {
0407     const char *fw_name[] = {"ifpp.bin", "ipue.bin"};
0408     const struct firmware *fw[FW_NB];
0409     char fw_path[37], *dir = NULL;
0410     int i, j, ret = 0, pe;
0411     int ipuesz, ifppsz, minifw = 0;
0412 
0413     if (priv->version == EIP197D_MRVL)
0414         dir = "eip197d";
0415     else if (priv->version == EIP197B_MRVL ||
0416          priv->version == EIP197_DEVBRD)
0417         dir = "eip197b";
0418     else
0419         return -ENODEV;
0420 
0421 retry_fw:
0422     for (i = 0; i < FW_NB; i++) {
0423         snprintf(fw_path, 37, "inside-secure/%s/%s", dir, fw_name[i]);
0424         ret = firmware_request_nowarn(&fw[i], fw_path, priv->dev);
0425         if (ret) {
0426             if (minifw || priv->version != EIP197B_MRVL)
0427                 goto release_fw;
0428 
0429             /* Fallback to the old firmware location for the
0430              * EIP197b.
0431              */
0432             ret = firmware_request_nowarn(&fw[i], fw_name[i],
0433                               priv->dev);
0434             if (ret)
0435                 goto release_fw;
0436         }
0437     }
0438 
0439     eip197_init_firmware(priv);
0440 
0441     ifppsz = eip197_write_firmware(priv, fw[FW_IFPP]);
0442 
0443     /* Enable access to IPUE program memories */
0444     for (pe = 0; pe < priv->config.pes; pe++)
0445         writel(EIP197_PE_ICE_RAM_CTRL_PUE_PROG_EN,
0446                EIP197_PE(priv) + EIP197_PE_ICE_RAM_CTRL(pe));
0447 
0448     ipuesz = eip197_write_firmware(priv, fw[FW_IPUE]);
0449 
0450     if (eip197_start_firmware(priv, ipuesz, ifppsz, minifw)) {
0451         dev_dbg(priv->dev, "Firmware loaded successfully\n");
0452         return 0;
0453     }
0454 
0455     ret = -ENODEV;
0456 
0457 release_fw:
0458     for (j = 0; j < i; j++)
0459         release_firmware(fw[j]);
0460 
0461     if (!minifw) {
0462         /* Retry with minifw path */
0463         dev_dbg(priv->dev, "Firmware set not (fully) present or init failed, falling back to BCLA mode\n");
0464         dir = "eip197_minifw";
0465         minifw = 1;
0466         goto retry_fw;
0467     }
0468 
0469     dev_dbg(priv->dev, "Firmware load failed.\n");
0470 
0471     return ret;
0472 }
0473 
0474 static int safexcel_hw_setup_cdesc_rings(struct safexcel_crypto_priv *priv)
0475 {
0476     u32 cd_size_rnd, val;
0477     int i, cd_fetch_cnt;
0478 
0479     cd_size_rnd  = (priv->config.cd_size +
0480             (BIT(priv->hwconfig.hwdataw) - 1)) >>
0481                priv->hwconfig.hwdataw;
0482     /* determine number of CD's we can fetch into the CD FIFO as 1 block */
0483     if (priv->flags & SAFEXCEL_HW_EIP197) {
0484         /* EIP197: try to fetch enough in 1 go to keep all pipes busy */
0485         cd_fetch_cnt = (1 << priv->hwconfig.hwcfsize) / cd_size_rnd;
0486         cd_fetch_cnt = min_t(uint, cd_fetch_cnt,
0487                      (priv->config.pes * EIP197_FETCH_DEPTH));
0488     } else {
0489         /* for the EIP97, just fetch all that fits minus 1 */
0490         cd_fetch_cnt = ((1 << priv->hwconfig.hwcfsize) /
0491                 cd_size_rnd) - 1;
0492     }
0493     /*
0494      * Since we're using command desc's way larger than formally specified,
0495      * we need to check whether we can fit even 1 for low-end EIP196's!
0496      */
0497     if (!cd_fetch_cnt) {
0498         dev_err(priv->dev, "Unable to fit even 1 command desc!\n");
0499         return -ENODEV;
0500     }
0501 
0502     for (i = 0; i < priv->config.rings; i++) {
0503         /* ring base address */
0504         writel(lower_32_bits(priv->ring[i].cdr.base_dma),
0505                EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO);
0506         writel(upper_32_bits(priv->ring[i].cdr.base_dma),
0507                EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI);
0508 
0509         writel(EIP197_xDR_DESC_MODE_64BIT | EIP197_CDR_DESC_MODE_ADCP |
0510                (priv->config.cd_offset << 14) | priv->config.cd_size,
0511                EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_DESC_SIZE);
0512         writel(((cd_fetch_cnt *
0513              (cd_size_rnd << priv->hwconfig.hwdataw)) << 16) |
0514                (cd_fetch_cnt * (priv->config.cd_offset / sizeof(u32))),
0515                EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_CFG);
0516 
0517         /* Configure DMA tx control */
0518         val = EIP197_HIA_xDR_CFG_WR_CACHE(WR_CACHE_3BITS);
0519         val |= EIP197_HIA_xDR_CFG_RD_CACHE(RD_CACHE_3BITS);
0520         writel(val, EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_DMA_CFG);
0521 
0522         /* clear any pending interrupt */
0523         writel(GENMASK(5, 0),
0524                EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_STAT);
0525     }
0526 
0527     return 0;
0528 }
0529 
0530 static int safexcel_hw_setup_rdesc_rings(struct safexcel_crypto_priv *priv)
0531 {
0532     u32 rd_size_rnd, val;
0533     int i, rd_fetch_cnt;
0534 
0535     /* determine number of RD's we can fetch into the FIFO as one block */
0536     rd_size_rnd = (EIP197_RD64_FETCH_SIZE +
0537                (BIT(priv->hwconfig.hwdataw) - 1)) >>
0538               priv->hwconfig.hwdataw;
0539     if (priv->flags & SAFEXCEL_HW_EIP197) {
0540         /* EIP197: try to fetch enough in 1 go to keep all pipes busy */
0541         rd_fetch_cnt = (1 << priv->hwconfig.hwrfsize) / rd_size_rnd;
0542         rd_fetch_cnt = min_t(uint, rd_fetch_cnt,
0543                      (priv->config.pes * EIP197_FETCH_DEPTH));
0544     } else {
0545         /* for the EIP97, just fetch all that fits minus 1 */
0546         rd_fetch_cnt = ((1 << priv->hwconfig.hwrfsize) /
0547                 rd_size_rnd) - 1;
0548     }
0549 
0550     for (i = 0; i < priv->config.rings; i++) {
0551         /* ring base address */
0552         writel(lower_32_bits(priv->ring[i].rdr.base_dma),
0553                EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO);
0554         writel(upper_32_bits(priv->ring[i].rdr.base_dma),
0555                EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI);
0556 
0557         writel(EIP197_xDR_DESC_MODE_64BIT | (priv->config.rd_offset << 14) |
0558                priv->config.rd_size,
0559                EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_DESC_SIZE);
0560 
0561         writel(((rd_fetch_cnt *
0562              (rd_size_rnd << priv->hwconfig.hwdataw)) << 16) |
0563                (rd_fetch_cnt * (priv->config.rd_offset / sizeof(u32))),
0564                EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_CFG);
0565 
0566         /* Configure DMA tx control */
0567         val = EIP197_HIA_xDR_CFG_WR_CACHE(WR_CACHE_3BITS);
0568         val |= EIP197_HIA_xDR_CFG_RD_CACHE(RD_CACHE_3BITS);
0569         val |= EIP197_HIA_xDR_WR_RES_BUF | EIP197_HIA_xDR_WR_CTRL_BUF;
0570         writel(val,
0571                EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_DMA_CFG);
0572 
0573         /* clear any pending interrupt */
0574         writel(GENMASK(7, 0),
0575                EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_STAT);
0576 
0577         /* enable ring interrupt */
0578         val = readl(EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLE_CTRL(i));
0579         val |= EIP197_RDR_IRQ(i);
0580         writel(val, EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLE_CTRL(i));
0581     }
0582 
0583     return 0;
0584 }
0585 
0586 static int safexcel_hw_init(struct safexcel_crypto_priv *priv)
0587 {
0588     u32 val;
0589     int i, ret, pe, opbuflo, opbufhi;
0590 
0591     dev_dbg(priv->dev, "HW init: using %d pipe(s) and %d ring(s)\n",
0592         priv->config.pes, priv->config.rings);
0593 
0594     /*
0595      * For EIP197's only set maximum number of TX commands to 2^5 = 32
0596      * Skip for the EIP97 as it does not have this field.
0597      */
0598     if (priv->flags & SAFEXCEL_HW_EIP197) {
0599         val = readl(EIP197_HIA_AIC(priv) + EIP197_HIA_MST_CTRL);
0600         val |= EIP197_MST_CTRL_TX_MAX_CMD(5);
0601         writel(val, EIP197_HIA_AIC(priv) + EIP197_HIA_MST_CTRL);
0602     }
0603 
0604     /* Configure wr/rd cache values */
0605     writel(EIP197_MST_CTRL_RD_CACHE(RD_CACHE_4BITS) |
0606            EIP197_MST_CTRL_WD_CACHE(WR_CACHE_4BITS),
0607            EIP197_HIA_GEN_CFG(priv) + EIP197_MST_CTRL);
0608 
0609     /* Interrupts reset */
0610 
0611     /* Disable all global interrupts */
0612     writel(0, EIP197_HIA_AIC_G(priv) + EIP197_HIA_AIC_G_ENABLE_CTRL);
0613 
0614     /* Clear any pending interrupt */
0615     writel(GENMASK(31, 0), EIP197_HIA_AIC_G(priv) + EIP197_HIA_AIC_G_ACK);
0616 
0617     /* Processing Engine configuration */
0618     for (pe = 0; pe < priv->config.pes; pe++) {
0619         /* Data Fetch Engine configuration */
0620 
0621         /* Reset all DFE threads */
0622         writel(EIP197_DxE_THR_CTRL_RESET_PE,
0623                EIP197_HIA_DFE_THR(priv) + EIP197_HIA_DFE_THR_CTRL(pe));
0624 
0625         if (priv->flags & EIP197_PE_ARB)
0626             /* Reset HIA input interface arbiter (if present) */
0627             writel(EIP197_HIA_RA_PE_CTRL_RESET,
0628                    EIP197_HIA_AIC(priv) + EIP197_HIA_RA_PE_CTRL(pe));
0629 
0630         /* DMA transfer size to use */
0631         val = EIP197_HIA_DFE_CFG_DIS_DEBUG;
0632         val |= EIP197_HIA_DxE_CFG_MIN_DATA_SIZE(6) |
0633                EIP197_HIA_DxE_CFG_MAX_DATA_SIZE(9);
0634         val |= EIP197_HIA_DxE_CFG_MIN_CTRL_SIZE(6) |
0635                EIP197_HIA_DxE_CFG_MAX_CTRL_SIZE(7);
0636         val |= EIP197_HIA_DxE_CFG_DATA_CACHE_CTRL(RD_CACHE_3BITS);
0637         val |= EIP197_HIA_DxE_CFG_CTRL_CACHE_CTRL(RD_CACHE_3BITS);
0638         writel(val, EIP197_HIA_DFE(priv) + EIP197_HIA_DFE_CFG(pe));
0639 
0640         /* Leave the DFE threads reset state */
0641         writel(0, EIP197_HIA_DFE_THR(priv) + EIP197_HIA_DFE_THR_CTRL(pe));
0642 
0643         /* Configure the processing engine thresholds */
0644         writel(EIP197_PE_IN_xBUF_THRES_MIN(6) |
0645                EIP197_PE_IN_xBUF_THRES_MAX(9),
0646                EIP197_PE(priv) + EIP197_PE_IN_DBUF_THRES(pe));
0647         writel(EIP197_PE_IN_xBUF_THRES_MIN(6) |
0648                EIP197_PE_IN_xBUF_THRES_MAX(7),
0649                EIP197_PE(priv) + EIP197_PE_IN_TBUF_THRES(pe));
0650 
0651         if (priv->flags & SAFEXCEL_HW_EIP197)
0652             /* enable HIA input interface arbiter and rings */
0653             writel(EIP197_HIA_RA_PE_CTRL_EN |
0654                    GENMASK(priv->config.rings - 1, 0),
0655                    EIP197_HIA_AIC(priv) + EIP197_HIA_RA_PE_CTRL(pe));
0656 
0657         /* Data Store Engine configuration */
0658 
0659         /* Reset all DSE threads */
0660         writel(EIP197_DxE_THR_CTRL_RESET_PE,
0661                EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_CTRL(pe));
0662 
0663         /* Wait for all DSE threads to complete */
0664         while ((readl(EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_STAT(pe)) &
0665             GENMASK(15, 12)) != GENMASK(15, 12))
0666             ;
0667 
0668         /* DMA transfer size to use */
0669         if (priv->hwconfig.hwnumpes > 4) {
0670             opbuflo = 9;
0671             opbufhi = 10;
0672         } else {
0673             opbuflo = 7;
0674             opbufhi = 8;
0675         }
0676         val = EIP197_HIA_DSE_CFG_DIS_DEBUG;
0677         val |= EIP197_HIA_DxE_CFG_MIN_DATA_SIZE(opbuflo) |
0678                EIP197_HIA_DxE_CFG_MAX_DATA_SIZE(opbufhi);
0679         val |= EIP197_HIA_DxE_CFG_DATA_CACHE_CTRL(WR_CACHE_3BITS);
0680         val |= EIP197_HIA_DSE_CFG_ALWAYS_BUFFERABLE;
0681         /* FIXME: instability issues can occur for EIP97 but disabling
0682          * it impacts performance.
0683          */
0684         if (priv->flags & SAFEXCEL_HW_EIP197)
0685             val |= EIP197_HIA_DSE_CFG_EN_SINGLE_WR;
0686         writel(val, EIP197_HIA_DSE(priv) + EIP197_HIA_DSE_CFG(pe));
0687 
0688         /* Leave the DSE threads reset state */
0689         writel(0, EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_CTRL(pe));
0690 
0691         /* Configure the processing engine thresholds */
0692         writel(EIP197_PE_OUT_DBUF_THRES_MIN(opbuflo) |
0693                EIP197_PE_OUT_DBUF_THRES_MAX(opbufhi),
0694                EIP197_PE(priv) + EIP197_PE_OUT_DBUF_THRES(pe));
0695 
0696         /* Processing Engine configuration */
0697 
0698         /* Token & context configuration */
0699         val = EIP197_PE_EIP96_TOKEN_CTRL_CTX_UPDATES |
0700               EIP197_PE_EIP96_TOKEN_CTRL_NO_TOKEN_WAIT |
0701               EIP197_PE_EIP96_TOKEN_CTRL_ENABLE_TIMEOUT;
0702         writel(val, EIP197_PE(priv) + EIP197_PE_EIP96_TOKEN_CTRL(pe));
0703 
0704         /* H/W capabilities selection: just enable everything */
0705         writel(EIP197_FUNCTION_ALL,
0706                EIP197_PE(priv) + EIP197_PE_EIP96_FUNCTION_EN(pe));
0707         writel(EIP197_FUNCTION_ALL,
0708                EIP197_PE(priv) + EIP197_PE_EIP96_FUNCTION2_EN(pe));
0709     }
0710 
0711     /* Command Descriptor Rings prepare */
0712     for (i = 0; i < priv->config.rings; i++) {
0713         /* Clear interrupts for this ring */
0714         writel(GENMASK(31, 0),
0715                EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLE_CLR(i));
0716 
0717         /* Disable external triggering */
0718         writel(0, EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_CFG);
0719 
0720         /* Clear the pending prepared counter */
0721         writel(EIP197_xDR_PREP_CLR_COUNT,
0722                EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PREP_COUNT);
0723 
0724         /* Clear the pending processed counter */
0725         writel(EIP197_xDR_PROC_CLR_COUNT,
0726                EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PROC_COUNT);
0727 
0728         writel(0,
0729                EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PREP_PNTR);
0730         writel(0,
0731                EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PROC_PNTR);
0732 
0733         writel((EIP197_DEFAULT_RING_SIZE * priv->config.cd_offset),
0734                EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_SIZE);
0735     }
0736 
0737     /* Result Descriptor Ring prepare */
0738     for (i = 0; i < priv->config.rings; i++) {
0739         /* Disable external triggering*/
0740         writel(0, EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_CFG);
0741 
0742         /* Clear the pending prepared counter */
0743         writel(EIP197_xDR_PREP_CLR_COUNT,
0744                EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PREP_COUNT);
0745 
0746         /* Clear the pending processed counter */
0747         writel(EIP197_xDR_PROC_CLR_COUNT,
0748                EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PROC_COUNT);
0749 
0750         writel(0,
0751                EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PREP_PNTR);
0752         writel(0,
0753                EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PROC_PNTR);
0754 
0755         /* Ring size */
0756         writel((EIP197_DEFAULT_RING_SIZE * priv->config.rd_offset),
0757                EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_SIZE);
0758     }
0759 
0760     for (pe = 0; pe < priv->config.pes; pe++) {
0761         /* Enable command descriptor rings */
0762         writel(EIP197_DxE_THR_CTRL_EN | GENMASK(priv->config.rings - 1, 0),
0763                EIP197_HIA_DFE_THR(priv) + EIP197_HIA_DFE_THR_CTRL(pe));
0764 
0765         /* Enable result descriptor rings */
0766         writel(EIP197_DxE_THR_CTRL_EN | GENMASK(priv->config.rings - 1, 0),
0767                EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_CTRL(pe));
0768     }
0769 
0770     /* Clear any HIA interrupt */
0771     writel(GENMASK(30, 20), EIP197_HIA_AIC_G(priv) + EIP197_HIA_AIC_G_ACK);
0772 
0773     if (priv->flags & EIP197_SIMPLE_TRC) {
0774         writel(EIP197_STRC_CONFIG_INIT |
0775                EIP197_STRC_CONFIG_LARGE_REC(EIP197_CS_TRC_REC_WC) |
0776                EIP197_STRC_CONFIG_SMALL_REC(EIP197_CS_TRC_REC_WC),
0777                priv->base + EIP197_STRC_CONFIG);
0778         writel(EIP197_PE_EIP96_TOKEN_CTRL2_CTX_DONE,
0779                EIP197_PE(priv) + EIP197_PE_EIP96_TOKEN_CTRL2(0));
0780     } else if (priv->flags & SAFEXCEL_HW_EIP197) {
0781         ret = eip197_trc_cache_init(priv);
0782         if (ret)
0783             return ret;
0784     }
0785 
0786     if (priv->flags & EIP197_ICE) {
0787         ret = eip197_load_firmwares(priv);
0788         if (ret)
0789             return ret;
0790     }
0791 
0792     return safexcel_hw_setup_cdesc_rings(priv) ?:
0793            safexcel_hw_setup_rdesc_rings(priv) ?:
0794            0;
0795 }
0796 
0797 /* Called with ring's lock taken */
0798 static void safexcel_try_push_requests(struct safexcel_crypto_priv *priv,
0799                        int ring)
0800 {
0801     int coal = min_t(int, priv->ring[ring].requests, EIP197_MAX_BATCH_SZ);
0802 
0803     if (!coal)
0804         return;
0805 
0806     /* Configure when we want an interrupt */
0807     writel(EIP197_HIA_RDR_THRESH_PKT_MODE |
0808            EIP197_HIA_RDR_THRESH_PROC_PKT(coal),
0809            EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_THRESH);
0810 }
0811 
0812 void safexcel_dequeue(struct safexcel_crypto_priv *priv, int ring)
0813 {
0814     struct crypto_async_request *req, *backlog;
0815     struct safexcel_context *ctx;
0816     int ret, nreq = 0, cdesc = 0, rdesc = 0, commands, results;
0817 
0818     /* If a request wasn't properly dequeued because of a lack of resources,
0819      * proceeded it first,
0820      */
0821     req = priv->ring[ring].req;
0822     backlog = priv->ring[ring].backlog;
0823     if (req)
0824         goto handle_req;
0825 
0826     while (true) {
0827         spin_lock_bh(&priv->ring[ring].queue_lock);
0828         backlog = crypto_get_backlog(&priv->ring[ring].queue);
0829         req = crypto_dequeue_request(&priv->ring[ring].queue);
0830         spin_unlock_bh(&priv->ring[ring].queue_lock);
0831 
0832         if (!req) {
0833             priv->ring[ring].req = NULL;
0834             priv->ring[ring].backlog = NULL;
0835             goto finalize;
0836         }
0837 
0838 handle_req:
0839         ctx = crypto_tfm_ctx(req->tfm);
0840         ret = ctx->send(req, ring, &commands, &results);
0841         if (ret)
0842             goto request_failed;
0843 
0844         if (backlog)
0845             backlog->complete(backlog, -EINPROGRESS);
0846 
0847         /* In case the send() helper did not issue any command to push
0848          * to the engine because the input data was cached, continue to
0849          * dequeue other requests as this is valid and not an error.
0850          */
0851         if (!commands && !results)
0852             continue;
0853 
0854         cdesc += commands;
0855         rdesc += results;
0856         nreq++;
0857     }
0858 
0859 request_failed:
0860     /* Not enough resources to handle all the requests. Bail out and save
0861      * the request and the backlog for the next dequeue call (per-ring).
0862      */
0863     priv->ring[ring].req = req;
0864     priv->ring[ring].backlog = backlog;
0865 
0866 finalize:
0867     if (!nreq)
0868         return;
0869 
0870     spin_lock_bh(&priv->ring[ring].lock);
0871 
0872     priv->ring[ring].requests += nreq;
0873 
0874     if (!priv->ring[ring].busy) {
0875         safexcel_try_push_requests(priv, ring);
0876         priv->ring[ring].busy = true;
0877     }
0878 
0879     spin_unlock_bh(&priv->ring[ring].lock);
0880 
0881     /* let the RDR know we have pending descriptors */
0882     writel((rdesc * priv->config.rd_offset),
0883            EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_PREP_COUNT);
0884 
0885     /* let the CDR know we have pending descriptors */
0886     writel((cdesc * priv->config.cd_offset),
0887            EIP197_HIA_CDR(priv, ring) + EIP197_HIA_xDR_PREP_COUNT);
0888 }
0889 
0890 inline int safexcel_rdesc_check_errors(struct safexcel_crypto_priv *priv,
0891                        void *rdp)
0892 {
0893     struct safexcel_result_desc *rdesc = rdp;
0894     struct result_data_desc *result_data = rdp + priv->config.res_offset;
0895 
0896     if (likely((!rdesc->last_seg) || /* Rest only valid if last seg! */
0897            ((!rdesc->descriptor_overflow) &&
0898             (!rdesc->buffer_overflow) &&
0899             (!result_data->error_code))))
0900         return 0;
0901 
0902     if (rdesc->descriptor_overflow)
0903         dev_err(priv->dev, "Descriptor overflow detected");
0904 
0905     if (rdesc->buffer_overflow)
0906         dev_err(priv->dev, "Buffer overflow detected");
0907 
0908     if (result_data->error_code & 0x4066) {
0909         /* Fatal error (bits 1,2,5,6 & 14) */
0910         dev_err(priv->dev,
0911             "result descriptor error (%x)",
0912             result_data->error_code);
0913 
0914         return -EIO;
0915     } else if (result_data->error_code &
0916            (BIT(7) | BIT(4) | BIT(3) | BIT(0))) {
0917         /*
0918          * Give priority over authentication fails:
0919          * Blocksize, length & overflow errors,
0920          * something wrong with the input!
0921          */
0922         return -EINVAL;
0923     } else if (result_data->error_code & BIT(9)) {
0924         /* Authentication failed */
0925         return -EBADMSG;
0926     }
0927 
0928     /* All other non-fatal errors */
0929     return -EINVAL;
0930 }
0931 
0932 inline void safexcel_rdr_req_set(struct safexcel_crypto_priv *priv,
0933                  int ring,
0934                  struct safexcel_result_desc *rdesc,
0935                  struct crypto_async_request *req)
0936 {
0937     int i = safexcel_ring_rdr_rdesc_index(priv, ring, rdesc);
0938 
0939     priv->ring[ring].rdr_req[i] = req;
0940 }
0941 
0942 inline struct crypto_async_request *
0943 safexcel_rdr_req_get(struct safexcel_crypto_priv *priv, int ring)
0944 {
0945     int i = safexcel_ring_first_rdr_index(priv, ring);
0946 
0947     return priv->ring[ring].rdr_req[i];
0948 }
0949 
0950 void safexcel_complete(struct safexcel_crypto_priv *priv, int ring)
0951 {
0952     struct safexcel_command_desc *cdesc;
0953 
0954     /* Acknowledge the command descriptors */
0955     do {
0956         cdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].cdr);
0957         if (IS_ERR(cdesc)) {
0958             dev_err(priv->dev,
0959                 "Could not retrieve the command descriptor\n");
0960             return;
0961         }
0962     } while (!cdesc->last_seg);
0963 }
0964 
0965 void safexcel_inv_complete(struct crypto_async_request *req, int error)
0966 {
0967     struct safexcel_inv_result *result = req->data;
0968 
0969     if (error == -EINPROGRESS)
0970         return;
0971 
0972     result->error = error;
0973     complete(&result->completion);
0974 }
0975 
0976 int safexcel_invalidate_cache(struct crypto_async_request *async,
0977                   struct safexcel_crypto_priv *priv,
0978                   dma_addr_t ctxr_dma, int ring)
0979 {
0980     struct safexcel_command_desc *cdesc;
0981     struct safexcel_result_desc *rdesc;
0982     struct safexcel_token  *dmmy;
0983     int ret = 0;
0984 
0985     /* Prepare command descriptor */
0986     cdesc = safexcel_add_cdesc(priv, ring, true, true, 0, 0, 0, ctxr_dma,
0987                    &dmmy);
0988     if (IS_ERR(cdesc))
0989         return PTR_ERR(cdesc);
0990 
0991     cdesc->control_data.type = EIP197_TYPE_EXTENDED;
0992     cdesc->control_data.options = 0;
0993     cdesc->control_data.context_lo &= ~EIP197_CONTEXT_SIZE_MASK;
0994     cdesc->control_data.control0 = CONTEXT_CONTROL_INV_TR;
0995 
0996     /* Prepare result descriptor */
0997     rdesc = safexcel_add_rdesc(priv, ring, true, true, 0, 0);
0998 
0999     if (IS_ERR(rdesc)) {
1000         ret = PTR_ERR(rdesc);
1001         goto cdesc_rollback;
1002     }
1003 
1004     safexcel_rdr_req_set(priv, ring, rdesc, async);
1005 
1006     return ret;
1007 
1008 cdesc_rollback:
1009     safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr);
1010 
1011     return ret;
1012 }
1013 
1014 static inline void safexcel_handle_result_descriptor(struct safexcel_crypto_priv *priv,
1015                              int ring)
1016 {
1017     struct crypto_async_request *req;
1018     struct safexcel_context *ctx;
1019     int ret, i, nreq, ndesc, tot_descs, handled = 0;
1020     bool should_complete;
1021 
1022 handle_results:
1023     tot_descs = 0;
1024 
1025     nreq = readl(EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_PROC_COUNT);
1026     nreq >>= EIP197_xDR_PROC_xD_PKT_OFFSET;
1027     nreq &= EIP197_xDR_PROC_xD_PKT_MASK;
1028     if (!nreq)
1029         goto requests_left;
1030 
1031     for (i = 0; i < nreq; i++) {
1032         req = safexcel_rdr_req_get(priv, ring);
1033 
1034         ctx = crypto_tfm_ctx(req->tfm);
1035         ndesc = ctx->handle_result(priv, ring, req,
1036                        &should_complete, &ret);
1037         if (ndesc < 0) {
1038             dev_err(priv->dev, "failed to handle result (%d)\n",
1039                 ndesc);
1040             goto acknowledge;
1041         }
1042 
1043         if (should_complete) {
1044             local_bh_disable();
1045             req->complete(req, ret);
1046             local_bh_enable();
1047         }
1048 
1049         tot_descs += ndesc;
1050         handled++;
1051     }
1052 
1053 acknowledge:
1054     if (i)
1055         writel(EIP197_xDR_PROC_xD_PKT(i) |
1056                (tot_descs * priv->config.rd_offset),
1057                EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_PROC_COUNT);
1058 
1059     /* If the number of requests overflowed the counter, try to proceed more
1060      * requests.
1061      */
1062     if (nreq == EIP197_xDR_PROC_xD_PKT_MASK)
1063         goto handle_results;
1064 
1065 requests_left:
1066     spin_lock_bh(&priv->ring[ring].lock);
1067 
1068     priv->ring[ring].requests -= handled;
1069     safexcel_try_push_requests(priv, ring);
1070 
1071     if (!priv->ring[ring].requests)
1072         priv->ring[ring].busy = false;
1073 
1074     spin_unlock_bh(&priv->ring[ring].lock);
1075 }
1076 
1077 static void safexcel_dequeue_work(struct work_struct *work)
1078 {
1079     struct safexcel_work_data *data =
1080             container_of(work, struct safexcel_work_data, work);
1081 
1082     safexcel_dequeue(data->priv, data->ring);
1083 }
1084 
1085 struct safexcel_ring_irq_data {
1086     struct safexcel_crypto_priv *priv;
1087     int ring;
1088 };
1089 
1090 static irqreturn_t safexcel_irq_ring(int irq, void *data)
1091 {
1092     struct safexcel_ring_irq_data *irq_data = data;
1093     struct safexcel_crypto_priv *priv = irq_data->priv;
1094     int ring = irq_data->ring, rc = IRQ_NONE;
1095     u32 status, stat;
1096 
1097     status = readl(EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLED_STAT(ring));
1098     if (!status)
1099         return rc;
1100 
1101     /* RDR interrupts */
1102     if (status & EIP197_RDR_IRQ(ring)) {
1103         stat = readl(EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_STAT);
1104 
1105         if (unlikely(stat & EIP197_xDR_ERR)) {
1106             /*
1107              * Fatal error, the RDR is unusable and must be
1108              * reinitialized. This should not happen under
1109              * normal circumstances.
1110              */
1111             dev_err(priv->dev, "RDR: fatal error.\n");
1112         } else if (likely(stat & EIP197_xDR_THRESH)) {
1113             rc = IRQ_WAKE_THREAD;
1114         }
1115 
1116         /* ACK the interrupts */
1117         writel(stat & 0xff,
1118                EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_STAT);
1119     }
1120 
1121     /* ACK the interrupts */
1122     writel(status, EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ACK(ring));
1123 
1124     return rc;
1125 }
1126 
1127 static irqreturn_t safexcel_irq_ring_thread(int irq, void *data)
1128 {
1129     struct safexcel_ring_irq_data *irq_data = data;
1130     struct safexcel_crypto_priv *priv = irq_data->priv;
1131     int ring = irq_data->ring;
1132 
1133     safexcel_handle_result_descriptor(priv, ring);
1134 
1135     queue_work(priv->ring[ring].workqueue,
1136            &priv->ring[ring].work_data.work);
1137 
1138     return IRQ_HANDLED;
1139 }
1140 
1141 static int safexcel_request_ring_irq(void *pdev, int irqid,
1142                      int is_pci_dev,
1143                      int ring_id,
1144                      irq_handler_t handler,
1145                      irq_handler_t threaded_handler,
1146                      struct safexcel_ring_irq_data *ring_irq_priv)
1147 {
1148     int ret, irq, cpu;
1149     struct device *dev;
1150 
1151     if (IS_ENABLED(CONFIG_PCI) && is_pci_dev) {
1152         struct pci_dev *pci_pdev = pdev;
1153 
1154         dev = &pci_pdev->dev;
1155         irq = pci_irq_vector(pci_pdev, irqid);
1156         if (irq < 0) {
1157             dev_err(dev, "unable to get device MSI IRQ %d (err %d)\n",
1158                 irqid, irq);
1159             return irq;
1160         }
1161     } else if (IS_ENABLED(CONFIG_OF)) {
1162         struct platform_device *plf_pdev = pdev;
1163         char irq_name[6] = {0}; /* "ringX\0" */
1164 
1165         snprintf(irq_name, 6, "ring%d", irqid);
1166         dev = &plf_pdev->dev;
1167         irq = platform_get_irq_byname(plf_pdev, irq_name);
1168 
1169         if (irq < 0)
1170             return irq;
1171     } else {
1172         return -ENXIO;
1173     }
1174 
1175     ret = devm_request_threaded_irq(dev, irq, handler,
1176                     threaded_handler, IRQF_ONESHOT,
1177                     dev_name(dev), ring_irq_priv);
1178     if (ret) {
1179         dev_err(dev, "unable to request IRQ %d\n", irq);
1180         return ret;
1181     }
1182 
1183     /* Set affinity */
1184     cpu = cpumask_local_spread(ring_id, NUMA_NO_NODE);
1185     irq_set_affinity_hint(irq, get_cpu_mask(cpu));
1186 
1187     return irq;
1188 }
1189 
1190 static struct safexcel_alg_template *safexcel_algs[] = {
1191     &safexcel_alg_ecb_des,
1192     &safexcel_alg_cbc_des,
1193     &safexcel_alg_ecb_des3_ede,
1194     &safexcel_alg_cbc_des3_ede,
1195     &safexcel_alg_ecb_aes,
1196     &safexcel_alg_cbc_aes,
1197     &safexcel_alg_cfb_aes,
1198     &safexcel_alg_ofb_aes,
1199     &safexcel_alg_ctr_aes,
1200     &safexcel_alg_md5,
1201     &safexcel_alg_sha1,
1202     &safexcel_alg_sha224,
1203     &safexcel_alg_sha256,
1204     &safexcel_alg_sha384,
1205     &safexcel_alg_sha512,
1206     &safexcel_alg_hmac_md5,
1207     &safexcel_alg_hmac_sha1,
1208     &safexcel_alg_hmac_sha224,
1209     &safexcel_alg_hmac_sha256,
1210     &safexcel_alg_hmac_sha384,
1211     &safexcel_alg_hmac_sha512,
1212     &safexcel_alg_authenc_hmac_sha1_cbc_aes,
1213     &safexcel_alg_authenc_hmac_sha224_cbc_aes,
1214     &safexcel_alg_authenc_hmac_sha256_cbc_aes,
1215     &safexcel_alg_authenc_hmac_sha384_cbc_aes,
1216     &safexcel_alg_authenc_hmac_sha512_cbc_aes,
1217     &safexcel_alg_authenc_hmac_sha1_cbc_des3_ede,
1218     &safexcel_alg_authenc_hmac_sha1_ctr_aes,
1219     &safexcel_alg_authenc_hmac_sha224_ctr_aes,
1220     &safexcel_alg_authenc_hmac_sha256_ctr_aes,
1221     &safexcel_alg_authenc_hmac_sha384_ctr_aes,
1222     &safexcel_alg_authenc_hmac_sha512_ctr_aes,
1223     &safexcel_alg_xts_aes,
1224     &safexcel_alg_gcm,
1225     &safexcel_alg_ccm,
1226     &safexcel_alg_crc32,
1227     &safexcel_alg_cbcmac,
1228     &safexcel_alg_xcbcmac,
1229     &safexcel_alg_cmac,
1230     &safexcel_alg_chacha20,
1231     &safexcel_alg_chachapoly,
1232     &safexcel_alg_chachapoly_esp,
1233     &safexcel_alg_sm3,
1234     &safexcel_alg_hmac_sm3,
1235     &safexcel_alg_ecb_sm4,
1236     &safexcel_alg_cbc_sm4,
1237     &safexcel_alg_ofb_sm4,
1238     &safexcel_alg_cfb_sm4,
1239     &safexcel_alg_ctr_sm4,
1240     &safexcel_alg_authenc_hmac_sha1_cbc_sm4,
1241     &safexcel_alg_authenc_hmac_sm3_cbc_sm4,
1242     &safexcel_alg_authenc_hmac_sha1_ctr_sm4,
1243     &safexcel_alg_authenc_hmac_sm3_ctr_sm4,
1244     &safexcel_alg_sha3_224,
1245     &safexcel_alg_sha3_256,
1246     &safexcel_alg_sha3_384,
1247     &safexcel_alg_sha3_512,
1248     &safexcel_alg_hmac_sha3_224,
1249     &safexcel_alg_hmac_sha3_256,
1250     &safexcel_alg_hmac_sha3_384,
1251     &safexcel_alg_hmac_sha3_512,
1252     &safexcel_alg_authenc_hmac_sha1_cbc_des,
1253     &safexcel_alg_authenc_hmac_sha256_cbc_des3_ede,
1254     &safexcel_alg_authenc_hmac_sha224_cbc_des3_ede,
1255     &safexcel_alg_authenc_hmac_sha512_cbc_des3_ede,
1256     &safexcel_alg_authenc_hmac_sha384_cbc_des3_ede,
1257     &safexcel_alg_authenc_hmac_sha256_cbc_des,
1258     &safexcel_alg_authenc_hmac_sha224_cbc_des,
1259     &safexcel_alg_authenc_hmac_sha512_cbc_des,
1260     &safexcel_alg_authenc_hmac_sha384_cbc_des,
1261     &safexcel_alg_rfc4106_gcm,
1262     &safexcel_alg_rfc4543_gcm,
1263     &safexcel_alg_rfc4309_ccm,
1264 };
1265 
1266 static int safexcel_register_algorithms(struct safexcel_crypto_priv *priv)
1267 {
1268     int i, j, ret = 0;
1269 
1270     for (i = 0; i < ARRAY_SIZE(safexcel_algs); i++) {
1271         safexcel_algs[i]->priv = priv;
1272 
1273         /* Do we have all required base algorithms available? */
1274         if ((safexcel_algs[i]->algo_mask & priv->hwconfig.algo_flags) !=
1275             safexcel_algs[i]->algo_mask)
1276             /* No, so don't register this ciphersuite */
1277             continue;
1278 
1279         if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_SKCIPHER)
1280             ret = crypto_register_skcipher(&safexcel_algs[i]->alg.skcipher);
1281         else if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_AEAD)
1282             ret = crypto_register_aead(&safexcel_algs[i]->alg.aead);
1283         else
1284             ret = crypto_register_ahash(&safexcel_algs[i]->alg.ahash);
1285 
1286         if (ret)
1287             goto fail;
1288     }
1289 
1290     return 0;
1291 
1292 fail:
1293     for (j = 0; j < i; j++) {
1294         /* Do we have all required base algorithms available? */
1295         if ((safexcel_algs[j]->algo_mask & priv->hwconfig.algo_flags) !=
1296             safexcel_algs[j]->algo_mask)
1297             /* No, so don't unregister this ciphersuite */
1298             continue;
1299 
1300         if (safexcel_algs[j]->type == SAFEXCEL_ALG_TYPE_SKCIPHER)
1301             crypto_unregister_skcipher(&safexcel_algs[j]->alg.skcipher);
1302         else if (safexcel_algs[j]->type == SAFEXCEL_ALG_TYPE_AEAD)
1303             crypto_unregister_aead(&safexcel_algs[j]->alg.aead);
1304         else
1305             crypto_unregister_ahash(&safexcel_algs[j]->alg.ahash);
1306     }
1307 
1308     return ret;
1309 }
1310 
1311 static void safexcel_unregister_algorithms(struct safexcel_crypto_priv *priv)
1312 {
1313     int i;
1314 
1315     for (i = 0; i < ARRAY_SIZE(safexcel_algs); i++) {
1316         /* Do we have all required base algorithms available? */
1317         if ((safexcel_algs[i]->algo_mask & priv->hwconfig.algo_flags) !=
1318             safexcel_algs[i]->algo_mask)
1319             /* No, so don't unregister this ciphersuite */
1320             continue;
1321 
1322         if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_SKCIPHER)
1323             crypto_unregister_skcipher(&safexcel_algs[i]->alg.skcipher);
1324         else if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_AEAD)
1325             crypto_unregister_aead(&safexcel_algs[i]->alg.aead);
1326         else
1327             crypto_unregister_ahash(&safexcel_algs[i]->alg.ahash);
1328     }
1329 }
1330 
1331 static void safexcel_configure(struct safexcel_crypto_priv *priv)
1332 {
1333     u32 mask = BIT(priv->hwconfig.hwdataw) - 1;
1334 
1335     priv->config.pes = priv->hwconfig.hwnumpes;
1336     priv->config.rings = min_t(u32, priv->hwconfig.hwnumrings, max_rings);
1337     /* Cannot currently support more rings than we have ring AICs! */
1338     priv->config.rings = min_t(u32, priv->config.rings,
1339                     priv->hwconfig.hwnumraic);
1340 
1341     priv->config.cd_size = EIP197_CD64_FETCH_SIZE;
1342     priv->config.cd_offset = (priv->config.cd_size + mask) & ~mask;
1343     priv->config.cdsh_offset = (EIP197_MAX_TOKENS + mask) & ~mask;
1344 
1345     /* res token is behind the descr, but ofs must be rounded to buswdth */
1346     priv->config.res_offset = (EIP197_RD64_FETCH_SIZE + mask) & ~mask;
1347     /* now the size of the descr is this 1st part plus the result struct */
1348     priv->config.rd_size    = priv->config.res_offset +
1349                   EIP197_RD64_RESULT_SIZE;
1350     priv->config.rd_offset = (priv->config.rd_size + mask) & ~mask;
1351 
1352     /* convert dwords to bytes */
1353     priv->config.cd_offset *= sizeof(u32);
1354     priv->config.cdsh_offset *= sizeof(u32);
1355     priv->config.rd_offset *= sizeof(u32);
1356     priv->config.res_offset *= sizeof(u32);
1357 }
1358 
1359 static void safexcel_init_register_offsets(struct safexcel_crypto_priv *priv)
1360 {
1361     struct safexcel_register_offsets *offsets = &priv->offsets;
1362 
1363     if (priv->flags & SAFEXCEL_HW_EIP197) {
1364         offsets->hia_aic    = EIP197_HIA_AIC_BASE;
1365         offsets->hia_aic_g  = EIP197_HIA_AIC_G_BASE;
1366         offsets->hia_aic_r  = EIP197_HIA_AIC_R_BASE;
1367         offsets->hia_aic_xdr    = EIP197_HIA_AIC_xDR_BASE;
1368         offsets->hia_dfe    = EIP197_HIA_DFE_BASE;
1369         offsets->hia_dfe_thr    = EIP197_HIA_DFE_THR_BASE;
1370         offsets->hia_dse    = EIP197_HIA_DSE_BASE;
1371         offsets->hia_dse_thr    = EIP197_HIA_DSE_THR_BASE;
1372         offsets->hia_gen_cfg    = EIP197_HIA_GEN_CFG_BASE;
1373         offsets->pe     = EIP197_PE_BASE;
1374         offsets->global     = EIP197_GLOBAL_BASE;
1375     } else {
1376         offsets->hia_aic    = EIP97_HIA_AIC_BASE;
1377         offsets->hia_aic_g  = EIP97_HIA_AIC_G_BASE;
1378         offsets->hia_aic_r  = EIP97_HIA_AIC_R_BASE;
1379         offsets->hia_aic_xdr    = EIP97_HIA_AIC_xDR_BASE;
1380         offsets->hia_dfe    = EIP97_HIA_DFE_BASE;
1381         offsets->hia_dfe_thr    = EIP97_HIA_DFE_THR_BASE;
1382         offsets->hia_dse    = EIP97_HIA_DSE_BASE;
1383         offsets->hia_dse_thr    = EIP97_HIA_DSE_THR_BASE;
1384         offsets->hia_gen_cfg    = EIP97_HIA_GEN_CFG_BASE;
1385         offsets->pe     = EIP97_PE_BASE;
1386         offsets->global     = EIP97_GLOBAL_BASE;
1387     }
1388 }
1389 
1390 /*
1391  * Generic part of probe routine, shared by platform and PCI driver
1392  *
1393  * Assumes IO resources have been mapped, private data mem has been allocated,
1394  * clocks have been enabled, device pointer has been assigned etc.
1395  *
1396  */
1397 static int safexcel_probe_generic(void *pdev,
1398                   struct safexcel_crypto_priv *priv,
1399                   int is_pci_dev)
1400 {
1401     struct device *dev = priv->dev;
1402     u32 peid, version, mask, val, hiaopt, hwopt, peopt;
1403     int i, ret, hwctg;
1404 
1405     priv->context_pool = dmam_pool_create("safexcel-context", dev,
1406                           sizeof(struct safexcel_context_record),
1407                           1, 0);
1408     if (!priv->context_pool)
1409         return -ENOMEM;
1410 
1411     /*
1412      * First try the EIP97 HIA version regs
1413      * For the EIP197, this is guaranteed to NOT return any of the test
1414      * values
1415      */
1416     version = readl(priv->base + EIP97_HIA_AIC_BASE + EIP197_HIA_VERSION);
1417 
1418     mask = 0;  /* do not swap */
1419     if (EIP197_REG_LO16(version) == EIP197_HIA_VERSION_LE) {
1420         priv->hwconfig.hiaver = EIP197_VERSION_MASK(version);
1421     } else if (EIP197_REG_HI16(version) == EIP197_HIA_VERSION_BE) {
1422         /* read back byte-swapped, so complement byte swap bits */
1423         mask = EIP197_MST_CTRL_BYTE_SWAP_BITS;
1424         priv->hwconfig.hiaver = EIP197_VERSION_SWAP(version);
1425     } else {
1426         /* So it wasn't an EIP97 ... maybe it's an EIP197? */
1427         version = readl(priv->base + EIP197_HIA_AIC_BASE +
1428                 EIP197_HIA_VERSION);
1429         if (EIP197_REG_LO16(version) == EIP197_HIA_VERSION_LE) {
1430             priv->hwconfig.hiaver = EIP197_VERSION_MASK(version);
1431             priv->flags |= SAFEXCEL_HW_EIP197;
1432         } else if (EIP197_REG_HI16(version) ==
1433                EIP197_HIA_VERSION_BE) {
1434             /* read back byte-swapped, so complement swap bits */
1435             mask = EIP197_MST_CTRL_BYTE_SWAP_BITS;
1436             priv->hwconfig.hiaver = EIP197_VERSION_SWAP(version);
1437             priv->flags |= SAFEXCEL_HW_EIP197;
1438         } else {
1439             return -ENODEV;
1440         }
1441     }
1442 
1443     /* Now initialize the reg offsets based on the probing info so far */
1444     safexcel_init_register_offsets(priv);
1445 
1446     /*
1447      * If the version was read byte-swapped, we need to flip the device
1448      * swapping Keep in mind here, though, that what we write will also be
1449      * byte-swapped ...
1450      */
1451     if (mask) {
1452         val = readl(EIP197_HIA_AIC(priv) + EIP197_HIA_MST_CTRL);
1453         val = val ^ (mask >> 24); /* toggle byte swap bits */
1454         writel(val, EIP197_HIA_AIC(priv) + EIP197_HIA_MST_CTRL);
1455     }
1456 
1457     /*
1458      * We're not done probing yet! We may fall through to here if no HIA
1459      * was found at all. So, with the endianness presumably correct now and
1460      * the offsets setup, *really* probe for the EIP97/EIP197.
1461      */
1462     version = readl(EIP197_GLOBAL(priv) + EIP197_VERSION);
1463     if (((priv->flags & SAFEXCEL_HW_EIP197) &&
1464          (EIP197_REG_LO16(version) != EIP197_VERSION_LE) &&
1465          (EIP197_REG_LO16(version) != EIP196_VERSION_LE)) ||
1466         ((!(priv->flags & SAFEXCEL_HW_EIP197) &&
1467          (EIP197_REG_LO16(version) != EIP97_VERSION_LE)))) {
1468         /*
1469          * We did not find the device that matched our initial probing
1470          * (or our initial probing failed) Report appropriate error.
1471          */
1472         dev_err(priv->dev, "Probing for EIP97/EIP19x failed - no such device (read %08x)\n",
1473             version);
1474         return -ENODEV;
1475     }
1476 
1477     priv->hwconfig.hwver = EIP197_VERSION_MASK(version);
1478     hwctg = version >> 28;
1479     peid = version & 255;
1480 
1481     /* Detect EIP206 processing pipe */
1482     version = readl(EIP197_PE(priv) + + EIP197_PE_VERSION(0));
1483     if (EIP197_REG_LO16(version) != EIP206_VERSION_LE) {
1484         dev_err(priv->dev, "EIP%d: EIP206 not detected\n", peid);
1485         return -ENODEV;
1486     }
1487     priv->hwconfig.ppver = EIP197_VERSION_MASK(version);
1488 
1489     /* Detect EIP96 packet engine and version */
1490     version = readl(EIP197_PE(priv) + EIP197_PE_EIP96_VERSION(0));
1491     if (EIP197_REG_LO16(version) != EIP96_VERSION_LE) {
1492         dev_err(dev, "EIP%d: EIP96 not detected.\n", peid);
1493         return -ENODEV;
1494     }
1495     priv->hwconfig.pever = EIP197_VERSION_MASK(version);
1496 
1497     hwopt = readl(EIP197_GLOBAL(priv) + EIP197_OPTIONS);
1498     hiaopt = readl(EIP197_HIA_AIC(priv) + EIP197_HIA_OPTIONS);
1499 
1500     priv->hwconfig.icever = 0;
1501     priv->hwconfig.ocever = 0;
1502     priv->hwconfig.psever = 0;
1503     if (priv->flags & SAFEXCEL_HW_EIP197) {
1504         /* EIP197 */
1505         peopt = readl(EIP197_PE(priv) + EIP197_PE_OPTIONS(0));
1506 
1507         priv->hwconfig.hwdataw  = (hiaopt >> EIP197_HWDATAW_OFFSET) &
1508                       EIP197_HWDATAW_MASK;
1509         priv->hwconfig.hwcfsize = ((hiaopt >> EIP197_CFSIZE_OFFSET) &
1510                        EIP197_CFSIZE_MASK) +
1511                       EIP197_CFSIZE_ADJUST;
1512         priv->hwconfig.hwrfsize = ((hiaopt >> EIP197_RFSIZE_OFFSET) &
1513                        EIP197_RFSIZE_MASK) +
1514                       EIP197_RFSIZE_ADJUST;
1515         priv->hwconfig.hwnumpes = (hiaopt >> EIP197_N_PES_OFFSET) &
1516                       EIP197_N_PES_MASK;
1517         priv->hwconfig.hwnumrings = (hiaopt >> EIP197_N_RINGS_OFFSET) &
1518                         EIP197_N_RINGS_MASK;
1519         if (hiaopt & EIP197_HIA_OPT_HAS_PE_ARB)
1520             priv->flags |= EIP197_PE_ARB;
1521         if (EIP206_OPT_ICE_TYPE(peopt) == 1) {
1522             priv->flags |= EIP197_ICE;
1523             /* Detect ICE EIP207 class. engine and version */
1524             version = readl(EIP197_PE(priv) +
1525                   EIP197_PE_ICE_VERSION(0));
1526             if (EIP197_REG_LO16(version) != EIP207_VERSION_LE) {
1527                 dev_err(dev, "EIP%d: ICE EIP207 not detected.\n",
1528                     peid);
1529                 return -ENODEV;
1530             }
1531             priv->hwconfig.icever = EIP197_VERSION_MASK(version);
1532         }
1533         if (EIP206_OPT_OCE_TYPE(peopt) == 1) {
1534             priv->flags |= EIP197_OCE;
1535             /* Detect EIP96PP packet stream editor and version */
1536             version = readl(EIP197_PE(priv) + EIP197_PE_PSE_VERSION(0));
1537             if (EIP197_REG_LO16(version) != EIP96_VERSION_LE) {
1538                 dev_err(dev, "EIP%d: EIP96PP not detected.\n", peid);
1539                 return -ENODEV;
1540             }
1541             priv->hwconfig.psever = EIP197_VERSION_MASK(version);
1542             /* Detect OCE EIP207 class. engine and version */
1543             version = readl(EIP197_PE(priv) +
1544                   EIP197_PE_ICE_VERSION(0));
1545             if (EIP197_REG_LO16(version) != EIP207_VERSION_LE) {
1546                 dev_err(dev, "EIP%d: OCE EIP207 not detected.\n",
1547                     peid);
1548                 return -ENODEV;
1549             }
1550             priv->hwconfig.ocever = EIP197_VERSION_MASK(version);
1551         }
1552         /* If not a full TRC, then assume simple TRC */
1553         if (!(hwopt & EIP197_OPT_HAS_TRC))
1554             priv->flags |= EIP197_SIMPLE_TRC;
1555         /* EIP197 always has SOME form of TRC */
1556         priv->flags |= EIP197_TRC_CACHE;
1557     } else {
1558         /* EIP97 */
1559         priv->hwconfig.hwdataw  = (hiaopt >> EIP197_HWDATAW_OFFSET) &
1560                       EIP97_HWDATAW_MASK;
1561         priv->hwconfig.hwcfsize = (hiaopt >> EIP97_CFSIZE_OFFSET) &
1562                       EIP97_CFSIZE_MASK;
1563         priv->hwconfig.hwrfsize = (hiaopt >> EIP97_RFSIZE_OFFSET) &
1564                       EIP97_RFSIZE_MASK;
1565         priv->hwconfig.hwnumpes = 1; /* by definition */
1566         priv->hwconfig.hwnumrings = (hiaopt >> EIP197_N_RINGS_OFFSET) &
1567                         EIP197_N_RINGS_MASK;
1568     }
1569 
1570     /* Scan for ring AIC's */
1571     for (i = 0; i < EIP197_MAX_RING_AIC; i++) {
1572         version = readl(EIP197_HIA_AIC_R(priv) +
1573                 EIP197_HIA_AIC_R_VERSION(i));
1574         if (EIP197_REG_LO16(version) != EIP201_VERSION_LE)
1575             break;
1576     }
1577     priv->hwconfig.hwnumraic = i;
1578     /* Low-end EIP196 may not have any ring AIC's ... */
1579     if (!priv->hwconfig.hwnumraic) {
1580         dev_err(priv->dev, "No ring interrupt controller present!\n");
1581         return -ENODEV;
1582     }
1583 
1584     /* Get supported algorithms from EIP96 transform engine */
1585     priv->hwconfig.algo_flags = readl(EIP197_PE(priv) +
1586                     EIP197_PE_EIP96_OPTIONS(0));
1587 
1588     /* Print single info line describing what we just detected */
1589     dev_info(priv->dev, "EIP%d:%x(%d,%d,%d,%d)-HIA:%x(%d,%d,%d),PE:%x/%x(alg:%08x)/%x/%x/%x\n",
1590          peid, priv->hwconfig.hwver, hwctg, priv->hwconfig.hwnumpes,
1591          priv->hwconfig.hwnumrings, priv->hwconfig.hwnumraic,
1592          priv->hwconfig.hiaver, priv->hwconfig.hwdataw,
1593          priv->hwconfig.hwcfsize, priv->hwconfig.hwrfsize,
1594          priv->hwconfig.ppver, priv->hwconfig.pever,
1595          priv->hwconfig.algo_flags, priv->hwconfig.icever,
1596          priv->hwconfig.ocever, priv->hwconfig.psever);
1597 
1598     safexcel_configure(priv);
1599 
1600     if (IS_ENABLED(CONFIG_PCI) && priv->version == EIP197_DEVBRD) {
1601         /*
1602          * Request MSI vectors for global + 1 per ring -
1603          * or just 1 for older dev images
1604          */
1605         struct pci_dev *pci_pdev = pdev;
1606 
1607         ret = pci_alloc_irq_vectors(pci_pdev,
1608                         priv->config.rings + 1,
1609                         priv->config.rings + 1,
1610                         PCI_IRQ_MSI | PCI_IRQ_MSIX);
1611         if (ret < 0) {
1612             dev_err(dev, "Failed to allocate PCI MSI interrupts\n");
1613             return ret;
1614         }
1615     }
1616 
1617     /* Register the ring IRQ handlers and configure the rings */
1618     priv->ring = devm_kcalloc(dev, priv->config.rings,
1619                   sizeof(*priv->ring),
1620                   GFP_KERNEL);
1621     if (!priv->ring)
1622         return -ENOMEM;
1623 
1624     for (i = 0; i < priv->config.rings; i++) {
1625         char wq_name[9] = {0};
1626         int irq;
1627         struct safexcel_ring_irq_data *ring_irq;
1628 
1629         ret = safexcel_init_ring_descriptors(priv,
1630                              &priv->ring[i].cdr,
1631                              &priv->ring[i].rdr);
1632         if (ret) {
1633             dev_err(dev, "Failed to initialize rings\n");
1634             return ret;
1635         }
1636 
1637         priv->ring[i].rdr_req = devm_kcalloc(dev,
1638             EIP197_DEFAULT_RING_SIZE,
1639             sizeof(*priv->ring[i].rdr_req),
1640             GFP_KERNEL);
1641         if (!priv->ring[i].rdr_req)
1642             return -ENOMEM;
1643 
1644         ring_irq = devm_kzalloc(dev, sizeof(*ring_irq), GFP_KERNEL);
1645         if (!ring_irq)
1646             return -ENOMEM;
1647 
1648         ring_irq->priv = priv;
1649         ring_irq->ring = i;
1650 
1651         irq = safexcel_request_ring_irq(pdev,
1652                         EIP197_IRQ_NUMBER(i, is_pci_dev),
1653                         is_pci_dev,
1654                         i,
1655                         safexcel_irq_ring,
1656                         safexcel_irq_ring_thread,
1657                         ring_irq);
1658         if (irq < 0) {
1659             dev_err(dev, "Failed to get IRQ ID for ring %d\n", i);
1660             return irq;
1661         }
1662 
1663         priv->ring[i].irq = irq;
1664         priv->ring[i].work_data.priv = priv;
1665         priv->ring[i].work_data.ring = i;
1666         INIT_WORK(&priv->ring[i].work_data.work,
1667               safexcel_dequeue_work);
1668 
1669         snprintf(wq_name, 9, "wq_ring%d", i);
1670         priv->ring[i].workqueue =
1671             create_singlethread_workqueue(wq_name);
1672         if (!priv->ring[i].workqueue)
1673             return -ENOMEM;
1674 
1675         priv->ring[i].requests = 0;
1676         priv->ring[i].busy = false;
1677 
1678         crypto_init_queue(&priv->ring[i].queue,
1679                   EIP197_DEFAULT_RING_SIZE);
1680 
1681         spin_lock_init(&priv->ring[i].lock);
1682         spin_lock_init(&priv->ring[i].queue_lock);
1683     }
1684 
1685     atomic_set(&priv->ring_used, 0);
1686 
1687     ret = safexcel_hw_init(priv);
1688     if (ret) {
1689         dev_err(dev, "HW init failed (%d)\n", ret);
1690         return ret;
1691     }
1692 
1693     ret = safexcel_register_algorithms(priv);
1694     if (ret) {
1695         dev_err(dev, "Failed to register algorithms (%d)\n", ret);
1696         return ret;
1697     }
1698 
1699     return 0;
1700 }
1701 
1702 static void safexcel_hw_reset_rings(struct safexcel_crypto_priv *priv)
1703 {
1704     int i;
1705 
1706     for (i = 0; i < priv->config.rings; i++) {
1707         /* clear any pending interrupt */
1708         writel(GENMASK(5, 0), EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_STAT);
1709         writel(GENMASK(7, 0), EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_STAT);
1710 
1711         /* Reset the CDR base address */
1712         writel(0, EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO);
1713         writel(0, EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI);
1714 
1715         /* Reset the RDR base address */
1716         writel(0, EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO);
1717         writel(0, EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI);
1718     }
1719 }
1720 
1721 /* for Device Tree platform driver */
1722 
1723 static int safexcel_probe(struct platform_device *pdev)
1724 {
1725     struct device *dev = &pdev->dev;
1726     struct safexcel_crypto_priv *priv;
1727     int ret;
1728 
1729     priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
1730     if (!priv)
1731         return -ENOMEM;
1732 
1733     priv->dev = dev;
1734     priv->version = (enum safexcel_eip_version)of_device_get_match_data(dev);
1735 
1736     platform_set_drvdata(pdev, priv);
1737 
1738     priv->base = devm_platform_ioremap_resource(pdev, 0);
1739     if (IS_ERR(priv->base)) {
1740         dev_err(dev, "failed to get resource\n");
1741         return PTR_ERR(priv->base);
1742     }
1743 
1744     priv->clk = devm_clk_get(&pdev->dev, NULL);
1745     ret = PTR_ERR_OR_ZERO(priv->clk);
1746     /* The clock isn't mandatory */
1747     if  (ret != -ENOENT) {
1748         if (ret)
1749             return ret;
1750 
1751         ret = clk_prepare_enable(priv->clk);
1752         if (ret) {
1753             dev_err(dev, "unable to enable clk (%d)\n", ret);
1754             return ret;
1755         }
1756     }
1757 
1758     priv->reg_clk = devm_clk_get(&pdev->dev, "reg");
1759     ret = PTR_ERR_OR_ZERO(priv->reg_clk);
1760     /* The clock isn't mandatory */
1761     if  (ret != -ENOENT) {
1762         if (ret)
1763             goto err_core_clk;
1764 
1765         ret = clk_prepare_enable(priv->reg_clk);
1766         if (ret) {
1767             dev_err(dev, "unable to enable reg clk (%d)\n", ret);
1768             goto err_core_clk;
1769         }
1770     }
1771 
1772     ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
1773     if (ret)
1774         goto err_reg_clk;
1775 
1776     /* Generic EIP97/EIP197 device probing */
1777     ret = safexcel_probe_generic(pdev, priv, 0);
1778     if (ret)
1779         goto err_reg_clk;
1780 
1781     return 0;
1782 
1783 err_reg_clk:
1784     clk_disable_unprepare(priv->reg_clk);
1785 err_core_clk:
1786     clk_disable_unprepare(priv->clk);
1787     return ret;
1788 }
1789 
1790 static int safexcel_remove(struct platform_device *pdev)
1791 {
1792     struct safexcel_crypto_priv *priv = platform_get_drvdata(pdev);
1793     int i;
1794 
1795     safexcel_unregister_algorithms(priv);
1796     safexcel_hw_reset_rings(priv);
1797 
1798     clk_disable_unprepare(priv->reg_clk);
1799     clk_disable_unprepare(priv->clk);
1800 
1801     for (i = 0; i < priv->config.rings; i++) {
1802         irq_set_affinity_hint(priv->ring[i].irq, NULL);
1803         destroy_workqueue(priv->ring[i].workqueue);
1804     }
1805 
1806     return 0;
1807 }
1808 
1809 static const struct of_device_id safexcel_of_match_table[] = {
1810     {
1811         .compatible = "inside-secure,safexcel-eip97ies",
1812         .data = (void *)EIP97IES_MRVL,
1813     },
1814     {
1815         .compatible = "inside-secure,safexcel-eip197b",
1816         .data = (void *)EIP197B_MRVL,
1817     },
1818     {
1819         .compatible = "inside-secure,safexcel-eip197d",
1820         .data = (void *)EIP197D_MRVL,
1821     },
1822     /* For backward compatibility and intended for generic use */
1823     {
1824         .compatible = "inside-secure,safexcel-eip97",
1825         .data = (void *)EIP97IES_MRVL,
1826     },
1827     {
1828         .compatible = "inside-secure,safexcel-eip197",
1829         .data = (void *)EIP197B_MRVL,
1830     },
1831     {},
1832 };
1833 
1834 MODULE_DEVICE_TABLE(of, safexcel_of_match_table);
1835 
1836 static struct platform_driver  crypto_safexcel = {
1837     .probe      = safexcel_probe,
1838     .remove     = safexcel_remove,
1839     .driver     = {
1840         .name   = "crypto-safexcel",
1841         .of_match_table = safexcel_of_match_table,
1842     },
1843 };
1844 
1845 /* PCIE devices - i.e. Inside Secure development boards */
1846 
1847 static int safexcel_pci_probe(struct pci_dev *pdev,
1848                    const struct pci_device_id *ent)
1849 {
1850     struct device *dev = &pdev->dev;
1851     struct safexcel_crypto_priv *priv;
1852     void __iomem *pciebase;
1853     int rc;
1854     u32 val;
1855 
1856     dev_dbg(dev, "Probing PCIE device: vendor %04x, device %04x, subv %04x, subdev %04x, ctxt %lx\n",
1857         ent->vendor, ent->device, ent->subvendor,
1858         ent->subdevice, ent->driver_data);
1859 
1860     priv = kzalloc(sizeof(*priv), GFP_KERNEL);
1861     if (!priv)
1862         return -ENOMEM;
1863 
1864     priv->dev = dev;
1865     priv->version = (enum safexcel_eip_version)ent->driver_data;
1866 
1867     pci_set_drvdata(pdev, priv);
1868 
1869     /* enable the device */
1870     rc = pcim_enable_device(pdev);
1871     if (rc) {
1872         dev_err(dev, "Failed to enable PCI device\n");
1873         return rc;
1874     }
1875 
1876     /* take ownership of PCI BAR0 */
1877     rc = pcim_iomap_regions(pdev, 1, "crypto_safexcel");
1878     if (rc) {
1879         dev_err(dev, "Failed to map IO region for BAR0\n");
1880         return rc;
1881     }
1882     priv->base = pcim_iomap_table(pdev)[0];
1883 
1884     if (priv->version == EIP197_DEVBRD) {
1885         dev_dbg(dev, "Device identified as FPGA based development board - applying HW reset\n");
1886 
1887         rc = pcim_iomap_regions(pdev, 4, "crypto_safexcel");
1888         if (rc) {
1889             dev_err(dev, "Failed to map IO region for BAR4\n");
1890             return rc;
1891         }
1892 
1893         pciebase = pcim_iomap_table(pdev)[2];
1894         val = readl(pciebase + EIP197_XLX_IRQ_BLOCK_ID_ADDR);
1895         if ((val >> 16) == EIP197_XLX_IRQ_BLOCK_ID_VALUE) {
1896             dev_dbg(dev, "Detected Xilinx PCIE IRQ block version %d, multiple MSI support enabled\n",
1897                 (val & 0xff));
1898 
1899             /* Setup MSI identity map mapping */
1900             writel(EIP197_XLX_USER_VECT_LUT0_IDENT,
1901                    pciebase + EIP197_XLX_USER_VECT_LUT0_ADDR);
1902             writel(EIP197_XLX_USER_VECT_LUT1_IDENT,
1903                    pciebase + EIP197_XLX_USER_VECT_LUT1_ADDR);
1904             writel(EIP197_XLX_USER_VECT_LUT2_IDENT,
1905                    pciebase + EIP197_XLX_USER_VECT_LUT2_ADDR);
1906             writel(EIP197_XLX_USER_VECT_LUT3_IDENT,
1907                    pciebase + EIP197_XLX_USER_VECT_LUT3_ADDR);
1908 
1909             /* Enable all device interrupts */
1910             writel(GENMASK(31, 0),
1911                    pciebase + EIP197_XLX_USER_INT_ENB_MSK);
1912         } else {
1913             dev_err(dev, "Unrecognised IRQ block identifier %x\n",
1914                 val);
1915             return -ENODEV;
1916         }
1917 
1918         /* HW reset FPGA dev board */
1919         /* assert reset */
1920         writel(1, priv->base + EIP197_XLX_GPIO_BASE);
1921         wmb(); /* maintain strict ordering for accesses here */
1922         /* deassert reset */
1923         writel(0, priv->base + EIP197_XLX_GPIO_BASE);
1924         wmb(); /* maintain strict ordering for accesses here */
1925     }
1926 
1927     /* enable bus mastering */
1928     pci_set_master(pdev);
1929 
1930     /* Generic EIP97/EIP197 device probing */
1931     rc = safexcel_probe_generic(pdev, priv, 1);
1932     return rc;
1933 }
1934 
1935 static void safexcel_pci_remove(struct pci_dev *pdev)
1936 {
1937     struct safexcel_crypto_priv *priv = pci_get_drvdata(pdev);
1938     int i;
1939 
1940     safexcel_unregister_algorithms(priv);
1941 
1942     for (i = 0; i < priv->config.rings; i++)
1943         destroy_workqueue(priv->ring[i].workqueue);
1944 
1945     safexcel_hw_reset_rings(priv);
1946 }
1947 
1948 static const struct pci_device_id safexcel_pci_ids[] = {
1949     {
1950         PCI_DEVICE_SUB(PCI_VENDOR_ID_XILINX, 0x9038,
1951                    0x16ae, 0xc522),
1952         .driver_data = EIP197_DEVBRD,
1953     },
1954     {},
1955 };
1956 
1957 MODULE_DEVICE_TABLE(pci, safexcel_pci_ids);
1958 
1959 static struct pci_driver safexcel_pci_driver = {
1960     .name          = "crypto-safexcel",
1961     .id_table      = safexcel_pci_ids,
1962     .probe         = safexcel_pci_probe,
1963     .remove        = safexcel_pci_remove,
1964 };
1965 
1966 static int __init safexcel_init(void)
1967 {
1968     int ret;
1969 
1970     /* Register PCI driver */
1971     ret = pci_register_driver(&safexcel_pci_driver);
1972 
1973     /* Register platform driver */
1974     if (IS_ENABLED(CONFIG_OF) && !ret) {
1975         ret = platform_driver_register(&crypto_safexcel);
1976         if (ret)
1977             pci_unregister_driver(&safexcel_pci_driver);
1978     }
1979 
1980     return ret;
1981 }
1982 
1983 static void __exit safexcel_exit(void)
1984 {
1985     /* Unregister platform driver */
1986     if (IS_ENABLED(CONFIG_OF))
1987         platform_driver_unregister(&crypto_safexcel);
1988 
1989     /* Unregister PCI driver if successfully registered before */
1990     pci_unregister_driver(&safexcel_pci_driver);
1991 }
1992 
1993 module_init(safexcel_init);
1994 module_exit(safexcel_exit);
1995 
1996 MODULE_AUTHOR("Antoine Tenart <antoine.tenart@free-electrons.com>");
1997 MODULE_AUTHOR("Ofer Heifetz <oferh@marvell.com>");
1998 MODULE_AUTHOR("Igal Liberman <igall@marvell.com>");
1999 MODULE_DESCRIPTION("Support for SafeXcel cryptographic engines: EIP97 & EIP197");
2000 MODULE_LICENSE("GPL v2");
2001 MODULE_IMPORT_NS(CRYPTO_INTERNAL);
2002 
2003 MODULE_FIRMWARE("ifpp.bin");
2004 MODULE_FIRMWARE("ipue.bin");
2005 MODULE_FIRMWARE("inside-secure/eip197b/ifpp.bin");
2006 MODULE_FIRMWARE("inside-secure/eip197b/ipue.bin");
2007 MODULE_FIRMWARE("inside-secure/eip197d/ifpp.bin");
2008 MODULE_FIRMWARE("inside-secure/eip197d/ipue.bin");
2009 MODULE_FIRMWARE("inside-secure/eip197_minifw/ifpp.bin");
2010 MODULE_FIRMWARE("inside-secure/eip197_minifw/ipue.bin");
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!