crypto/inside-secure/safexcel_ring.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * Copyright (C) 2017 Marvell
0004  *
0005  * Antoine Tenart <antoine.tenart@free-electrons.com>
0006  */
0007
0008 #include <linux/dma-mapping.h>
0009 #include <linux/spinlock.h>
0010
0011 #include "safexcel.h"
0012
0013 int safexcel_init_ring_descriptors(struct safexcel_crypto_priv *priv,
0014                    struct safexcel_desc_ring *cdr,
0015                    struct safexcel_desc_ring *rdr)
0016 {
0017     int i;
0018     struct safexcel_command_desc *cdesc;
0019     dma_addr_t atok;
0020
0021     /* Actual command descriptor ring */
0022     cdr->offset = priv->config.cd_offset;
0023     cdr->base = dmam_alloc_coherent(priv->dev,
0024                     cdr->offset * EIP197_DEFAULT_RING_SIZE,
0025                     &cdr->base_dma, GFP_KERNEL);
0026     if (!cdr->base)
0027         return -ENOMEM;
0028     cdr->write = cdr->base;
0029     cdr->base_end = cdr->base + cdr->offset * (EIP197_DEFAULT_RING_SIZE - 1);
0030     cdr->read = cdr->base;
0031
0032     /* Command descriptor shadow ring for storing additional token data */
0033     cdr->shoffset = priv->config.cdsh_offset;
0034     cdr->shbase = dmam_alloc_coherent(priv->dev,
0035                       cdr->shoffset *
0036                       EIP197_DEFAULT_RING_SIZE,
0037                       &cdr->shbase_dma, GFP_KERNEL);
0038     if (!cdr->shbase)
0039         return -ENOMEM;
0040     cdr->shwrite = cdr->shbase;
0041     cdr->shbase_end = cdr->shbase + cdr->shoffset *
0042                     (EIP197_DEFAULT_RING_SIZE - 1);
0043
0044     /*
0045      * Populate command descriptors with physical pointers to shadow descs.
0046      * Note that we only need to do this once if we don't overwrite them.
0047      */
0048     cdesc = cdr->base;
0049     atok = cdr->shbase_dma;
0050     for (i = 0; i < EIP197_DEFAULT_RING_SIZE; i++) {
0051         cdesc->atok_lo = lower_32_bits(atok);
0052         cdesc->atok_hi = upper_32_bits(atok);
0053         cdesc = (void *)cdesc + cdr->offset;
0054         atok += cdr->shoffset;
0055     }
0056
0057     rdr->offset = priv->config.rd_offset;
0058     /* Use shoffset for result token offset here */
0059     rdr->shoffset = priv->config.res_offset;
0060     rdr->base = dmam_alloc_coherent(priv->dev,
0061                     rdr->offset * EIP197_DEFAULT_RING_SIZE,
0062                     &rdr->base_dma, GFP_KERNEL);
0063     if (!rdr->base)
0064         return -ENOMEM;
0065     rdr->write = rdr->base;
0066     rdr->base_end = rdr->base + rdr->offset  * (EIP197_DEFAULT_RING_SIZE - 1);
0067     rdr->read = rdr->base;
0068
0069     return 0;
0070 }
0071
0072 inline int safexcel_select_ring(struct safexcel_crypto_priv *priv)
0073 {
0074     return (atomic_inc_return(&priv->ring_used) % priv->config.rings);
0075 }
0076
0077 static void *safexcel_ring_next_cwptr(struct safexcel_crypto_priv *priv,
0078                      struct safexcel_desc_ring *ring,
0079                      bool first,
0080                      struct safexcel_token **atoken)
0081 {
0082     void *ptr = ring->write;
0083
0084     if (first)
0085         *atoken = ring->shwrite;
0086
0087     if ((ring->write == ring->read - ring->offset) ||
0088         (ring->read == ring->base && ring->write == ring->base_end))
0089         return ERR_PTR(-ENOMEM);
0090
0091     if (ring->write == ring->base_end) {
0092         ring->write = ring->base;
0093         ring->shwrite = ring->shbase;
0094     } else {
0095         ring->write += ring->offset;
0096         ring->shwrite += ring->shoffset;
0097     }
0098
0099     return ptr;
0100 }
0101
0102 static void *safexcel_ring_next_rwptr(struct safexcel_crypto_priv *priv,
0103                      struct safexcel_desc_ring *ring,
0104                      struct result_data_desc **rtoken)
0105 {
0106     void *ptr = ring->write;
0107
0108     /* Result token at relative offset shoffset */
0109     *rtoken = ring->write + ring->shoffset;
0110
0111     if ((ring->write == ring->read - ring->offset) ||
0112         (ring->read == ring->base && ring->write == ring->base_end))
0113         return ERR_PTR(-ENOMEM);
0114
0115     if (ring->write == ring->base_end)
0116         ring->write = ring->base;
0117     else
0118         ring->write += ring->offset;
0119
0120     return ptr;
0121 }
0122
0123 void *safexcel_ring_next_rptr(struct safexcel_crypto_priv *priv,
0124                   struct safexcel_desc_ring *ring)
0125 {
0126     void *ptr = ring->read;
0127
0128     if (ring->write == ring->read)
0129         return ERR_PTR(-ENOENT);
0130
0131     if (ring->read == ring->base_end)
0132         ring->read = ring->base;
0133     else
0134         ring->read += ring->offset;
0135
0136     return ptr;
0137 }
0138
0139 inline void *safexcel_ring_curr_rptr(struct safexcel_crypto_priv *priv,
0140                      int ring)
0141 {
0142     struct safexcel_desc_ring *rdr = &priv->ring[ring].rdr;
0143
0144     return rdr->read;
0145 }
0146
0147 inline int safexcel_ring_first_rdr_index(struct safexcel_crypto_priv *priv,
0148                      int ring)
0149 {
0150     struct safexcel_desc_ring *rdr = &priv->ring[ring].rdr;
0151
0152     return (rdr->read - rdr->base) / rdr->offset;
0153 }
0154
0155 inline int safexcel_ring_rdr_rdesc_index(struct safexcel_crypto_priv *priv,
0156                      int ring,
0157                      struct safexcel_result_desc *rdesc)
0158 {
0159     struct safexcel_desc_ring *rdr = &priv->ring[ring].rdr;
0160
0161     return ((void *)rdesc - rdr->base) / rdr->offset;
0162 }
0163
0164 void safexcel_ring_rollback_wptr(struct safexcel_crypto_priv *priv,
0165                  struct safexcel_desc_ring *ring)
0166 {
0167     if (ring->write == ring->read)
0168         return;
0169
0170     if (ring->write == ring->base) {
0171         ring->write = ring->base_end;
0172         ring->shwrite = ring->shbase_end;
0173     } else {
0174         ring->write -= ring->offset;
0175         ring->shwrite -= ring->shoffset;
0176     }
0177 }
0178
0179 struct safexcel_command_desc *safexcel_add_cdesc(struct safexcel_crypto_priv *priv,
0180                          int ring_id,
0181                          bool first, bool last,
0182                          dma_addr_t data, u32 data_len,
0183                          u32 full_data_len,
0184                          dma_addr_t context,
0185                          struct safexcel_token **atoken)
0186 {
0187     struct safexcel_command_desc *cdesc;
0188
0189     cdesc = safexcel_ring_next_cwptr(priv, &priv->ring[ring_id].cdr,
0190                      first, atoken);
0191     if (IS_ERR(cdesc))
0192         return cdesc;
0193
0194     cdesc->particle_size = data_len;
0195     cdesc->rsvd0 = 0;
0196     cdesc->last_seg = last;
0197     cdesc->first_seg = first;
0198     cdesc->additional_cdata_size = 0;
0199     cdesc->rsvd1 = 0;
0200     cdesc->data_lo = lower_32_bits(data);
0201     cdesc->data_hi = upper_32_bits(data);
0202
0203     if (first) {
0204         /*
0205          * Note that the length here MUST be >0 or else the EIP(1)97
0206          * may hang. Newer EIP197 firmware actually incorporates this
0207          * fix already, but that doesn't help the EIP97 and we may
0208          * also be running older firmware.
0209          */
0210         cdesc->control_data.packet_length = full_data_len ?: 1;
0211         cdesc->control_data.options = EIP197_OPTION_MAGIC_VALUE |
0212                           EIP197_OPTION_64BIT_CTX |
0213                           EIP197_OPTION_CTX_CTRL_IN_CMD |
0214                           EIP197_OPTION_RC_AUTO;
0215         cdesc->control_data.type = EIP197_TYPE_BCLA;
0216         cdesc->control_data.context_lo = lower_32_bits(context) |
0217                          EIP197_CONTEXT_SMALL;
0218         cdesc->control_data.context_hi = upper_32_bits(context);
0219     }
0220
0221     return cdesc;
0222 }
0223
0224 struct safexcel_result_desc *safexcel_add_rdesc(struct safexcel_crypto_priv *priv,
0225                         int ring_id,
0226                         bool first, bool last,
0227                         dma_addr_t data, u32 len)
0228 {
0229     struct safexcel_result_desc *rdesc;
0230     struct result_data_desc *rtoken;
0231
0232     rdesc = safexcel_ring_next_rwptr(priv, &priv->ring[ring_id].rdr,
0233                      &rtoken);
0234     if (IS_ERR(rdesc))
0235         return rdesc;
0236
0237     rdesc->particle_size = len;
0238     rdesc->rsvd0 = 0;
0239     rdesc->descriptor_overflow = 1; /* assume error */
0240     rdesc->buffer_overflow = 1;     /* assume error */
0241     rdesc->last_seg = last;
0242     rdesc->first_seg = first;
0243     rdesc->result_size = EIP197_RD64_RESULT_SIZE;
0244     rdesc->rsvd1 = 0;
0245     rdesc->data_lo = lower_32_bits(data);
0246     rdesc->data_hi = upper_32_bits(data);
0247
0248     /* Clear length in result token */
0249     rtoken->packet_length = 0;
0250     /* Assume errors - HW will clear if not the case */
0251     rtoken->error_code = 0x7fff;
0252
0253     return rdesc;
0254 }