drivers/crypto/talitos.c

0001 // SPDX-License-Identifier: GPL-2.0-or-later
0002 /*
0003  * talitos - Freescale Integrated Security Engine (SEC) device driver
0004  *
0005  * Copyright (c) 2008-2011 Freescale Semiconductor, Inc.
0006  *
0007  * Scatterlist Crypto API glue code copied from files with the following:
0008  * Copyright (c) 2006-2007 Herbert Xu <herbert@gondor.apana.org.au>
0009  *
0010  * Crypto algorithm registration code copied from hifn driver:
0011  * 2007+ Copyright (c) Evgeniy Polyakov <johnpol@2ka.mipt.ru>
0012  * All rights reserved.
0013  */
0014
0015 #include <linux/kernel.h>
0016 #include <linux/module.h>
0017 #include <linux/mod_devicetable.h>
0018 #include <linux/device.h>
0019 #include <linux/interrupt.h>
0020 #include <linux/crypto.h>
0021 #include <linux/hw_random.h>
0022 #include <linux/of_address.h>
0023 #include <linux/of_irq.h>
0024 #include <linux/of_platform.h>
0025 #include <linux/dma-mapping.h>
0026 #include <linux/io.h>
0027 #include <linux/spinlock.h>
0028 #include <linux/rtnetlink.h>
0029 #include <linux/slab.h>
0030
0031 #include <crypto/algapi.h>
0032 #include <crypto/aes.h>
0033 #include <crypto/internal/des.h>
0034 #include <crypto/sha1.h>
0035 #include <crypto/sha2.h>
0036 #include <crypto/md5.h>
0037 #include <crypto/internal/aead.h>
0038 #include <crypto/authenc.h>
0039 #include <crypto/internal/skcipher.h>
0040 #include <crypto/hash.h>
0041 #include <crypto/internal/hash.h>
0042 #include <crypto/scatterwalk.h>
0043
0044 #include "talitos.h"
0045
0046 static void to_talitos_ptr(struct talitos_ptr *ptr, dma_addr_t dma_addr,
0047                unsigned int len, bool is_sec1)
0048 {
0049     ptr->ptr = cpu_to_be32(lower_32_bits(dma_addr));
0050     if (is_sec1) {
0051         ptr->len1 = cpu_to_be16(len);
0052     } else {
0053         ptr->len = cpu_to_be16(len);
0054         ptr->eptr = upper_32_bits(dma_addr);
0055     }
0056 }
0057
0058 static void copy_talitos_ptr(struct talitos_ptr *dst_ptr,
0059                  struct talitos_ptr *src_ptr, bool is_sec1)
0060 {
0061     dst_ptr->ptr = src_ptr->ptr;
0062     if (is_sec1) {
0063         dst_ptr->len1 = src_ptr->len1;
0064     } else {
0065         dst_ptr->len = src_ptr->len;
0066         dst_ptr->eptr = src_ptr->eptr;
0067     }
0068 }
0069
0070 static unsigned short from_talitos_ptr_len(struct talitos_ptr *ptr,
0071                        bool is_sec1)
0072 {
0073     if (is_sec1)
0074         return be16_to_cpu(ptr->len1);
0075     else
0076         return be16_to_cpu(ptr->len);
0077 }
0078
0079 static void to_talitos_ptr_ext_set(struct talitos_ptr *ptr, u8 val,
0080                    bool is_sec1)
0081 {
0082     if (!is_sec1)
0083         ptr->j_extent = val;
0084 }
0085
0086 static void to_talitos_ptr_ext_or(struct talitos_ptr *ptr, u8 val, bool is_sec1)
0087 {
0088     if (!is_sec1)
0089         ptr->j_extent |= val;
0090 }
0091
0092 /*
0093  * map virtual single (contiguous) pointer to h/w descriptor pointer
0094  */
0095 static void __map_single_talitos_ptr(struct device *dev,
0096                      struct talitos_ptr *ptr,
0097                      unsigned int len, void *data,
0098                      enum dma_data_direction dir,
0099                      unsigned long attrs)
0100 {
0101     dma_addr_t dma_addr = dma_map_single_attrs(dev, data, len, dir, attrs);
0102     struct talitos_private *priv = dev_get_drvdata(dev);
0103     bool is_sec1 = has_ftr_sec1(priv);
0104
0105     to_talitos_ptr(ptr, dma_addr, len, is_sec1);
0106 }
0107
0108 static void map_single_talitos_ptr(struct device *dev,
0109                    struct talitos_ptr *ptr,
0110                    unsigned int len, void *data,
0111                    enum dma_data_direction dir)
0112 {
0113     __map_single_talitos_ptr(dev, ptr, len, data, dir, 0);
0114 }
0115
0116 static void map_single_talitos_ptr_nosync(struct device *dev,
0117                       struct talitos_ptr *ptr,
0118                       unsigned int len, void *data,
0119                       enum dma_data_direction dir)
0120 {
0121     __map_single_talitos_ptr(dev, ptr, len, data, dir,
0122                  DMA_ATTR_SKIP_CPU_SYNC);
0123 }
0124
0125 /*
0126  * unmap bus single (contiguous) h/w descriptor pointer
0127  */
0128 static void unmap_single_talitos_ptr(struct device *dev,
0129                      struct talitos_ptr *ptr,
0130                      enum dma_data_direction dir)
0131 {
0132     struct talitos_private *priv = dev_get_drvdata(dev);
0133     bool is_sec1 = has_ftr_sec1(priv);
0134
0135     dma_unmap_single(dev, be32_to_cpu(ptr->ptr),
0136              from_talitos_ptr_len(ptr, is_sec1), dir);
0137 }
0138
0139 static int reset_channel(struct device *dev, int ch)
0140 {
0141     struct talitos_private *priv = dev_get_drvdata(dev);
0142     unsigned int timeout = TALITOS_TIMEOUT;
0143     bool is_sec1 = has_ftr_sec1(priv);
0144
0145     if (is_sec1) {
0146         setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO,
0147               TALITOS1_CCCR_LO_RESET);
0148
0149         while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR_LO) &
0150             TALITOS1_CCCR_LO_RESET) && --timeout)
0151             cpu_relax();
0152     } else {
0153         setbits32(priv->chan[ch].reg + TALITOS_CCCR,
0154               TALITOS2_CCCR_RESET);
0155
0156         while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) &
0157             TALITOS2_CCCR_RESET) && --timeout)
0158             cpu_relax();
0159     }
0160
0161     if (timeout == 0) {
0162         dev_err(dev, "failed to reset channel %d\n", ch);
0163         return -EIO;
0164     }
0165
0166     /* set 36-bit addressing, done writeback enable and done IRQ enable */
0167     setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, TALITOS_CCCR_LO_EAE |
0168           TALITOS_CCCR_LO_CDWE | TALITOS_CCCR_LO_CDIE);
0169     /* enable chaining descriptors */
0170     if (is_sec1)
0171         setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO,
0172               TALITOS_CCCR_LO_NE);
0173
0174     /* and ICCR writeback, if available */
0175     if (priv->features & TALITOS_FTR_HW_AUTH_CHECK)
0176         setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO,
0177                   TALITOS_CCCR_LO_IWSE);
0178
0179     return 0;
0180 }
0181
0182 static int reset_device(struct device *dev)
0183 {
0184     struct talitos_private *priv = dev_get_drvdata(dev);
0185     unsigned int timeout = TALITOS_TIMEOUT;
0186     bool is_sec1 = has_ftr_sec1(priv);
0187     u32 mcr = is_sec1 ? TALITOS1_MCR_SWR : TALITOS2_MCR_SWR;
0188
0189     setbits32(priv->reg + TALITOS_MCR, mcr);
0190
0191     while ((in_be32(priv->reg + TALITOS_MCR) & mcr)
0192            && --timeout)
0193         cpu_relax();
0194
0195     if (priv->irq[1]) {
0196         mcr = TALITOS_MCR_RCA1 | TALITOS_MCR_RCA3;
0197         setbits32(priv->reg + TALITOS_MCR, mcr);
0198     }
0199
0200     if (timeout == 0) {
0201         dev_err(dev, "failed to reset device\n");
0202         return -EIO;
0203     }
0204
0205     return 0;
0206 }
0207
0208 /*
0209  * Reset and initialize the device
0210  */
0211 static int init_device(struct device *dev)
0212 {
0213     struct talitos_private *priv = dev_get_drvdata(dev);
0214     int ch, err;
0215     bool is_sec1 = has_ftr_sec1(priv);
0216
0217     /*
0218      * Master reset
0219      * errata documentation: warning: certain SEC interrupts
0220      * are not fully cleared by writing the MCR:SWR bit,
0221      * set bit twice to completely reset
0222      */
0223     err = reset_device(dev);
0224     if (err)
0225         return err;
0226
0227     err = reset_device(dev);
0228     if (err)
0229         return err;
0230
0231     /* reset channels */
0232     for (ch = 0; ch < priv->num_channels; ch++) {
0233         err = reset_channel(dev, ch);
0234         if (err)
0235             return err;
0236     }
0237
0238     /* enable channel done and error interrupts */
0239     if (is_sec1) {
0240         clrbits32(priv->reg + TALITOS_IMR, TALITOS1_IMR_INIT);
0241         clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT);
0242         /* disable parity error check in DEU (erroneous? test vect.) */
0243         setbits32(priv->reg_deu + TALITOS_EUICR, TALITOS1_DEUICR_KPE);
0244     } else {
0245         setbits32(priv->reg + TALITOS_IMR, TALITOS2_IMR_INIT);
0246         setbits32(priv->reg + TALITOS_IMR_LO, TALITOS2_IMR_LO_INIT);
0247     }
0248
0249     /* disable integrity check error interrupts (use writeback instead) */
0250     if (priv->features & TALITOS_FTR_HW_AUTH_CHECK)
0251         setbits32(priv->reg_mdeu + TALITOS_EUICR_LO,
0252                   TALITOS_MDEUICR_LO_ICE);
0253
0254     return 0;
0255 }
0256
0257 /**
0258  * talitos_submit - submits a descriptor to the device for processing
0259  * @dev:    the SEC device to be used
0260  * @ch:     the SEC device channel to be used
0261  * @desc:   the descriptor to be processed by the device
0262  * @callback:   whom to call when processing is complete
0263  * @context:    a handle for use by caller (optional)
0264  *
0265  * desc must contain valid dma-mapped (bus physical) address pointers.
0266  * callback must check err and feedback in descriptor header
0267  * for device processing status.
0268  */
0269 static int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc,
0270               void (*callback)(struct device *dev,
0271                        struct talitos_desc *desc,
0272                        void *context, int error),
0273               void *context)
0274 {
0275     struct talitos_private *priv = dev_get_drvdata(dev);
0276     struct talitos_request *request;
0277     unsigned long flags;
0278     int head;
0279     bool is_sec1 = has_ftr_sec1(priv);
0280
0281     spin_lock_irqsave(&priv->chan[ch].head_lock, flags);
0282
0283     if (!atomic_inc_not_zero(&priv->chan[ch].submit_count)) {
0284         /* h/w fifo is full */
0285         spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags);
0286         return -EAGAIN;
0287     }
0288
0289     head = priv->chan[ch].head;
0290     request = &priv->chan[ch].fifo[head];
0291
0292     /* map descriptor and save caller data */
0293     if (is_sec1) {
0294         desc->hdr1 = desc->hdr;
0295         request->dma_desc = dma_map_single(dev, &desc->hdr1,
0296                            TALITOS_DESC_SIZE,
0297                            DMA_BIDIRECTIONAL);
0298     } else {
0299         request->dma_desc = dma_map_single(dev, desc,
0300                            TALITOS_DESC_SIZE,
0301                            DMA_BIDIRECTIONAL);
0302     }
0303     request->callback = callback;
0304     request->context = context;
0305
0306     /* increment fifo head */
0307     priv->chan[ch].head = (priv->chan[ch].head + 1) & (priv->fifo_len - 1);
0308
0309     smp_wmb();
0310     request->desc = desc;
0311
0312     /* GO! */
0313     wmb();
0314     out_be32(priv->chan[ch].reg + TALITOS_FF,
0315          upper_32_bits(request->dma_desc));
0316     out_be32(priv->chan[ch].reg + TALITOS_FF_LO,
0317          lower_32_bits(request->dma_desc));
0318
0319     spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags);
0320
0321     return -EINPROGRESS;
0322 }
0323
0324 static __be32 get_request_hdr(struct talitos_request *request, bool is_sec1)
0325 {
0326     struct talitos_edesc *edesc;
0327
0328     if (!is_sec1)
0329         return request->desc->hdr;
0330
0331     if (!request->desc->next_desc)
0332         return request->desc->hdr1;
0333
0334     edesc = container_of(request->desc, struct talitos_edesc, desc);
0335
0336     return ((struct talitos_desc *)(edesc->buf + edesc->dma_len))->hdr1;
0337 }
0338
0339 /*
0340  * process what was done, notify callback of error if not
0341  */
0342 static void flush_channel(struct device *dev, int ch, int error, int reset_ch)
0343 {
0344     struct talitos_private *priv = dev_get_drvdata(dev);
0345     struct talitos_request *request, saved_req;
0346     unsigned long flags;
0347     int tail, status;
0348     bool is_sec1 = has_ftr_sec1(priv);
0349
0350     spin_lock_irqsave(&priv->chan[ch].tail_lock, flags);
0351
0352     tail = priv->chan[ch].tail;
0353     while (priv->chan[ch].fifo[tail].desc) {
0354         __be32 hdr;
0355
0356         request = &priv->chan[ch].fifo[tail];
0357
0358         /* descriptors with their done bits set don't get the error */
0359         rmb();
0360         hdr = get_request_hdr(request, is_sec1);
0361
0362         if ((hdr & DESC_HDR_DONE) == DESC_HDR_DONE)
0363             status = 0;
0364         else
0365             if (!error)
0366                 break;
0367             else
0368                 status = error;
0369
0370         dma_unmap_single(dev, request->dma_desc,
0371                  TALITOS_DESC_SIZE,
0372                  DMA_BIDIRECTIONAL);
0373
0374         /* copy entries so we can call callback outside lock */
0375         saved_req.desc = request->desc;
0376         saved_req.callback = request->callback;
0377         saved_req.context = request->context;
0378
0379         /* release request entry in fifo */
0380         smp_wmb();
0381         request->desc = NULL;
0382
0383         /* increment fifo tail */
0384         priv->chan[ch].tail = (tail + 1) & (priv->fifo_len - 1);
0385
0386         spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags);
0387
0388         atomic_dec(&priv->chan[ch].submit_count);
0389
0390         saved_req.callback(dev, saved_req.desc, saved_req.context,
0391                    status);
0392         /* channel may resume processing in single desc error case */
0393         if (error && !reset_ch && status == error)
0394             return;
0395         spin_lock_irqsave(&priv->chan[ch].tail_lock, flags);
0396         tail = priv->chan[ch].tail;
0397     }
0398
0399     spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags);
0400 }
0401
0402 /*
0403  * process completed requests for channels that have done status
0404  */
0405 #define DEF_TALITOS1_DONE(name, ch_done_mask)               \
0406 static void talitos1_done_##name(unsigned long data)            \
0407 {                                   \
0408     struct device *dev = (struct device *)data;         \
0409     struct talitos_private *priv = dev_get_drvdata(dev);        \
0410     unsigned long flags;                        \
0411                                     \
0412     if (ch_done_mask & 0x10000000)                  \
0413         flush_channel(dev, 0, 0, 0);            \
0414     if (ch_done_mask & 0x40000000)                  \
0415         flush_channel(dev, 1, 0, 0);            \
0416     if (ch_done_mask & 0x00010000)                  \
0417         flush_channel(dev, 2, 0, 0);            \
0418     if (ch_done_mask & 0x00040000)                  \
0419         flush_channel(dev, 3, 0, 0);            \
0420                                     \
0421     /* At this point, all completed channels have been processed */ \
0422     /* Unmask done interrupts for channels completed later on. */   \
0423     spin_lock_irqsave(&priv->reg_lock, flags);          \
0424     clrbits32(priv->reg + TALITOS_IMR, ch_done_mask);       \
0425     clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT);    \
0426     spin_unlock_irqrestore(&priv->reg_lock, flags);         \
0427 }
0428
0429 DEF_TALITOS1_DONE(4ch, TALITOS1_ISR_4CHDONE)
0430 DEF_TALITOS1_DONE(ch0, TALITOS1_ISR_CH_0_DONE)
0431
0432 #define DEF_TALITOS2_DONE(name, ch_done_mask)               \
0433 static void talitos2_done_##name(unsigned long data)            \
0434 {                                   \
0435     struct device *dev = (struct device *)data;         \
0436     struct talitos_private *priv = dev_get_drvdata(dev);        \
0437     unsigned long flags;                        \
0438                                     \
0439     if (ch_done_mask & 1)                       \
0440         flush_channel(dev, 0, 0, 0);                \
0441     if (ch_done_mask & (1 << 2))                    \
0442         flush_channel(dev, 1, 0, 0);                \
0443     if (ch_done_mask & (1 << 4))                    \
0444         flush_channel(dev, 2, 0, 0);                \
0445     if (ch_done_mask & (1 << 6))                    \
0446         flush_channel(dev, 3, 0, 0);                \
0447                                     \
0448     /* At this point, all completed channels have been processed */ \
0449     /* Unmask done interrupts for channels completed later on. */   \
0450     spin_lock_irqsave(&priv->reg_lock, flags);          \
0451     setbits32(priv->reg + TALITOS_IMR, ch_done_mask);       \
0452     setbits32(priv->reg + TALITOS_IMR_LO, TALITOS2_IMR_LO_INIT);    \
0453     spin_unlock_irqrestore(&priv->reg_lock, flags);         \
0454 }
0455
0456 DEF_TALITOS2_DONE(4ch, TALITOS2_ISR_4CHDONE)
0457 DEF_TALITOS2_DONE(ch0, TALITOS2_ISR_CH_0_DONE)
0458 DEF_TALITOS2_DONE(ch0_2, TALITOS2_ISR_CH_0_2_DONE)
0459 DEF_TALITOS2_DONE(ch1_3, TALITOS2_ISR_CH_1_3_DONE)
0460
0461 /*
0462  * locate current (offending) descriptor
0463  */
0464 static __be32 current_desc_hdr(struct device *dev, int ch)
0465 {
0466     struct talitos_private *priv = dev_get_drvdata(dev);
0467     int tail, iter;
0468     dma_addr_t cur_desc;
0469
0470     cur_desc = ((u64)in_be32(priv->chan[ch].reg + TALITOS_CDPR)) << 32;
0471     cur_desc |= in_be32(priv->chan[ch].reg + TALITOS_CDPR_LO);
0472
0473     if (!cur_desc) {
0474         dev_err(dev, "CDPR is NULL, giving up search for offending descriptor\n");
0475         return 0;
0476     }
0477
0478     tail = priv->chan[ch].tail;
0479
0480     iter = tail;
0481     while (priv->chan[ch].fifo[iter].dma_desc != cur_desc &&
0482            priv->chan[ch].fifo[iter].desc->next_desc != cpu_to_be32(cur_desc)) {
0483         iter = (iter + 1) & (priv->fifo_len - 1);
0484         if (iter == tail) {
0485             dev_err(dev, "couldn't locate current descriptor\n");
0486             return 0;
0487         }
0488     }
0489
0490     if (priv->chan[ch].fifo[iter].desc->next_desc == cpu_to_be32(cur_desc)) {
0491         struct talitos_edesc *edesc;
0492
0493         edesc = container_of(priv->chan[ch].fifo[iter].desc,
0494                      struct talitos_edesc, desc);
0495         return ((struct talitos_desc *)
0496             (edesc->buf + edesc->dma_len))->hdr;
0497     }
0498
0499     return priv->chan[ch].fifo[iter].desc->hdr;
0500 }
0501
0502 /*
0503  * user diagnostics; report root cause of error based on execution unit status
0504  */
0505 static void report_eu_error(struct device *dev, int ch, __be32 desc_hdr)
0506 {
0507     struct talitos_private *priv = dev_get_drvdata(dev);
0508     int i;
0509
0510     if (!desc_hdr)
0511         desc_hdr = cpu_to_be32(in_be32(priv->chan[ch].reg + TALITOS_DESCBUF));
0512
0513     switch (desc_hdr & DESC_HDR_SEL0_MASK) {
0514     case DESC_HDR_SEL0_AFEU:
0515         dev_err(dev, "AFEUISR 0x%08x_%08x\n",
0516             in_be32(priv->reg_afeu + TALITOS_EUISR),
0517             in_be32(priv->reg_afeu + TALITOS_EUISR_LO));
0518         break;
0519     case DESC_HDR_SEL0_DEU:
0520         dev_err(dev, "DEUISR 0x%08x_%08x\n",
0521             in_be32(priv->reg_deu + TALITOS_EUISR),
0522             in_be32(priv->reg_deu + TALITOS_EUISR_LO));
0523         break;
0524     case DESC_HDR_SEL0_MDEUA:
0525     case DESC_HDR_SEL0_MDEUB:
0526         dev_err(dev, "MDEUISR 0x%08x_%08x\n",
0527             in_be32(priv->reg_mdeu + TALITOS_EUISR),
0528             in_be32(priv->reg_mdeu + TALITOS_EUISR_LO));
0529         break;
0530     case DESC_HDR_SEL0_RNG:
0531         dev_err(dev, "RNGUISR 0x%08x_%08x\n",
0532             in_be32(priv->reg_rngu + TALITOS_ISR),
0533             in_be32(priv->reg_rngu + TALITOS_ISR_LO));
0534         break;
0535     case DESC_HDR_SEL0_PKEU:
0536         dev_err(dev, "PKEUISR 0x%08x_%08x\n",
0537             in_be32(priv->reg_pkeu + TALITOS_EUISR),
0538             in_be32(priv->reg_pkeu + TALITOS_EUISR_LO));
0539         break;
0540     case DESC_HDR_SEL0_AESU:
0541         dev_err(dev, "AESUISR 0x%08x_%08x\n",
0542             in_be32(priv->reg_aesu + TALITOS_EUISR),
0543             in_be32(priv->reg_aesu + TALITOS_EUISR_LO));
0544         break;
0545     case DESC_HDR_SEL0_CRCU:
0546         dev_err(dev, "CRCUISR 0x%08x_%08x\n",
0547             in_be32(priv->reg_crcu + TALITOS_EUISR),
0548             in_be32(priv->reg_crcu + TALITOS_EUISR_LO));
0549         break;
0550     case DESC_HDR_SEL0_KEU:
0551         dev_err(dev, "KEUISR 0x%08x_%08x\n",
0552             in_be32(priv->reg_pkeu + TALITOS_EUISR),
0553             in_be32(priv->reg_pkeu + TALITOS_EUISR_LO));
0554         break;
0555     }
0556
0557     switch (desc_hdr & DESC_HDR_SEL1_MASK) {
0558     case DESC_HDR_SEL1_MDEUA:
0559     case DESC_HDR_SEL1_MDEUB:
0560         dev_err(dev, "MDEUISR 0x%08x_%08x\n",
0561             in_be32(priv->reg_mdeu + TALITOS_EUISR),
0562             in_be32(priv->reg_mdeu + TALITOS_EUISR_LO));
0563         break;
0564     case DESC_HDR_SEL1_CRCU:
0565         dev_err(dev, "CRCUISR 0x%08x_%08x\n",
0566             in_be32(priv->reg_crcu + TALITOS_EUISR),
0567             in_be32(priv->reg_crcu + TALITOS_EUISR_LO));
0568         break;
0569     }
0570
0571     for (i = 0; i < 8; i++)
0572         dev_err(dev, "DESCBUF 0x%08x_%08x\n",
0573             in_be32(priv->chan[ch].reg + TALITOS_DESCBUF + 8*i),
0574             in_be32(priv->chan[ch].reg + TALITOS_DESCBUF_LO + 8*i));
0575 }
0576
0577 /*
0578  * recover from error interrupts
0579  */
0580 static void talitos_error(struct device *dev, u32 isr, u32 isr_lo)
0581 {
0582     struct talitos_private *priv = dev_get_drvdata(dev);
0583     unsigned int timeout = TALITOS_TIMEOUT;
0584     int ch, error, reset_dev = 0;
0585     u32 v_lo;
0586     bool is_sec1 = has_ftr_sec1(priv);
0587     int reset_ch = is_sec1 ? 1 : 0; /* only SEC2 supports continuation */
0588
0589     for (ch = 0; ch < priv->num_channels; ch++) {
0590         /* skip channels without errors */
0591         if (is_sec1) {
0592             /* bits 29, 31, 17, 19 */
0593             if (!(isr & (1 << (29 + (ch & 1) * 2 - (ch & 2) * 6))))
0594                 continue;
0595         } else {
0596             if (!(isr & (1 << (ch * 2 + 1))))
0597                 continue;
0598         }
0599
0600         error = -EINVAL;
0601
0602         v_lo = in_be32(priv->chan[ch].reg + TALITOS_CCPSR_LO);
0603
0604         if (v_lo & TALITOS_CCPSR_LO_DOF) {
0605             dev_err(dev, "double fetch fifo overflow error\n");
0606             error = -EAGAIN;
0607             reset_ch = 1;
0608         }
0609         if (v_lo & TALITOS_CCPSR_LO_SOF) {
0610             /* h/w dropped descriptor */
0611             dev_err(dev, "single fetch fifo overflow error\n");
0612             error = -EAGAIN;
0613         }
0614         if (v_lo & TALITOS_CCPSR_LO_MDTE)
0615             dev_err(dev, "master data transfer error\n");
0616         if (v_lo & TALITOS_CCPSR_LO_SGDLZ)
0617             dev_err(dev, is_sec1 ? "pointer not complete error\n"
0618                          : "s/g data length zero error\n");
0619         if (v_lo & TALITOS_CCPSR_LO_FPZ)
0620             dev_err(dev, is_sec1 ? "parity error\n"
0621                          : "fetch pointer zero error\n");
0622         if (v_lo & TALITOS_CCPSR_LO_IDH)
0623             dev_err(dev, "illegal descriptor header error\n");
0624         if (v_lo & TALITOS_CCPSR_LO_IEU)
0625             dev_err(dev, is_sec1 ? "static assignment error\n"
0626                          : "invalid exec unit error\n");
0627         if (v_lo & TALITOS_CCPSR_LO_EU)
0628             report_eu_error(dev, ch, current_desc_hdr(dev, ch));
0629         if (!is_sec1) {
0630             if (v_lo & TALITOS_CCPSR_LO_GB)
0631                 dev_err(dev, "gather boundary error\n");
0632             if (v_lo & TALITOS_CCPSR_LO_GRL)
0633                 dev_err(dev, "gather return/length error\n");
0634             if (v_lo & TALITOS_CCPSR_LO_SB)
0635                 dev_err(dev, "scatter boundary error\n");
0636             if (v_lo & TALITOS_CCPSR_LO_SRL)
0637                 dev_err(dev, "scatter return/length error\n");
0638         }
0639
0640         flush_channel(dev, ch, error, reset_ch);
0641
0642         if (reset_ch) {
0643             reset_channel(dev, ch);
0644         } else {
0645             setbits32(priv->chan[ch].reg + TALITOS_CCCR,
0646                   TALITOS2_CCCR_CONT);
0647             setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, 0);
0648             while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) &
0649                    TALITOS2_CCCR_CONT) && --timeout)
0650                 cpu_relax();
0651             if (timeout == 0) {
0652                 dev_err(dev, "failed to restart channel %d\n",
0653                     ch);
0654                 reset_dev = 1;
0655             }
0656         }
0657     }
0658     if (reset_dev || (is_sec1 && isr & ~TALITOS1_ISR_4CHERR) ||
0659         (!is_sec1 && isr & ~TALITOS2_ISR_4CHERR) || isr_lo) {
0660         if (is_sec1 && (isr_lo & TALITOS1_ISR_TEA_ERR))
0661             dev_err(dev, "TEA error: ISR 0x%08x_%08x\n",
0662                 isr, isr_lo);
0663         else
0664             dev_err(dev, "done overflow, internal time out, or "
0665                 "rngu error: ISR 0x%08x_%08x\n", isr, isr_lo);
0666
0667         /* purge request queues */
0668         for (ch = 0; ch < priv->num_channels; ch++)
0669             flush_channel(dev, ch, -EIO, 1);
0670
0671         /* reset and reinitialize the device */
0672         init_device(dev);
0673     }
0674 }
0675
0676 #define DEF_TALITOS1_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet)          \
0677 static irqreturn_t talitos1_interrupt_##name(int irq, void *data)          \
0678 {                                          \
0679     struct device *dev = data;                         \
0680     struct talitos_private *priv = dev_get_drvdata(dev);               \
0681     u32 isr, isr_lo;                               \
0682     unsigned long flags;                               \
0683                                            \
0684     spin_lock_irqsave(&priv->reg_lock, flags);                 \
0685     isr = in_be32(priv->reg + TALITOS_ISR);                    \
0686     isr_lo = in_be32(priv->reg + TALITOS_ISR_LO);                  \
0687     /* Acknowledge interrupt */                        \
0688     out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \
0689     out_be32(priv->reg + TALITOS_ICR_LO, isr_lo);                  \
0690                                            \
0691     if (unlikely(isr & ch_err_mask || isr_lo & TALITOS1_IMR_LO_INIT)) {    \
0692         spin_unlock_irqrestore(&priv->reg_lock, flags);            \
0693         talitos_error(dev, isr & ch_err_mask, isr_lo);             \
0694     }                                      \
0695     else {                                     \
0696         if (likely(isr & ch_done_mask)) {                  \
0697             /* mask further done interrupts. */            \
0698             setbits32(priv->reg + TALITOS_IMR, ch_done_mask);      \
0699             /* done_task will unmask done interrupts at exit */    \
0700             tasklet_schedule(&priv->done_task[tlet]);          \
0701         }                                  \
0702         spin_unlock_irqrestore(&priv->reg_lock, flags);            \
0703     }                                      \
0704                                            \
0705     return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED :  \
0706                                 IRQ_NONE;      \
0707 }
0708
0709 DEF_TALITOS1_INTERRUPT(4ch, TALITOS1_ISR_4CHDONE, TALITOS1_ISR_4CHERR, 0)
0710
0711 #define DEF_TALITOS2_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet)          \
0712 static irqreturn_t talitos2_interrupt_##name(int irq, void *data)          \
0713 {                                          \
0714     struct device *dev = data;                         \
0715     struct talitos_private *priv = dev_get_drvdata(dev);               \
0716     u32 isr, isr_lo;                               \
0717     unsigned long flags;                               \
0718                                            \
0719     spin_lock_irqsave(&priv->reg_lock, flags);                 \
0720     isr = in_be32(priv->reg + TALITOS_ISR);                    \
0721     isr_lo = in_be32(priv->reg + TALITOS_ISR_LO);                  \
0722     /* Acknowledge interrupt */                        \
0723     out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \
0724     out_be32(priv->reg + TALITOS_ICR_LO, isr_lo);                  \
0725                                            \
0726     if (unlikely(isr & ch_err_mask || isr_lo)) {                   \
0727         spin_unlock_irqrestore(&priv->reg_lock, flags);            \
0728         talitos_error(dev, isr & ch_err_mask, isr_lo);             \
0729     }                                      \
0730     else {                                     \
0731         if (likely(isr & ch_done_mask)) {                  \
0732             /* mask further done interrupts. */            \
0733             clrbits32(priv->reg + TALITOS_IMR, ch_done_mask);      \
0734             /* done_task will unmask done interrupts at exit */    \
0735             tasklet_schedule(&priv->done_task[tlet]);          \
0736         }                                  \
0737         spin_unlock_irqrestore(&priv->reg_lock, flags);            \
0738     }                                      \
0739                                            \
0740     return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED :  \
0741                                 IRQ_NONE;      \
0742 }
0743
0744 DEF_TALITOS2_INTERRUPT(4ch, TALITOS2_ISR_4CHDONE, TALITOS2_ISR_4CHERR, 0)
0745 DEF_TALITOS2_INTERRUPT(ch0_2, TALITOS2_ISR_CH_0_2_DONE, TALITOS2_ISR_CH_0_2_ERR,
0746                0)
0747 DEF_TALITOS2_INTERRUPT(ch1_3, TALITOS2_ISR_CH_1_3_DONE, TALITOS2_ISR_CH_1_3_ERR,
0748                1)
0749
0750 /*
0751  * hwrng
0752  */
0753 static int talitos_rng_data_present(struct hwrng *rng, int wait)
0754 {
0755     struct device *dev = (struct device *)rng->priv;
0756     struct talitos_private *priv = dev_get_drvdata(dev);
0757     u32 ofl;
0758     int i;
0759
0760     for (i = 0; i < 20; i++) {
0761         ofl = in_be32(priv->reg_rngu + TALITOS_EUSR_LO) &
0762               TALITOS_RNGUSR_LO_OFL;
0763         if (ofl || !wait)
0764             break;
0765         udelay(10);
0766     }
0767
0768     return !!ofl;
0769 }
0770
0771 static int talitos_rng_data_read(struct hwrng *rng, u32 *data)
0772 {
0773     struct device *dev = (struct device *)rng->priv;
0774     struct talitos_private *priv = dev_get_drvdata(dev);
0775
0776     /* rng fifo requires 64-bit accesses */
0777     *data = in_be32(priv->reg_rngu + TALITOS_EU_FIFO);
0778     *data = in_be32(priv->reg_rngu + TALITOS_EU_FIFO_LO);
0779
0780     return sizeof(u32);
0781 }
0782
0783 static int talitos_rng_init(struct hwrng *rng)
0784 {
0785     struct device *dev = (struct device *)rng->priv;
0786     struct talitos_private *priv = dev_get_drvdata(dev);
0787     unsigned int timeout = TALITOS_TIMEOUT;
0788
0789     setbits32(priv->reg_rngu + TALITOS_EURCR_LO, TALITOS_RNGURCR_LO_SR);
0790     while (!(in_be32(priv->reg_rngu + TALITOS_EUSR_LO)
0791          & TALITOS_RNGUSR_LO_RD)
0792            && --timeout)
0793         cpu_relax();
0794     if (timeout == 0) {
0795         dev_err(dev, "failed to reset rng hw\n");
0796         return -ENODEV;
0797     }
0798
0799     /* start generating */
0800     setbits32(priv->reg_rngu + TALITOS_EUDSR_LO, 0);
0801
0802     return 0;
0803 }
0804
0805 static int talitos_register_rng(struct device *dev)
0806 {
0807     struct talitos_private *priv = dev_get_drvdata(dev);
0808     int err;
0809
0810     priv->rng.name      = dev_driver_string(dev);
0811     priv->rng.init      = talitos_rng_init;
0812     priv->rng.data_present  = talitos_rng_data_present;
0813     priv->rng.data_read = talitos_rng_data_read;
0814     priv->rng.priv      = (unsigned long)dev;
0815
0816     err = hwrng_register(&priv->rng);
0817     if (!err)
0818         priv->rng_registered = true;
0819
0820     return err;
0821 }
0822
0823 static void talitos_unregister_rng(struct device *dev)
0824 {
0825     struct talitos_private *priv = dev_get_drvdata(dev);
0826
0827     if (!priv->rng_registered)
0828         return;
0829
0830     hwrng_unregister(&priv->rng);
0831     priv->rng_registered = false;
0832 }
0833
0834 /*
0835  * crypto alg
0836  */
0837 #define TALITOS_CRA_PRIORITY        3000
0838 /*
0839  * Defines a priority for doing AEAD with descriptors type
0840  * HMAC_SNOOP_NO_AFEA (HSNA) instead of type IPSEC_ESP
0841  */
0842 #define TALITOS_CRA_PRIORITY_AEAD_HSNA  (TALITOS_CRA_PRIORITY - 1)
0843 #ifdef CONFIG_CRYPTO_DEV_TALITOS2
0844 #define TALITOS_MAX_KEY_SIZE        (AES_MAX_KEY_SIZE + SHA512_BLOCK_SIZE)
0845 #else
0846 #define TALITOS_MAX_KEY_SIZE        (AES_MAX_KEY_SIZE + SHA256_BLOCK_SIZE)
0847 #endif
0848 #define TALITOS_MAX_IV_LENGTH       16 /* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */
0849
0850 struct talitos_ctx {
0851     struct device *dev;
0852     int ch;
0853     __be32 desc_hdr_template;
0854     u8 key[TALITOS_MAX_KEY_SIZE];
0855     u8 iv[TALITOS_MAX_IV_LENGTH];
0856     dma_addr_t dma_key;
0857     unsigned int keylen;
0858     unsigned int enckeylen;
0859     unsigned int authkeylen;
0860 };
0861
0862 #define HASH_MAX_BLOCK_SIZE     SHA512_BLOCK_SIZE
0863 #define TALITOS_MDEU_MAX_CONTEXT_SIZE   TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512
0864
0865 struct talitos_ahash_req_ctx {
0866     u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)];
0867     unsigned int hw_context_size;
0868     u8 buf[2][HASH_MAX_BLOCK_SIZE];
0869     int buf_idx;
0870     unsigned int swinit;
0871     unsigned int first;
0872     unsigned int last;
0873     unsigned int to_hash_later;
0874     unsigned int nbuf;
0875     struct scatterlist bufsl[2];
0876     struct scatterlist *psrc;
0877 };
0878
0879 struct talitos_export_state {
0880     u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)];
0881     u8 buf[HASH_MAX_BLOCK_SIZE];
0882     unsigned int swinit;
0883     unsigned int first;
0884     unsigned int last;
0885     unsigned int to_hash_later;
0886     unsigned int nbuf;
0887 };
0888
0889 static int aead_setkey(struct crypto_aead *authenc,
0890                const u8 *key, unsigned int keylen)
0891 {
0892     struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
0893     struct device *dev = ctx->dev;
0894     struct crypto_authenc_keys keys;
0895
0896     if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
0897         goto badkey;
0898
0899     if (keys.authkeylen + keys.enckeylen > TALITOS_MAX_KEY_SIZE)
0900         goto badkey;
0901
0902     if (ctx->keylen)
0903         dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE);
0904
0905     memcpy(ctx->key, keys.authkey, keys.authkeylen);
0906     memcpy(&ctx->key[keys.authkeylen], keys.enckey, keys.enckeylen);
0907
0908     ctx->keylen = keys.authkeylen + keys.enckeylen;
0909     ctx->enckeylen = keys.enckeylen;
0910     ctx->authkeylen = keys.authkeylen;
0911     ctx->dma_key = dma_map_single(dev, ctx->key, ctx->keylen,
0912                       DMA_TO_DEVICE);
0913
0914     memzero_explicit(&keys, sizeof(keys));
0915     return 0;
0916
0917 badkey:
0918     memzero_explicit(&keys, sizeof(keys));
0919     return -EINVAL;
0920 }
0921
0922 static int aead_des3_setkey(struct crypto_aead *authenc,
0923                 const u8 *key, unsigned int keylen)
0924 {
0925     struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
0926     struct device *dev = ctx->dev;
0927     struct crypto_authenc_keys keys;
0928     int err;
0929
0930     err = crypto_authenc_extractkeys(&keys, key, keylen);
0931     if (unlikely(err))
0932         goto out;
0933
0934     err = -EINVAL;
0935     if (keys.authkeylen + keys.enckeylen > TALITOS_MAX_KEY_SIZE)
0936         goto out;
0937
0938     err = verify_aead_des3_key(authenc, keys.enckey, keys.enckeylen);
0939     if (err)
0940         goto out;
0941
0942     if (ctx->keylen)
0943         dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE);
0944
0945     memcpy(ctx->key, keys.authkey, keys.authkeylen);
0946     memcpy(&ctx->key[keys.authkeylen], keys.enckey, keys.enckeylen);
0947
0948     ctx->keylen = keys.authkeylen + keys.enckeylen;
0949     ctx->enckeylen = keys.enckeylen;
0950     ctx->authkeylen = keys.authkeylen;
0951     ctx->dma_key = dma_map_single(dev, ctx->key, ctx->keylen,
0952                       DMA_TO_DEVICE);
0953
0954 out:
0955     memzero_explicit(&keys, sizeof(keys));
0956     return err;
0957 }
0958
0959 static void talitos_sg_unmap(struct device *dev,
0960                  struct talitos_edesc *edesc,
0961                  struct scatterlist *src,
0962                  struct scatterlist *dst,
0963                  unsigned int len, unsigned int offset)
0964 {
0965     struct talitos_private *priv = dev_get_drvdata(dev);
0966     bool is_sec1 = has_ftr_sec1(priv);
0967     unsigned int src_nents = edesc->src_nents ? : 1;
0968     unsigned int dst_nents = edesc->dst_nents ? : 1;
0969
0970     if (is_sec1 && dst && dst_nents > 1) {
0971         dma_sync_single_for_device(dev, edesc->dma_link_tbl + offset,
0972                        len, DMA_FROM_DEVICE);
0973         sg_pcopy_from_buffer(dst, dst_nents, edesc->buf + offset, len,
0974                      offset);
0975     }
0976     if (src != dst) {
0977         if (src_nents == 1 || !is_sec1)
0978             dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE);
0979
0980         if (dst && (dst_nents == 1 || !is_sec1))
0981             dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE);
0982     } else if (src_nents == 1 || !is_sec1) {
0983         dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL);
0984     }
0985 }
0986
0987 static void ipsec_esp_unmap(struct device *dev,
0988                 struct talitos_edesc *edesc,
0989                 struct aead_request *areq, bool encrypt)
0990 {
0991     struct crypto_aead *aead = crypto_aead_reqtfm(areq);
0992     struct talitos_ctx *ctx = crypto_aead_ctx(aead);
0993     unsigned int ivsize = crypto_aead_ivsize(aead);
0994     unsigned int authsize = crypto_aead_authsize(aead);
0995     unsigned int cryptlen = areq->cryptlen - (encrypt ? 0 : authsize);
0996     bool is_ipsec_esp = edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP;
0997     struct talitos_ptr *civ_ptr = &edesc->desc.ptr[is_ipsec_esp ? 2 : 3];
0998
0999     if (is_ipsec_esp)
1000         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[6],
1001                      DMA_FROM_DEVICE);
1002     unmap_single_talitos_ptr(dev, civ_ptr, DMA_TO_DEVICE);
1003
1004     talitos_sg_unmap(dev, edesc, areq->src, areq->dst,
1005              cryptlen + authsize, areq->assoclen);
1006
1007     if (edesc->dma_len)
1008         dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
1009                  DMA_BIDIRECTIONAL);
1010
1011     if (!is_ipsec_esp) {
1012         unsigned int dst_nents = edesc->dst_nents ? : 1;
1013
1014         sg_pcopy_to_buffer(areq->dst, dst_nents, ctx->iv, ivsize,
1015                    areq->assoclen + cryptlen - ivsize);
1016     }
1017 }
1018
1019 /*
1020  * ipsec_esp descriptor callbacks
1021  */
1022 static void ipsec_esp_encrypt_done(struct device *dev,
1023                    struct talitos_desc *desc, void *context,
1024                    int err)
1025 {
1026     struct aead_request *areq = context;
1027     struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
1028     unsigned int ivsize = crypto_aead_ivsize(authenc);
1029     struct talitos_edesc *edesc;
1030
1031     edesc = container_of(desc, struct talitos_edesc, desc);
1032
1033     ipsec_esp_unmap(dev, edesc, areq, true);
1034
1035     dma_unmap_single(dev, edesc->iv_dma, ivsize, DMA_TO_DEVICE);
1036
1037     kfree(edesc);
1038
1039     aead_request_complete(areq, err);
1040 }
1041
1042 static void ipsec_esp_decrypt_swauth_done(struct device *dev,
1043                       struct talitos_desc *desc,
1044                       void *context, int err)
1045 {
1046     struct aead_request *req = context;
1047     struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1048     unsigned int authsize = crypto_aead_authsize(authenc);
1049     struct talitos_edesc *edesc;
1050     char *oicv, *icv;
1051
1052     edesc = container_of(desc, struct talitos_edesc, desc);
1053
1054     ipsec_esp_unmap(dev, edesc, req, false);
1055
1056     if (!err) {
1057         /* auth check */
1058         oicv = edesc->buf + edesc->dma_len;
1059         icv = oicv - authsize;
1060
1061         err = crypto_memneq(oicv, icv, authsize) ? -EBADMSG : 0;
1062     }
1063
1064     kfree(edesc);
1065
1066     aead_request_complete(req, err);
1067 }
1068
1069 static void ipsec_esp_decrypt_hwauth_done(struct device *dev,
1070                       struct talitos_desc *desc,
1071                       void *context, int err)
1072 {
1073     struct aead_request *req = context;
1074     struct talitos_edesc *edesc;
1075
1076     edesc = container_of(desc, struct talitos_edesc, desc);
1077
1078     ipsec_esp_unmap(dev, edesc, req, false);
1079
1080     /* check ICV auth status */
1081     if (!err && ((desc->hdr_lo & DESC_HDR_LO_ICCR1_MASK) !=
1082              DESC_HDR_LO_ICCR1_PASS))
1083         err = -EBADMSG;
1084
1085     kfree(edesc);
1086
1087     aead_request_complete(req, err);
1088 }
1089
1090 /*
1091  * convert scatterlist to SEC h/w link table format
1092  * stop at cryptlen bytes
1093  */
1094 static int sg_to_link_tbl_offset(struct scatterlist *sg, int sg_count,
1095                  unsigned int offset, int datalen, int elen,
1096                  struct talitos_ptr *link_tbl_ptr, int align)
1097 {
1098     int n_sg = elen ? sg_count + 1 : sg_count;
1099     int count = 0;
1100     int cryptlen = datalen + elen;
1101     int padding = ALIGN(cryptlen, align) - cryptlen;
1102
1103     while (cryptlen && sg && n_sg--) {
1104         unsigned int len = sg_dma_len(sg);
1105
1106         if (offset >= len) {
1107             offset -= len;
1108             goto next;
1109         }
1110
1111         len -= offset;
1112
1113         if (len > cryptlen)
1114             len = cryptlen;
1115
1116         if (datalen > 0 && len > datalen) {
1117             to_talitos_ptr(link_tbl_ptr + count,
1118                        sg_dma_address(sg) + offset, datalen, 0);
1119             to_talitos_ptr_ext_set(link_tbl_ptr + count, 0, 0);
1120             count++;
1121             len -= datalen;
1122             offset += datalen;
1123         }
1124         to_talitos_ptr(link_tbl_ptr + count,
1125                    sg_dma_address(sg) + offset, sg_next(sg) ? len : len + padding, 0);
1126         to_talitos_ptr_ext_set(link_tbl_ptr + count, 0, 0);
1127         count++;
1128         cryptlen -= len;
1129         datalen -= len;
1130         offset = 0;
1131
1132 next:
1133         sg = sg_next(sg);
1134     }
1135
1136     /* tag end of link table */
1137     if (count > 0)
1138         to_talitos_ptr_ext_set(link_tbl_ptr + count - 1,
1139                        DESC_PTR_LNKTBL_RET, 0);
1140
1141     return count;
1142 }
1143
1144 static int talitos_sg_map_ext(struct device *dev, struct scatterlist *src,
1145                   unsigned int len, struct talitos_edesc *edesc,
1146                   struct talitos_ptr *ptr, int sg_count,
1147                   unsigned int offset, int tbl_off, int elen,
1148                   bool force, int align)
1149 {
1150     struct talitos_private *priv = dev_get_drvdata(dev);
1151     bool is_sec1 = has_ftr_sec1(priv);
1152     int aligned_len = ALIGN(len, align);
1153
1154     if (!src) {
1155         to_talitos_ptr(ptr, 0, 0, is_sec1);
1156         return 1;
1157     }
1158     to_talitos_ptr_ext_set(ptr, elen, is_sec1);
1159     if (sg_count == 1 && !force) {
1160         to_talitos_ptr(ptr, sg_dma_address(src) + offset, aligned_len, is_sec1);
1161         return sg_count;
1162     }
1163     if (is_sec1) {
1164         to_talitos_ptr(ptr, edesc->dma_link_tbl + offset, aligned_len, is_sec1);
1165         return sg_count;
1166     }
1167     sg_count = sg_to_link_tbl_offset(src, sg_count, offset, len, elen,
1168                      &edesc->link_tbl[tbl_off], align);
1169     if (sg_count == 1 && !force) {
1170         /* Only one segment now, so no link tbl needed*/
1171         copy_talitos_ptr(ptr, &edesc->link_tbl[tbl_off], is_sec1);
1172         return sg_count;
1173     }
1174     to_talitos_ptr(ptr, edesc->dma_link_tbl +
1175                 tbl_off * sizeof(struct talitos_ptr), aligned_len, is_sec1);
1176     to_talitos_ptr_ext_or(ptr, DESC_PTR_LNKTBL_JUMP, is_sec1);
1177
1178     return sg_count;
1179 }
1180
1181 static int talitos_sg_map(struct device *dev, struct scatterlist *src,
1182               unsigned int len, struct talitos_edesc *edesc,
1183               struct talitos_ptr *ptr, int sg_count,
1184               unsigned int offset, int tbl_off)
1185 {
1186     return talitos_sg_map_ext(dev, src, len, edesc, ptr, sg_count, offset,
1187                   tbl_off, 0, false, 1);
1188 }
1189
1190 /*
1191  * fill in and submit ipsec_esp descriptor
1192  */
1193 static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq,
1194              bool encrypt,
1195              void (*callback)(struct device *dev,
1196                       struct talitos_desc *desc,
1197                       void *context, int error))
1198 {
1199     struct crypto_aead *aead = crypto_aead_reqtfm(areq);
1200     unsigned int authsize = crypto_aead_authsize(aead);
1201     struct talitos_ctx *ctx = crypto_aead_ctx(aead);
1202     struct device *dev = ctx->dev;
1203     struct talitos_desc *desc = &edesc->desc;
1204     unsigned int cryptlen = areq->cryptlen - (encrypt ? 0 : authsize);
1205     unsigned int ivsize = crypto_aead_ivsize(aead);
1206     int tbl_off = 0;
1207     int sg_count, ret;
1208     int elen = 0;
1209     bool sync_needed = false;
1210     struct talitos_private *priv = dev_get_drvdata(dev);
1211     bool is_sec1 = has_ftr_sec1(priv);
1212     bool is_ipsec_esp = desc->hdr & DESC_HDR_TYPE_IPSEC_ESP;
1213     struct talitos_ptr *civ_ptr = &desc->ptr[is_ipsec_esp ? 2 : 3];
1214     struct talitos_ptr *ckey_ptr = &desc->ptr[is_ipsec_esp ? 3 : 2];
1215     dma_addr_t dma_icv = edesc->dma_link_tbl + edesc->dma_len - authsize;
1216
1217     /* hmac key */
1218     to_talitos_ptr(&desc->ptr[0], ctx->dma_key, ctx->authkeylen, is_sec1);
1219
1220     sg_count = edesc->src_nents ?: 1;
1221     if (is_sec1 && sg_count > 1)
1222         sg_copy_to_buffer(areq->src, sg_count, edesc->buf,
1223                   areq->assoclen + cryptlen);
1224     else
1225         sg_count = dma_map_sg(dev, areq->src, sg_count,
1226                       (areq->src == areq->dst) ?
1227                       DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1228
1229     /* hmac data */
1230     ret = talitos_sg_map(dev, areq->src, areq->assoclen, edesc,
1231                  &desc->ptr[1], sg_count, 0, tbl_off);
1232
1233     if (ret > 1) {
1234         tbl_off += ret;
1235         sync_needed = true;
1236     }
1237
1238     /* cipher iv */
1239     to_talitos_ptr(civ_ptr, edesc->iv_dma, ivsize, is_sec1);
1240
1241     /* cipher key */
1242     to_talitos_ptr(ckey_ptr, ctx->dma_key  + ctx->authkeylen,
1243                ctx->enckeylen, is_sec1);
1244
1245     /*
1246      * cipher in
1247      * map and adjust cipher len to aead request cryptlen.
1248      * extent is bytes of HMAC postpended to ciphertext,
1249      * typically 12 for ipsec
1250      */
1251     if (is_ipsec_esp && (desc->hdr & DESC_HDR_MODE1_MDEU_CICV))
1252         elen = authsize;
1253
1254     ret = talitos_sg_map_ext(dev, areq->src, cryptlen, edesc, &desc->ptr[4],
1255                  sg_count, areq->assoclen, tbl_off, elen,
1256                  false, 1);
1257
1258     if (ret > 1) {
1259         tbl_off += ret;
1260         sync_needed = true;
1261     }
1262
1263     /* cipher out */
1264     if (areq->src != areq->dst) {
1265         sg_count = edesc->dst_nents ? : 1;
1266         if (!is_sec1 || sg_count == 1)
1267             dma_map_sg(dev, areq->dst, sg_count, DMA_FROM_DEVICE);
1268     }
1269
1270     if (is_ipsec_esp && encrypt)
1271         elen = authsize;
1272     else
1273         elen = 0;
1274     ret = talitos_sg_map_ext(dev, areq->dst, cryptlen, edesc, &desc->ptr[5],
1275                  sg_count, areq->assoclen, tbl_off, elen,
1276                  is_ipsec_esp && !encrypt, 1);
1277     tbl_off += ret;
1278
1279     if (!encrypt && is_ipsec_esp) {
1280         struct talitos_ptr *tbl_ptr = &edesc->link_tbl[tbl_off];
1281
1282         /* Add an entry to the link table for ICV data */
1283         to_talitos_ptr_ext_set(tbl_ptr - 1, 0, is_sec1);
1284         to_talitos_ptr_ext_set(tbl_ptr, DESC_PTR_LNKTBL_RET, is_sec1);
1285
1286         /* icv data follows link tables */
1287         to_talitos_ptr(tbl_ptr, dma_icv, authsize, is_sec1);
1288         to_talitos_ptr_ext_or(&desc->ptr[5], authsize, is_sec1);
1289         sync_needed = true;
1290     } else if (!encrypt) {
1291         to_talitos_ptr(&desc->ptr[6], dma_icv, authsize, is_sec1);
1292         sync_needed = true;
1293     } else if (!is_ipsec_esp) {
1294         talitos_sg_map(dev, areq->dst, authsize, edesc, &desc->ptr[6],
1295                    sg_count, areq->assoclen + cryptlen, tbl_off);
1296     }
1297
1298     /* iv out */
1299     if (is_ipsec_esp)
1300         map_single_talitos_ptr(dev, &desc->ptr[6], ivsize, ctx->iv,
1301                        DMA_FROM_DEVICE);
1302
1303     if (sync_needed)
1304         dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1305                        edesc->dma_len,
1306                        DMA_BIDIRECTIONAL);
1307
1308     ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
1309     if (ret != -EINPROGRESS) {
1310         ipsec_esp_unmap(dev, edesc, areq, encrypt);
1311         kfree(edesc);
1312     }
1313     return ret;
1314 }
1315
1316 /*
1317  * allocate and map the extended descriptor
1318  */
1319 static struct talitos_edesc *talitos_edesc_alloc(struct device *dev,
1320                          struct scatterlist *src,
1321                          struct scatterlist *dst,
1322                          u8 *iv,
1323                          unsigned int assoclen,
1324                          unsigned int cryptlen,
1325                          unsigned int authsize,
1326                          unsigned int ivsize,
1327                          int icv_stashing,
1328                          u32 cryptoflags,
1329                          bool encrypt)
1330 {
1331     struct talitos_edesc *edesc;
1332     int src_nents, dst_nents, alloc_len, dma_len, src_len, dst_len;
1333     dma_addr_t iv_dma = 0;
1334     gfp_t flags = cryptoflags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
1335               GFP_ATOMIC;
1336     struct talitos_private *priv = dev_get_drvdata(dev);
1337     bool is_sec1 = has_ftr_sec1(priv);
1338     int max_len = is_sec1 ? TALITOS1_MAX_DATA_LEN : TALITOS2_MAX_DATA_LEN;
1339
1340     if (cryptlen + authsize > max_len) {
1341         dev_err(dev, "length exceeds h/w max limit\n");
1342         return ERR_PTR(-EINVAL);
1343     }
1344
1345     if (!dst || dst == src) {
1346         src_len = assoclen + cryptlen + authsize;
1347         src_nents = sg_nents_for_len(src, src_len);
1348         if (src_nents < 0) {
1349             dev_err(dev, "Invalid number of src SG.\n");
1350             return ERR_PTR(-EINVAL);
1351         }
1352         src_nents = (src_nents == 1) ? 0 : src_nents;
1353         dst_nents = dst ? src_nents : 0;
1354         dst_len = 0;
1355     } else { /* dst && dst != src*/
1356         src_len = assoclen + cryptlen + (encrypt ? 0 : authsize);
1357         src_nents = sg_nents_for_len(src, src_len);
1358         if (src_nents < 0) {
1359             dev_err(dev, "Invalid number of src SG.\n");
1360             return ERR_PTR(-EINVAL);
1361         }
1362         src_nents = (src_nents == 1) ? 0 : src_nents;
1363         dst_len = assoclen + cryptlen + (encrypt ? authsize : 0);
1364         dst_nents = sg_nents_for_len(dst, dst_len);
1365         if (dst_nents < 0) {
1366             dev_err(dev, "Invalid number of dst SG.\n");
1367             return ERR_PTR(-EINVAL);
1368         }
1369         dst_nents = (dst_nents == 1) ? 0 : dst_nents;
1370     }
1371
1372     /*
1373      * allocate space for base edesc plus the link tables,
1374      * allowing for two separate entries for AD and generated ICV (+ 2),
1375      * and space for two sets of ICVs (stashed and generated)
1376      */
1377     alloc_len = sizeof(struct talitos_edesc);
1378     if (src_nents || dst_nents || !encrypt) {
1379         if (is_sec1)
1380             dma_len = (src_nents ? src_len : 0) +
1381                   (dst_nents ? dst_len : 0) + authsize;
1382         else
1383             dma_len = (src_nents + dst_nents + 2) *
1384                   sizeof(struct talitos_ptr) + authsize;
1385         alloc_len += dma_len;
1386     } else {
1387         dma_len = 0;
1388     }
1389     alloc_len += icv_stashing ? authsize : 0;
1390
1391     /* if its a ahash, add space for a second desc next to the first one */
1392     if (is_sec1 && !dst)
1393         alloc_len += sizeof(struct talitos_desc);
1394     alloc_len += ivsize;
1395
1396     edesc = kmalloc(alloc_len, GFP_DMA | flags);
1397     if (!edesc)
1398         return ERR_PTR(-ENOMEM);
1399     if (ivsize) {
1400         iv = memcpy(((u8 *)edesc) + alloc_len - ivsize, iv, ivsize);
1401         iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE);
1402     }
1403     memset(&edesc->desc, 0, sizeof(edesc->desc));
1404
1405     edesc->src_nents = src_nents;
1406     edesc->dst_nents = dst_nents;
1407     edesc->iv_dma = iv_dma;
1408     edesc->dma_len = dma_len;
1409     if (dma_len)
1410         edesc->dma_link_tbl = dma_map_single(dev, &edesc->link_tbl[0],
1411                              edesc->dma_len,
1412                              DMA_BIDIRECTIONAL);
1413
1414     return edesc;
1415 }
1416
1417 static struct talitos_edesc *aead_edesc_alloc(struct aead_request *areq, u8 *iv,
1418                           int icv_stashing, bool encrypt)
1419 {
1420     struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
1421     unsigned int authsize = crypto_aead_authsize(authenc);
1422     struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1423     unsigned int ivsize = crypto_aead_ivsize(authenc);
1424     unsigned int cryptlen = areq->cryptlen - (encrypt ? 0 : authsize);
1425
1426     return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst,
1427                    iv, areq->assoclen, cryptlen,
1428                    authsize, ivsize, icv_stashing,
1429                    areq->base.flags, encrypt);
1430 }
1431
1432 static int aead_encrypt(struct aead_request *req)
1433 {
1434     struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1435     struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1436     struct talitos_edesc *edesc;
1437
1438     /* allocate extended descriptor */
1439     edesc = aead_edesc_alloc(req, req->iv, 0, true);
1440     if (IS_ERR(edesc))
1441         return PTR_ERR(edesc);
1442
1443     /* set encrypt */
1444     edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
1445
1446     return ipsec_esp(edesc, req, true, ipsec_esp_encrypt_done);
1447 }
1448
1449 static int aead_decrypt(struct aead_request *req)
1450 {
1451     struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1452     unsigned int authsize = crypto_aead_authsize(authenc);
1453     struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1454     struct talitos_private *priv = dev_get_drvdata(ctx->dev);
1455     struct talitos_edesc *edesc;
1456     void *icvdata;
1457
1458     /* allocate extended descriptor */
1459     edesc = aead_edesc_alloc(req, req->iv, 1, false);
1460     if (IS_ERR(edesc))
1461         return PTR_ERR(edesc);
1462
1463     if ((edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP) &&
1464         (priv->features & TALITOS_FTR_HW_AUTH_CHECK) &&
1465         ((!edesc->src_nents && !edesc->dst_nents) ||
1466          priv->features & TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT)) {
1467
1468         /* decrypt and check the ICV */
1469         edesc->desc.hdr = ctx->desc_hdr_template |
1470                   DESC_HDR_DIR_INBOUND |
1471                   DESC_HDR_MODE1_MDEU_CICV;
1472
1473         /* reset integrity check result bits */
1474
1475         return ipsec_esp(edesc, req, false,
1476                  ipsec_esp_decrypt_hwauth_done);
1477     }
1478
1479     /* Have to check the ICV with software */
1480     edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;
1481
1482     /* stash incoming ICV for later cmp with ICV generated by the h/w */
1483     icvdata = edesc->buf + edesc->dma_len;
1484
1485     sg_pcopy_to_buffer(req->src, edesc->src_nents ? : 1, icvdata, authsize,
1486                req->assoclen + req->cryptlen - authsize);
1487
1488     return ipsec_esp(edesc, req, false, ipsec_esp_decrypt_swauth_done);
1489 }
1490
1491 static int skcipher_setkey(struct crypto_skcipher *cipher,
1492                  const u8 *key, unsigned int keylen)
1493 {
1494     struct talitos_ctx *ctx = crypto_skcipher_ctx(cipher);
1495     struct device *dev = ctx->dev;
1496
1497     if (ctx->keylen)
1498         dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE);
1499
1500     memcpy(&ctx->key, key, keylen);
1501     ctx->keylen = keylen;
1502
1503     ctx->dma_key = dma_map_single(dev, ctx->key, keylen, DMA_TO_DEVICE);
1504
1505     return 0;
1506 }
1507
1508 static int skcipher_des_setkey(struct crypto_skcipher *cipher,
1509                  const u8 *key, unsigned int keylen)
1510 {
1511     return verify_skcipher_des_key(cipher, key) ?:
1512            skcipher_setkey(cipher, key, keylen);
1513 }
1514
1515 static int skcipher_des3_setkey(struct crypto_skcipher *cipher,
1516                   const u8 *key, unsigned int keylen)
1517 {
1518     return verify_skcipher_des3_key(cipher, key) ?:
1519            skcipher_setkey(cipher, key, keylen);
1520 }
1521
1522 static int skcipher_aes_setkey(struct crypto_skcipher *cipher,
1523                   const u8 *key, unsigned int keylen)
1524 {
1525     if (keylen == AES_KEYSIZE_128 || keylen == AES_KEYSIZE_192 ||
1526         keylen == AES_KEYSIZE_256)
1527         return skcipher_setkey(cipher, key, keylen);
1528
1529     return -EINVAL;
1530 }
1531
1532 static void common_nonsnoop_unmap(struct device *dev,
1533                   struct talitos_edesc *edesc,
1534                   struct skcipher_request *areq)
1535 {
1536     unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE);
1537
1538     talitos_sg_unmap(dev, edesc, areq->src, areq->dst, areq->cryptlen, 0);
1539     unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1], DMA_TO_DEVICE);
1540
1541     if (edesc->dma_len)
1542         dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
1543                  DMA_BIDIRECTIONAL);
1544 }
1545
1546 static void skcipher_done(struct device *dev,
1547                 struct talitos_desc *desc, void *context,
1548                 int err)
1549 {
1550     struct skcipher_request *areq = context;
1551     struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq);
1552     struct talitos_ctx *ctx = crypto_skcipher_ctx(cipher);
1553     unsigned int ivsize = crypto_skcipher_ivsize(cipher);
1554     struct talitos_edesc *edesc;
1555
1556     edesc = container_of(desc, struct talitos_edesc, desc);
1557
1558     common_nonsnoop_unmap(dev, edesc, areq);
1559     memcpy(areq->iv, ctx->iv, ivsize);
1560
1561     kfree(edesc);
1562
1563     areq->base.complete(&areq->base, err);
1564 }
1565
1566 static int common_nonsnoop(struct talitos_edesc *edesc,
1567                struct skcipher_request *areq,
1568                void (*callback) (struct device *dev,
1569                          struct talitos_desc *desc,
1570                          void *context, int error))
1571 {
1572     struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq);
1573     struct talitos_ctx *ctx = crypto_skcipher_ctx(cipher);
1574     struct device *dev = ctx->dev;
1575     struct talitos_desc *desc = &edesc->desc;
1576     unsigned int cryptlen = areq->cryptlen;
1577     unsigned int ivsize = crypto_skcipher_ivsize(cipher);
1578     int sg_count, ret;
1579     bool sync_needed = false;
1580     struct talitos_private *priv = dev_get_drvdata(dev);
1581     bool is_sec1 = has_ftr_sec1(priv);
1582     bool is_ctr = (desc->hdr & DESC_HDR_SEL0_MASK) == DESC_HDR_SEL0_AESU &&
1583               (desc->hdr & DESC_HDR_MODE0_AESU_MASK) == DESC_HDR_MODE0_AESU_CTR;
1584
1585     /* first DWORD empty */
1586
1587     /* cipher iv */
1588     to_talitos_ptr(&desc->ptr[1], edesc->iv_dma, ivsize, is_sec1);
1589
1590     /* cipher key */
1591     to_talitos_ptr(&desc->ptr[2], ctx->dma_key, ctx->keylen, is_sec1);
1592
1593     sg_count = edesc->src_nents ?: 1;
1594     if (is_sec1 && sg_count > 1)
1595         sg_copy_to_buffer(areq->src, sg_count, edesc->buf,
1596                   cryptlen);
1597     else
1598         sg_count = dma_map_sg(dev, areq->src, sg_count,
1599                       (areq->src == areq->dst) ?
1600                       DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1601     /*
1602      * cipher in
1603      */
1604     sg_count = talitos_sg_map_ext(dev, areq->src, cryptlen, edesc, &desc->ptr[3],
1605                       sg_count, 0, 0, 0, false, is_ctr ? 16 : 1);
1606     if (sg_count > 1)
1607         sync_needed = true;
1608
1609     /* cipher out */
1610     if (areq->src != areq->dst) {
1611         sg_count = edesc->dst_nents ? : 1;
1612         if (!is_sec1 || sg_count == 1)
1613             dma_map_sg(dev, areq->dst, sg_count, DMA_FROM_DEVICE);
1614     }
1615
1616     ret = talitos_sg_map(dev, areq->dst, cryptlen, edesc, &desc->ptr[4],
1617                  sg_count, 0, (edesc->src_nents + 1));
1618     if (ret > 1)
1619         sync_needed = true;
1620
1621     /* iv out */
1622     map_single_talitos_ptr(dev, &desc->ptr[5], ivsize, ctx->iv,
1623                    DMA_FROM_DEVICE);
1624
1625     /* last DWORD empty */
1626
1627     if (sync_needed)
1628         dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1629                        edesc->dma_len, DMA_BIDIRECTIONAL);
1630
1631     ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
1632     if (ret != -EINPROGRESS) {
1633         common_nonsnoop_unmap(dev, edesc, areq);
1634         kfree(edesc);
1635     }
1636     return ret;
1637 }
1638
1639 static struct talitos_edesc *skcipher_edesc_alloc(struct skcipher_request *
1640                             areq, bool encrypt)
1641 {
1642     struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq);
1643     struct talitos_ctx *ctx = crypto_skcipher_ctx(cipher);
1644     unsigned int ivsize = crypto_skcipher_ivsize(cipher);
1645
1646     return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst,
1647                    areq->iv, 0, areq->cryptlen, 0, ivsize, 0,
1648                    areq->base.flags, encrypt);
1649 }
1650
1651 static int skcipher_encrypt(struct skcipher_request *areq)
1652 {
1653     struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq);
1654     struct talitos_ctx *ctx = crypto_skcipher_ctx(cipher);
1655     struct talitos_edesc *edesc;
1656     unsigned int blocksize =
1657             crypto_tfm_alg_blocksize(crypto_skcipher_tfm(cipher));
1658
1659     if (!areq->cryptlen)
1660         return 0;
1661
1662     if (areq->cryptlen % blocksize)
1663         return -EINVAL;
1664
1665     /* allocate extended descriptor */
1666     edesc = skcipher_edesc_alloc(areq, true);
1667     if (IS_ERR(edesc))
1668         return PTR_ERR(edesc);
1669
1670     /* set encrypt */
1671     edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
1672
1673     return common_nonsnoop(edesc, areq, skcipher_done);
1674 }
1675
1676 static int skcipher_decrypt(struct skcipher_request *areq)
1677 {
1678     struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq);
1679     struct talitos_ctx *ctx = crypto_skcipher_ctx(cipher);
1680     struct talitos_edesc *edesc;
1681     unsigned int blocksize =
1682             crypto_tfm_alg_blocksize(crypto_skcipher_tfm(cipher));
1683
1684     if (!areq->cryptlen)
1685         return 0;
1686
1687     if (areq->cryptlen % blocksize)
1688         return -EINVAL;
1689
1690     /* allocate extended descriptor */
1691     edesc = skcipher_edesc_alloc(areq, false);
1692     if (IS_ERR(edesc))
1693         return PTR_ERR(edesc);
1694
1695     edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;
1696
1697     return common_nonsnoop(edesc, areq, skcipher_done);
1698 }
1699
1700 static void common_nonsnoop_hash_unmap(struct device *dev,
1701                        struct talitos_edesc *edesc,
1702                        struct ahash_request *areq)
1703 {
1704     struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1705     struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1706     struct talitos_private *priv = dev_get_drvdata(dev);
1707     bool is_sec1 = has_ftr_sec1(priv);
1708     struct talitos_desc *desc = &edesc->desc;
1709     struct talitos_desc *desc2 = (struct talitos_desc *)
1710                      (edesc->buf + edesc->dma_len);
1711
1712     unmap_single_talitos_ptr(dev, &desc->ptr[5], DMA_FROM_DEVICE);
1713     if (desc->next_desc &&
1714         desc->ptr[5].ptr != desc2->ptr[5].ptr)
1715         unmap_single_talitos_ptr(dev, &desc2->ptr[5], DMA_FROM_DEVICE);
1716     if (req_ctx->last)
1717         memcpy(areq->result, req_ctx->hw_context,
1718                crypto_ahash_digestsize(tfm));
1719
1720     if (req_ctx->psrc)
1721         talitos_sg_unmap(dev, edesc, req_ctx->psrc, NULL, 0, 0);
1722
1723     /* When using hashctx-in, must unmap it. */
1724     if (from_talitos_ptr_len(&desc->ptr[1], is_sec1))
1725         unmap_single_talitos_ptr(dev, &desc->ptr[1],
1726                      DMA_TO_DEVICE);
1727     else if (desc->next_desc)
1728         unmap_single_talitos_ptr(dev, &desc2->ptr[1],
1729                      DMA_TO_DEVICE);
1730
1731     if (is_sec1 && req_ctx->nbuf)
1732         unmap_single_talitos_ptr(dev, &desc->ptr[3],
1733                      DMA_TO_DEVICE);
1734
1735     if (edesc->dma_len)
1736         dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
1737                  DMA_BIDIRECTIONAL);
1738
1739     if (desc->next_desc)
1740         dma_unmap_single(dev, be32_to_cpu(desc->next_desc),
1741                  TALITOS_DESC_SIZE, DMA_BIDIRECTIONAL);
1742 }
1743
1744 static void ahash_done(struct device *dev,
1745                struct talitos_desc *desc, void *context,
1746                int err)
1747 {
1748     struct ahash_request *areq = context;
1749     struct talitos_edesc *edesc =
1750          container_of(desc, struct talitos_edesc, desc);
1751     struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1752
1753     if (!req_ctx->last && req_ctx->to_hash_later) {
1754         /* Position any partial block for next update/final/finup */
1755         req_ctx->buf_idx = (req_ctx->buf_idx + 1) & 1;
1756         req_ctx->nbuf = req_ctx->to_hash_later;
1757     }
1758     common_nonsnoop_hash_unmap(dev, edesc, areq);
1759
1760     kfree(edesc);
1761
1762     areq->base.complete(&areq->base, err);
1763 }
1764
1765 /*
1766  * SEC1 doesn't like hashing of 0 sized message, so we do the padding
1767  * ourself and submit a padded block
1768  */
1769 static void talitos_handle_buggy_hash(struct talitos_ctx *ctx,
1770                    struct talitos_edesc *edesc,
1771                    struct talitos_ptr *ptr)
1772 {
1773     static u8 padded_hash[64] = {
1774         0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1775         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1776         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1777         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1778     };
1779
1780     pr_err_once("Bug in SEC1, padding ourself\n");
1781     edesc->desc.hdr &= ~DESC_HDR_MODE0_MDEU_PAD;
1782     map_single_talitos_ptr(ctx->dev, ptr, sizeof(padded_hash),
1783                    (char *)padded_hash, DMA_TO_DEVICE);
1784 }
1785
1786 static int common_nonsnoop_hash(struct talitos_edesc *edesc,
1787                 struct ahash_request *areq, unsigned int length,
1788                 void (*callback) (struct device *dev,
1789                           struct talitos_desc *desc,
1790                           void *context, int error))
1791 {
1792     struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1793     struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1794     struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1795     struct device *dev = ctx->dev;
1796     struct talitos_desc *desc = &edesc->desc;
1797     int ret;
1798     bool sync_needed = false;
1799     struct talitos_private *priv = dev_get_drvdata(dev);
1800     bool is_sec1 = has_ftr_sec1(priv);
1801     int sg_count;
1802
1803     /* first DWORD empty */
1804
1805     /* hash context in */
1806     if (!req_ctx->first || req_ctx->swinit) {
1807         map_single_talitos_ptr_nosync(dev, &desc->ptr[1],
1808                           req_ctx->hw_context_size,
1809                           req_ctx->hw_context,
1810                           DMA_TO_DEVICE);
1811         req_ctx->swinit = 0;
1812     }
1813     /* Indicate next op is not the first. */
1814     req_ctx->first = 0;
1815
1816     /* HMAC key */
1817     if (ctx->keylen)
1818         to_talitos_ptr(&desc->ptr[2], ctx->dma_key, ctx->keylen,
1819                    is_sec1);
1820
1821     if (is_sec1 && req_ctx->nbuf)
1822         length -= req_ctx->nbuf;
1823
1824     sg_count = edesc->src_nents ?: 1;
1825     if (is_sec1 && sg_count > 1)
1826         sg_copy_to_buffer(req_ctx->psrc, sg_count, edesc->buf, length);
1827     else if (length)
1828         sg_count = dma_map_sg(dev, req_ctx->psrc, sg_count,
1829                       DMA_TO_DEVICE);
1830     /*
1831      * data in
1832      */
1833     if (is_sec1 && req_ctx->nbuf) {
1834         map_single_talitos_ptr(dev, &desc->ptr[3], req_ctx->nbuf,
1835                        req_ctx->buf[req_ctx->buf_idx],
1836                        DMA_TO_DEVICE);
1837     } else {
1838         sg_count = talitos_sg_map(dev, req_ctx->psrc, length, edesc,
1839                       &desc->ptr[3], sg_count, 0, 0);
1840         if (sg_count > 1)
1841             sync_needed = true;
1842     }
1843
1844     /* fifth DWORD empty */
1845
1846     /* hash/HMAC out -or- hash context out */
1847     if (req_ctx->last)
1848         map_single_talitos_ptr(dev, &desc->ptr[5],
1849                        crypto_ahash_digestsize(tfm),
1850                        req_ctx->hw_context, DMA_FROM_DEVICE);
1851     else
1852         map_single_talitos_ptr_nosync(dev, &desc->ptr[5],
1853                           req_ctx->hw_context_size,
1854                           req_ctx->hw_context,
1855                           DMA_FROM_DEVICE);
1856
1857     /* last DWORD empty */
1858
1859     if (is_sec1 && from_talitos_ptr_len(&desc->ptr[3], true) == 0)
1860         talitos_handle_buggy_hash(ctx, edesc, &desc->ptr[3]);
1861
1862     if (is_sec1 && req_ctx->nbuf && length) {
1863         struct talitos_desc *desc2 = (struct talitos_desc *)
1864                          (edesc->buf + edesc->dma_len);
1865         dma_addr_t next_desc;
1866
1867         memset(desc2, 0, sizeof(*desc2));
1868         desc2->hdr = desc->hdr;
1869         desc2->hdr &= ~DESC_HDR_MODE0_MDEU_INIT;
1870         desc2->hdr1 = desc2->hdr;
1871         desc->hdr &= ~DESC_HDR_MODE0_MDEU_PAD;
1872         desc->hdr |= DESC_HDR_MODE0_MDEU_CONT;
1873         desc->hdr &= ~DESC_HDR_DONE_NOTIFY;
1874
1875         if (desc->ptr[1].ptr)
1876             copy_talitos_ptr(&desc2->ptr[1], &desc->ptr[1],
1877                      is_sec1);
1878         else
1879             map_single_talitos_ptr_nosync(dev, &desc2->ptr[1],
1880                               req_ctx->hw_context_size,
1881                               req_ctx->hw_context,
1882                               DMA_TO_DEVICE);
1883         copy_talitos_ptr(&desc2->ptr[2], &desc->ptr[2], is_sec1);
1884         sg_count = talitos_sg_map(dev, req_ctx->psrc, length, edesc,
1885                       &desc2->ptr[3], sg_count, 0, 0);
1886         if (sg_count > 1)
1887             sync_needed = true;
1888         copy_talitos_ptr(&desc2->ptr[5], &desc->ptr[5], is_sec1);
1889         if (req_ctx->last)
1890             map_single_talitos_ptr_nosync(dev, &desc->ptr[5],
1891                               req_ctx->hw_context_size,
1892                               req_ctx->hw_context,
1893                               DMA_FROM_DEVICE);
1894
1895         next_desc = dma_map_single(dev, &desc2->hdr1, TALITOS_DESC_SIZE,
1896                        DMA_BIDIRECTIONAL);
1897         desc->next_desc = cpu_to_be32(next_desc);
1898     }
1899
1900     if (sync_needed)
1901         dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1902                        edesc->dma_len, DMA_BIDIRECTIONAL);
1903
1904     ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
1905     if (ret != -EINPROGRESS) {
1906         common_nonsnoop_hash_unmap(dev, edesc, areq);
1907         kfree(edesc);
1908     }
1909     return ret;
1910 }
1911
1912 static struct talitos_edesc *ahash_edesc_alloc(struct ahash_request *areq,
1913                            unsigned int nbytes)
1914 {
1915     struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1916     struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1917     struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1918     struct talitos_private *priv = dev_get_drvdata(ctx->dev);
1919     bool is_sec1 = has_ftr_sec1(priv);
1920
1921     if (is_sec1)
1922         nbytes -= req_ctx->nbuf;
1923
1924     return talitos_edesc_alloc(ctx->dev, req_ctx->psrc, NULL, NULL, 0,
1925                    nbytes, 0, 0, 0, areq->base.flags, false);
1926 }
1927
1928 static int ahash_init(struct ahash_request *areq)
1929 {
1930     struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1931     struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1932     struct device *dev = ctx->dev;
1933     struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1934     unsigned int size;
1935     dma_addr_t dma;
1936
1937     /* Initialize the context */
1938     req_ctx->buf_idx = 0;
1939     req_ctx->nbuf = 0;
1940     req_ctx->first = 1; /* first indicates h/w must init its context */
1941     req_ctx->swinit = 0; /* assume h/w init of context */
1942     size =  (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE)
1943             ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256
1944             : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512;
1945     req_ctx->hw_context_size = size;
1946
1947     dma = dma_map_single(dev, req_ctx->hw_context, req_ctx->hw_context_size,
1948                  DMA_TO_DEVICE);
1949     dma_unmap_single(dev, dma, req_ctx->hw_context_size, DMA_TO_DEVICE);
1950
1951     return 0;
1952 }
1953
1954 /*
1955  * on h/w without explicit sha224 support, we initialize h/w context
1956  * manually with sha224 constants, and tell it to run sha256.
1957  */
1958 static int ahash_init_sha224_swinit(struct ahash_request *areq)
1959 {
1960     struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1961
1962     req_ctx->hw_context[0] = SHA224_H0;
1963     req_ctx->hw_context[1] = SHA224_H1;
1964     req_ctx->hw_context[2] = SHA224_H2;
1965     req_ctx->hw_context[3] = SHA224_H3;
1966     req_ctx->hw_context[4] = SHA224_H4;
1967     req_ctx->hw_context[5] = SHA224_H5;
1968     req_ctx->hw_context[6] = SHA224_H6;
1969     req_ctx->hw_context[7] = SHA224_H7;
1970
1971     /* init 64-bit count */
1972     req_ctx->hw_context[8] = 0;
1973     req_ctx->hw_context[9] = 0;
1974
1975     ahash_init(areq);
1976     req_ctx->swinit = 1;/* prevent h/w initting context with sha256 values*/
1977
1978     return 0;
1979 }
1980
1981 static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes)
1982 {
1983     struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1984     struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1985     struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1986     struct talitos_edesc *edesc;
1987     unsigned int blocksize =
1988             crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
1989     unsigned int nbytes_to_hash;
1990     unsigned int to_hash_later;
1991     unsigned int nsg;
1992     int nents;
1993     struct device *dev = ctx->dev;
1994     struct talitos_private *priv = dev_get_drvdata(dev);
1995     bool is_sec1 = has_ftr_sec1(priv);
1996     u8 *ctx_buf = req_ctx->buf[req_ctx->buf_idx];
1997
1998     if (!req_ctx->last && (nbytes + req_ctx->nbuf <= blocksize)) {
1999         /* Buffer up to one whole block */
2000         nents = sg_nents_for_len(areq->src, nbytes);
2001         if (nents < 0) {
2002             dev_err(ctx->dev, "Invalid number of src SG.\n");
2003             return nents;
2004         }
2005         sg_copy_to_buffer(areq->src, nents,
2006                   ctx_buf + req_ctx->nbuf, nbytes);
2007         req_ctx->nbuf += nbytes;
2008         return 0;
2009     }
2010
2011     /* At least (blocksize + 1) bytes are available to hash */
2012     nbytes_to_hash = nbytes + req_ctx->nbuf;
2013     to_hash_later = nbytes_to_hash & (blocksize - 1);
2014
2015     if (req_ctx->last)
2016         to_hash_later = 0;
2017     else if (to_hash_later)
2018         /* There is a partial block. Hash the full block(s) now */
2019         nbytes_to_hash -= to_hash_later;
2020     else {
2021         /* Keep one block buffered */
2022         nbytes_to_hash -= blocksize;
2023         to_hash_later = blocksize;
2024     }
2025
2026     /* Chain in any previously buffered data */
2027     if (!is_sec1 && req_ctx->nbuf) {
2028         nsg = (req_ctx->nbuf < nbytes_to_hash) ? 2 : 1;
2029         sg_init_table(req_ctx->bufsl, nsg);
2030         sg_set_buf(req_ctx->bufsl, ctx_buf, req_ctx->nbuf);
2031         if (nsg > 1)
2032             sg_chain(req_ctx->bufsl, 2, areq->src);
2033         req_ctx->psrc = req_ctx->bufsl;
2034     } else if (is_sec1 && req_ctx->nbuf && req_ctx->nbuf < blocksize) {
2035         int offset;
2036
2037         if (nbytes_to_hash > blocksize)
2038             offset = blocksize - req_ctx->nbuf;
2039         else
2040             offset = nbytes_to_hash - req_ctx->nbuf;
2041         nents = sg_nents_for_len(areq->src, offset);
2042         if (nents < 0) {
2043             dev_err(ctx->dev, "Invalid number of src SG.\n");
2044             return nents;
2045         }
2046         sg_copy_to_buffer(areq->src, nents,
2047                   ctx_buf + req_ctx->nbuf, offset);
2048         req_ctx->nbuf += offset;
2049         req_ctx->psrc = scatterwalk_ffwd(req_ctx->bufsl, areq->src,
2050                          offset);
2051     } else
2052         req_ctx->psrc = areq->src;
2053
2054     if (to_hash_later) {
2055         nents = sg_nents_for_len(areq->src, nbytes);
2056         if (nents < 0) {
2057             dev_err(ctx->dev, "Invalid number of src SG.\n");
2058             return nents;
2059         }
2060         sg_pcopy_to_buffer(areq->src, nents,
2061                    req_ctx->buf[(req_ctx->buf_idx + 1) & 1],
2062                       to_hash_later,
2063                       nbytes - to_hash_later);
2064     }
2065     req_ctx->to_hash_later = to_hash_later;
2066
2067     /* Allocate extended descriptor */
2068     edesc = ahash_edesc_alloc(areq, nbytes_to_hash);
2069     if (IS_ERR(edesc))
2070         return PTR_ERR(edesc);
2071
2072     edesc->desc.hdr = ctx->desc_hdr_template;
2073
2074     /* On last one, request SEC to pad; otherwise continue */
2075     if (req_ctx->last)
2076         edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_PAD;
2077     else
2078         edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_CONT;
2079
2080     /* request SEC to INIT hash. */
2081     if (req_ctx->first && !req_ctx->swinit)
2082         edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_INIT;
2083
2084     /* When the tfm context has a keylen, it's an HMAC.
2085      * A first or last (ie. not middle) descriptor must request HMAC.
2086      */
2087     if (ctx->keylen && (req_ctx->first || req_ctx->last))
2088         edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_HMAC;
2089
2090     return common_nonsnoop_hash(edesc, areq, nbytes_to_hash, ahash_done);
2091 }
2092
2093 static int ahash_update(struct ahash_request *areq)
2094 {
2095     struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2096
2097     req_ctx->last = 0;
2098
2099     return ahash_process_req(areq, areq->nbytes);
2100 }
2101
2102 static int ahash_final(struct ahash_request *areq)
2103 {
2104     struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2105
2106     req_ctx->last = 1;
2107
2108     return ahash_process_req(areq, 0);
2109 }
2110
2111 static int ahash_finup(struct ahash_request *areq)
2112 {
2113     struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2114
2115     req_ctx->last = 1;
2116
2117     return ahash_process_req(areq, areq->nbytes);
2118 }
2119
2120 static int ahash_digest(struct ahash_request *areq)
2121 {
2122     struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2123     struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
2124
2125     ahash->init(areq);
2126     req_ctx->last = 1;
2127
2128     return ahash_process_req(areq, areq->nbytes);
2129 }
2130
2131 static int ahash_export(struct ahash_request *areq, void *out)
2132 {
2133     struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2134     struct talitos_export_state *export = out;
2135     struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2136     struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
2137     struct device *dev = ctx->dev;
2138     dma_addr_t dma;
2139
2140     dma = dma_map_single(dev, req_ctx->hw_context, req_ctx->hw_context_size,
2141                  DMA_FROM_DEVICE);
2142     dma_unmap_single(dev, dma, req_ctx->hw_context_size, DMA_FROM_DEVICE);
2143
2144     memcpy(export->hw_context, req_ctx->hw_context,
2145            req_ctx->hw_context_size);
2146     memcpy(export->buf, req_ctx->buf[req_ctx->buf_idx], req_ctx->nbuf);
2147     export->swinit = req_ctx->swinit;
2148     export->first = req_ctx->first;
2149     export->last = req_ctx->last;
2150     export->to_hash_later = req_ctx->to_hash_later;
2151     export->nbuf = req_ctx->nbuf;
2152
2153     return 0;
2154 }
2155
2156 static int ahash_import(struct ahash_request *areq, const void *in)
2157 {
2158     struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2159     struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2160     struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
2161     struct device *dev = ctx->dev;
2162     const struct talitos_export_state *export = in;
2163     unsigned int size;
2164     dma_addr_t dma;
2165
2166     memset(req_ctx, 0, sizeof(*req_ctx));
2167     size = (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE)
2168             ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256
2169             : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512;
2170     req_ctx->hw_context_size = size;
2171     memcpy(req_ctx->hw_context, export->hw_context, size);
2172     memcpy(req_ctx->buf[0], export->buf, export->nbuf);
2173     req_ctx->swinit = export->swinit;
2174     req_ctx->first = export->first;
2175     req_ctx->last = export->last;
2176     req_ctx->to_hash_later = export->to_hash_later;
2177     req_ctx->nbuf = export->nbuf;
2178
2179     dma = dma_map_single(dev, req_ctx->hw_context, req_ctx->hw_context_size,
2180                  DMA_TO_DEVICE);
2181     dma_unmap_single(dev, dma, req_ctx->hw_context_size, DMA_TO_DEVICE);
2182
2183     return 0;
2184 }
2185
2186 static int keyhash(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen,
2187            u8 *hash)
2188 {
2189     struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
2190
2191     struct scatterlist sg[1];
2192     struct ahash_request *req;
2193     struct crypto_wait wait;
2194     int ret;
2195
2196     crypto_init_wait(&wait);
2197
2198     req = ahash_request_alloc(tfm, GFP_KERNEL);
2199     if (!req)
2200         return -ENOMEM;
2201
2202     /* Keep tfm keylen == 0 during hash of the long key */
2203     ctx->keylen = 0;
2204     ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
2205                    crypto_req_done, &wait);
2206
2207     sg_init_one(&sg[0], key, keylen);
2208
2209     ahash_request_set_crypt(req, sg, hash, keylen);
2210     ret = crypto_wait_req(crypto_ahash_digest(req), &wait);
2211
2212     ahash_request_free(req);
2213
2214     return ret;
2215 }
2216
2217 static int ahash_setkey(struct crypto_ahash *tfm, const u8 *key,
2218             unsigned int keylen)
2219 {
2220     struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
2221     struct device *dev = ctx->dev;
2222     unsigned int blocksize =
2223             crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
2224     unsigned int digestsize = crypto_ahash_digestsize(tfm);
2225     unsigned int keysize = keylen;
2226     u8 hash[SHA512_DIGEST_SIZE];
2227     int ret;
2228
2229     if (keylen <= blocksize)
2230         memcpy(ctx->key, key, keysize);
2231     else {
2232         /* Must get the hash of the long key */
2233         ret = keyhash(tfm, key, keylen, hash);
2234
2235         if (ret)
2236             return -EINVAL;
2237
2238         keysize = digestsize;
2239         memcpy(ctx->key, hash, digestsize);
2240     }
2241
2242     if (ctx->keylen)
2243         dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE);
2244
2245     ctx->keylen = keysize;
2246     ctx->dma_key = dma_map_single(dev, ctx->key, keysize, DMA_TO_DEVICE);
2247
2248     return 0;
2249 }
2250
2251
2252 struct talitos_alg_template {
2253     u32 type;
2254     u32 priority;
2255     union {
2256         struct skcipher_alg skcipher;
2257         struct ahash_alg hash;
2258         struct aead_alg aead;
2259     } alg;
2260     __be32 desc_hdr_template;
2261 };
2262
2263 static struct talitos_alg_template driver_algs[] = {
2264     /* AEAD algorithms.  These use a single-pass ipsec_esp descriptor */
2265     {   .type = CRYPTO_ALG_TYPE_AEAD,
2266         .alg.aead = {
2267             .base = {
2268                 .cra_name = "authenc(hmac(sha1),cbc(aes))",
2269                 .cra_driver_name = "authenc-hmac-sha1-"
2270                            "cbc-aes-talitos",
2271                 .cra_blocksize = AES_BLOCK_SIZE,
2272                 .cra_flags = CRYPTO_ALG_ASYNC |
2273                          CRYPTO_ALG_ALLOCATES_MEMORY,
2274             },
2275             .ivsize = AES_BLOCK_SIZE,
2276             .maxauthsize = SHA1_DIGEST_SIZE,
2277         },
2278         .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2279                          DESC_HDR_SEL0_AESU |
2280                              DESC_HDR_MODE0_AESU_CBC |
2281                              DESC_HDR_SEL1_MDEUA |
2282                              DESC_HDR_MODE1_MDEU_INIT |
2283                              DESC_HDR_MODE1_MDEU_PAD |
2284                              DESC_HDR_MODE1_MDEU_SHA1_HMAC,
2285     },
2286     {   .type = CRYPTO_ALG_TYPE_AEAD,
2287         .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2288         .alg.aead = {
2289             .base = {
2290                 .cra_name = "authenc(hmac(sha1),cbc(aes))",
2291                 .cra_driver_name = "authenc-hmac-sha1-"
2292                            "cbc-aes-talitos-hsna",
2293                 .cra_blocksize = AES_BLOCK_SIZE,
2294                 .cra_flags = CRYPTO_ALG_ASYNC |
2295                          CRYPTO_ALG_ALLOCATES_MEMORY,
2296             },
2297             .ivsize = AES_BLOCK_SIZE,
2298             .maxauthsize = SHA1_DIGEST_SIZE,
2299         },
2300         .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2301                      DESC_HDR_SEL0_AESU |
2302                      DESC_HDR_MODE0_AESU_CBC |
2303                      DESC_HDR_SEL1_MDEUA |
2304                      DESC_HDR_MODE1_MDEU_INIT |
2305                      DESC_HDR_MODE1_MDEU_PAD |
2306                      DESC_HDR_MODE1_MDEU_SHA1_HMAC,
2307     },
2308     {   .type = CRYPTO_ALG_TYPE_AEAD,
2309         .alg.aead = {
2310             .base = {
2311                 .cra_name = "authenc(hmac(sha1),"
2312                         "cbc(des3_ede))",
2313                 .cra_driver_name = "authenc-hmac-sha1-"
2314                            "cbc-3des-talitos",
2315                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2316                 .cra_flags = CRYPTO_ALG_ASYNC |
2317                          CRYPTO_ALG_ALLOCATES_MEMORY,
2318             },
2319             .ivsize = DES3_EDE_BLOCK_SIZE,
2320             .maxauthsize = SHA1_DIGEST_SIZE,
2321             .setkey = aead_des3_setkey,
2322         },
2323         .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2324                          DESC_HDR_SEL0_DEU |
2325                              DESC_HDR_MODE0_DEU_CBC |
2326                              DESC_HDR_MODE0_DEU_3DES |
2327                              DESC_HDR_SEL1_MDEUA |
2328                              DESC_HDR_MODE1_MDEU_INIT |
2329                              DESC_HDR_MODE1_MDEU_PAD |
2330                              DESC_HDR_MODE1_MDEU_SHA1_HMAC,
2331     },
2332     {   .type = CRYPTO_ALG_TYPE_AEAD,
2333         .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2334         .alg.aead = {
2335             .base = {
2336                 .cra_name = "authenc(hmac(sha1),"
2337                         "cbc(des3_ede))",
2338                 .cra_driver_name = "authenc-hmac-sha1-"
2339                            "cbc-3des-talitos-hsna",
2340                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2341                 .cra_flags = CRYPTO_ALG_ASYNC |
2342                          CRYPTO_ALG_ALLOCATES_MEMORY,
2343             },
2344             .ivsize = DES3_EDE_BLOCK_SIZE,
2345             .maxauthsize = SHA1_DIGEST_SIZE,
2346             .setkey = aead_des3_setkey,
2347         },
2348         .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2349                      DESC_HDR_SEL0_DEU |
2350                      DESC_HDR_MODE0_DEU_CBC |
2351                      DESC_HDR_MODE0_DEU_3DES |
2352                      DESC_HDR_SEL1_MDEUA |
2353                      DESC_HDR_MODE1_MDEU_INIT |
2354                      DESC_HDR_MODE1_MDEU_PAD |
2355                      DESC_HDR_MODE1_MDEU_SHA1_HMAC,
2356     },
2357     {       .type = CRYPTO_ALG_TYPE_AEAD,
2358         .alg.aead = {
2359             .base = {
2360                 .cra_name = "authenc(hmac(sha224),cbc(aes))",
2361                 .cra_driver_name = "authenc-hmac-sha224-"
2362                            "cbc-aes-talitos",
2363                 .cra_blocksize = AES_BLOCK_SIZE,
2364                 .cra_flags = CRYPTO_ALG_ASYNC |
2365                          CRYPTO_ALG_ALLOCATES_MEMORY,
2366             },
2367             .ivsize = AES_BLOCK_SIZE,
2368             .maxauthsize = SHA224_DIGEST_SIZE,
2369         },
2370         .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2371                      DESC_HDR_SEL0_AESU |
2372                      DESC_HDR_MODE0_AESU_CBC |
2373                      DESC_HDR_SEL1_MDEUA |
2374                      DESC_HDR_MODE1_MDEU_INIT |
2375                      DESC_HDR_MODE1_MDEU_PAD |
2376                      DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2377     },
2378     {       .type = CRYPTO_ALG_TYPE_AEAD,
2379         .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2380         .alg.aead = {
2381             .base = {
2382                 .cra_name = "authenc(hmac(sha224),cbc(aes))",
2383                 .cra_driver_name = "authenc-hmac-sha224-"
2384                            "cbc-aes-talitos-hsna",
2385                 .cra_blocksize = AES_BLOCK_SIZE,
2386                 .cra_flags = CRYPTO_ALG_ASYNC |
2387                          CRYPTO_ALG_ALLOCATES_MEMORY,
2388             },
2389             .ivsize = AES_BLOCK_SIZE,
2390             .maxauthsize = SHA224_DIGEST_SIZE,
2391         },
2392         .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2393                      DESC_HDR_SEL0_AESU |
2394                      DESC_HDR_MODE0_AESU_CBC |
2395                      DESC_HDR_SEL1_MDEUA |
2396                      DESC_HDR_MODE1_MDEU_INIT |
2397                      DESC_HDR_MODE1_MDEU_PAD |
2398                      DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2399     },
2400     {   .type = CRYPTO_ALG_TYPE_AEAD,
2401         .alg.aead = {
2402             .base = {
2403                 .cra_name = "authenc(hmac(sha224),"
2404                         "cbc(des3_ede))",
2405                 .cra_driver_name = "authenc-hmac-sha224-"
2406                            "cbc-3des-talitos",
2407                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2408                 .cra_flags = CRYPTO_ALG_ASYNC |
2409                          CRYPTO_ALG_ALLOCATES_MEMORY,
2410             },
2411             .ivsize = DES3_EDE_BLOCK_SIZE,
2412             .maxauthsize = SHA224_DIGEST_SIZE,
2413             .setkey = aead_des3_setkey,
2414         },
2415         .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2416                          DESC_HDR_SEL0_DEU |
2417                              DESC_HDR_MODE0_DEU_CBC |
2418                              DESC_HDR_MODE0_DEU_3DES |
2419                              DESC_HDR_SEL1_MDEUA |
2420                              DESC_HDR_MODE1_MDEU_INIT |
2421                              DESC_HDR_MODE1_MDEU_PAD |
2422                              DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2423     },
2424     {   .type = CRYPTO_ALG_TYPE_AEAD,
2425         .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2426         .alg.aead = {
2427             .base = {
2428                 .cra_name = "authenc(hmac(sha224),"
2429                         "cbc(des3_ede))",
2430                 .cra_driver_name = "authenc-hmac-sha224-"
2431                            "cbc-3des-talitos-hsna",
2432                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2433                 .cra_flags = CRYPTO_ALG_ASYNC |
2434                          CRYPTO_ALG_ALLOCATES_MEMORY,
2435             },
2436             .ivsize = DES3_EDE_BLOCK_SIZE,
2437             .maxauthsize = SHA224_DIGEST_SIZE,
2438             .setkey = aead_des3_setkey,
2439         },
2440         .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2441                      DESC_HDR_SEL0_DEU |
2442                      DESC_HDR_MODE0_DEU_CBC |
2443                      DESC_HDR_MODE0_DEU_3DES |
2444                      DESC_HDR_SEL1_MDEUA |
2445                      DESC_HDR_MODE1_MDEU_INIT |
2446                      DESC_HDR_MODE1_MDEU_PAD |
2447                      DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2448     },
2449     {   .type = CRYPTO_ALG_TYPE_AEAD,
2450         .alg.aead = {
2451             .base = {
2452                 .cra_name = "authenc(hmac(sha256),cbc(aes))",
2453                 .cra_driver_name = "authenc-hmac-sha256-"
2454                            "cbc-aes-talitos",
2455                 .cra_blocksize = AES_BLOCK_SIZE,
2456                 .cra_flags = CRYPTO_ALG_ASYNC |
2457                          CRYPTO_ALG_ALLOCATES_MEMORY,
2458             },
2459             .ivsize = AES_BLOCK_SIZE,
2460             .maxauthsize = SHA256_DIGEST_SIZE,
2461         },
2462         .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2463                          DESC_HDR_SEL0_AESU |
2464                              DESC_HDR_MODE0_AESU_CBC |
2465                              DESC_HDR_SEL1_MDEUA |
2466                              DESC_HDR_MODE1_MDEU_INIT |
2467                              DESC_HDR_MODE1_MDEU_PAD |
2468                              DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2469     },
2470     {   .type = CRYPTO_ALG_TYPE_AEAD,
2471         .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2472         .alg.aead = {
2473             .base = {
2474                 .cra_name = "authenc(hmac(sha256),cbc(aes))",
2475                 .cra_driver_name = "authenc-hmac-sha256-"
2476                            "cbc-aes-talitos-hsna",
2477                 .cra_blocksize = AES_BLOCK_SIZE,
2478                 .cra_flags = CRYPTO_ALG_ASYNC |
2479                          CRYPTO_ALG_ALLOCATES_MEMORY,
2480             },
2481             .ivsize = AES_BLOCK_SIZE,
2482             .maxauthsize = SHA256_DIGEST_SIZE,
2483         },
2484         .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2485                      DESC_HDR_SEL0_AESU |
2486                      DESC_HDR_MODE0_AESU_CBC |
2487                      DESC_HDR_SEL1_MDEUA |
2488                      DESC_HDR_MODE1_MDEU_INIT |
2489                      DESC_HDR_MODE1_MDEU_PAD |
2490                      DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2491     },
2492     {   .type = CRYPTO_ALG_TYPE_AEAD,
2493         .alg.aead = {
2494             .base = {
2495                 .cra_name = "authenc(hmac(sha256),"
2496                         "cbc(des3_ede))",
2497                 .cra_driver_name = "authenc-hmac-sha256-"
2498                            "cbc-3des-talitos",
2499                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2500                 .cra_flags = CRYPTO_ALG_ASYNC |
2501                          CRYPTO_ALG_ALLOCATES_MEMORY,
2502             },
2503             .ivsize = DES3_EDE_BLOCK_SIZE,
2504             .maxauthsize = SHA256_DIGEST_SIZE,
2505             .setkey = aead_des3_setkey,
2506         },
2507         .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2508                          DESC_HDR_SEL0_DEU |
2509                              DESC_HDR_MODE0_DEU_CBC |
2510                              DESC_HDR_MODE0_DEU_3DES |
2511                              DESC_HDR_SEL1_MDEUA |
2512                              DESC_HDR_MODE1_MDEU_INIT |
2513                              DESC_HDR_MODE1_MDEU_PAD |
2514                              DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2515     },
2516     {   .type = CRYPTO_ALG_TYPE_AEAD,
2517         .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2518         .alg.aead = {
2519             .base = {
2520                 .cra_name = "authenc(hmac(sha256),"
2521                         "cbc(des3_ede))",
2522                 .cra_driver_name = "authenc-hmac-sha256-"
2523                            "cbc-3des-talitos-hsna",
2524                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2525                 .cra_flags = CRYPTO_ALG_ASYNC |
2526                          CRYPTO_ALG_ALLOCATES_MEMORY,
2527             },
2528             .ivsize = DES3_EDE_BLOCK_SIZE,
2529             .maxauthsize = SHA256_DIGEST_SIZE,
2530             .setkey = aead_des3_setkey,
2531         },
2532         .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2533                      DESC_HDR_SEL0_DEU |
2534                      DESC_HDR_MODE0_DEU_CBC |
2535                      DESC_HDR_MODE0_DEU_3DES |
2536                      DESC_HDR_SEL1_MDEUA |
2537                      DESC_HDR_MODE1_MDEU_INIT |
2538                      DESC_HDR_MODE1_MDEU_PAD |
2539                      DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2540     },
2541     {   .type = CRYPTO_ALG_TYPE_AEAD,
2542         .alg.aead = {
2543             .base = {
2544                 .cra_name = "authenc(hmac(sha384),cbc(aes))",
2545                 .cra_driver_name = "authenc-hmac-sha384-"
2546                            "cbc-aes-talitos",
2547                 .cra_blocksize = AES_BLOCK_SIZE,
2548                 .cra_flags = CRYPTO_ALG_ASYNC |
2549                          CRYPTO_ALG_ALLOCATES_MEMORY,
2550             },
2551             .ivsize = AES_BLOCK_SIZE,
2552             .maxauthsize = SHA384_DIGEST_SIZE,
2553         },
2554         .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2555                          DESC_HDR_SEL0_AESU |
2556                              DESC_HDR_MODE0_AESU_CBC |
2557                              DESC_HDR_SEL1_MDEUB |
2558                              DESC_HDR_MODE1_MDEU_INIT |
2559                              DESC_HDR_MODE1_MDEU_PAD |
2560                              DESC_HDR_MODE1_MDEUB_SHA384_HMAC,
2561     },
2562     {   .type = CRYPTO_ALG_TYPE_AEAD,
2563         .alg.aead = {
2564             .base = {
2565                 .cra_name = "authenc(hmac(sha384),"
2566                         "cbc(des3_ede))",
2567                 .cra_driver_name = "authenc-hmac-sha384-"
2568                            "cbc-3des-talitos",
2569                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2570                 .cra_flags = CRYPTO_ALG_ASYNC |
2571                          CRYPTO_ALG_ALLOCATES_MEMORY,
2572             },
2573             .ivsize = DES3_EDE_BLOCK_SIZE,
2574             .maxauthsize = SHA384_DIGEST_SIZE,
2575             .setkey = aead_des3_setkey,
2576         },
2577         .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2578                          DESC_HDR_SEL0_DEU |
2579                              DESC_HDR_MODE0_DEU_CBC |
2580                              DESC_HDR_MODE0_DEU_3DES |
2581                              DESC_HDR_SEL1_MDEUB |
2582                              DESC_HDR_MODE1_MDEU_INIT |
2583                              DESC_HDR_MODE1_MDEU_PAD |
2584                              DESC_HDR_MODE1_MDEUB_SHA384_HMAC,
2585     },
2586     {   .type = CRYPTO_ALG_TYPE_AEAD,
2587         .alg.aead = {
2588             .base = {
2589                 .cra_name = "authenc(hmac(sha512),cbc(aes))",
2590                 .cra_driver_name = "authenc-hmac-sha512-"
2591                            "cbc-aes-talitos",
2592                 .cra_blocksize = AES_BLOCK_SIZE,
2593                 .cra_flags = CRYPTO_ALG_ASYNC |
2594                          CRYPTO_ALG_ALLOCATES_MEMORY,
2595             },
2596             .ivsize = AES_BLOCK_SIZE,
2597             .maxauthsize = SHA512_DIGEST_SIZE,
2598         },
2599         .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2600                          DESC_HDR_SEL0_AESU |
2601                              DESC_HDR_MODE0_AESU_CBC |
2602                              DESC_HDR_SEL1_MDEUB |
2603                              DESC_HDR_MODE1_MDEU_INIT |
2604                              DESC_HDR_MODE1_MDEU_PAD |
2605                              DESC_HDR_MODE1_MDEUB_SHA512_HMAC,
2606     },
2607     {   .type = CRYPTO_ALG_TYPE_AEAD,
2608         .alg.aead = {
2609             .base = {
2610                 .cra_name = "authenc(hmac(sha512),"
2611                         "cbc(des3_ede))",
2612                 .cra_driver_name = "authenc-hmac-sha512-"
2613                            "cbc-3des-talitos",
2614                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2615                 .cra_flags = CRYPTO_ALG_ASYNC |
2616                          CRYPTO_ALG_ALLOCATES_MEMORY,
2617             },
2618             .ivsize = DES3_EDE_BLOCK_SIZE,
2619             .maxauthsize = SHA512_DIGEST_SIZE,
2620             .setkey = aead_des3_setkey,
2621         },
2622         .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2623                          DESC_HDR_SEL0_DEU |
2624                              DESC_HDR_MODE0_DEU_CBC |
2625                              DESC_HDR_MODE0_DEU_3DES |
2626                              DESC_HDR_SEL1_MDEUB |
2627                              DESC_HDR_MODE1_MDEU_INIT |
2628                              DESC_HDR_MODE1_MDEU_PAD |
2629                              DESC_HDR_MODE1_MDEUB_SHA512_HMAC,
2630     },
2631     {   .type = CRYPTO_ALG_TYPE_AEAD,
2632         .alg.aead = {
2633             .base = {
2634                 .cra_name = "authenc(hmac(md5),cbc(aes))",
2635                 .cra_driver_name = "authenc-hmac-md5-"
2636                            "cbc-aes-talitos",
2637                 .cra_blocksize = AES_BLOCK_SIZE,
2638                 .cra_flags = CRYPTO_ALG_ASYNC |
2639                          CRYPTO_ALG_ALLOCATES_MEMORY,
2640             },
2641             .ivsize = AES_BLOCK_SIZE,
2642             .maxauthsize = MD5_DIGEST_SIZE,
2643         },
2644         .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2645                          DESC_HDR_SEL0_AESU |
2646                              DESC_HDR_MODE0_AESU_CBC |
2647                              DESC_HDR_SEL1_MDEUA |
2648                              DESC_HDR_MODE1_MDEU_INIT |
2649                              DESC_HDR_MODE1_MDEU_PAD |
2650                              DESC_HDR_MODE1_MDEU_MD5_HMAC,
2651     },
2652     {   .type = CRYPTO_ALG_TYPE_AEAD,
2653         .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2654         .alg.aead = {
2655             .base = {
2656                 .cra_name = "authenc(hmac(md5),cbc(aes))",
2657                 .cra_driver_name = "authenc-hmac-md5-"
2658                            "cbc-aes-talitos-hsna",
2659                 .cra_blocksize = AES_BLOCK_SIZE,
2660                 .cra_flags = CRYPTO_ALG_ASYNC |
2661                          CRYPTO_ALG_ALLOCATES_MEMORY,
2662             },
2663             .ivsize = AES_BLOCK_SIZE,
2664             .maxauthsize = MD5_DIGEST_SIZE,
2665         },
2666         .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2667                      DESC_HDR_SEL0_AESU |
2668                      DESC_HDR_MODE0_AESU_CBC |
2669                      DESC_HDR_SEL1_MDEUA |
2670                      DESC_HDR_MODE1_MDEU_INIT |
2671                      DESC_HDR_MODE1_MDEU_PAD |
2672                      DESC_HDR_MODE1_MDEU_MD5_HMAC,
2673     },
2674     {   .type = CRYPTO_ALG_TYPE_AEAD,
2675         .alg.aead = {
2676             .base = {
2677                 .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
2678                 .cra_driver_name = "authenc-hmac-md5-"
2679                            "cbc-3des-talitos",
2680                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2681                 .cra_flags = CRYPTO_ALG_ASYNC |
2682                          CRYPTO_ALG_ALLOCATES_MEMORY,
2683             },
2684             .ivsize = DES3_EDE_BLOCK_SIZE,
2685             .maxauthsize = MD5_DIGEST_SIZE,
2686             .setkey = aead_des3_setkey,
2687         },
2688         .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2689                          DESC_HDR_SEL0_DEU |
2690                              DESC_HDR_MODE0_DEU_CBC |
2691                              DESC_HDR_MODE0_DEU_3DES |
2692                              DESC_HDR_SEL1_MDEUA |
2693                              DESC_HDR_MODE1_MDEU_INIT |
2694                              DESC_HDR_MODE1_MDEU_PAD |
2695                              DESC_HDR_MODE1_MDEU_MD5_HMAC,
2696     },
2697     {   .type = CRYPTO_ALG_TYPE_AEAD,
2698         .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2699         .alg.aead = {
2700             .base = {
2701                 .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
2702                 .cra_driver_name = "authenc-hmac-md5-"
2703                            "cbc-3des-talitos-hsna",
2704                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2705                 .cra_flags = CRYPTO_ALG_ASYNC |
2706                          CRYPTO_ALG_ALLOCATES_MEMORY,
2707             },
2708             .ivsize = DES3_EDE_BLOCK_SIZE,
2709             .maxauthsize = MD5_DIGEST_SIZE,
2710             .setkey = aead_des3_setkey,
2711         },
2712         .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2713                      DESC_HDR_SEL0_DEU |
2714                      DESC_HDR_MODE0_DEU_CBC |
2715                      DESC_HDR_MODE0_DEU_3DES |
2716                      DESC_HDR_SEL1_MDEUA |
2717                      DESC_HDR_MODE1_MDEU_INIT |
2718                      DESC_HDR_MODE1_MDEU_PAD |
2719                      DESC_HDR_MODE1_MDEU_MD5_HMAC,
2720     },
2721     /* SKCIPHER algorithms. */
2722     {   .type = CRYPTO_ALG_TYPE_SKCIPHER,
2723         .alg.skcipher = {
2724             .base.cra_name = "ecb(aes)",
2725             .base.cra_driver_name = "ecb-aes-talitos",
2726             .base.cra_blocksize = AES_BLOCK_SIZE,
2727             .base.cra_flags = CRYPTO_ALG_ASYNC |
2728                       CRYPTO_ALG_ALLOCATES_MEMORY,
2729             .min_keysize = AES_MIN_KEY_SIZE,
2730             .max_keysize = AES_MAX_KEY_SIZE,
2731             .setkey = skcipher_aes_setkey,
2732         },
2733         .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2734                      DESC_HDR_SEL0_AESU,
2735     },
2736     {   .type = CRYPTO_ALG_TYPE_SKCIPHER,
2737         .alg.skcipher = {
2738             .base.cra_name = "cbc(aes)",
2739             .base.cra_driver_name = "cbc-aes-talitos",
2740             .base.cra_blocksize = AES_BLOCK_SIZE,
2741             .base.cra_flags = CRYPTO_ALG_ASYNC |
2742                       CRYPTO_ALG_ALLOCATES_MEMORY,
2743             .min_keysize = AES_MIN_KEY_SIZE,
2744             .max_keysize = AES_MAX_KEY_SIZE,
2745             .ivsize = AES_BLOCK_SIZE,
2746             .setkey = skcipher_aes_setkey,
2747         },
2748         .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2749                      DESC_HDR_SEL0_AESU |
2750                      DESC_HDR_MODE0_AESU_CBC,
2751     },
2752     {   .type = CRYPTO_ALG_TYPE_SKCIPHER,
2753         .alg.skcipher = {
2754             .base.cra_name = "ctr(aes)",
2755             .base.cra_driver_name = "ctr-aes-talitos",
2756             .base.cra_blocksize = 1,
2757             .base.cra_flags = CRYPTO_ALG_ASYNC |
2758                       CRYPTO_ALG_ALLOCATES_MEMORY,
2759             .min_keysize = AES_MIN_KEY_SIZE,
2760             .max_keysize = AES_MAX_KEY_SIZE,
2761             .ivsize = AES_BLOCK_SIZE,
2762             .setkey = skcipher_aes_setkey,
2763         },
2764         .desc_hdr_template = DESC_HDR_TYPE_AESU_CTR_NONSNOOP |
2765                      DESC_HDR_SEL0_AESU |
2766                      DESC_HDR_MODE0_AESU_CTR,
2767     },
2768     {   .type = CRYPTO_ALG_TYPE_SKCIPHER,
2769         .alg.skcipher = {
2770             .base.cra_name = "ctr(aes)",
2771             .base.cra_driver_name = "ctr-aes-talitos",
2772             .base.cra_blocksize = 1,
2773             .base.cra_flags = CRYPTO_ALG_ASYNC |
2774                       CRYPTO_ALG_ALLOCATES_MEMORY,
2775             .min_keysize = AES_MIN_KEY_SIZE,
2776             .max_keysize = AES_MAX_KEY_SIZE,
2777             .ivsize = AES_BLOCK_SIZE,
2778             .setkey = skcipher_aes_setkey,
2779         },
2780         .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2781                      DESC_HDR_SEL0_AESU |
2782                      DESC_HDR_MODE0_AESU_CTR,
2783     },
2784     {   .type = CRYPTO_ALG_TYPE_SKCIPHER,
2785         .alg.skcipher = {
2786             .base.cra_name = "ecb(des)",
2787             .base.cra_driver_name = "ecb-des-talitos",
2788             .base.cra_blocksize = DES_BLOCK_SIZE,
2789             .base.cra_flags = CRYPTO_ALG_ASYNC |
2790                       CRYPTO_ALG_ALLOCATES_MEMORY,
2791             .min_keysize = DES_KEY_SIZE,
2792             .max_keysize = DES_KEY_SIZE,
2793             .setkey = skcipher_des_setkey,
2794         },
2795         .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2796                      DESC_HDR_SEL0_DEU,
2797     },
2798     {   .type = CRYPTO_ALG_TYPE_SKCIPHER,
2799         .alg.skcipher = {
2800             .base.cra_name = "cbc(des)",
2801             .base.cra_driver_name = "cbc-des-talitos",
2802             .base.cra_blocksize = DES_BLOCK_SIZE,
2803             .base.cra_flags = CRYPTO_ALG_ASYNC |
2804                       CRYPTO_ALG_ALLOCATES_MEMORY,
2805             .min_keysize = DES_KEY_SIZE,
2806             .max_keysize = DES_KEY_SIZE,
2807             .ivsize = DES_BLOCK_SIZE,
2808             .setkey = skcipher_des_setkey,
2809         },
2810         .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2811                      DESC_HDR_SEL0_DEU |
2812                      DESC_HDR_MODE0_DEU_CBC,
2813     },
2814     {   .type = CRYPTO_ALG_TYPE_SKCIPHER,
2815         .alg.skcipher = {
2816             .base.cra_name = "ecb(des3_ede)",
2817             .base.cra_driver_name = "ecb-3des-talitos",
2818             .base.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2819             .base.cra_flags = CRYPTO_ALG_ASYNC |
2820                       CRYPTO_ALG_ALLOCATES_MEMORY,
2821             .min_keysize = DES3_EDE_KEY_SIZE,
2822             .max_keysize = DES3_EDE_KEY_SIZE,
2823             .setkey = skcipher_des3_setkey,
2824         },
2825         .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2826                      DESC_HDR_SEL0_DEU |
2827                      DESC_HDR_MODE0_DEU_3DES,
2828     },
2829     {   .type = CRYPTO_ALG_TYPE_SKCIPHER,
2830         .alg.skcipher = {
2831             .base.cra_name = "cbc(des3_ede)",
2832             .base.cra_driver_name = "cbc-3des-talitos",
2833             .base.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2834             .base.cra_flags = CRYPTO_ALG_ASYNC |
2835                       CRYPTO_ALG_ALLOCATES_MEMORY,
2836             .min_keysize = DES3_EDE_KEY_SIZE,
2837             .max_keysize = DES3_EDE_KEY_SIZE,
2838             .ivsize = DES3_EDE_BLOCK_SIZE,
2839             .setkey = skcipher_des3_setkey,
2840         },
2841         .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2842                          DESC_HDR_SEL0_DEU |
2843                              DESC_HDR_MODE0_DEU_CBC |
2844                              DESC_HDR_MODE0_DEU_3DES,
2845     },
2846     /* AHASH algorithms. */
2847     {   .type = CRYPTO_ALG_TYPE_AHASH,
2848         .alg.hash = {
2849             .halg.digestsize = MD5_DIGEST_SIZE,
2850             .halg.statesize = sizeof(struct talitos_export_state),
2851             .halg.base = {
2852                 .cra_name = "md5",
2853                 .cra_driver_name = "md5-talitos",
2854                 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
2855                 .cra_flags = CRYPTO_ALG_ASYNC |
2856                          CRYPTO_ALG_ALLOCATES_MEMORY,
2857             }
2858         },
2859         .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2860                      DESC_HDR_SEL0_MDEUA |
2861                      DESC_HDR_MODE0_MDEU_MD5,
2862     },
2863     {   .type = CRYPTO_ALG_TYPE_AHASH,
2864         .alg.hash = {
2865             .halg.digestsize = SHA1_DIGEST_SIZE,
2866             .halg.statesize = sizeof(struct talitos_export_state),
2867             .halg.base = {
2868                 .cra_name = "sha1",
2869                 .cra_driver_name = "sha1-talitos",
2870                 .cra_blocksize = SHA1_BLOCK_SIZE,
2871                 .cra_flags = CRYPTO_ALG_ASYNC |
2872                          CRYPTO_ALG_ALLOCATES_MEMORY,
2873             }
2874         },
2875         .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2876                      DESC_HDR_SEL0_MDEUA |
2877                      DESC_HDR_MODE0_MDEU_SHA1,
2878     },
2879     {   .type = CRYPTO_ALG_TYPE_AHASH,
2880         .alg.hash = {
2881             .halg.digestsize = SHA224_DIGEST_SIZE,
2882             .halg.statesize = sizeof(struct talitos_export_state),
2883             .halg.base = {
2884                 .cra_name = "sha224",
2885                 .cra_driver_name = "sha224-talitos",
2886                 .cra_blocksize = SHA224_BLOCK_SIZE,
2887                 .cra_flags = CRYPTO_ALG_ASYNC |
2888                          CRYPTO_ALG_ALLOCATES_MEMORY,
2889             }
2890         },
2891         .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2892                      DESC_HDR_SEL0_MDEUA |
2893                      DESC_HDR_MODE0_MDEU_SHA224,
2894     },
2895     {   .type = CRYPTO_ALG_TYPE_AHASH,
2896         .alg.hash = {
2897             .halg.digestsize = SHA256_DIGEST_SIZE,
2898             .halg.statesize = sizeof(struct talitos_export_state),
2899             .halg.base = {
2900                 .cra_name = "sha256",
2901                 .cra_driver_name = "sha256-talitos",
2902                 .cra_blocksize = SHA256_BLOCK_SIZE,
2903                 .cra_flags = CRYPTO_ALG_ASYNC |
2904                          CRYPTO_ALG_ALLOCATES_MEMORY,
2905             }
2906         },
2907         .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2908                      DESC_HDR_SEL0_MDEUA |
2909                      DESC_HDR_MODE0_MDEU_SHA256,
2910     },
2911     {   .type = CRYPTO_ALG_TYPE_AHASH,
2912         .alg.hash = {
2913             .halg.digestsize = SHA384_DIGEST_SIZE,
2914             .halg.statesize = sizeof(struct talitos_export_state),
2915             .halg.base = {
2916                 .cra_name = "sha384",
2917                 .cra_driver_name = "sha384-talitos",
2918                 .cra_blocksize = SHA384_BLOCK_SIZE,
2919                 .cra_flags = CRYPTO_ALG_ASYNC |
2920                          CRYPTO_ALG_ALLOCATES_MEMORY,
2921             }
2922         },
2923         .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2924                      DESC_HDR_SEL0_MDEUB |
2925                      DESC_HDR_MODE0_MDEUB_SHA384,
2926     },
2927     {   .type = CRYPTO_ALG_TYPE_AHASH,
2928         .alg.hash = {
2929             .halg.digestsize = SHA512_DIGEST_SIZE,
2930             .halg.statesize = sizeof(struct talitos_export_state),
2931             .halg.base = {
2932                 .cra_name = "sha512",
2933                 .cra_driver_name = "sha512-talitos",
2934                 .cra_blocksize = SHA512_BLOCK_SIZE,
2935                 .cra_flags = CRYPTO_ALG_ASYNC |
2936                          CRYPTO_ALG_ALLOCATES_MEMORY,
2937             }
2938         },
2939         .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2940                      DESC_HDR_SEL0_MDEUB |
2941                      DESC_HDR_MODE0_MDEUB_SHA512,
2942     },
2943     {   .type = CRYPTO_ALG_TYPE_AHASH,
2944         .alg.hash = {
2945             .halg.digestsize = MD5_DIGEST_SIZE,
2946             .halg.statesize = sizeof(struct talitos_export_state),
2947             .halg.base = {
2948                 .cra_name = "hmac(md5)",
2949                 .cra_driver_name = "hmac-md5-talitos",
2950                 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
2951                 .cra_flags = CRYPTO_ALG_ASYNC |
2952                          CRYPTO_ALG_ALLOCATES_MEMORY,
2953             }
2954         },
2955         .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2956                      DESC_HDR_SEL0_MDEUA |
2957                      DESC_HDR_MODE0_MDEU_MD5,
2958     },
2959     {   .type = CRYPTO_ALG_TYPE_AHASH,
2960         .alg.hash = {
2961             .halg.digestsize = SHA1_DIGEST_SIZE,
2962             .halg.statesize = sizeof(struct talitos_export_state),
2963             .halg.base = {
2964                 .cra_name = "hmac(sha1)",
2965                 .cra_driver_name = "hmac-sha1-talitos",
2966                 .cra_blocksize = SHA1_BLOCK_SIZE,
2967                 .cra_flags = CRYPTO_ALG_ASYNC |
2968                          CRYPTO_ALG_ALLOCATES_MEMORY,
2969             }
2970         },
2971         .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2972                      DESC_HDR_SEL0_MDEUA |
2973                      DESC_HDR_MODE0_MDEU_SHA1,
2974     },
2975     {   .type = CRYPTO_ALG_TYPE_AHASH,
2976         .alg.hash = {
2977             .halg.digestsize = SHA224_DIGEST_SIZE,
2978             .halg.statesize = sizeof(struct talitos_export_state),
2979             .halg.base = {
2980                 .cra_name = "hmac(sha224)",
2981                 .cra_driver_name = "hmac-sha224-talitos",
2982                 .cra_blocksize = SHA224_BLOCK_SIZE,
2983                 .cra_flags = CRYPTO_ALG_ASYNC |
2984                          CRYPTO_ALG_ALLOCATES_MEMORY,
2985             }
2986         },
2987         .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2988                      DESC_HDR_SEL0_MDEUA |
2989                      DESC_HDR_MODE0_MDEU_SHA224,
2990     },
2991     {   .type = CRYPTO_ALG_TYPE_AHASH,
2992         .alg.hash = {
2993             .halg.digestsize = SHA256_DIGEST_SIZE,
2994             .halg.statesize = sizeof(struct talitos_export_state),
2995             .halg.base = {
2996                 .cra_name = "hmac(sha256)",
2997                 .cra_driver_name = "hmac-sha256-talitos",
2998                 .cra_blocksize = SHA256_BLOCK_SIZE,
2999                 .cra_flags = CRYPTO_ALG_ASYNC |
3000                          CRYPTO_ALG_ALLOCATES_MEMORY,
3001             }
3002         },
3003         .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
3004                      DESC_HDR_SEL0_MDEUA |
3005                      DESC_HDR_MODE0_MDEU_SHA256,
3006     },
3007     {   .type = CRYPTO_ALG_TYPE_AHASH,
3008         .alg.hash = {
3009             .halg.digestsize = SHA384_DIGEST_SIZE,
3010             .halg.statesize = sizeof(struct talitos_export_state),
3011             .halg.base = {
3012                 .cra_name = "hmac(sha384)",
3013                 .cra_driver_name = "hmac-sha384-talitos",
3014                 .cra_blocksize = SHA384_BLOCK_SIZE,
3015                 .cra_flags = CRYPTO_ALG_ASYNC |
3016                          CRYPTO_ALG_ALLOCATES_MEMORY,
3017             }
3018         },
3019         .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
3020                      DESC_HDR_SEL0_MDEUB |
3021                      DESC_HDR_MODE0_MDEUB_SHA384,
3022     },
3023     {   .type = CRYPTO_ALG_TYPE_AHASH,
3024         .alg.hash = {
3025             .halg.digestsize = SHA512_DIGEST_SIZE,
3026             .halg.statesize = sizeof(struct talitos_export_state),
3027             .halg.base = {
3028                 .cra_name = "hmac(sha512)",
3029                 .cra_driver_name = "hmac-sha512-talitos",
3030                 .cra_blocksize = SHA512_BLOCK_SIZE,
3031                 .cra_flags = CRYPTO_ALG_ASYNC |
3032                          CRYPTO_ALG_ALLOCATES_MEMORY,
3033             }
3034         },
3035         .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
3036                      DESC_HDR_SEL0_MDEUB |
3037                      DESC_HDR_MODE0_MDEUB_SHA512,
3038     }
3039 };
3040
3041 struct talitos_crypto_alg {
3042     struct list_head entry;
3043     struct device *dev;
3044     struct talitos_alg_template algt;
3045 };
3046
3047 static int talitos_init_common(struct talitos_ctx *ctx,
3048                    struct talitos_crypto_alg *talitos_alg)
3049 {
3050     struct talitos_private *priv;
3051
3052     /* update context with ptr to dev */
3053     ctx->dev = talitos_alg->dev;
3054
3055     /* assign SEC channel to tfm in round-robin fashion */
3056     priv = dev_get_drvdata(ctx->dev);
3057     ctx->ch = atomic_inc_return(&priv->last_chan) &
3058           (priv->num_channels - 1);
3059
3060     /* copy descriptor header template value */
3061     ctx->desc_hdr_template = talitos_alg->algt.desc_hdr_template;
3062
3063     /* select done notification */
3064     ctx->desc_hdr_template |= DESC_HDR_DONE_NOTIFY;
3065
3066     return 0;
3067 }
3068
3069 static int talitos_cra_init_aead(struct crypto_aead *tfm)
3070 {
3071     struct aead_alg *alg = crypto_aead_alg(tfm);
3072     struct talitos_crypto_alg *talitos_alg;
3073     struct talitos_ctx *ctx = crypto_aead_ctx(tfm);
3074
3075     talitos_alg = container_of(alg, struct talitos_crypto_alg,
3076                    algt.alg.aead);
3077
3078     return talitos_init_common(ctx, talitos_alg);
3079 }
3080
3081 static int talitos_cra_init_skcipher(struct crypto_skcipher *tfm)
3082 {
3083     struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
3084     struct talitos_crypto_alg *talitos_alg;
3085     struct talitos_ctx *ctx = crypto_skcipher_ctx(tfm);
3086
3087     talitos_alg = container_of(alg, struct talitos_crypto_alg,
3088                    algt.alg.skcipher);
3089
3090     return talitos_init_common(ctx, talitos_alg);
3091 }
3092
3093 static int talitos_cra_init_ahash(struct crypto_tfm *tfm)
3094 {
3095     struct crypto_alg *alg = tfm->__crt_alg;
3096     struct talitos_crypto_alg *talitos_alg;
3097     struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
3098
3099     talitos_alg = container_of(__crypto_ahash_alg(alg),
3100                    struct talitos_crypto_alg,
3101                    algt.alg.hash);
3102
3103     ctx->keylen = 0;
3104     crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
3105                  sizeof(struct talitos_ahash_req_ctx));
3106
3107     return talitos_init_common(ctx, talitos_alg);
3108 }
3109
3110 static void talitos_cra_exit(struct crypto_tfm *tfm)
3111 {
3112     struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
3113     struct device *dev = ctx->dev;
3114
3115     if (ctx->keylen)
3116         dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE);
3117 }
3118
3119 /*
3120  * given the alg's descriptor header template, determine whether descriptor
3121  * type and primary/secondary execution units required match the hw
3122  * capabilities description provided in the device tree node.
3123  */
3124 static int hw_supports(struct device *dev, __be32 desc_hdr_template)
3125 {
3126     struct talitos_private *priv = dev_get_drvdata(dev);
3127     int ret;
3128
3129     ret = (1 << DESC_TYPE(desc_hdr_template) & priv->desc_types) &&
3130           (1 << PRIMARY_EU(desc_hdr_template) & priv->exec_units);
3131
3132     if (SECONDARY_EU(desc_hdr_template))
3133         ret = ret && (1 << SECONDARY_EU(desc_hdr_template)
3134                       & priv->exec_units);
3135
3136     return ret;
3137 }
3138
3139 static int talitos_remove(struct platform_device *ofdev)
3140 {
3141     struct device *dev = &ofdev->dev;
3142     struct talitos_private *priv = dev_get_drvdata(dev);
3143     struct talitos_crypto_alg *t_alg, *n;
3144     int i;
3145
3146     list_for_each_entry_safe(t_alg, n, &priv->alg_list, entry) {
3147         switch (t_alg->algt.type) {
3148         case CRYPTO_ALG_TYPE_SKCIPHER:
3149             crypto_unregister_skcipher(&t_alg->algt.alg.skcipher);
3150             break;
3151         case CRYPTO_ALG_TYPE_AEAD:
3152             crypto_unregister_aead(&t_alg->algt.alg.aead);
3153             break;
3154         case CRYPTO_ALG_TYPE_AHASH:
3155             crypto_unregister_ahash(&t_alg->algt.alg.hash);
3156             break;
3157         }
3158         list_del(&t_alg->entry);
3159     }
3160
3161     if (hw_supports(dev, DESC_HDR_SEL0_RNG))
3162         talitos_unregister_rng(dev);
3163
3164     for (i = 0; i < 2; i++)
3165         if (priv->irq[i]) {
3166             free_irq(priv->irq[i], dev);
3167             irq_dispose_mapping(priv->irq[i]);
3168         }
3169
3170     tasklet_kill(&priv->done_task[0]);
3171     if (priv->irq[1])
3172         tasklet_kill(&priv->done_task[1]);
3173
3174     return 0;
3175 }
3176
3177 static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev,
3178                             struct talitos_alg_template
3179                                    *template)
3180 {
3181     struct talitos_private *priv = dev_get_drvdata(dev);
3182     struct talitos_crypto_alg *t_alg;
3183     struct crypto_alg *alg;
3184
3185     t_alg = devm_kzalloc(dev, sizeof(struct talitos_crypto_alg),
3186                  GFP_KERNEL);
3187     if (!t_alg)
3188         return ERR_PTR(-ENOMEM);
3189
3190     t_alg->algt = *template;
3191
3192     switch (t_alg->algt.type) {
3193     case CRYPTO_ALG_TYPE_SKCIPHER:
3194         alg = &t_alg->algt.alg.skcipher.base;
3195         alg->cra_exit = talitos_cra_exit;
3196         t_alg->algt.alg.skcipher.init = talitos_cra_init_skcipher;
3197         t_alg->algt.alg.skcipher.setkey =
3198             t_alg->algt.alg.skcipher.setkey ?: skcipher_setkey;
3199         t_alg->algt.alg.skcipher.encrypt = skcipher_encrypt;
3200         t_alg->algt.alg.skcipher.decrypt = skcipher_decrypt;
3201         if (!strcmp(alg->cra_name, "ctr(aes)") && !has_ftr_sec1(priv) &&
3202             DESC_TYPE(t_alg->algt.desc_hdr_template) !=
3203             DESC_TYPE(DESC_HDR_TYPE_AESU_CTR_NONSNOOP)) {
3204             devm_kfree(dev, t_alg);
3205             return ERR_PTR(-ENOTSUPP);
3206         }
3207         break;
3208     case CRYPTO_ALG_TYPE_AEAD:
3209         alg = &t_alg->algt.alg.aead.base;
3210         alg->cra_exit = talitos_cra_exit;
3211         t_alg->algt.alg.aead.init = talitos_cra_init_aead;
3212         t_alg->algt.alg.aead.setkey = t_alg->algt.alg.aead.setkey ?:
3213                           aead_setkey;
3214         t_alg->algt.alg.aead.encrypt = aead_encrypt;
3215         t_alg->algt.alg.aead.decrypt = aead_decrypt;
3216         if (!(priv->features & TALITOS_FTR_SHA224_HWINIT) &&
3217             !strncmp(alg->cra_name, "authenc(hmac(sha224)", 20)) {
3218             devm_kfree(dev, t_alg);
3219             return ERR_PTR(-ENOTSUPP);
3220         }
3221         break;
3222     case CRYPTO_ALG_TYPE_AHASH:
3223         alg = &t_alg->algt.alg.hash.halg.base;
3224         alg->cra_init = talitos_cra_init_ahash;
3225         alg->cra_exit = talitos_cra_exit;
3226         t_alg->algt.alg.hash.init = ahash_init;
3227         t_alg->algt.alg.hash.update = ahash_update;
3228         t_alg->algt.alg.hash.final = ahash_final;
3229         t_alg->algt.alg.hash.finup = ahash_finup;
3230         t_alg->algt.alg.hash.digest = ahash_digest;
3231         if (!strncmp(alg->cra_name, "hmac", 4))
3232             t_alg->algt.alg.hash.setkey = ahash_setkey;
3233         t_alg->algt.alg.hash.import = ahash_import;
3234         t_alg->algt.alg.hash.export = ahash_export;
3235
3236         if (!(priv->features & TALITOS_FTR_HMAC_OK) &&
3237             !strncmp(alg->cra_name, "hmac", 4)) {
3238             devm_kfree(dev, t_alg);
3239             return ERR_PTR(-ENOTSUPP);
3240         }
3241         if (!(priv->features & TALITOS_FTR_SHA224_HWINIT) &&
3242             (!strcmp(alg->cra_name, "sha224") ||
3243              !strcmp(alg->cra_name, "hmac(sha224)"))) {
3244             t_alg->algt.alg.hash.init = ahash_init_sha224_swinit;
3245             t_alg->algt.desc_hdr_template =
3246                     DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
3247                     DESC_HDR_SEL0_MDEUA |
3248                     DESC_HDR_MODE0_MDEU_SHA256;
3249         }
3250         break;
3251     default:
3252         dev_err(dev, "unknown algorithm type %d\n", t_alg->algt.type);
3253         devm_kfree(dev, t_alg);
3254         return ERR_PTR(-EINVAL);
3255     }
3256
3257     alg->cra_module = THIS_MODULE;
3258     if (t_alg->algt.priority)
3259         alg->cra_priority = t_alg->algt.priority;
3260     else
3261         alg->cra_priority = TALITOS_CRA_PRIORITY;
3262     if (has_ftr_sec1(priv))
3263         alg->cra_alignmask = 3;
3264     else
3265         alg->cra_alignmask = 0;
3266     alg->cra_ctxsize = sizeof(struct talitos_ctx);
3267     alg->cra_flags |= CRYPTO_ALG_KERN_DRIVER_ONLY;
3268
3269     t_alg->dev = dev;
3270
3271     return t_alg;
3272 }
3273
3274 static int talitos_probe_irq(struct platform_device *ofdev)
3275 {
3276     struct device *dev = &ofdev->dev;
3277     struct device_node *np = ofdev->dev.of_node;
3278     struct talitos_private *priv = dev_get_drvdata(dev);
3279     int err;
3280     bool is_sec1 = has_ftr_sec1(priv);
3281
3282     priv->irq[0] = irq_of_parse_and_map(np, 0);
3283     if (!priv->irq[0]) {
3284         dev_err(dev, "failed to map irq\n");
3285         return -EINVAL;
3286     }
3287     if (is_sec1) {
3288         err = request_irq(priv->irq[0], talitos1_interrupt_4ch, 0,
3289                   dev_driver_string(dev), dev);
3290         goto primary_out;
3291     }
3292
3293     priv->irq[1] = irq_of_parse_and_map(np, 1);
3294
3295     /* get the primary irq line */
3296     if (!priv->irq[1]) {
3297         err = request_irq(priv->irq[0], talitos2_interrupt_4ch, 0,
3298                   dev_driver_string(dev), dev);
3299         goto primary_out;
3300     }
3301
3302     err = request_irq(priv->irq[0], talitos2_interrupt_ch0_2, 0,
3303               dev_driver_string(dev), dev);
3304     if (err)
3305         goto primary_out;
3306
3307     /* get the secondary irq line */
3308     err = request_irq(priv->irq[1], talitos2_interrupt_ch1_3, 0,
3309               dev_driver_string(dev), dev);
3310     if (err) {
3311         dev_err(dev, "failed to request secondary irq\n");
3312         irq_dispose_mapping(priv->irq[1]);
3313         priv->irq[1] = 0;
3314     }
3315
3316     return err;
3317
3318 primary_out:
3319     if (err) {
3320         dev_err(dev, "failed to request primary irq\n");
3321         irq_dispose_mapping(priv->irq[0]);
3322         priv->irq[0] = 0;
3323     }
3324
3325     return err;
3326 }
3327
3328 static int talitos_probe(struct platform_device *ofdev)
3329 {
3330     struct device *dev = &ofdev->dev;
3331     struct device_node *np = ofdev->dev.of_node;
3332     struct talitos_private *priv;
3333     int i, err;
3334     int stride;
3335     struct resource *res;
3336
3337     priv = devm_kzalloc(dev, sizeof(struct talitos_private), GFP_KERNEL);
3338     if (!priv)
3339         return -ENOMEM;
3340
3341     INIT_LIST_HEAD(&priv->alg_list);
3342
3343     dev_set_drvdata(dev, priv);
3344
3345     priv->ofdev = ofdev;
3346
3347     spin_lock_init(&priv->reg_lock);
3348
3349     res = platform_get_resource(ofdev, IORESOURCE_MEM, 0);
3350     if (!res)
3351         return -ENXIO;
3352     priv->reg = devm_ioremap(dev, res->start, resource_size(res));
3353     if (!priv->reg) {
3354         dev_err(dev, "failed to of_iomap\n");
3355         err = -ENOMEM;
3356         goto err_out;
3357     }
3358
3359     /* get SEC version capabilities from device tree */
3360     of_property_read_u32(np, "fsl,num-channels", &priv->num_channels);
3361     of_property_read_u32(np, "fsl,channel-fifo-len", &priv->chfifo_len);
3362     of_property_read_u32(np, "fsl,exec-units-mask", &priv->exec_units);
3363     of_property_read_u32(np, "fsl,descriptor-types-mask",
3364                  &priv->desc_types);
3365
3366     if (!is_power_of_2(priv->num_channels) || !priv->chfifo_len ||
3367         !priv->exec_units || !priv->desc_types) {
3368         dev_err(dev, "invalid property data in device tree node\n");
3369         err = -EINVAL;
3370         goto err_out;
3371     }
3372
3373     if (of_device_is_compatible(np, "fsl,sec3.0"))
3374         priv->features |= TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT;
3375
3376     if (of_device_is_compatible(np, "fsl,sec2.1"))
3377         priv->features |= TALITOS_FTR_HW_AUTH_CHECK |
3378                   TALITOS_FTR_SHA224_HWINIT |
3379                   TALITOS_FTR_HMAC_OK;
3380
3381     if (of_device_is_compatible(np, "fsl,sec1.0"))
3382         priv->features |= TALITOS_FTR_SEC1;
3383
3384     if (of_device_is_compatible(np, "fsl,sec1.2")) {
3385         priv->reg_deu = priv->reg + TALITOS12_DEU;
3386         priv->reg_aesu = priv->reg + TALITOS12_AESU;
3387         priv->reg_mdeu = priv->reg + TALITOS12_MDEU;
3388         stride = TALITOS1_CH_STRIDE;
3389     } else if (of_device_is_compatible(np, "fsl,sec1.0")) {
3390         priv->reg_deu = priv->reg + TALITOS10_DEU;
3391         priv->reg_aesu = priv->reg + TALITOS10_AESU;
3392         priv->reg_mdeu = priv->reg + TALITOS10_MDEU;
3393         priv->reg_afeu = priv->reg + TALITOS10_AFEU;
3394         priv->reg_rngu = priv->reg + TALITOS10_RNGU;
3395         priv->reg_pkeu = priv->reg + TALITOS10_PKEU;
3396         stride = TALITOS1_CH_STRIDE;
3397     } else {
3398         priv->reg_deu = priv->reg + TALITOS2_DEU;
3399         priv->reg_aesu = priv->reg + TALITOS2_AESU;
3400         priv->reg_mdeu = priv->reg + TALITOS2_MDEU;
3401         priv->reg_afeu = priv->reg + TALITOS2_AFEU;
3402         priv->reg_rngu = priv->reg + TALITOS2_RNGU;
3403         priv->reg_pkeu = priv->reg + TALITOS2_PKEU;
3404         priv->reg_keu = priv->reg + TALITOS2_KEU;
3405         priv->reg_crcu = priv->reg + TALITOS2_CRCU;
3406         stride = TALITOS2_CH_STRIDE;
3407     }
3408
3409     err = talitos_probe_irq(ofdev);
3410     if (err)
3411         goto err_out;
3412
3413     if (has_ftr_sec1(priv)) {
3414         if (priv->num_channels == 1)
3415             tasklet_init(&priv->done_task[0], talitos1_done_ch0,
3416                      (unsigned long)dev);
3417         else
3418             tasklet_init(&priv->done_task[0], talitos1_done_4ch,
3419                      (unsigned long)dev);
3420     } else {
3421         if (priv->irq[1]) {
3422             tasklet_init(&priv->done_task[0], talitos2_done_ch0_2,
3423                      (unsigned long)dev);
3424             tasklet_init(&priv->done_task[1], talitos2_done_ch1_3,
3425                      (unsigned long)dev);
3426         } else if (priv->num_channels == 1) {
3427             tasklet_init(&priv->done_task[0], talitos2_done_ch0,
3428                      (unsigned long)dev);
3429         } else {
3430             tasklet_init(&priv->done_task[0], talitos2_done_4ch,
3431                      (unsigned long)dev);
3432         }
3433     }
3434
3435     priv->chan = devm_kcalloc(dev,
3436                   priv->num_channels,
3437                   sizeof(struct talitos_channel),
3438                   GFP_KERNEL);
3439     if (!priv->chan) {
3440         dev_err(dev, "failed to allocate channel management space\n");
3441         err = -ENOMEM;
3442         goto err_out;
3443     }
3444
3445     priv->fifo_len = roundup_pow_of_two(priv->chfifo_len);
3446
3447     for (i = 0; i < priv->num_channels; i++) {
3448         priv->chan[i].reg = priv->reg + stride * (i + 1);
3449         if (!priv->irq[1] || !(i & 1))
3450             priv->chan[i].reg += TALITOS_CH_BASE_OFFSET;
3451
3452         spin_lock_init(&priv->chan[i].head_lock);
3453         spin_lock_init(&priv->chan[i].tail_lock);
3454
3455         priv->chan[i].fifo = devm_kcalloc(dev,
3456                         priv->fifo_len,
3457                         sizeof(struct talitos_request),
3458                         GFP_KERNEL);
3459         if (!priv->chan[i].fifo) {
3460             dev_err(dev, "failed to allocate request fifo %d\n", i);
3461             err = -ENOMEM;
3462             goto err_out;
3463         }
3464
3465         atomic_set(&priv->chan[i].submit_count,
3466                -(priv->chfifo_len - 1));
3467     }
3468
3469     dma_set_mask(dev, DMA_BIT_MASK(36));
3470
3471     /* reset and initialize the h/w */
3472     err = init_device(dev);
3473     if (err) {
3474         dev_err(dev, "failed to initialize device\n");
3475         goto err_out;
3476     }
3477
3478     /* register the RNG, if available */
3479     if (hw_supports(dev, DESC_HDR_SEL0_RNG)) {
3480         err = talitos_register_rng(dev);
3481         if (err) {
3482             dev_err(dev, "failed to register hwrng: %d\n", err);
3483             goto err_out;
3484         } else
3485             dev_info(dev, "hwrng\n");
3486     }
3487
3488     /* register crypto algorithms the device supports */
3489     for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
3490         if (hw_supports(dev, driver_algs[i].desc_hdr_template)) {
3491             struct talitos_crypto_alg *t_alg;
3492             struct crypto_alg *alg = NULL;
3493
3494             t_alg = talitos_alg_alloc(dev, &driver_algs[i]);
3495             if (IS_ERR(t_alg)) {
3496                 err = PTR_ERR(t_alg);
3497                 if (err == -ENOTSUPP)
3498                     continue;
3499                 goto err_out;
3500             }
3501
3502             switch (t_alg->algt.type) {
3503             case CRYPTO_ALG_TYPE_SKCIPHER:
3504                 err = crypto_register_skcipher(
3505                         &t_alg->algt.alg.skcipher);
3506                 alg = &t_alg->algt.alg.skcipher.base;
3507                 break;
3508
3509             case CRYPTO_ALG_TYPE_AEAD:
3510                 err = crypto_register_aead(
3511                     &t_alg->algt.alg.aead);
3512                 alg = &t_alg->algt.alg.aead.base;
3513                 break;
3514
3515             case CRYPTO_ALG_TYPE_AHASH:
3516                 err = crypto_register_ahash(
3517                         &t_alg->algt.alg.hash);
3518                 alg = &t_alg->algt.alg.hash.halg.base;
3519                 break;
3520             }
3521             if (err) {
3522                 dev_err(dev, "%s alg registration failed\n",
3523                     alg->cra_driver_name);
3524                 devm_kfree(dev, t_alg);
3525             } else
3526                 list_add_tail(&t_alg->entry, &priv->alg_list);
3527         }
3528     }
3529     if (!list_empty(&priv->alg_list))
3530         dev_info(dev, "%s algorithms registered in /proc/crypto\n",
3531              (char *)of_get_property(np, "compatible", NULL));
3532
3533     return 0;
3534
3535 err_out:
3536     talitos_remove(ofdev);
3537
3538     return err;
3539 }
3540
3541 static const struct of_device_id talitos_match[] = {
3542 #ifdef CONFIG_CRYPTO_DEV_TALITOS1
3543     {
3544         .compatible = "fsl,sec1.0",
3545     },
3546 #endif
3547 #ifdef CONFIG_CRYPTO_DEV_TALITOS2
3548     {
3549         .compatible = "fsl,sec2.0",
3550     },
3551 #endif
3552     {},
3553 };
3554 MODULE_DEVICE_TABLE(of, talitos_match);
3555
3556 static struct platform_driver talitos_driver = {
3557     .driver = {
3558         .name = "talitos",
3559         .of_match_table = talitos_match,
3560     },
3561     .probe = talitos_probe,
3562     .remove = talitos_remove,
3563 };
3564
3565 module_platform_driver(talitos_driver);
3566
3567 MODULE_LICENSE("GPL");
3568 MODULE_AUTHOR("Kim Phillips <kim.phillips@freescale.com>");
3569 MODULE_DESCRIPTION("Freescale integrated security engine (SEC) driver");