drivers/crypto/geode-aes.c

0001 // SPDX-License-Identifier: GPL-2.0-or-later
0002  /* Copyright (C) 2004-2006, Advanced Micro Devices, Inc.
0003   */
0004
0005 #include <linux/module.h>
0006 #include <linux/kernel.h>
0007 #include <linux/pci.h>
0008 #include <linux/pci_ids.h>
0009 #include <linux/crypto.h>
0010 #include <linux/spinlock.h>
0011 #include <crypto/algapi.h>
0012 #include <crypto/aes.h>
0013 #include <crypto/internal/cipher.h>
0014 #include <crypto/internal/skcipher.h>
0015
0016 #include <linux/io.h>
0017 #include <linux/delay.h>
0018
0019 #include "geode-aes.h"
0020
0021 /* Static structures */
0022
0023 static void __iomem *_iobase;
0024 static DEFINE_SPINLOCK(lock);
0025
0026 /* Write a 128 bit field (either a writable key or IV) */
0027 static inline void
0028 _writefield(u32 offset, const void *value)
0029 {
0030     int i;
0031
0032     for (i = 0; i < 4; i++)
0033         iowrite32(((const u32 *) value)[i], _iobase + offset + (i * 4));
0034 }
0035
0036 /* Read a 128 bit field (either a writable key or IV) */
0037 static inline void
0038 _readfield(u32 offset, void *value)
0039 {
0040     int i;
0041
0042     for (i = 0; i < 4; i++)
0043         ((u32 *) value)[i] = ioread32(_iobase + offset + (i * 4));
0044 }
0045
0046 static int
0047 do_crypt(const void *src, void *dst, u32 len, u32 flags)
0048 {
0049     u32 status;
0050     u32 counter = AES_OP_TIMEOUT;
0051
0052     iowrite32(virt_to_phys((void *)src), _iobase + AES_SOURCEA_REG);
0053     iowrite32(virt_to_phys(dst), _iobase + AES_DSTA_REG);
0054     iowrite32(len,  _iobase + AES_LENA_REG);
0055
0056     /* Start the operation */
0057     iowrite32(AES_CTRL_START | flags, _iobase + AES_CTRLA_REG);
0058
0059     do {
0060         status = ioread32(_iobase + AES_INTR_REG);
0061         cpu_relax();
0062     } while (!(status & AES_INTRA_PENDING) && --counter);
0063
0064     /* Clear the event */
0065     iowrite32((status & 0xFF) | AES_INTRA_PENDING, _iobase + AES_INTR_REG);
0066     return counter ? 0 : 1;
0067 }
0068
0069 static void
0070 geode_aes_crypt(const struct geode_aes_tfm_ctx *tctx, const void *src,
0071         void *dst, u32 len, u8 *iv, int mode, int dir)
0072 {
0073     u32 flags = 0;
0074     unsigned long iflags;
0075     int ret;
0076
0077     /* If the source and destination is the same, then
0078      * we need to turn on the coherent flags, otherwise
0079      * we don't need to worry
0080      */
0081
0082     flags |= (AES_CTRL_DCA | AES_CTRL_SCA);
0083
0084     if (dir == AES_DIR_ENCRYPT)
0085         flags |= AES_CTRL_ENCRYPT;
0086
0087     /* Start the critical section */
0088
0089     spin_lock_irqsave(&lock, iflags);
0090
0091     if (mode == AES_MODE_CBC) {
0092         flags |= AES_CTRL_CBC;
0093         _writefield(AES_WRITEIV0_REG, iv);
0094     }
0095
0096     flags |= AES_CTRL_WRKEY;
0097     _writefield(AES_WRITEKEY0_REG, tctx->key);
0098
0099     ret = do_crypt(src, dst, len, flags);
0100     BUG_ON(ret);
0101
0102     if (mode == AES_MODE_CBC)
0103         _readfield(AES_WRITEIV0_REG, iv);
0104
0105     spin_unlock_irqrestore(&lock, iflags);
0106 }
0107
0108 /* CRYPTO-API Functions */
0109
0110 static int geode_setkey_cip(struct crypto_tfm *tfm, const u8 *key,
0111         unsigned int len)
0112 {
0113     struct geode_aes_tfm_ctx *tctx = crypto_tfm_ctx(tfm);
0114
0115     tctx->keylen = len;
0116
0117     if (len == AES_KEYSIZE_128) {
0118         memcpy(tctx->key, key, len);
0119         return 0;
0120     }
0121
0122     if (len != AES_KEYSIZE_192 && len != AES_KEYSIZE_256)
0123         /* not supported at all */
0124         return -EINVAL;
0125
0126     /*
0127      * The requested key size is not supported by HW, do a fallback
0128      */
0129     tctx->fallback.cip->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK;
0130     tctx->fallback.cip->base.crt_flags |=
0131         (tfm->crt_flags & CRYPTO_TFM_REQ_MASK);
0132
0133     return crypto_cipher_setkey(tctx->fallback.cip, key, len);
0134 }
0135
0136 static int geode_setkey_skcipher(struct crypto_skcipher *tfm, const u8 *key,
0137                  unsigned int len)
0138 {
0139     struct geode_aes_tfm_ctx *tctx = crypto_skcipher_ctx(tfm);
0140
0141     tctx->keylen = len;
0142
0143     if (len == AES_KEYSIZE_128) {
0144         memcpy(tctx->key, key, len);
0145         return 0;
0146     }
0147
0148     if (len != AES_KEYSIZE_192 && len != AES_KEYSIZE_256)
0149         /* not supported at all */
0150         return -EINVAL;
0151
0152     /*
0153      * The requested key size is not supported by HW, do a fallback
0154      */
0155     crypto_skcipher_clear_flags(tctx->fallback.skcipher,
0156                     CRYPTO_TFM_REQ_MASK);
0157     crypto_skcipher_set_flags(tctx->fallback.skcipher,
0158                   crypto_skcipher_get_flags(tfm) &
0159                   CRYPTO_TFM_REQ_MASK);
0160     return crypto_skcipher_setkey(tctx->fallback.skcipher, key, len);
0161 }
0162
0163 static void
0164 geode_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
0165 {
0166     const struct geode_aes_tfm_ctx *tctx = crypto_tfm_ctx(tfm);
0167
0168     if (unlikely(tctx->keylen != AES_KEYSIZE_128)) {
0169         crypto_cipher_encrypt_one(tctx->fallback.cip, out, in);
0170         return;
0171     }
0172
0173     geode_aes_crypt(tctx, in, out, AES_BLOCK_SIZE, NULL,
0174             AES_MODE_ECB, AES_DIR_ENCRYPT);
0175 }
0176
0177
0178 static void
0179 geode_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
0180 {
0181     const struct geode_aes_tfm_ctx *tctx = crypto_tfm_ctx(tfm);
0182
0183     if (unlikely(tctx->keylen != AES_KEYSIZE_128)) {
0184         crypto_cipher_decrypt_one(tctx->fallback.cip, out, in);
0185         return;
0186     }
0187
0188     geode_aes_crypt(tctx, in, out, AES_BLOCK_SIZE, NULL,
0189             AES_MODE_ECB, AES_DIR_DECRYPT);
0190 }
0191
0192 static int fallback_init_cip(struct crypto_tfm *tfm)
0193 {
0194     const char *name = crypto_tfm_alg_name(tfm);
0195     struct geode_aes_tfm_ctx *tctx = crypto_tfm_ctx(tfm);
0196
0197     tctx->fallback.cip = crypto_alloc_cipher(name, 0,
0198                          CRYPTO_ALG_NEED_FALLBACK);
0199
0200     if (IS_ERR(tctx->fallback.cip)) {
0201         printk(KERN_ERR "Error allocating fallback algo %s\n", name);
0202         return PTR_ERR(tctx->fallback.cip);
0203     }
0204
0205     return 0;
0206 }
0207
0208 static void fallback_exit_cip(struct crypto_tfm *tfm)
0209 {
0210     struct geode_aes_tfm_ctx *tctx = crypto_tfm_ctx(tfm);
0211
0212     crypto_free_cipher(tctx->fallback.cip);
0213 }
0214
0215 static struct crypto_alg geode_alg = {
0216     .cra_name           =   "aes",
0217     .cra_driver_name    =   "geode-aes",
0218     .cra_priority       =   300,
0219     .cra_alignmask      =   15,
0220     .cra_flags          =   CRYPTO_ALG_TYPE_CIPHER |
0221                             CRYPTO_ALG_NEED_FALLBACK,
0222     .cra_init           =   fallback_init_cip,
0223     .cra_exit           =   fallback_exit_cip,
0224     .cra_blocksize      =   AES_BLOCK_SIZE,
0225     .cra_ctxsize        =   sizeof(struct geode_aes_tfm_ctx),
0226     .cra_module         =   THIS_MODULE,
0227     .cra_u              =   {
0228         .cipher =   {
0229             .cia_min_keysize    =   AES_MIN_KEY_SIZE,
0230             .cia_max_keysize    =   AES_MAX_KEY_SIZE,
0231             .cia_setkey         =   geode_setkey_cip,
0232             .cia_encrypt        =   geode_encrypt,
0233             .cia_decrypt        =   geode_decrypt
0234         }
0235     }
0236 };
0237
0238 static int geode_init_skcipher(struct crypto_skcipher *tfm)
0239 {
0240     const char *name = crypto_tfm_alg_name(&tfm->base);
0241     struct geode_aes_tfm_ctx *tctx = crypto_skcipher_ctx(tfm);
0242
0243     tctx->fallback.skcipher =
0244         crypto_alloc_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK |
0245                       CRYPTO_ALG_ASYNC);
0246     if (IS_ERR(tctx->fallback.skcipher)) {
0247         printk(KERN_ERR "Error allocating fallback algo %s\n", name);
0248         return PTR_ERR(tctx->fallback.skcipher);
0249     }
0250
0251     crypto_skcipher_set_reqsize(tfm, sizeof(struct skcipher_request) +
0252                     crypto_skcipher_reqsize(tctx->fallback.skcipher));
0253     return 0;
0254 }
0255
0256 static void geode_exit_skcipher(struct crypto_skcipher *tfm)
0257 {
0258     struct geode_aes_tfm_ctx *tctx = crypto_skcipher_ctx(tfm);
0259
0260     crypto_free_skcipher(tctx->fallback.skcipher);
0261 }
0262
0263 static int geode_skcipher_crypt(struct skcipher_request *req, int mode, int dir)
0264 {
0265     struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
0266     const struct geode_aes_tfm_ctx *tctx = crypto_skcipher_ctx(tfm);
0267     struct skcipher_walk walk;
0268     unsigned int nbytes;
0269     int err;
0270
0271     if (unlikely(tctx->keylen != AES_KEYSIZE_128)) {
0272         struct skcipher_request *subreq = skcipher_request_ctx(req);
0273
0274         *subreq = *req;
0275         skcipher_request_set_tfm(subreq, tctx->fallback.skcipher);
0276         if (dir == AES_DIR_DECRYPT)
0277             return crypto_skcipher_decrypt(subreq);
0278         else
0279             return crypto_skcipher_encrypt(subreq);
0280     }
0281
0282     err = skcipher_walk_virt(&walk, req, false);
0283
0284     while ((nbytes = walk.nbytes) != 0) {
0285         geode_aes_crypt(tctx, walk.src.virt.addr, walk.dst.virt.addr,
0286                 round_down(nbytes, AES_BLOCK_SIZE),
0287                 walk.iv, mode, dir);
0288         err = skcipher_walk_done(&walk, nbytes % AES_BLOCK_SIZE);
0289     }
0290
0291     return err;
0292 }
0293
0294 static int geode_cbc_encrypt(struct skcipher_request *req)
0295 {
0296     return geode_skcipher_crypt(req, AES_MODE_CBC, AES_DIR_ENCRYPT);
0297 }
0298
0299 static int geode_cbc_decrypt(struct skcipher_request *req)
0300 {
0301     return geode_skcipher_crypt(req, AES_MODE_CBC, AES_DIR_DECRYPT);
0302 }
0303
0304 static int geode_ecb_encrypt(struct skcipher_request *req)
0305 {
0306     return geode_skcipher_crypt(req, AES_MODE_ECB, AES_DIR_ENCRYPT);
0307 }
0308
0309 static int geode_ecb_decrypt(struct skcipher_request *req)
0310 {
0311     return geode_skcipher_crypt(req, AES_MODE_ECB, AES_DIR_DECRYPT);
0312 }
0313
0314 static struct skcipher_alg geode_skcipher_algs[] = {
0315     {
0316         .base.cra_name      = "cbc(aes)",
0317         .base.cra_driver_name   = "cbc-aes-geode",
0318         .base.cra_priority  = 400,
0319         .base.cra_flags     = CRYPTO_ALG_KERN_DRIVER_ONLY |
0320                       CRYPTO_ALG_NEED_FALLBACK,
0321         .base.cra_blocksize = AES_BLOCK_SIZE,
0322         .base.cra_ctxsize   = sizeof(struct geode_aes_tfm_ctx),
0323         .base.cra_alignmask = 15,
0324         .base.cra_module    = THIS_MODULE,
0325         .init           = geode_init_skcipher,
0326         .exit           = geode_exit_skcipher,
0327         .setkey         = geode_setkey_skcipher,
0328         .encrypt        = geode_cbc_encrypt,
0329         .decrypt        = geode_cbc_decrypt,
0330         .min_keysize        = AES_MIN_KEY_SIZE,
0331         .max_keysize        = AES_MAX_KEY_SIZE,
0332         .ivsize         = AES_BLOCK_SIZE,
0333     }, {
0334         .base.cra_name      = "ecb(aes)",
0335         .base.cra_driver_name   = "ecb-aes-geode",
0336         .base.cra_priority  = 400,
0337         .base.cra_flags     = CRYPTO_ALG_KERN_DRIVER_ONLY |
0338                       CRYPTO_ALG_NEED_FALLBACK,
0339         .base.cra_blocksize = AES_BLOCK_SIZE,
0340         .base.cra_ctxsize   = sizeof(struct geode_aes_tfm_ctx),
0341         .base.cra_alignmask = 15,
0342         .base.cra_module    = THIS_MODULE,
0343         .init           = geode_init_skcipher,
0344         .exit           = geode_exit_skcipher,
0345         .setkey         = geode_setkey_skcipher,
0346         .encrypt        = geode_ecb_encrypt,
0347         .decrypt        = geode_ecb_decrypt,
0348         .min_keysize        = AES_MIN_KEY_SIZE,
0349         .max_keysize        = AES_MAX_KEY_SIZE,
0350     },
0351 };
0352
0353 static void geode_aes_remove(struct pci_dev *dev)
0354 {
0355     crypto_unregister_alg(&geode_alg);
0356     crypto_unregister_skciphers(geode_skcipher_algs,
0357                     ARRAY_SIZE(geode_skcipher_algs));
0358
0359     pci_iounmap(dev, _iobase);
0360     _iobase = NULL;
0361
0362     pci_release_regions(dev);
0363     pci_disable_device(dev);
0364 }
0365
0366
0367 static int geode_aes_probe(struct pci_dev *dev, const struct pci_device_id *id)
0368 {
0369     int ret;
0370
0371     ret = pci_enable_device(dev);
0372     if (ret)
0373         return ret;
0374
0375     ret = pci_request_regions(dev, "geode-aes");
0376     if (ret)
0377         goto eenable;
0378
0379     _iobase = pci_iomap(dev, 0, 0);
0380
0381     if (_iobase == NULL) {
0382         ret = -ENOMEM;
0383         goto erequest;
0384     }
0385
0386     /* Clear any pending activity */
0387     iowrite32(AES_INTR_PENDING | AES_INTR_MASK, _iobase + AES_INTR_REG);
0388
0389     ret = crypto_register_alg(&geode_alg);
0390     if (ret)
0391         goto eiomap;
0392
0393     ret = crypto_register_skciphers(geode_skcipher_algs,
0394                     ARRAY_SIZE(geode_skcipher_algs));
0395     if (ret)
0396         goto ealg;
0397
0398     dev_notice(&dev->dev, "GEODE AES engine enabled.\n");
0399     return 0;
0400
0401  ealg:
0402     crypto_unregister_alg(&geode_alg);
0403
0404  eiomap:
0405     pci_iounmap(dev, _iobase);
0406
0407  erequest:
0408     pci_release_regions(dev);
0409
0410  eenable:
0411     pci_disable_device(dev);
0412
0413     dev_err(&dev->dev, "GEODE AES initialization failed.\n");
0414     return ret;
0415 }
0416
0417 static struct pci_device_id geode_aes_tbl[] = {
0418     { PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_LX_AES), },
0419     { 0, }
0420 };
0421
0422 MODULE_DEVICE_TABLE(pci, geode_aes_tbl);
0423
0424 static struct pci_driver geode_aes_driver = {
0425     .name = "Geode LX AES",
0426     .id_table = geode_aes_tbl,
0427     .probe = geode_aes_probe,
0428     .remove = geode_aes_remove,
0429 };
0430
0431 module_pci_driver(geode_aes_driver);
0432
0433 MODULE_AUTHOR("Advanced Micro Devices, Inc.");
0434 MODULE_DESCRIPTION("Geode LX Hardware AES driver");
0435 MODULE_LICENSE("GPL");
0436 MODULE_IMPORT_NS(CRYPTO_INTERNAL);