Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0-or-later
0002 /*
0003  * Glue code for AES implementation for SPE instructions (PPC)
0004  *
0005  * Based on generic implementation. The assembler module takes care
0006  * about the SPE registers so it can run from interrupt context.
0007  *
0008  * Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de>
0009  */
0010 
0011 #include <crypto/aes.h>
0012 #include <linux/module.h>
0013 #include <linux/init.h>
0014 #include <linux/types.h>
0015 #include <linux/errno.h>
0016 #include <linux/crypto.h>
0017 #include <asm/byteorder.h>
0018 #include <asm/switch_to.h>
0019 #include <crypto/algapi.h>
0020 #include <crypto/internal/skcipher.h>
0021 #include <crypto/xts.h>
0022 #include <crypto/gf128mul.h>
0023 #include <crypto/scatterwalk.h>
0024 
0025 /*
0026  * MAX_BYTES defines the number of bytes that are allowed to be processed
0027  * between preempt_disable() and preempt_enable(). e500 cores can issue two
0028  * instructions per clock cycle using one 32/64 bit unit (SU1) and one 32
0029  * bit unit (SU2). One of these can be a memory access that is executed via
0030  * a single load and store unit (LSU). XTS-AES-256 takes ~780 operations per
0031  * 16 byte block or 25 cycles per byte. Thus 768 bytes of input data
0032  * will need an estimated maximum of 20,000 cycles. Headroom for cache misses
0033  * included. Even with the low end model clocked at 667 MHz this equals to a
0034  * critical time window of less than 30us. The value has been chosen to
0035  * process a 512 byte disk block in one or a large 1400 bytes IPsec network
0036  * packet in two runs.
0037  *
0038  */
0039 #define MAX_BYTES 768
0040 
0041 struct ppc_aes_ctx {
0042     u32 key_enc[AES_MAX_KEYLENGTH_U32];
0043     u32 key_dec[AES_MAX_KEYLENGTH_U32];
0044     u32 rounds;
0045 };
0046 
0047 struct ppc_xts_ctx {
0048     u32 key_enc[AES_MAX_KEYLENGTH_U32];
0049     u32 key_dec[AES_MAX_KEYLENGTH_U32];
0050     u32 key_twk[AES_MAX_KEYLENGTH_U32];
0051     u32 rounds;
0052 };
0053 
0054 extern void ppc_encrypt_aes(u8 *out, const u8 *in, u32 *key_enc, u32 rounds);
0055 extern void ppc_decrypt_aes(u8 *out, const u8 *in, u32 *key_dec, u32 rounds);
0056 extern void ppc_encrypt_ecb(u8 *out, const u8 *in, u32 *key_enc, u32 rounds,
0057                 u32 bytes);
0058 extern void ppc_decrypt_ecb(u8 *out, const u8 *in, u32 *key_dec, u32 rounds,
0059                 u32 bytes);
0060 extern void ppc_encrypt_cbc(u8 *out, const u8 *in, u32 *key_enc, u32 rounds,
0061                 u32 bytes, u8 *iv);
0062 extern void ppc_decrypt_cbc(u8 *out, const u8 *in, u32 *key_dec, u32 rounds,
0063                 u32 bytes, u8 *iv);
0064 extern void ppc_crypt_ctr  (u8 *out, const u8 *in, u32 *key_enc, u32 rounds,
0065                 u32 bytes, u8 *iv);
0066 extern void ppc_encrypt_xts(u8 *out, const u8 *in, u32 *key_enc, u32 rounds,
0067                 u32 bytes, u8 *iv, u32 *key_twk);
0068 extern void ppc_decrypt_xts(u8 *out, const u8 *in, u32 *key_dec, u32 rounds,
0069                 u32 bytes, u8 *iv, u32 *key_twk);
0070 
0071 extern void ppc_expand_key_128(u32 *key_enc, const u8 *key);
0072 extern void ppc_expand_key_192(u32 *key_enc, const u8 *key);
0073 extern void ppc_expand_key_256(u32 *key_enc, const u8 *key);
0074 
0075 extern void ppc_generate_decrypt_key(u32 *key_dec,u32 *key_enc,
0076                      unsigned int key_len);
0077 
0078 static void spe_begin(void)
0079 {
0080     /* disable preemption and save users SPE registers if required */
0081     preempt_disable();
0082     enable_kernel_spe();
0083 }
0084 
0085 static void spe_end(void)
0086 {
0087     disable_kernel_spe();
0088     /* reenable preemption */
0089     preempt_enable();
0090 }
0091 
0092 static int ppc_aes_setkey(struct crypto_tfm *tfm, const u8 *in_key,
0093         unsigned int key_len)
0094 {
0095     struct ppc_aes_ctx *ctx = crypto_tfm_ctx(tfm);
0096 
0097     switch (key_len) {
0098     case AES_KEYSIZE_128:
0099         ctx->rounds = 4;
0100         ppc_expand_key_128(ctx->key_enc, in_key);
0101         break;
0102     case AES_KEYSIZE_192:
0103         ctx->rounds = 5;
0104         ppc_expand_key_192(ctx->key_enc, in_key);
0105         break;
0106     case AES_KEYSIZE_256:
0107         ctx->rounds = 6;
0108         ppc_expand_key_256(ctx->key_enc, in_key);
0109         break;
0110     default:
0111         return -EINVAL;
0112     }
0113 
0114     ppc_generate_decrypt_key(ctx->key_dec, ctx->key_enc, key_len);
0115 
0116     return 0;
0117 }
0118 
0119 static int ppc_aes_setkey_skcipher(struct crypto_skcipher *tfm,
0120                    const u8 *in_key, unsigned int key_len)
0121 {
0122     return ppc_aes_setkey(crypto_skcipher_tfm(tfm), in_key, key_len);
0123 }
0124 
0125 static int ppc_xts_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
0126            unsigned int key_len)
0127 {
0128     struct ppc_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
0129     int err;
0130 
0131     err = xts_verify_key(tfm, in_key, key_len);
0132     if (err)
0133         return err;
0134 
0135     key_len >>= 1;
0136 
0137     switch (key_len) {
0138     case AES_KEYSIZE_128:
0139         ctx->rounds = 4;
0140         ppc_expand_key_128(ctx->key_enc, in_key);
0141         ppc_expand_key_128(ctx->key_twk, in_key + AES_KEYSIZE_128);
0142         break;
0143     case AES_KEYSIZE_192:
0144         ctx->rounds = 5;
0145         ppc_expand_key_192(ctx->key_enc, in_key);
0146         ppc_expand_key_192(ctx->key_twk, in_key + AES_KEYSIZE_192);
0147         break;
0148     case AES_KEYSIZE_256:
0149         ctx->rounds = 6;
0150         ppc_expand_key_256(ctx->key_enc, in_key);
0151         ppc_expand_key_256(ctx->key_twk, in_key + AES_KEYSIZE_256);
0152         break;
0153     default:
0154         return -EINVAL;
0155     }
0156 
0157     ppc_generate_decrypt_key(ctx->key_dec, ctx->key_enc, key_len);
0158 
0159     return 0;
0160 }
0161 
0162 static void ppc_aes_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
0163 {
0164     struct ppc_aes_ctx *ctx = crypto_tfm_ctx(tfm);
0165 
0166     spe_begin();
0167     ppc_encrypt_aes(out, in, ctx->key_enc, ctx->rounds);
0168     spe_end();
0169 }
0170 
0171 static void ppc_aes_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
0172 {
0173     struct ppc_aes_ctx *ctx = crypto_tfm_ctx(tfm);
0174 
0175     spe_begin();
0176     ppc_decrypt_aes(out, in, ctx->key_dec, ctx->rounds);
0177     spe_end();
0178 }
0179 
0180 static int ppc_ecb_crypt(struct skcipher_request *req, bool enc)
0181 {
0182     struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
0183     struct ppc_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
0184     struct skcipher_walk walk;
0185     unsigned int nbytes;
0186     int err;
0187 
0188     err = skcipher_walk_virt(&walk, req, false);
0189 
0190     while ((nbytes = walk.nbytes) != 0) {
0191         nbytes = min_t(unsigned int, nbytes, MAX_BYTES);
0192         nbytes = round_down(nbytes, AES_BLOCK_SIZE);
0193 
0194         spe_begin();
0195         if (enc)
0196             ppc_encrypt_ecb(walk.dst.virt.addr, walk.src.virt.addr,
0197                     ctx->key_enc, ctx->rounds, nbytes);
0198         else
0199             ppc_decrypt_ecb(walk.dst.virt.addr, walk.src.virt.addr,
0200                     ctx->key_dec, ctx->rounds, nbytes);
0201         spe_end();
0202 
0203         err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
0204     }
0205 
0206     return err;
0207 }
0208 
0209 static int ppc_ecb_encrypt(struct skcipher_request *req)
0210 {
0211     return ppc_ecb_crypt(req, true);
0212 }
0213 
0214 static int ppc_ecb_decrypt(struct skcipher_request *req)
0215 {
0216     return ppc_ecb_crypt(req, false);
0217 }
0218 
0219 static int ppc_cbc_crypt(struct skcipher_request *req, bool enc)
0220 {
0221     struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
0222     struct ppc_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
0223     struct skcipher_walk walk;
0224     unsigned int nbytes;
0225     int err;
0226 
0227     err = skcipher_walk_virt(&walk, req, false);
0228 
0229     while ((nbytes = walk.nbytes) != 0) {
0230         nbytes = min_t(unsigned int, nbytes, MAX_BYTES);
0231         nbytes = round_down(nbytes, AES_BLOCK_SIZE);
0232 
0233         spe_begin();
0234         if (enc)
0235             ppc_encrypt_cbc(walk.dst.virt.addr, walk.src.virt.addr,
0236                     ctx->key_enc, ctx->rounds, nbytes,
0237                     walk.iv);
0238         else
0239             ppc_decrypt_cbc(walk.dst.virt.addr, walk.src.virt.addr,
0240                     ctx->key_dec, ctx->rounds, nbytes,
0241                     walk.iv);
0242         spe_end();
0243 
0244         err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
0245     }
0246 
0247     return err;
0248 }
0249 
0250 static int ppc_cbc_encrypt(struct skcipher_request *req)
0251 {
0252     return ppc_cbc_crypt(req, true);
0253 }
0254 
0255 static int ppc_cbc_decrypt(struct skcipher_request *req)
0256 {
0257     return ppc_cbc_crypt(req, false);
0258 }
0259 
0260 static int ppc_ctr_crypt(struct skcipher_request *req)
0261 {
0262     struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
0263     struct ppc_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
0264     struct skcipher_walk walk;
0265     unsigned int nbytes;
0266     int err;
0267 
0268     err = skcipher_walk_virt(&walk, req, false);
0269 
0270     while ((nbytes = walk.nbytes) != 0) {
0271         nbytes = min_t(unsigned int, nbytes, MAX_BYTES);
0272         if (nbytes < walk.total)
0273             nbytes = round_down(nbytes, AES_BLOCK_SIZE);
0274 
0275         spe_begin();
0276         ppc_crypt_ctr(walk.dst.virt.addr, walk.src.virt.addr,
0277                   ctx->key_enc, ctx->rounds, nbytes, walk.iv);
0278         spe_end();
0279 
0280         err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
0281     }
0282 
0283     return err;
0284 }
0285 
0286 static int ppc_xts_crypt(struct skcipher_request *req, bool enc)
0287 {
0288     struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
0289     struct ppc_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
0290     struct skcipher_walk walk;
0291     unsigned int nbytes;
0292     int err;
0293     u32 *twk;
0294 
0295     err = skcipher_walk_virt(&walk, req, false);
0296     twk = ctx->key_twk;
0297 
0298     while ((nbytes = walk.nbytes) != 0) {
0299         nbytes = min_t(unsigned int, nbytes, MAX_BYTES);
0300         nbytes = round_down(nbytes, AES_BLOCK_SIZE);
0301 
0302         spe_begin();
0303         if (enc)
0304             ppc_encrypt_xts(walk.dst.virt.addr, walk.src.virt.addr,
0305                     ctx->key_enc, ctx->rounds, nbytes,
0306                     walk.iv, twk);
0307         else
0308             ppc_decrypt_xts(walk.dst.virt.addr, walk.src.virt.addr,
0309                     ctx->key_dec, ctx->rounds, nbytes,
0310                     walk.iv, twk);
0311         spe_end();
0312 
0313         twk = NULL;
0314         err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
0315     }
0316 
0317     return err;
0318 }
0319 
0320 static int ppc_xts_encrypt(struct skcipher_request *req)
0321 {
0322     struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
0323     struct ppc_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
0324     int tail = req->cryptlen % AES_BLOCK_SIZE;
0325     int offset = req->cryptlen - tail - AES_BLOCK_SIZE;
0326     struct skcipher_request subreq;
0327     u8 b[2][AES_BLOCK_SIZE];
0328     int err;
0329 
0330     if (req->cryptlen < AES_BLOCK_SIZE)
0331         return -EINVAL;
0332 
0333     if (tail) {
0334         subreq = *req;
0335         skcipher_request_set_crypt(&subreq, req->src, req->dst,
0336                        req->cryptlen - tail, req->iv);
0337         req = &subreq;
0338     }
0339 
0340     err = ppc_xts_crypt(req, true);
0341     if (err || !tail)
0342         return err;
0343 
0344     scatterwalk_map_and_copy(b[0], req->dst, offset, AES_BLOCK_SIZE, 0);
0345     memcpy(b[1], b[0], tail);
0346     scatterwalk_map_and_copy(b[0], req->src, offset + AES_BLOCK_SIZE, tail, 0);
0347 
0348     spe_begin();
0349     ppc_encrypt_xts(b[0], b[0], ctx->key_enc, ctx->rounds, AES_BLOCK_SIZE,
0350             req->iv, NULL);
0351     spe_end();
0352 
0353     scatterwalk_map_and_copy(b[0], req->dst, offset, AES_BLOCK_SIZE + tail, 1);
0354 
0355     return 0;
0356 }
0357 
0358 static int ppc_xts_decrypt(struct skcipher_request *req)
0359 {
0360     struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
0361     struct ppc_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
0362     int tail = req->cryptlen % AES_BLOCK_SIZE;
0363     int offset = req->cryptlen - tail - AES_BLOCK_SIZE;
0364     struct skcipher_request subreq;
0365     u8 b[3][AES_BLOCK_SIZE];
0366     le128 twk;
0367     int err;
0368 
0369     if (req->cryptlen < AES_BLOCK_SIZE)
0370         return -EINVAL;
0371 
0372     if (tail) {
0373         subreq = *req;
0374         skcipher_request_set_crypt(&subreq, req->src, req->dst,
0375                        offset, req->iv);
0376         req = &subreq;
0377     }
0378 
0379     err = ppc_xts_crypt(req, false);
0380     if (err || !tail)
0381         return err;
0382 
0383     scatterwalk_map_and_copy(b[1], req->src, offset, AES_BLOCK_SIZE + tail, 0);
0384 
0385     spe_begin();
0386     if (!offset)
0387         ppc_encrypt_ecb(req->iv, req->iv, ctx->key_twk, ctx->rounds,
0388                 AES_BLOCK_SIZE);
0389 
0390     gf128mul_x_ble(&twk, (le128 *)req->iv);
0391 
0392     ppc_decrypt_xts(b[1], b[1], ctx->key_dec, ctx->rounds, AES_BLOCK_SIZE,
0393             (u8 *)&twk, NULL);
0394     memcpy(b[0], b[2], tail);
0395     memcpy(b[0] + tail, b[1] + tail, AES_BLOCK_SIZE - tail);
0396     ppc_decrypt_xts(b[0], b[0], ctx->key_dec, ctx->rounds, AES_BLOCK_SIZE,
0397             req->iv, NULL);
0398     spe_end();
0399 
0400     scatterwalk_map_and_copy(b[0], req->dst, offset, AES_BLOCK_SIZE + tail, 1);
0401 
0402     return 0;
0403 }
0404 
0405 /*
0406  * Algorithm definitions. Disabling alignment (cra_alignmask=0) was chosen
0407  * because the e500 platform can handle unaligned reads/writes very efficiently.
0408  * This improves IPsec thoughput by another few percent. Additionally we assume
0409  * that AES context is always aligned to at least 8 bytes because it is created
0410  * with kmalloc() in the crypto infrastructure
0411  */
0412 
0413 static struct crypto_alg aes_cipher_alg = {
0414     .cra_name       =   "aes",
0415     .cra_driver_name    =   "aes-ppc-spe",
0416     .cra_priority       =   300,
0417     .cra_flags      =   CRYPTO_ALG_TYPE_CIPHER,
0418     .cra_blocksize      =   AES_BLOCK_SIZE,
0419     .cra_ctxsize        =   sizeof(struct ppc_aes_ctx),
0420     .cra_alignmask      =   0,
0421     .cra_module     =   THIS_MODULE,
0422     .cra_u          =   {
0423         .cipher = {
0424             .cia_min_keysize    =   AES_MIN_KEY_SIZE,
0425             .cia_max_keysize    =   AES_MAX_KEY_SIZE,
0426             .cia_setkey     =   ppc_aes_setkey,
0427             .cia_encrypt        =   ppc_aes_encrypt,
0428             .cia_decrypt        =   ppc_aes_decrypt
0429         }
0430     }
0431 };
0432 
0433 static struct skcipher_alg aes_skcipher_algs[] = {
0434     {
0435         .base.cra_name      =   "ecb(aes)",
0436         .base.cra_driver_name   =   "ecb-ppc-spe",
0437         .base.cra_priority  =   300,
0438         .base.cra_blocksize =   AES_BLOCK_SIZE,
0439         .base.cra_ctxsize   =   sizeof(struct ppc_aes_ctx),
0440         .base.cra_module    =   THIS_MODULE,
0441         .min_keysize        =   AES_MIN_KEY_SIZE,
0442         .max_keysize        =   AES_MAX_KEY_SIZE,
0443         .setkey         =   ppc_aes_setkey_skcipher,
0444         .encrypt        =   ppc_ecb_encrypt,
0445         .decrypt        =   ppc_ecb_decrypt,
0446     }, {
0447         .base.cra_name      =   "cbc(aes)",
0448         .base.cra_driver_name   =   "cbc-ppc-spe",
0449         .base.cra_priority  =   300,
0450         .base.cra_blocksize =   AES_BLOCK_SIZE,
0451         .base.cra_ctxsize   =   sizeof(struct ppc_aes_ctx),
0452         .base.cra_module    =   THIS_MODULE,
0453         .min_keysize        =   AES_MIN_KEY_SIZE,
0454         .max_keysize        =   AES_MAX_KEY_SIZE,
0455         .ivsize         =   AES_BLOCK_SIZE,
0456         .setkey         =   ppc_aes_setkey_skcipher,
0457         .encrypt        =   ppc_cbc_encrypt,
0458         .decrypt        =   ppc_cbc_decrypt,
0459     }, {
0460         .base.cra_name      =   "ctr(aes)",
0461         .base.cra_driver_name   =   "ctr-ppc-spe",
0462         .base.cra_priority  =   300,
0463         .base.cra_blocksize =   1,
0464         .base.cra_ctxsize   =   sizeof(struct ppc_aes_ctx),
0465         .base.cra_module    =   THIS_MODULE,
0466         .min_keysize        =   AES_MIN_KEY_SIZE,
0467         .max_keysize        =   AES_MAX_KEY_SIZE,
0468         .ivsize         =   AES_BLOCK_SIZE,
0469         .setkey         =   ppc_aes_setkey_skcipher,
0470         .encrypt        =   ppc_ctr_crypt,
0471         .decrypt        =   ppc_ctr_crypt,
0472         .chunksize      =   AES_BLOCK_SIZE,
0473     }, {
0474         .base.cra_name      =   "xts(aes)",
0475         .base.cra_driver_name   =   "xts-ppc-spe",
0476         .base.cra_priority  =   300,
0477         .base.cra_blocksize =   AES_BLOCK_SIZE,
0478         .base.cra_ctxsize   =   sizeof(struct ppc_xts_ctx),
0479         .base.cra_module    =   THIS_MODULE,
0480         .min_keysize        =   AES_MIN_KEY_SIZE * 2,
0481         .max_keysize        =   AES_MAX_KEY_SIZE * 2,
0482         .ivsize         =   AES_BLOCK_SIZE,
0483         .setkey         =   ppc_xts_setkey,
0484         .encrypt        =   ppc_xts_encrypt,
0485         .decrypt        =   ppc_xts_decrypt,
0486     }
0487 };
0488 
0489 static int __init ppc_aes_mod_init(void)
0490 {
0491     int err;
0492 
0493     err = crypto_register_alg(&aes_cipher_alg);
0494     if (err)
0495         return err;
0496 
0497     err = crypto_register_skciphers(aes_skcipher_algs,
0498                     ARRAY_SIZE(aes_skcipher_algs));
0499     if (err)
0500         crypto_unregister_alg(&aes_cipher_alg);
0501     return err;
0502 }
0503 
0504 static void __exit ppc_aes_mod_fini(void)
0505 {
0506     crypto_unregister_alg(&aes_cipher_alg);
0507     crypto_unregister_skciphers(aes_skcipher_algs,
0508                     ARRAY_SIZE(aes_skcipher_algs));
0509 }
0510 
0511 module_init(ppc_aes_mod_init);
0512 module_exit(ppc_aes_mod_fini);
0513 
0514 MODULE_LICENSE("GPL");
0515 MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS, SPE optimized");
0516 
0517 MODULE_ALIAS_CRYPTO("aes");
0518 MODULE_ALIAS_CRYPTO("ecb(aes)");
0519 MODULE_ALIAS_CRYPTO("cbc(aes)");
0520 MODULE_ALIAS_CRYPTO("ctr(aes)");
0521 MODULE_ALIAS_CRYPTO("xts(aes)");
0522 MODULE_ALIAS_CRYPTO("aes-ppc-spe");
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!