Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /* Glue code for AES encryption optimized for sparc64 crypto opcodes.
0003  *
0004  * This is based largely upon arch/x86/crypto/aesni-intel_glue.c
0005  *
0006  * Copyright (C) 2008, Intel Corp.
0007  *    Author: Huang Ying <ying.huang@intel.com>
0008  *
0009  * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD
0010  * interface for 64-bit kernels.
0011  *    Authors: Adrian Hoban <adrian.hoban@intel.com>
0012  *             Gabriele Paoloni <gabriele.paoloni@intel.com>
0013  *             Tadeusz Struk (tadeusz.struk@intel.com)
0014  *             Aidan O'Mahony (aidan.o.mahony@intel.com)
0015  *    Copyright (c) 2010, Intel Corporation.
0016  */
0017 
0018 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
0019 
0020 #include <linux/crypto.h>
0021 #include <linux/init.h>
0022 #include <linux/module.h>
0023 #include <linux/mm.h>
0024 #include <linux/types.h>
0025 #include <crypto/algapi.h>
0026 #include <crypto/aes.h>
0027 #include <crypto/internal/skcipher.h>
0028 
0029 #include <asm/fpumacro.h>
0030 #include <asm/pstate.h>
0031 #include <asm/elf.h>
0032 
0033 #include "opcodes.h"
0034 
0035 struct aes_ops {
0036     void (*encrypt)(const u64 *key, const u32 *input, u32 *output);
0037     void (*decrypt)(const u64 *key, const u32 *input, u32 *output);
0038     void (*load_encrypt_keys)(const u64 *key);
0039     void (*load_decrypt_keys)(const u64 *key);
0040     void (*ecb_encrypt)(const u64 *key, const u64 *input, u64 *output,
0041                 unsigned int len);
0042     void (*ecb_decrypt)(const u64 *key, const u64 *input, u64 *output,
0043                 unsigned int len);
0044     void (*cbc_encrypt)(const u64 *key, const u64 *input, u64 *output,
0045                 unsigned int len, u64 *iv);
0046     void (*cbc_decrypt)(const u64 *key, const u64 *input, u64 *output,
0047                 unsigned int len, u64 *iv);
0048     void (*ctr_crypt)(const u64 *key, const u64 *input, u64 *output,
0049               unsigned int len, u64 *iv);
0050 };
0051 
0052 struct crypto_sparc64_aes_ctx {
0053     struct aes_ops *ops;
0054     u64 key[AES_MAX_KEYLENGTH / sizeof(u64)];
0055     u32 key_length;
0056     u32 expanded_key_length;
0057 };
0058 
0059 extern void aes_sparc64_encrypt_128(const u64 *key, const u32 *input,
0060                     u32 *output);
0061 extern void aes_sparc64_encrypt_192(const u64 *key, const u32 *input,
0062                     u32 *output);
0063 extern void aes_sparc64_encrypt_256(const u64 *key, const u32 *input,
0064                     u32 *output);
0065 
0066 extern void aes_sparc64_decrypt_128(const u64 *key, const u32 *input,
0067                     u32 *output);
0068 extern void aes_sparc64_decrypt_192(const u64 *key, const u32 *input,
0069                     u32 *output);
0070 extern void aes_sparc64_decrypt_256(const u64 *key, const u32 *input,
0071                     u32 *output);
0072 
0073 extern void aes_sparc64_load_encrypt_keys_128(const u64 *key);
0074 extern void aes_sparc64_load_encrypt_keys_192(const u64 *key);
0075 extern void aes_sparc64_load_encrypt_keys_256(const u64 *key);
0076 
0077 extern void aes_sparc64_load_decrypt_keys_128(const u64 *key);
0078 extern void aes_sparc64_load_decrypt_keys_192(const u64 *key);
0079 extern void aes_sparc64_load_decrypt_keys_256(const u64 *key);
0080 
0081 extern void aes_sparc64_ecb_encrypt_128(const u64 *key, const u64 *input,
0082                     u64 *output, unsigned int len);
0083 extern void aes_sparc64_ecb_encrypt_192(const u64 *key, const u64 *input,
0084                     u64 *output, unsigned int len);
0085 extern void aes_sparc64_ecb_encrypt_256(const u64 *key, const u64 *input,
0086                     u64 *output, unsigned int len);
0087 
0088 extern void aes_sparc64_ecb_decrypt_128(const u64 *key, const u64 *input,
0089                     u64 *output, unsigned int len);
0090 extern void aes_sparc64_ecb_decrypt_192(const u64 *key, const u64 *input,
0091                     u64 *output, unsigned int len);
0092 extern void aes_sparc64_ecb_decrypt_256(const u64 *key, const u64 *input,
0093                     u64 *output, unsigned int len);
0094 
0095 extern void aes_sparc64_cbc_encrypt_128(const u64 *key, const u64 *input,
0096                     u64 *output, unsigned int len,
0097                     u64 *iv);
0098 
0099 extern void aes_sparc64_cbc_encrypt_192(const u64 *key, const u64 *input,
0100                     u64 *output, unsigned int len,
0101                     u64 *iv);
0102 
0103 extern void aes_sparc64_cbc_encrypt_256(const u64 *key, const u64 *input,
0104                     u64 *output, unsigned int len,
0105                     u64 *iv);
0106 
0107 extern void aes_sparc64_cbc_decrypt_128(const u64 *key, const u64 *input,
0108                     u64 *output, unsigned int len,
0109                     u64 *iv);
0110 
0111 extern void aes_sparc64_cbc_decrypt_192(const u64 *key, const u64 *input,
0112                     u64 *output, unsigned int len,
0113                     u64 *iv);
0114 
0115 extern void aes_sparc64_cbc_decrypt_256(const u64 *key, const u64 *input,
0116                     u64 *output, unsigned int len,
0117                     u64 *iv);
0118 
0119 extern void aes_sparc64_ctr_crypt_128(const u64 *key, const u64 *input,
0120                       u64 *output, unsigned int len,
0121                       u64 *iv);
0122 extern void aes_sparc64_ctr_crypt_192(const u64 *key, const u64 *input,
0123                       u64 *output, unsigned int len,
0124                       u64 *iv);
0125 extern void aes_sparc64_ctr_crypt_256(const u64 *key, const u64 *input,
0126                       u64 *output, unsigned int len,
0127                       u64 *iv);
0128 
0129 static struct aes_ops aes128_ops = {
0130     .encrypt        = aes_sparc64_encrypt_128,
0131     .decrypt        = aes_sparc64_decrypt_128,
0132     .load_encrypt_keys  = aes_sparc64_load_encrypt_keys_128,
0133     .load_decrypt_keys  = aes_sparc64_load_decrypt_keys_128,
0134     .ecb_encrypt        = aes_sparc64_ecb_encrypt_128,
0135     .ecb_decrypt        = aes_sparc64_ecb_decrypt_128,
0136     .cbc_encrypt        = aes_sparc64_cbc_encrypt_128,
0137     .cbc_decrypt        = aes_sparc64_cbc_decrypt_128,
0138     .ctr_crypt      = aes_sparc64_ctr_crypt_128,
0139 };
0140 
0141 static struct aes_ops aes192_ops = {
0142     .encrypt        = aes_sparc64_encrypt_192,
0143     .decrypt        = aes_sparc64_decrypt_192,
0144     .load_encrypt_keys  = aes_sparc64_load_encrypt_keys_192,
0145     .load_decrypt_keys  = aes_sparc64_load_decrypt_keys_192,
0146     .ecb_encrypt        = aes_sparc64_ecb_encrypt_192,
0147     .ecb_decrypt        = aes_sparc64_ecb_decrypt_192,
0148     .cbc_encrypt        = aes_sparc64_cbc_encrypt_192,
0149     .cbc_decrypt        = aes_sparc64_cbc_decrypt_192,
0150     .ctr_crypt      = aes_sparc64_ctr_crypt_192,
0151 };
0152 
0153 static struct aes_ops aes256_ops = {
0154     .encrypt        = aes_sparc64_encrypt_256,
0155     .decrypt        = aes_sparc64_decrypt_256,
0156     .load_encrypt_keys  = aes_sparc64_load_encrypt_keys_256,
0157     .load_decrypt_keys  = aes_sparc64_load_decrypt_keys_256,
0158     .ecb_encrypt        = aes_sparc64_ecb_encrypt_256,
0159     .ecb_decrypt        = aes_sparc64_ecb_decrypt_256,
0160     .cbc_encrypt        = aes_sparc64_cbc_encrypt_256,
0161     .cbc_decrypt        = aes_sparc64_cbc_decrypt_256,
0162     .ctr_crypt      = aes_sparc64_ctr_crypt_256,
0163 };
0164 
0165 extern void aes_sparc64_key_expand(const u32 *in_key, u64 *output_key,
0166                    unsigned int key_len);
0167 
0168 static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
0169                unsigned int key_len)
0170 {
0171     struct crypto_sparc64_aes_ctx *ctx = crypto_tfm_ctx(tfm);
0172 
0173     switch (key_len) {
0174     case AES_KEYSIZE_128:
0175         ctx->expanded_key_length = 0xb0;
0176         ctx->ops = &aes128_ops;
0177         break;
0178 
0179     case AES_KEYSIZE_192:
0180         ctx->expanded_key_length = 0xd0;
0181         ctx->ops = &aes192_ops;
0182         break;
0183 
0184     case AES_KEYSIZE_256:
0185         ctx->expanded_key_length = 0xf0;
0186         ctx->ops = &aes256_ops;
0187         break;
0188 
0189     default:
0190         return -EINVAL;
0191     }
0192 
0193     aes_sparc64_key_expand((const u32 *)in_key, &ctx->key[0], key_len);
0194     ctx->key_length = key_len;
0195 
0196     return 0;
0197 }
0198 
0199 static int aes_set_key_skcipher(struct crypto_skcipher *tfm, const u8 *in_key,
0200                 unsigned int key_len)
0201 {
0202     return aes_set_key(crypto_skcipher_tfm(tfm), in_key, key_len);
0203 }
0204 
0205 static void crypto_aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
0206 {
0207     struct crypto_sparc64_aes_ctx *ctx = crypto_tfm_ctx(tfm);
0208 
0209     ctx->ops->encrypt(&ctx->key[0], (const u32 *) src, (u32 *) dst);
0210 }
0211 
0212 static void crypto_aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
0213 {
0214     struct crypto_sparc64_aes_ctx *ctx = crypto_tfm_ctx(tfm);
0215 
0216     ctx->ops->decrypt(&ctx->key[0], (const u32 *) src, (u32 *) dst);
0217 }
0218 
0219 static int ecb_encrypt(struct skcipher_request *req)
0220 {
0221     struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
0222     const struct crypto_sparc64_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
0223     struct skcipher_walk walk;
0224     unsigned int nbytes;
0225     int err;
0226 
0227     err = skcipher_walk_virt(&walk, req, true);
0228     if (err)
0229         return err;
0230 
0231     ctx->ops->load_encrypt_keys(&ctx->key[0]);
0232     while ((nbytes = walk.nbytes) != 0) {
0233         ctx->ops->ecb_encrypt(&ctx->key[0], walk.src.virt.addr,
0234                       walk.dst.virt.addr,
0235                       round_down(nbytes, AES_BLOCK_SIZE));
0236         err = skcipher_walk_done(&walk, nbytes % AES_BLOCK_SIZE);
0237     }
0238     fprs_write(0);
0239     return err;
0240 }
0241 
0242 static int ecb_decrypt(struct skcipher_request *req)
0243 {
0244     struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
0245     const struct crypto_sparc64_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
0246     const u64 *key_end;
0247     struct skcipher_walk walk;
0248     unsigned int nbytes;
0249     int err;
0250 
0251     err = skcipher_walk_virt(&walk, req, true);
0252     if (err)
0253         return err;
0254 
0255     ctx->ops->load_decrypt_keys(&ctx->key[0]);
0256     key_end = &ctx->key[ctx->expanded_key_length / sizeof(u64)];
0257     while ((nbytes = walk.nbytes) != 0) {
0258         ctx->ops->ecb_decrypt(key_end, walk.src.virt.addr,
0259                       walk.dst.virt.addr,
0260                       round_down(nbytes, AES_BLOCK_SIZE));
0261         err = skcipher_walk_done(&walk, nbytes % AES_BLOCK_SIZE);
0262     }
0263     fprs_write(0);
0264 
0265     return err;
0266 }
0267 
0268 static int cbc_encrypt(struct skcipher_request *req)
0269 {
0270     struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
0271     const struct crypto_sparc64_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
0272     struct skcipher_walk walk;
0273     unsigned int nbytes;
0274     int err;
0275 
0276     err = skcipher_walk_virt(&walk, req, true);
0277     if (err)
0278         return err;
0279 
0280     ctx->ops->load_encrypt_keys(&ctx->key[0]);
0281     while ((nbytes = walk.nbytes) != 0) {
0282         ctx->ops->cbc_encrypt(&ctx->key[0], walk.src.virt.addr,
0283                       walk.dst.virt.addr,
0284                       round_down(nbytes, AES_BLOCK_SIZE),
0285                       walk.iv);
0286         err = skcipher_walk_done(&walk, nbytes % AES_BLOCK_SIZE);
0287     }
0288     fprs_write(0);
0289     return err;
0290 }
0291 
0292 static int cbc_decrypt(struct skcipher_request *req)
0293 {
0294     struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
0295     const struct crypto_sparc64_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
0296     const u64 *key_end;
0297     struct skcipher_walk walk;
0298     unsigned int nbytes;
0299     int err;
0300 
0301     err = skcipher_walk_virt(&walk, req, true);
0302     if (err)
0303         return err;
0304 
0305     ctx->ops->load_decrypt_keys(&ctx->key[0]);
0306     key_end = &ctx->key[ctx->expanded_key_length / sizeof(u64)];
0307     while ((nbytes = walk.nbytes) != 0) {
0308         ctx->ops->cbc_decrypt(key_end, walk.src.virt.addr,
0309                       walk.dst.virt.addr,
0310                       round_down(nbytes, AES_BLOCK_SIZE),
0311                       walk.iv);
0312         err = skcipher_walk_done(&walk, nbytes % AES_BLOCK_SIZE);
0313     }
0314     fprs_write(0);
0315 
0316     return err;
0317 }
0318 
0319 static void ctr_crypt_final(const struct crypto_sparc64_aes_ctx *ctx,
0320                 struct skcipher_walk *walk)
0321 {
0322     u8 *ctrblk = walk->iv;
0323     u64 keystream[AES_BLOCK_SIZE / sizeof(u64)];
0324     u8 *src = walk->src.virt.addr;
0325     u8 *dst = walk->dst.virt.addr;
0326     unsigned int nbytes = walk->nbytes;
0327 
0328     ctx->ops->ecb_encrypt(&ctx->key[0], (const u64 *)ctrblk,
0329                   keystream, AES_BLOCK_SIZE);
0330     crypto_xor_cpy(dst, (u8 *) keystream, src, nbytes);
0331     crypto_inc(ctrblk, AES_BLOCK_SIZE);
0332 }
0333 
0334 static int ctr_crypt(struct skcipher_request *req)
0335 {
0336     struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
0337     const struct crypto_sparc64_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
0338     struct skcipher_walk walk;
0339     unsigned int nbytes;
0340     int err;
0341 
0342     err = skcipher_walk_virt(&walk, req, true);
0343     if (err)
0344         return err;
0345 
0346     ctx->ops->load_encrypt_keys(&ctx->key[0]);
0347     while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) {
0348         ctx->ops->ctr_crypt(&ctx->key[0], walk.src.virt.addr,
0349                     walk.dst.virt.addr,
0350                     round_down(nbytes, AES_BLOCK_SIZE),
0351                     walk.iv);
0352         err = skcipher_walk_done(&walk, nbytes % AES_BLOCK_SIZE);
0353     }
0354     if (walk.nbytes) {
0355         ctr_crypt_final(ctx, &walk);
0356         err = skcipher_walk_done(&walk, 0);
0357     }
0358     fprs_write(0);
0359     return err;
0360 }
0361 
0362 static struct crypto_alg cipher_alg = {
0363     .cra_name       = "aes",
0364     .cra_driver_name    = "aes-sparc64",
0365     .cra_priority       = SPARC_CR_OPCODE_PRIORITY,
0366     .cra_flags      = CRYPTO_ALG_TYPE_CIPHER,
0367     .cra_blocksize      = AES_BLOCK_SIZE,
0368     .cra_ctxsize        = sizeof(struct crypto_sparc64_aes_ctx),
0369     .cra_alignmask      = 3,
0370     .cra_module     = THIS_MODULE,
0371     .cra_u  = {
0372         .cipher = {
0373             .cia_min_keysize    = AES_MIN_KEY_SIZE,
0374             .cia_max_keysize    = AES_MAX_KEY_SIZE,
0375             .cia_setkey     = aes_set_key,
0376             .cia_encrypt        = crypto_aes_encrypt,
0377             .cia_decrypt        = crypto_aes_decrypt
0378         }
0379     }
0380 };
0381 
0382 static struct skcipher_alg skcipher_algs[] = {
0383     {
0384         .base.cra_name      = "ecb(aes)",
0385         .base.cra_driver_name   = "ecb-aes-sparc64",
0386         .base.cra_priority  = SPARC_CR_OPCODE_PRIORITY,
0387         .base.cra_blocksize = AES_BLOCK_SIZE,
0388         .base.cra_ctxsize   = sizeof(struct crypto_sparc64_aes_ctx),
0389         .base.cra_alignmask = 7,
0390         .base.cra_module    = THIS_MODULE,
0391         .min_keysize        = AES_MIN_KEY_SIZE,
0392         .max_keysize        = AES_MAX_KEY_SIZE,
0393         .setkey         = aes_set_key_skcipher,
0394         .encrypt        = ecb_encrypt,
0395         .decrypt        = ecb_decrypt,
0396     }, {
0397         .base.cra_name      = "cbc(aes)",
0398         .base.cra_driver_name   = "cbc-aes-sparc64",
0399         .base.cra_priority  = SPARC_CR_OPCODE_PRIORITY,
0400         .base.cra_blocksize = AES_BLOCK_SIZE,
0401         .base.cra_ctxsize   = sizeof(struct crypto_sparc64_aes_ctx),
0402         .base.cra_alignmask = 7,
0403         .base.cra_module    = THIS_MODULE,
0404         .min_keysize        = AES_MIN_KEY_SIZE,
0405         .max_keysize        = AES_MAX_KEY_SIZE,
0406         .ivsize         = AES_BLOCK_SIZE,
0407         .setkey         = aes_set_key_skcipher,
0408         .encrypt        = cbc_encrypt,
0409         .decrypt        = cbc_decrypt,
0410     }, {
0411         .base.cra_name      = "ctr(aes)",
0412         .base.cra_driver_name   = "ctr-aes-sparc64",
0413         .base.cra_priority  = SPARC_CR_OPCODE_PRIORITY,
0414         .base.cra_blocksize = 1,
0415         .base.cra_ctxsize   = sizeof(struct crypto_sparc64_aes_ctx),
0416         .base.cra_alignmask = 7,
0417         .base.cra_module    = THIS_MODULE,
0418         .min_keysize        = AES_MIN_KEY_SIZE,
0419         .max_keysize        = AES_MAX_KEY_SIZE,
0420         .ivsize         = AES_BLOCK_SIZE,
0421         .setkey         = aes_set_key_skcipher,
0422         .encrypt        = ctr_crypt,
0423         .decrypt        = ctr_crypt,
0424         .chunksize      = AES_BLOCK_SIZE,
0425     }
0426 };
0427 
0428 static bool __init sparc64_has_aes_opcode(void)
0429 {
0430     unsigned long cfr;
0431 
0432     if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO))
0433         return false;
0434 
0435     __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr));
0436     if (!(cfr & CFR_AES))
0437         return false;
0438 
0439     return true;
0440 }
0441 
0442 static int __init aes_sparc64_mod_init(void)
0443 {
0444     int err;
0445 
0446     if (!sparc64_has_aes_opcode()) {
0447         pr_info("sparc64 aes opcodes not available.\n");
0448         return -ENODEV;
0449     }
0450     pr_info("Using sparc64 aes opcodes optimized AES implementation\n");
0451     err = crypto_register_alg(&cipher_alg);
0452     if (err)
0453         return err;
0454     err = crypto_register_skciphers(skcipher_algs,
0455                     ARRAY_SIZE(skcipher_algs));
0456     if (err)
0457         crypto_unregister_alg(&cipher_alg);
0458     return err;
0459 }
0460 
0461 static void __exit aes_sparc64_mod_fini(void)
0462 {
0463     crypto_unregister_alg(&cipher_alg);
0464     crypto_unregister_skciphers(skcipher_algs, ARRAY_SIZE(skcipher_algs));
0465 }
0466 
0467 module_init(aes_sparc64_mod_init);
0468 module_exit(aes_sparc64_mod_fini);
0469 
0470 MODULE_LICENSE("GPL");
0471 MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, sparc64 aes opcode accelerated");
0472 
0473 MODULE_ALIAS_CRYPTO("aes");
0474 
0475 #include "crop_devid.c"
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!