arm64/crypto/crct10dif-ce-glue.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * Accelerated CRC-T10DIF using arm64 NEON and Crypto Extensions instructions
0004  *
0005  * Copyright (C) 2016 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
0006  */
0007
0008 #include <linux/cpufeature.h>
0009 #include <linux/crc-t10dif.h>
0010 #include <linux/init.h>
0011 #include <linux/kernel.h>
0012 #include <linux/module.h>
0013 #include <linux/string.h>
0014
0015 #include <crypto/internal/hash.h>
0016 #include <crypto/internal/simd.h>
0017
0018 #include <asm/neon.h>
0019 #include <asm/simd.h>
0020
0021 #define CRC_T10DIF_PMULL_CHUNK_SIZE 16U
0022
0023 asmlinkage u16 crc_t10dif_pmull_p8(u16 init_crc, const u8 *buf, size_t len);
0024 asmlinkage u16 crc_t10dif_pmull_p64(u16 init_crc, const u8 *buf, size_t len);
0025
0026 static int crct10dif_init(struct shash_desc *desc)
0027 {
0028     u16 *crc = shash_desc_ctx(desc);
0029
0030     *crc = 0;
0031     return 0;
0032 }
0033
0034 static int crct10dif_update_pmull_p8(struct shash_desc *desc, const u8 *data,
0035                 unsigned int length)
0036 {
0037     u16 *crc = shash_desc_ctx(desc);
0038
0039     if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE && crypto_simd_usable()) {
0040         do {
0041             unsigned int chunk = length;
0042
0043             if (chunk > SZ_4K + CRC_T10DIF_PMULL_CHUNK_SIZE)
0044                 chunk = SZ_4K;
0045
0046             kernel_neon_begin();
0047             *crc = crc_t10dif_pmull_p8(*crc, data, chunk);
0048             kernel_neon_end();
0049             data += chunk;
0050             length -= chunk;
0051         } while (length);
0052     } else {
0053         *crc = crc_t10dif_generic(*crc, data, length);
0054     }
0055
0056     return 0;
0057 }
0058
0059 static int crct10dif_update_pmull_p64(struct shash_desc *desc, const u8 *data,
0060                 unsigned int length)
0061 {
0062     u16 *crc = shash_desc_ctx(desc);
0063
0064     if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE && crypto_simd_usable()) {
0065         do {
0066             unsigned int chunk = length;
0067
0068             if (chunk > SZ_4K + CRC_T10DIF_PMULL_CHUNK_SIZE)
0069                 chunk = SZ_4K;
0070
0071             kernel_neon_begin();
0072             *crc = crc_t10dif_pmull_p64(*crc, data, chunk);
0073             kernel_neon_end();
0074             data += chunk;
0075             length -= chunk;
0076         } while (length);
0077     } else {
0078         *crc = crc_t10dif_generic(*crc, data, length);
0079     }
0080
0081     return 0;
0082 }
0083
0084 static int crct10dif_final(struct shash_desc *desc, u8 *out)
0085 {
0086     u16 *crc = shash_desc_ctx(desc);
0087
0088     *(u16 *)out = *crc;
0089     return 0;
0090 }
0091
0092 static struct shash_alg crc_t10dif_alg[] = {{
0093     .digestsize     = CRC_T10DIF_DIGEST_SIZE,
0094     .init           = crct10dif_init,
0095     .update         = crct10dif_update_pmull_p8,
0096     .final          = crct10dif_final,
0097     .descsize       = CRC_T10DIF_DIGEST_SIZE,
0098
0099     .base.cra_name      = "crct10dif",
0100     .base.cra_driver_name   = "crct10dif-arm64-neon",
0101     .base.cra_priority  = 100,
0102     .base.cra_blocksize = CRC_T10DIF_BLOCK_SIZE,
0103     .base.cra_module    = THIS_MODULE,
0104 }, {
0105     .digestsize     = CRC_T10DIF_DIGEST_SIZE,
0106     .init           = crct10dif_init,
0107     .update         = crct10dif_update_pmull_p64,
0108     .final          = crct10dif_final,
0109     .descsize       = CRC_T10DIF_DIGEST_SIZE,
0110
0111     .base.cra_name      = "crct10dif",
0112     .base.cra_driver_name   = "crct10dif-arm64-ce",
0113     .base.cra_priority  = 200,
0114     .base.cra_blocksize = CRC_T10DIF_BLOCK_SIZE,
0115     .base.cra_module    = THIS_MODULE,
0116 }};
0117
0118 static int __init crc_t10dif_mod_init(void)
0119 {
0120     if (cpu_have_named_feature(PMULL))
0121         return crypto_register_shashes(crc_t10dif_alg,
0122                            ARRAY_SIZE(crc_t10dif_alg));
0123     else
0124         /* only register the first array element */
0125         return crypto_register_shash(crc_t10dif_alg);
0126 }
0127
0128 static void __exit crc_t10dif_mod_exit(void)
0129 {
0130     if (cpu_have_named_feature(PMULL))
0131         crypto_unregister_shashes(crc_t10dif_alg,
0132                       ARRAY_SIZE(crc_t10dif_alg));
0133     else
0134         crypto_unregister_shash(crc_t10dif_alg);
0135 }
0136
0137 module_cpu_feature_match(ASIMD, crc_t10dif_mod_init);
0138 module_exit(crc_t10dif_mod_exit);
0139
0140 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
0141 MODULE_LICENSE("GPL v2");
0142 MODULE_ALIAS_CRYPTO("crct10dif");
0143 MODULE_ALIAS_CRYPTO("crct10dif-arm64-ce");