powerpc/crypto/crc32c-vpmsum_glue.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 #include <linux/crc32.h>
0003 #include <crypto/internal/hash.h>
0004 #include <crypto/internal/simd.h>
0005 #include <linux/init.h>
0006 #include <linux/module.h>
0007 #include <linux/string.h>
0008 #include <linux/kernel.h>
0009 #include <linux/cpufeature.h>
0010 #include <asm/simd.h>
0011 #include <asm/switch_to.h>
0012
0013 #define CHKSUM_BLOCK_SIZE   1
0014 #define CHKSUM_DIGEST_SIZE  4
0015
0016 #define VMX_ALIGN       16
0017 #define VMX_ALIGN_MASK      (VMX_ALIGN-1)
0018
0019 #define VECTOR_BREAKPOINT   512
0020
0021 u32 __crc32c_vpmsum(u32 crc, unsigned char const *p, size_t len);
0022
0023 static u32 crc32c_vpmsum(u32 crc, unsigned char const *p, size_t len)
0024 {
0025     unsigned int prealign;
0026     unsigned int tail;
0027
0028     if (len < (VECTOR_BREAKPOINT + VMX_ALIGN) || !crypto_simd_usable())
0029         return __crc32c_le(crc, p, len);
0030
0031     if ((unsigned long)p & VMX_ALIGN_MASK) {
0032         prealign = VMX_ALIGN - ((unsigned long)p & VMX_ALIGN_MASK);
0033         crc = __crc32c_le(crc, p, prealign);
0034         len -= prealign;
0035         p += prealign;
0036     }
0037
0038     if (len & ~VMX_ALIGN_MASK) {
0039         preempt_disable();
0040         pagefault_disable();
0041         enable_kernel_altivec();
0042         crc = __crc32c_vpmsum(crc, p, len & ~VMX_ALIGN_MASK);
0043         disable_kernel_altivec();
0044         pagefault_enable();
0045         preempt_enable();
0046     }
0047
0048     tail = len & VMX_ALIGN_MASK;
0049     if (tail) {
0050         p += len & ~VMX_ALIGN_MASK;
0051         crc = __crc32c_le(crc, p, tail);
0052     }
0053
0054     return crc;
0055 }
0056
0057 static int crc32c_vpmsum_cra_init(struct crypto_tfm *tfm)
0058 {
0059     u32 *key = crypto_tfm_ctx(tfm);
0060
0061     *key = ~0;
0062
0063     return 0;
0064 }
0065
0066 /*
0067  * Setting the seed allows arbitrary accumulators and flexible XOR policy
0068  * If your algorithm starts with ~0, then XOR with ~0 before you set
0069  * the seed.
0070  */
0071 static int crc32c_vpmsum_setkey(struct crypto_shash *hash, const u8 *key,
0072                    unsigned int keylen)
0073 {
0074     u32 *mctx = crypto_shash_ctx(hash);
0075
0076     if (keylen != sizeof(u32))
0077         return -EINVAL;
0078     *mctx = le32_to_cpup((__le32 *)key);
0079     return 0;
0080 }
0081
0082 static int crc32c_vpmsum_init(struct shash_desc *desc)
0083 {
0084     u32 *mctx = crypto_shash_ctx(desc->tfm);
0085     u32 *crcp = shash_desc_ctx(desc);
0086
0087     *crcp = *mctx;
0088
0089     return 0;
0090 }
0091
0092 static int crc32c_vpmsum_update(struct shash_desc *desc, const u8 *data,
0093                    unsigned int len)
0094 {
0095     u32 *crcp = shash_desc_ctx(desc);
0096
0097     *crcp = crc32c_vpmsum(*crcp, data, len);
0098
0099     return 0;
0100 }
0101
0102 static int __crc32c_vpmsum_finup(u32 *crcp, const u8 *data, unsigned int len,
0103                 u8 *out)
0104 {
0105     *(__le32 *)out = ~cpu_to_le32(crc32c_vpmsum(*crcp, data, len));
0106
0107     return 0;
0108 }
0109
0110 static int crc32c_vpmsum_finup(struct shash_desc *desc, const u8 *data,
0111                   unsigned int len, u8 *out)
0112 {
0113     return __crc32c_vpmsum_finup(shash_desc_ctx(desc), data, len, out);
0114 }
0115
0116 static int crc32c_vpmsum_final(struct shash_desc *desc, u8 *out)
0117 {
0118     u32 *crcp = shash_desc_ctx(desc);
0119
0120     *(__le32 *)out = ~cpu_to_le32p(crcp);
0121
0122     return 0;
0123 }
0124
0125 static int crc32c_vpmsum_digest(struct shash_desc *desc, const u8 *data,
0126                    unsigned int len, u8 *out)
0127 {
0128     return __crc32c_vpmsum_finup(crypto_shash_ctx(desc->tfm), data, len,
0129                      out);
0130 }
0131
0132 static struct shash_alg alg = {
0133     .setkey     = crc32c_vpmsum_setkey,
0134     .init       = crc32c_vpmsum_init,
0135     .update     = crc32c_vpmsum_update,
0136     .final      = crc32c_vpmsum_final,
0137     .finup      = crc32c_vpmsum_finup,
0138     .digest     = crc32c_vpmsum_digest,
0139     .descsize   = sizeof(u32),
0140     .digestsize = CHKSUM_DIGEST_SIZE,
0141     .base       = {
0142         .cra_name       = "crc32c",
0143         .cra_driver_name    = "crc32c-vpmsum",
0144         .cra_priority       = 200,
0145         .cra_flags      = CRYPTO_ALG_OPTIONAL_KEY,
0146         .cra_blocksize      = CHKSUM_BLOCK_SIZE,
0147         .cra_ctxsize        = sizeof(u32),
0148         .cra_module     = THIS_MODULE,
0149         .cra_init       = crc32c_vpmsum_cra_init,
0150     }
0151 };
0152
0153 static int __init crc32c_vpmsum_mod_init(void)
0154 {
0155     if (!cpu_has_feature(CPU_FTR_ARCH_207S))
0156         return -ENODEV;
0157
0158     return crypto_register_shash(&alg);
0159 }
0160
0161 static void __exit crc32c_vpmsum_mod_fini(void)
0162 {
0163     crypto_unregister_shash(&alg);
0164 }
0165
0166 module_cpu_feature_match(PPC_MODULE_FEATURE_VEC_CRYPTO, crc32c_vpmsum_mod_init);
0167 module_exit(crc32c_vpmsum_mod_fini);
0168
0169 MODULE_AUTHOR("Anton Blanchard <anton@samba.org>");
0170 MODULE_DESCRIPTION("CRC32C using vector polynomial multiply-sum instructions");
0171 MODULE_LICENSE("GPL");
0172 MODULE_ALIAS_CRYPTO("crc32c");
0173 MODULE_ALIAS_CRYPTO("crc32c-vpmsum");