powerpc/crypto/crct10dif-vpmsum_glue.c

0001 // SPDX-License-Identifier: GPL-2.0-or-later
0002 /*
0003  * Calculate a CRC T10-DIF with vpmsum acceleration
0004  *
0005  * Copyright 2017, Daniel Axtens, IBM Corporation.
0006  * [based on crc32c-vpmsum_glue.c]
0007  */
0008
0009 #include <linux/crc-t10dif.h>
0010 #include <crypto/internal/hash.h>
0011 #include <crypto/internal/simd.h>
0012 #include <linux/init.h>
0013 #include <linux/module.h>
0014 #include <linux/string.h>
0015 #include <linux/kernel.h>
0016 #include <linux/cpufeature.h>
0017 #include <asm/simd.h>
0018 #include <asm/switch_to.h>
0019
0020 #define VMX_ALIGN       16
0021 #define VMX_ALIGN_MASK      (VMX_ALIGN-1)
0022
0023 #define VECTOR_BREAKPOINT   64
0024
0025 u32 __crct10dif_vpmsum(u32 crc, unsigned char const *p, size_t len);
0026
0027 static u16 crct10dif_vpmsum(u16 crci, unsigned char const *p, size_t len)
0028 {
0029     unsigned int prealign;
0030     unsigned int tail;
0031     u32 crc = crci;
0032
0033     if (len < (VECTOR_BREAKPOINT + VMX_ALIGN) || !crypto_simd_usable())
0034         return crc_t10dif_generic(crc, p, len);
0035
0036     if ((unsigned long)p & VMX_ALIGN_MASK) {
0037         prealign = VMX_ALIGN - ((unsigned long)p & VMX_ALIGN_MASK);
0038         crc = crc_t10dif_generic(crc, p, prealign);
0039         len -= prealign;
0040         p += prealign;
0041     }
0042
0043     if (len & ~VMX_ALIGN_MASK) {
0044         crc <<= 16;
0045         preempt_disable();
0046         pagefault_disable();
0047         enable_kernel_altivec();
0048         crc = __crct10dif_vpmsum(crc, p, len & ~VMX_ALIGN_MASK);
0049         disable_kernel_altivec();
0050         pagefault_enable();
0051         preempt_enable();
0052         crc >>= 16;
0053     }
0054
0055     tail = len & VMX_ALIGN_MASK;
0056     if (tail) {
0057         p += len & ~VMX_ALIGN_MASK;
0058         crc = crc_t10dif_generic(crc, p, tail);
0059     }
0060
0061     return crc & 0xffff;
0062 }
0063
0064 static int crct10dif_vpmsum_init(struct shash_desc *desc)
0065 {
0066     u16 *crc = shash_desc_ctx(desc);
0067
0068     *crc = 0;
0069     return 0;
0070 }
0071
0072 static int crct10dif_vpmsum_update(struct shash_desc *desc, const u8 *data,
0073                 unsigned int length)
0074 {
0075     u16 *crc = shash_desc_ctx(desc);
0076
0077     *crc = crct10dif_vpmsum(*crc, data, length);
0078
0079     return 0;
0080 }
0081
0082
0083 static int crct10dif_vpmsum_final(struct shash_desc *desc, u8 *out)
0084 {
0085     u16 *crcp = shash_desc_ctx(desc);
0086
0087     *(u16 *)out = *crcp;
0088     return 0;
0089 }
0090
0091 static struct shash_alg alg = {
0092     .init       = crct10dif_vpmsum_init,
0093     .update     = crct10dif_vpmsum_update,
0094     .final      = crct10dif_vpmsum_final,
0095     .descsize   = CRC_T10DIF_DIGEST_SIZE,
0096     .digestsize = CRC_T10DIF_DIGEST_SIZE,
0097     .base       = {
0098         .cra_name       = "crct10dif",
0099         .cra_driver_name    = "crct10dif-vpmsum",
0100         .cra_priority       = 200,
0101         .cra_blocksize      = CRC_T10DIF_BLOCK_SIZE,
0102         .cra_module     = THIS_MODULE,
0103     }
0104 };
0105
0106 static int __init crct10dif_vpmsum_mod_init(void)
0107 {
0108     if (!cpu_has_feature(CPU_FTR_ARCH_207S))
0109         return -ENODEV;
0110
0111     return crypto_register_shash(&alg);
0112 }
0113
0114 static void __exit crct10dif_vpmsum_mod_fini(void)
0115 {
0116     crypto_unregister_shash(&alg);
0117 }
0118
0119 module_cpu_feature_match(PPC_MODULE_FEATURE_VEC_CRYPTO, crct10dif_vpmsum_mod_init);
0120 module_exit(crct10dif_vpmsum_mod_fini);
0121
0122 MODULE_AUTHOR("Daniel Axtens <dja@axtens.net>");
0123 MODULE_DESCRIPTION("CRCT10DIF using vector polynomial multiply-sum instructions");
0124 MODULE_LICENSE("GPL");
0125 MODULE_ALIAS_CRYPTO("crct10dif");
0126 MODULE_ALIAS_CRYPTO("crct10dif-vpmsum");