s390/crypto/crc32-vx.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * Crypto-API module for CRC-32 algorithms implemented with the
0004  * z/Architecture Vector Extension Facility.
0005  *
0006  * Copyright IBM Corp. 2015
0007  * Author(s): Hendrik Brueckner <brueckner@linux.vnet.ibm.com>
0008  */
0009 #define KMSG_COMPONENT  "crc32-vx"
0010 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
0011
0012 #include <linux/module.h>
0013 #include <linux/cpufeature.h>
0014 #include <linux/crc32.h>
0015 #include <crypto/internal/hash.h>
0016 #include <asm/fpu/api.h>
0017
0018
0019 #define CRC32_BLOCK_SIZE    1
0020 #define CRC32_DIGEST_SIZE   4
0021
0022 #define VX_MIN_LEN      64
0023 #define VX_ALIGNMENT        16L
0024 #define VX_ALIGN_MASK       (VX_ALIGNMENT - 1)
0025
0026 struct crc_ctx {
0027     u32 key;
0028 };
0029
0030 struct crc_desc_ctx {
0031     u32 crc;
0032 };
0033
0034 /* Prototypes for functions in assembly files */
0035 u32 crc32_le_vgfm_16(u32 crc, unsigned char const *buf, size_t size);
0036 u32 crc32_be_vgfm_16(u32 crc, unsigned char const *buf, size_t size);
0037 u32 crc32c_le_vgfm_16(u32 crc, unsigned char const *buf, size_t size);
0038
0039 /*
0040  * DEFINE_CRC32_VX() - Define a CRC-32 function using the vector extension
0041  *
0042  * Creates a function to perform a particular CRC-32 computation. Depending
0043  * on the message buffer, the hardware-accelerated or software implementation
0044  * is used.   Note that the message buffer is aligned to improve fetch
0045  * operations of VECTOR LOAD MULTIPLE instructions.
0046  *
0047  */
0048 #define DEFINE_CRC32_VX(___fname, ___crc32_vx, ___crc32_sw)         \
0049     static u32 __pure ___fname(u32 crc,                 \
0050                 unsigned char const *data, size_t datalen)  \
0051     {                                   \
0052         struct kernel_fpu vxstate;                  \
0053         unsigned long prealign, aligned, remaining;         \
0054                                         \
0055         if (datalen < VX_MIN_LEN + VX_ALIGN_MASK)           \
0056             return ___crc32_sw(crc, data, datalen);         \
0057                                         \
0058         if ((unsigned long)data & VX_ALIGN_MASK) {          \
0059             prealign = VX_ALIGNMENT -               \
0060                   ((unsigned long)data & VX_ALIGN_MASK);    \
0061             datalen -= prealign;                    \
0062             crc = ___crc32_sw(crc, data, prealign);         \
0063             data = (void *)((unsigned long)data + prealign);    \
0064         }                               \
0065                                         \
0066         aligned = datalen & ~VX_ALIGN_MASK;             \
0067         remaining = datalen & VX_ALIGN_MASK;                \
0068                                         \
0069         kernel_fpu_begin(&vxstate, KERNEL_VXR_LOW);         \
0070         crc = ___crc32_vx(crc, data, aligned);              \
0071         kernel_fpu_end(&vxstate, KERNEL_VXR_LOW);           \
0072                                         \
0073         if (remaining)                          \
0074             crc = ___crc32_sw(crc, data + aligned, remaining);  \
0075                                         \
0076         return crc;                         \
0077     }
0078
0079 DEFINE_CRC32_VX(crc32_le_vx, crc32_le_vgfm_16, crc32_le)
0080 DEFINE_CRC32_VX(crc32_be_vx, crc32_be_vgfm_16, crc32_be)
0081 DEFINE_CRC32_VX(crc32c_le_vx, crc32c_le_vgfm_16, __crc32c_le)
0082
0083
0084 static int crc32_vx_cra_init_zero(struct crypto_tfm *tfm)
0085 {
0086     struct crc_ctx *mctx = crypto_tfm_ctx(tfm);
0087
0088     mctx->key = 0;
0089     return 0;
0090 }
0091
0092 static int crc32_vx_cra_init_invert(struct crypto_tfm *tfm)
0093 {
0094     struct crc_ctx *mctx = crypto_tfm_ctx(tfm);
0095
0096     mctx->key = ~0;
0097     return 0;
0098 }
0099
0100 static int crc32_vx_init(struct shash_desc *desc)
0101 {
0102     struct crc_ctx *mctx = crypto_shash_ctx(desc->tfm);
0103     struct crc_desc_ctx *ctx = shash_desc_ctx(desc);
0104
0105     ctx->crc = mctx->key;
0106     return 0;
0107 }
0108
0109 static int crc32_vx_setkey(struct crypto_shash *tfm, const u8 *newkey,
0110                unsigned int newkeylen)
0111 {
0112     struct crc_ctx *mctx = crypto_shash_ctx(tfm);
0113
0114     if (newkeylen != sizeof(mctx->key))
0115         return -EINVAL;
0116     mctx->key = le32_to_cpu(*(__le32 *)newkey);
0117     return 0;
0118 }
0119
0120 static int crc32be_vx_setkey(struct crypto_shash *tfm, const u8 *newkey,
0121                  unsigned int newkeylen)
0122 {
0123     struct crc_ctx *mctx = crypto_shash_ctx(tfm);
0124
0125     if (newkeylen != sizeof(mctx->key))
0126         return -EINVAL;
0127     mctx->key = be32_to_cpu(*(__be32 *)newkey);
0128     return 0;
0129 }
0130
0131 static int crc32le_vx_final(struct shash_desc *desc, u8 *out)
0132 {
0133     struct crc_desc_ctx *ctx = shash_desc_ctx(desc);
0134
0135     *(__le32 *)out = cpu_to_le32p(&ctx->crc);
0136     return 0;
0137 }
0138
0139 static int crc32be_vx_final(struct shash_desc *desc, u8 *out)
0140 {
0141     struct crc_desc_ctx *ctx = shash_desc_ctx(desc);
0142
0143     *(__be32 *)out = cpu_to_be32p(&ctx->crc);
0144     return 0;
0145 }
0146
0147 static int crc32c_vx_final(struct shash_desc *desc, u8 *out)
0148 {
0149     struct crc_desc_ctx *ctx = shash_desc_ctx(desc);
0150
0151     /*
0152      * Perform a final XOR with 0xFFFFFFFF to be in sync
0153      * with the generic crc32c shash implementation.
0154      */
0155     *(__le32 *)out = ~cpu_to_le32p(&ctx->crc);
0156     return 0;
0157 }
0158
0159 static int __crc32le_vx_finup(u32 *crc, const u8 *data, unsigned int len,
0160                   u8 *out)
0161 {
0162     *(__le32 *)out = cpu_to_le32(crc32_le_vx(*crc, data, len));
0163     return 0;
0164 }
0165
0166 static int __crc32be_vx_finup(u32 *crc, const u8 *data, unsigned int len,
0167                   u8 *out)
0168 {
0169     *(__be32 *)out = cpu_to_be32(crc32_be_vx(*crc, data, len));
0170     return 0;
0171 }
0172
0173 static int __crc32c_vx_finup(u32 *crc, const u8 *data, unsigned int len,
0174                  u8 *out)
0175 {
0176     /*
0177      * Perform a final XOR with 0xFFFFFFFF to be in sync
0178      * with the generic crc32c shash implementation.
0179      */
0180     *(__le32 *)out = ~cpu_to_le32(crc32c_le_vx(*crc, data, len));
0181     return 0;
0182 }
0183
0184
0185 #define CRC32_VX_FINUP(alg, func)                         \
0186     static int alg ## _vx_finup(struct shash_desc *desc, const u8 *data,  \
0187                    unsigned int datalen, u8 *out)         \
0188     {                                     \
0189         return __ ## alg ## _vx_finup(shash_desc_ctx(desc),       \
0190                           data, datalen, out);        \
0191     }
0192
0193 CRC32_VX_FINUP(crc32le, crc32_le_vx)
0194 CRC32_VX_FINUP(crc32be, crc32_be_vx)
0195 CRC32_VX_FINUP(crc32c, crc32c_le_vx)
0196
0197 #define CRC32_VX_DIGEST(alg, func)                        \
0198     static int alg ## _vx_digest(struct shash_desc *desc, const u8 *data, \
0199                      unsigned int len, u8 *out)           \
0200     {                                     \
0201         return __ ## alg ## _vx_finup(crypto_shash_ctx(desc->tfm),    \
0202                           data, len, out);            \
0203     }
0204
0205 CRC32_VX_DIGEST(crc32le, crc32_le_vx)
0206 CRC32_VX_DIGEST(crc32be, crc32_be_vx)
0207 CRC32_VX_DIGEST(crc32c, crc32c_le_vx)
0208
0209 #define CRC32_VX_UPDATE(alg, func)                        \
0210     static int alg ## _vx_update(struct shash_desc *desc, const u8 *data, \
0211                      unsigned int datalen)            \
0212     {                                     \
0213         struct crc_desc_ctx *ctx = shash_desc_ctx(desc);          \
0214         ctx->crc = func(ctx->crc, data, datalen);             \
0215         return 0;                             \
0216     }
0217
0218 CRC32_VX_UPDATE(crc32le, crc32_le_vx)
0219 CRC32_VX_UPDATE(crc32be, crc32_be_vx)
0220 CRC32_VX_UPDATE(crc32c, crc32c_le_vx)
0221
0222
0223 static struct shash_alg crc32_vx_algs[] = {
0224     /* CRC-32 LE */
0225     {
0226         .init       =   crc32_vx_init,
0227         .setkey     =   crc32_vx_setkey,
0228         .update     =   crc32le_vx_update,
0229         .final      =   crc32le_vx_final,
0230         .finup      =   crc32le_vx_finup,
0231         .digest     =   crc32le_vx_digest,
0232         .descsize   =   sizeof(struct crc_desc_ctx),
0233         .digestsize =   CRC32_DIGEST_SIZE,
0234         .base       =   {
0235             .cra_name    = "crc32",
0236             .cra_driver_name = "crc32-vx",
0237             .cra_priority    = 200,
0238             .cra_flags   = CRYPTO_ALG_OPTIONAL_KEY,
0239             .cra_blocksize   = CRC32_BLOCK_SIZE,
0240             .cra_ctxsize     = sizeof(struct crc_ctx),
0241             .cra_module  = THIS_MODULE,
0242             .cra_init    = crc32_vx_cra_init_zero,
0243         },
0244     },
0245     /* CRC-32 BE */
0246     {
0247         .init       =   crc32_vx_init,
0248         .setkey     =   crc32be_vx_setkey,
0249         .update     =   crc32be_vx_update,
0250         .final      =   crc32be_vx_final,
0251         .finup      =   crc32be_vx_finup,
0252         .digest     =   crc32be_vx_digest,
0253         .descsize   =   sizeof(struct crc_desc_ctx),
0254         .digestsize =   CRC32_DIGEST_SIZE,
0255         .base       =   {
0256             .cra_name    = "crc32be",
0257             .cra_driver_name = "crc32be-vx",
0258             .cra_priority    = 200,
0259             .cra_flags   = CRYPTO_ALG_OPTIONAL_KEY,
0260             .cra_blocksize   = CRC32_BLOCK_SIZE,
0261             .cra_ctxsize     = sizeof(struct crc_ctx),
0262             .cra_module  = THIS_MODULE,
0263             .cra_init    = crc32_vx_cra_init_zero,
0264         },
0265     },
0266     /* CRC-32C LE */
0267     {
0268         .init       =   crc32_vx_init,
0269         .setkey     =   crc32_vx_setkey,
0270         .update     =   crc32c_vx_update,
0271         .final      =   crc32c_vx_final,
0272         .finup      =   crc32c_vx_finup,
0273         .digest     =   crc32c_vx_digest,
0274         .descsize   =   sizeof(struct crc_desc_ctx),
0275         .digestsize =   CRC32_DIGEST_SIZE,
0276         .base       =   {
0277             .cra_name    = "crc32c",
0278             .cra_driver_name = "crc32c-vx",
0279             .cra_priority    = 200,
0280             .cra_flags   = CRYPTO_ALG_OPTIONAL_KEY,
0281             .cra_blocksize   = CRC32_BLOCK_SIZE,
0282             .cra_ctxsize     = sizeof(struct crc_ctx),
0283             .cra_module  = THIS_MODULE,
0284             .cra_init    = crc32_vx_cra_init_invert,
0285         },
0286     },
0287 };
0288
0289
0290 static int __init crc_vx_mod_init(void)
0291 {
0292     return crypto_register_shashes(crc32_vx_algs,
0293                        ARRAY_SIZE(crc32_vx_algs));
0294 }
0295
0296 static void __exit crc_vx_mod_exit(void)
0297 {
0298     crypto_unregister_shashes(crc32_vx_algs, ARRAY_SIZE(crc32_vx_algs));
0299 }
0300
0301 module_cpu_feature_match(S390_CPU_FEATURE_VXRS, crc_vx_mod_init);
0302 module_exit(crc_vx_mod_exit);
0303
0304 MODULE_AUTHOR("Hendrik Brueckner <brueckner@linux.vnet.ibm.com>");
0305 MODULE_LICENSE("GPL");
0306
0307 MODULE_ALIAS_CRYPTO("crc32");
0308 MODULE_ALIAS_CRYPTO("crc32-vx");
0309 MODULE_ALIAS_CRYPTO("crc32c");
0310 MODULE_ALIAS_CRYPTO("crc32c-vx");