powerpc/crypto/sha256-spe-glue.c

0001 // SPDX-License-Identifier: GPL-2.0-or-later
0002 /*
0003  * Glue code for SHA-256 implementation for SPE instructions (PPC)
0004  *
0005  * Based on generic implementation. The assembler module takes care
0006  * about the SPE registers so it can run from interrupt context.
0007  *
0008  * Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de>
0009  */
0010
0011 #include <crypto/internal/hash.h>
0012 #include <linux/init.h>
0013 #include <linux/module.h>
0014 #include <linux/mm.h>
0015 #include <linux/types.h>
0016 #include <crypto/sha2.h>
0017 #include <crypto/sha256_base.h>
0018 #include <asm/byteorder.h>
0019 #include <asm/switch_to.h>
0020 #include <linux/hardirq.h>
0021
0022 /*
0023  * MAX_BYTES defines the number of bytes that are allowed to be processed
0024  * between preempt_disable() and preempt_enable(). SHA256 takes ~2,000
0025  * operations per 64 bytes. e500 cores can issue two arithmetic instructions
0026  * per clock cycle using one 32/64 bit unit (SU1) and one 32 bit unit (SU2).
0027  * Thus 1KB of input data will need an estimated maximum of 18,000 cycles.
0028  * Headroom for cache misses included. Even with the low end model clocked
0029  * at 667 MHz this equals to a critical time window of less than 27us.
0030  *
0031  */
0032 #define MAX_BYTES 1024
0033
0034 extern void ppc_spe_sha256_transform(u32 *state, const u8 *src, u32 blocks);
0035
0036 static void spe_begin(void)
0037 {
0038     /* We just start SPE operations and will save SPE registers later. */
0039     preempt_disable();
0040     enable_kernel_spe();
0041 }
0042
0043 static void spe_end(void)
0044 {
0045     disable_kernel_spe();
0046     /* reenable preemption */
0047     preempt_enable();
0048 }
0049
0050 static inline void ppc_sha256_clear_context(struct sha256_state *sctx)
0051 {
0052     int count = sizeof(struct sha256_state) >> 2;
0053     u32 *ptr = (u32 *)sctx;
0054
0055     /* make sure we can clear the fast way */
0056     BUILD_BUG_ON(sizeof(struct sha256_state) % 4);
0057     do { *ptr++ = 0; } while (--count);
0058 }
0059
0060 static int ppc_spe_sha256_update(struct shash_desc *desc, const u8 *data,
0061             unsigned int len)
0062 {
0063     struct sha256_state *sctx = shash_desc_ctx(desc);
0064     const unsigned int offset = sctx->count & 0x3f;
0065     const unsigned int avail = 64 - offset;
0066     unsigned int bytes;
0067     const u8 *src = data;
0068
0069     if (avail > len) {
0070         sctx->count += len;
0071         memcpy((char *)sctx->buf + offset, src, len);
0072         return 0;
0073     }
0074
0075     sctx->count += len;
0076
0077     if (offset) {
0078         memcpy((char *)sctx->buf + offset, src, avail);
0079
0080         spe_begin();
0081         ppc_spe_sha256_transform(sctx->state, (const u8 *)sctx->buf, 1);
0082         spe_end();
0083
0084         len -= avail;
0085         src += avail;
0086     }
0087
0088     while (len > 63) {
0089         /* cut input data into smaller blocks */
0090         bytes = (len > MAX_BYTES) ? MAX_BYTES : len;
0091         bytes = bytes & ~0x3f;
0092
0093         spe_begin();
0094         ppc_spe_sha256_transform(sctx->state, src, bytes >> 6);
0095         spe_end();
0096
0097         src += bytes;
0098         len -= bytes;
0099     }
0100
0101     memcpy((char *)sctx->buf, src, len);
0102     return 0;
0103 }
0104
0105 static int ppc_spe_sha256_final(struct shash_desc *desc, u8 *out)
0106 {
0107     struct sha256_state *sctx = shash_desc_ctx(desc);
0108     const unsigned int offset = sctx->count & 0x3f;
0109     char *p = (char *)sctx->buf + offset;
0110     int padlen;
0111     __be64 *pbits = (__be64 *)(((char *)&sctx->buf) + 56);
0112     __be32 *dst = (__be32 *)out;
0113
0114     padlen = 55 - offset;
0115     *p++ = 0x80;
0116
0117     spe_begin();
0118
0119     if (padlen < 0) {
0120         memset(p, 0x00, padlen + sizeof (u64));
0121         ppc_spe_sha256_transform(sctx->state, sctx->buf, 1);
0122         p = (char *)sctx->buf;
0123         padlen = 56;
0124     }
0125
0126     memset(p, 0, padlen);
0127     *pbits = cpu_to_be64(sctx->count << 3);
0128     ppc_spe_sha256_transform(sctx->state, sctx->buf, 1);
0129
0130     spe_end();
0131
0132     dst[0] = cpu_to_be32(sctx->state[0]);
0133     dst[1] = cpu_to_be32(sctx->state[1]);
0134     dst[2] = cpu_to_be32(sctx->state[2]);
0135     dst[3] = cpu_to_be32(sctx->state[3]);
0136     dst[4] = cpu_to_be32(sctx->state[4]);
0137     dst[5] = cpu_to_be32(sctx->state[5]);
0138     dst[6] = cpu_to_be32(sctx->state[6]);
0139     dst[7] = cpu_to_be32(sctx->state[7]);
0140
0141     ppc_sha256_clear_context(sctx);
0142     return 0;
0143 }
0144
0145 static int ppc_spe_sha224_final(struct shash_desc *desc, u8 *out)
0146 {
0147     __be32 D[SHA256_DIGEST_SIZE >> 2];
0148     __be32 *dst = (__be32 *)out;
0149
0150     ppc_spe_sha256_final(desc, (u8 *)D);
0151
0152     /* avoid bytewise memcpy */
0153     dst[0] = D[0];
0154     dst[1] = D[1];
0155     dst[2] = D[2];
0156     dst[3] = D[3];
0157     dst[4] = D[4];
0158     dst[5] = D[5];
0159     dst[6] = D[6];
0160
0161     /* clear sensitive data */
0162     memzero_explicit(D, SHA256_DIGEST_SIZE);
0163     return 0;
0164 }
0165
0166 static int ppc_spe_sha256_export(struct shash_desc *desc, void *out)
0167 {
0168     struct sha256_state *sctx = shash_desc_ctx(desc);
0169
0170     memcpy(out, sctx, sizeof(*sctx));
0171     return 0;
0172 }
0173
0174 static int ppc_spe_sha256_import(struct shash_desc *desc, const void *in)
0175 {
0176     struct sha256_state *sctx = shash_desc_ctx(desc);
0177
0178     memcpy(sctx, in, sizeof(*sctx));
0179     return 0;
0180 }
0181
0182 static struct shash_alg algs[2] = { {
0183     .digestsize =   SHA256_DIGEST_SIZE,
0184     .init       =   sha256_base_init,
0185     .update     =   ppc_spe_sha256_update,
0186     .final      =   ppc_spe_sha256_final,
0187     .export     =   ppc_spe_sha256_export,
0188     .import     =   ppc_spe_sha256_import,
0189     .descsize   =   sizeof(struct sha256_state),
0190     .statesize  =   sizeof(struct sha256_state),
0191     .base       =   {
0192         .cra_name   =   "sha256",
0193         .cra_driver_name=   "sha256-ppc-spe",
0194         .cra_priority   =   300,
0195         .cra_blocksize  =   SHA256_BLOCK_SIZE,
0196         .cra_module =   THIS_MODULE,
0197     }
0198 }, {
0199     .digestsize =   SHA224_DIGEST_SIZE,
0200     .init       =   sha224_base_init,
0201     .update     =   ppc_spe_sha256_update,
0202     .final      =   ppc_spe_sha224_final,
0203     .export     =   ppc_spe_sha256_export,
0204     .import     =   ppc_spe_sha256_import,
0205     .descsize   =   sizeof(struct sha256_state),
0206     .statesize  =   sizeof(struct sha256_state),
0207     .base       =   {
0208         .cra_name   =   "sha224",
0209         .cra_driver_name=   "sha224-ppc-spe",
0210         .cra_priority   =   300,
0211         .cra_blocksize  =   SHA224_BLOCK_SIZE,
0212         .cra_module =   THIS_MODULE,
0213     }
0214 } };
0215
0216 static int __init ppc_spe_sha256_mod_init(void)
0217 {
0218     return crypto_register_shashes(algs, ARRAY_SIZE(algs));
0219 }
0220
0221 static void __exit ppc_spe_sha256_mod_fini(void)
0222 {
0223     crypto_unregister_shashes(algs, ARRAY_SIZE(algs));
0224 }
0225
0226 module_init(ppc_spe_sha256_mod_init);
0227 module_exit(ppc_spe_sha256_mod_fini);
0228
0229 MODULE_LICENSE("GPL");
0230 MODULE_DESCRIPTION("SHA-224 and SHA-256 Secure Hash Algorithm, SPE optimized");
0231
0232 MODULE_ALIAS_CRYPTO("sha224");
0233 MODULE_ALIAS_CRYPTO("sha224-ppc-spe");
0234 MODULE_ALIAS_CRYPTO("sha256");
0235 MODULE_ALIAS_CRYPTO("sha256-ppc-spe");