x86/crypto/blake2s-glue.c

0001 // SPDX-License-Identifier: GPL-2.0 OR MIT
0002 /*
0003  * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
0004  */
0005
0006 #include <crypto/internal/blake2s.h>
0007
0008 #include <linux/types.h>
0009 #include <linux/jump_label.h>
0010 #include <linux/kernel.h>
0011 #include <linux/module.h>
0012 #include <linux/sizes.h>
0013
0014 #include <asm/cpufeature.h>
0015 #include <asm/fpu/api.h>
0016 #include <asm/processor.h>
0017 #include <asm/simd.h>
0018
0019 asmlinkage void blake2s_compress_ssse3(struct blake2s_state *state,
0020                        const u8 *block, const size_t nblocks,
0021                        const u32 inc);
0022 asmlinkage void blake2s_compress_avx512(struct blake2s_state *state,
0023                     const u8 *block, const size_t nblocks,
0024                     const u32 inc);
0025
0026 static __ro_after_init DEFINE_STATIC_KEY_FALSE(blake2s_use_ssse3);
0027 static __ro_after_init DEFINE_STATIC_KEY_FALSE(blake2s_use_avx512);
0028
0029 void blake2s_compress(struct blake2s_state *state, const u8 *block,
0030               size_t nblocks, const u32 inc)
0031 {
0032     /* SIMD disables preemption, so relax after processing each page. */
0033     BUILD_BUG_ON(SZ_4K / BLAKE2S_BLOCK_SIZE < 8);
0034
0035     if (!static_branch_likely(&blake2s_use_ssse3) || !may_use_simd()) {
0036         blake2s_compress_generic(state, block, nblocks, inc);
0037         return;
0038     }
0039
0040     do {
0041         const size_t blocks = min_t(size_t, nblocks,
0042                         SZ_4K / BLAKE2S_BLOCK_SIZE);
0043
0044         kernel_fpu_begin();
0045         if (IS_ENABLED(CONFIG_AS_AVX512) &&
0046             static_branch_likely(&blake2s_use_avx512))
0047             blake2s_compress_avx512(state, block, blocks, inc);
0048         else
0049             blake2s_compress_ssse3(state, block, blocks, inc);
0050         kernel_fpu_end();
0051
0052         nblocks -= blocks;
0053         block += blocks * BLAKE2S_BLOCK_SIZE;
0054     } while (nblocks);
0055 }
0056 EXPORT_SYMBOL(blake2s_compress);
0057
0058 static int __init blake2s_mod_init(void)
0059 {
0060     if (boot_cpu_has(X86_FEATURE_SSSE3))
0061         static_branch_enable(&blake2s_use_ssse3);
0062
0063     if (IS_ENABLED(CONFIG_AS_AVX512) &&
0064         boot_cpu_has(X86_FEATURE_AVX) &&
0065         boot_cpu_has(X86_FEATURE_AVX2) &&
0066         boot_cpu_has(X86_FEATURE_AVX512F) &&
0067         boot_cpu_has(X86_FEATURE_AVX512VL) &&
0068         cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM |
0069                   XFEATURE_MASK_AVX512, NULL))
0070         static_branch_enable(&blake2s_use_avx512);
0071
0072     return 0;
0073 }
0074
0075 module_init(blake2s_mod_init);
0076
0077 MODULE_LICENSE("GPL v2");