Back to home page

LXR

 
 

    


0001 /*
0002  * Modified to interface to the Linux kernel
0003  * Copyright (c) 2009, Intel Corporation.
0004  *
0005  * This program is free software; you can redistribute it and/or modify it
0006  * under the terms and conditions of the GNU General Public License,
0007  * version 2, as published by the Free Software Foundation.
0008  *
0009  * This program is distributed in the hope it will be useful, but WITHOUT
0010  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
0011  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
0012  * more details.
0013  *
0014  * You should have received a copy of the GNU General Public License along with
0015  * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
0016  * Place - Suite 330, Boston, MA 02111-1307 USA.
0017  */
0018 
0019 /* --------------------------------------------------------------------------
0020  * VMAC and VHASH Implementation by Ted Krovetz (tdk@acm.org) and Wei Dai.
0021  * This implementation is herby placed in the public domain.
0022  * The authors offers no warranty. Use at your own risk.
0023  * Please send bug reports to the authors.
0024  * Last modified: 17 APR 08, 1700 PDT
0025  * ----------------------------------------------------------------------- */
0026 
0027 #include <linux/init.h>
0028 #include <linux/types.h>
0029 #include <linux/crypto.h>
0030 #include <linux/module.h>
0031 #include <linux/scatterlist.h>
0032 #include <asm/byteorder.h>
0033 #include <crypto/scatterwalk.h>
0034 #include <crypto/vmac.h>
0035 #include <crypto/internal/hash.h>
0036 
0037 /*
0038  * Constants and masks
0039  */
0040 #define UINT64_C(x) x##ULL
0041 static const u64 p64   = UINT64_C(0xfffffffffffffeff);  /* 2^64 - 257 prime  */
0042 static const u64 m62   = UINT64_C(0x3fffffffffffffff);  /* 62-bit mask       */
0043 static const u64 m63   = UINT64_C(0x7fffffffffffffff);  /* 63-bit mask       */
0044 static const u64 m64   = UINT64_C(0xffffffffffffffff);  /* 64-bit mask       */
0045 static const u64 mpoly = UINT64_C(0x1fffffff1fffffff);  /* Poly key mask     */
0046 
0047 #define pe64_to_cpup le64_to_cpup       /* Prefer little endian */
0048 
0049 #ifdef __LITTLE_ENDIAN
0050 #define INDEX_HIGH 1
0051 #define INDEX_LOW 0
0052 #else
0053 #define INDEX_HIGH 0
0054 #define INDEX_LOW 1
0055 #endif
0056 
0057 /*
0058  * The following routines are used in this implementation. They are
0059  * written via macros to simulate zero-overhead call-by-reference.
0060  *
0061  * MUL64: 64x64->128-bit multiplication
0062  * PMUL64: assumes top bits cleared on inputs
0063  * ADD128: 128x128->128-bit addition
0064  */
0065 
0066 #define ADD128(rh, rl, ih, il)                      \
0067     do {                                \
0068         u64 _il = (il);                     \
0069         (rl) += (_il);                      \
0070         if ((rl) < (_il))                   \
0071             (rh)++;                     \
0072         (rh) += (ih);                       \
0073     } while (0)
0074 
0075 #define MUL32(i1, i2)   ((u64)(u32)(i1)*(u32)(i2))
0076 
0077 #define PMUL64(rh, rl, i1, i2)  /* Assumes m doesn't overflow */    \
0078     do {                                \
0079         u64 _i1 = (i1), _i2 = (i2);             \
0080         u64 m = MUL32(_i1, _i2>>32) + MUL32(_i1>>32, _i2);  \
0081         rh = MUL32(_i1>>32, _i2>>32);               \
0082         rl = MUL32(_i1, _i2);                   \
0083         ADD128(rh, rl, (m >> 32), (m << 32));           \
0084     } while (0)
0085 
0086 #define MUL64(rh, rl, i1, i2)                       \
0087     do {                                \
0088         u64 _i1 = (i1), _i2 = (i2);             \
0089         u64 m1 = MUL32(_i1, _i2>>32);               \
0090         u64 m2 = MUL32(_i1>>32, _i2);               \
0091         rh = MUL32(_i1>>32, _i2>>32);               \
0092         rl = MUL32(_i1, _i2);                   \
0093         ADD128(rh, rl, (m1 >> 32), (m1 << 32));         \
0094         ADD128(rh, rl, (m2 >> 32), (m2 << 32));         \
0095     } while (0)
0096 
0097 /*
0098  * For highest performance the L1 NH and L2 polynomial hashes should be
0099  * carefully implemented to take advantage of one's target architecture.
0100  * Here these two hash functions are defined multiple time; once for
0101  * 64-bit architectures, once for 32-bit SSE2 architectures, and once
0102  * for the rest (32-bit) architectures.
0103  * For each, nh_16 *must* be defined (works on multiples of 16 bytes).
0104  * Optionally, nh_vmac_nhbytes can be defined (for multiples of
0105  * VMAC_NHBYTES), and nh_16_2 and nh_vmac_nhbytes_2 (versions that do two
0106  * NH computations at once).
0107  */
0108 
0109 #ifdef CONFIG_64BIT
0110 
0111 #define nh_16(mp, kp, nw, rh, rl)                   \
0112     do {                                \
0113         int i; u64 th, tl;                  \
0114         rh = rl = 0;                        \
0115         for (i = 0; i < nw; i += 2) {               \
0116             MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i], \
0117                 pe64_to_cpup((mp)+i+1)+(kp)[i+1]);  \
0118             ADD128(rh, rl, th, tl);             \
0119         }                           \
0120     } while (0)
0121 
0122 #define nh_16_2(mp, kp, nw, rh, rl, rh1, rl1)               \
0123     do {                                \
0124         int i; u64 th, tl;                  \
0125         rh1 = rl1 = rh = rl = 0;                \
0126         for (i = 0; i < nw; i += 2) {               \
0127             MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i], \
0128                 pe64_to_cpup((mp)+i+1)+(kp)[i+1]);  \
0129             ADD128(rh, rl, th, tl);             \
0130             MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i+2],   \
0131                 pe64_to_cpup((mp)+i+1)+(kp)[i+3]);  \
0132             ADD128(rh1, rl1, th, tl);           \
0133         }                           \
0134     } while (0)
0135 
0136 #if (VMAC_NHBYTES >= 64) /* These versions do 64-bytes of message at a time */
0137 #define nh_vmac_nhbytes(mp, kp, nw, rh, rl)             \
0138     do {                                \
0139         int i; u64 th, tl;                  \
0140         rh = rl = 0;                        \
0141         for (i = 0; i < nw; i += 8) {               \
0142             MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i], \
0143                 pe64_to_cpup((mp)+i+1)+(kp)[i+1]);  \
0144             ADD128(rh, rl, th, tl);             \
0145             MUL64(th, tl, pe64_to_cpup((mp)+i+2)+(kp)[i+2], \
0146                 pe64_to_cpup((mp)+i+3)+(kp)[i+3]);  \
0147             ADD128(rh, rl, th, tl);             \
0148             MUL64(th, tl, pe64_to_cpup((mp)+i+4)+(kp)[i+4], \
0149                 pe64_to_cpup((mp)+i+5)+(kp)[i+5]);  \
0150             ADD128(rh, rl, th, tl);             \
0151             MUL64(th, tl, pe64_to_cpup((mp)+i+6)+(kp)[i+6], \
0152                 pe64_to_cpup((mp)+i+7)+(kp)[i+7]);  \
0153             ADD128(rh, rl, th, tl);             \
0154         }                           \
0155     } while (0)
0156 
0157 #define nh_vmac_nhbytes_2(mp, kp, nw, rh, rl, rh1, rl1)         \
0158     do {                                \
0159         int i; u64 th, tl;                  \
0160         rh1 = rl1 = rh = rl = 0;                \
0161         for (i = 0; i < nw; i += 8) {               \
0162             MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i], \
0163                 pe64_to_cpup((mp)+i+1)+(kp)[i+1]);  \
0164             ADD128(rh, rl, th, tl);             \
0165             MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i+2],   \
0166                 pe64_to_cpup((mp)+i+1)+(kp)[i+3]);  \
0167             ADD128(rh1, rl1, th, tl);           \
0168             MUL64(th, tl, pe64_to_cpup((mp)+i+2)+(kp)[i+2], \
0169                 pe64_to_cpup((mp)+i+3)+(kp)[i+3]);  \
0170             ADD128(rh, rl, th, tl);             \
0171             MUL64(th, tl, pe64_to_cpup((mp)+i+2)+(kp)[i+4], \
0172                 pe64_to_cpup((mp)+i+3)+(kp)[i+5]);  \
0173             ADD128(rh1, rl1, th, tl);           \
0174             MUL64(th, tl, pe64_to_cpup((mp)+i+4)+(kp)[i+4], \
0175                 pe64_to_cpup((mp)+i+5)+(kp)[i+5]);  \
0176             ADD128(rh, rl, th, tl);             \
0177             MUL64(th, tl, pe64_to_cpup((mp)+i+4)+(kp)[i+6], \
0178                 pe64_to_cpup((mp)+i+5)+(kp)[i+7]);  \
0179             ADD128(rh1, rl1, th, tl);           \
0180             MUL64(th, tl, pe64_to_cpup((mp)+i+6)+(kp)[i+6], \
0181                 pe64_to_cpup((mp)+i+7)+(kp)[i+7]);  \
0182             ADD128(rh, rl, th, tl);             \
0183             MUL64(th, tl, pe64_to_cpup((mp)+i+6)+(kp)[i+8], \
0184                 pe64_to_cpup((mp)+i+7)+(kp)[i+9]);  \
0185             ADD128(rh1, rl1, th, tl);           \
0186         }                           \
0187     } while (0)
0188 #endif
0189 
0190 #define poly_step(ah, al, kh, kl, mh, ml)               \
0191     do {                                \
0192         u64 t1h, t1l, t2h, t2l, t3h, t3l, z = 0;        \
0193         /* compute ab*cd, put bd into result registers */   \
0194         PMUL64(t3h, t3l, al, kh);               \
0195         PMUL64(t2h, t2l, ah, kl);               \
0196         PMUL64(t1h, t1l, ah, 2*kh);             \
0197         PMUL64(ah, al, al, kl);                 \
0198         /* add 2 * ac to result */              \
0199         ADD128(ah, al, t1h, t1l);               \
0200         /* add together ad + bc */              \
0201         ADD128(t2h, t2l, t3h, t3l);             \
0202         /* now (ah,al), (t2l,2*t2h) need summing */     \
0203         /* first add the high registers, carrying into t2h */   \
0204         ADD128(t2h, ah, z, t2l);                \
0205         /* double t2h and add top bit of ah */          \
0206         t2h = 2 * t2h + (ah >> 63);             \
0207         ah &= m63;                      \
0208         /* now add the low registers */             \
0209         ADD128(ah, al, mh, ml);                 \
0210         ADD128(ah, al, z, t2h);                 \
0211     } while (0)
0212 
0213 #else /* ! CONFIG_64BIT */
0214 
0215 #ifndef nh_16
0216 #define nh_16(mp, kp, nw, rh, rl)                   \
0217     do {                                \
0218         u64 t1, t2, m1, m2, t;                  \
0219         int i;                          \
0220         rh = rl = t = 0;                    \
0221         for (i = 0; i < nw; i += 2)  {              \
0222             t1 = pe64_to_cpup(mp+i) + kp[i];        \
0223             t2 = pe64_to_cpup(mp+i+1) + kp[i+1];        \
0224             m2 = MUL32(t1 >> 32, t2);           \
0225             m1 = MUL32(t1, t2 >> 32);           \
0226             ADD128(rh, rl, MUL32(t1 >> 32, t2 >> 32),   \
0227                 MUL32(t1, t2));             \
0228             rh += (u64)(u32)(m1 >> 32)          \
0229                 + (u32)(m2 >> 32);          \
0230             t += (u64)(u32)m1 + (u32)m2;            \
0231         }                           \
0232         ADD128(rh, rl, (t >> 32), (t << 32));           \
0233     } while (0)
0234 #endif
0235 
0236 static void poly_step_func(u64 *ahi, u64 *alo,
0237             const u64 *kh, const u64 *kl,
0238             const u64 *mh, const u64 *ml)
0239 {
0240 #define a0 (*(((u32 *)alo)+INDEX_LOW))
0241 #define a1 (*(((u32 *)alo)+INDEX_HIGH))
0242 #define a2 (*(((u32 *)ahi)+INDEX_LOW))
0243 #define a3 (*(((u32 *)ahi)+INDEX_HIGH))
0244 #define k0 (*(((u32 *)kl)+INDEX_LOW))
0245 #define k1 (*(((u32 *)kl)+INDEX_HIGH))
0246 #define k2 (*(((u32 *)kh)+INDEX_LOW))
0247 #define k3 (*(((u32 *)kh)+INDEX_HIGH))
0248 
0249     u64 p, q, t;
0250     u32 t2;
0251 
0252     p = MUL32(a3, k3);
0253     p += p;
0254     p += *(u64 *)mh;
0255     p += MUL32(a0, k2);
0256     p += MUL32(a1, k1);
0257     p += MUL32(a2, k0);
0258     t = (u32)(p);
0259     p >>= 32;
0260     p += MUL32(a0, k3);
0261     p += MUL32(a1, k2);
0262     p += MUL32(a2, k1);
0263     p += MUL32(a3, k0);
0264     t |= ((u64)((u32)p & 0x7fffffff)) << 32;
0265     p >>= 31;
0266     p += (u64)(((u32 *)ml)[INDEX_LOW]);
0267     p += MUL32(a0, k0);
0268     q =  MUL32(a1, k3);
0269     q += MUL32(a2, k2);
0270     q += MUL32(a3, k1);
0271     q += q;
0272     p += q;
0273     t2 = (u32)(p);
0274     p >>= 32;
0275     p += (u64)(((u32 *)ml)[INDEX_HIGH]);
0276     p += MUL32(a0, k1);
0277     p += MUL32(a1, k0);
0278     q =  MUL32(a2, k3);
0279     q += MUL32(a3, k2);
0280     q += q;
0281     p += q;
0282     *(u64 *)(alo) = (p << 32) | t2;
0283     p >>= 32;
0284     *(u64 *)(ahi) = p + t;
0285 
0286 #undef a0
0287 #undef a1
0288 #undef a2
0289 #undef a3
0290 #undef k0
0291 #undef k1
0292 #undef k2
0293 #undef k3
0294 }
0295 
0296 #define poly_step(ah, al, kh, kl, mh, ml)               \
0297     poly_step_func(&(ah), &(al), &(kh), &(kl), &(mh), &(ml))
0298 
0299 #endif  /* end of specialized NH and poly definitions */
0300 
0301 /* At least nh_16 is defined. Defined others as needed here */
0302 #ifndef nh_16_2
0303 #define nh_16_2(mp, kp, nw, rh, rl, rh2, rl2)               \
0304     do {                                \
0305         nh_16(mp, kp, nw, rh, rl);              \
0306         nh_16(mp, ((kp)+2), nw, rh2, rl2);          \
0307     } while (0)
0308 #endif
0309 #ifndef nh_vmac_nhbytes
0310 #define nh_vmac_nhbytes(mp, kp, nw, rh, rl)             \
0311     nh_16(mp, kp, nw, rh, rl)
0312 #endif
0313 #ifndef nh_vmac_nhbytes_2
0314 #define nh_vmac_nhbytes_2(mp, kp, nw, rh, rl, rh2, rl2)         \
0315     do {                                \
0316         nh_vmac_nhbytes(mp, kp, nw, rh, rl);            \
0317         nh_vmac_nhbytes(mp, ((kp)+2), nw, rh2, rl2);        \
0318     } while (0)
0319 #endif
0320 
0321 static void vhash_abort(struct vmac_ctx *ctx)
0322 {
0323     ctx->polytmp[0] = ctx->polykey[0] ;
0324     ctx->polytmp[1] = ctx->polykey[1] ;
0325     ctx->first_block_processed = 0;
0326 }
0327 
0328 static u64 l3hash(u64 p1, u64 p2, u64 k1, u64 k2, u64 len)
0329 {
0330     u64 rh, rl, t, z = 0;
0331 
0332     /* fully reduce (p1,p2)+(len,0) mod p127 */
0333     t = p1 >> 63;
0334     p1 &= m63;
0335     ADD128(p1, p2, len, t);
0336     /* At this point, (p1,p2) is at most 2^127+(len<<64) */
0337     t = (p1 > m63) + ((p1 == m63) && (p2 == m64));
0338     ADD128(p1, p2, z, t);
0339     p1 &= m63;
0340 
0341     /* compute (p1,p2)/(2^64-2^32) and (p1,p2)%(2^64-2^32) */
0342     t = p1 + (p2 >> 32);
0343     t += (t >> 32);
0344     t += (u32)t > 0xfffffffeu;
0345     p1 += (t >> 32);
0346     p2 += (p1 << 32);
0347 
0348     /* compute (p1+k1)%p64 and (p2+k2)%p64 */
0349     p1 += k1;
0350     p1 += (0 - (p1 < k1)) & 257;
0351     p2 += k2;
0352     p2 += (0 - (p2 < k2)) & 257;
0353 
0354     /* compute (p1+k1)*(p2+k2)%p64 */
0355     MUL64(rh, rl, p1, p2);
0356     t = rh >> 56;
0357     ADD128(t, rl, z, rh);
0358     rh <<= 8;
0359     ADD128(t, rl, z, rh);
0360     t += t << 8;
0361     rl += t;
0362     rl += (0 - (rl < t)) & 257;
0363     rl += (0 - (rl > p64-1)) & 257;
0364     return rl;
0365 }
0366 
0367 static void vhash_update(const unsigned char *m,
0368             unsigned int mbytes, /* Pos multiple of VMAC_NHBYTES */
0369             struct vmac_ctx *ctx)
0370 {
0371     u64 rh, rl, *mptr;
0372     const u64 *kptr = (u64 *)ctx->nhkey;
0373     int i;
0374     u64 ch, cl;
0375     u64 pkh = ctx->polykey[0];
0376     u64 pkl = ctx->polykey[1];
0377 
0378     if (!mbytes)
0379         return;
0380 
0381     BUG_ON(mbytes % VMAC_NHBYTES);
0382 
0383     mptr = (u64 *)m;
0384     i = mbytes / VMAC_NHBYTES;  /* Must be non-zero */
0385 
0386     ch = ctx->polytmp[0];
0387     cl = ctx->polytmp[1];
0388 
0389     if (!ctx->first_block_processed) {
0390         ctx->first_block_processed = 1;
0391         nh_vmac_nhbytes(mptr, kptr, VMAC_NHBYTES/8, rh, rl);
0392         rh &= m62;
0393         ADD128(ch, cl, rh, rl);
0394         mptr += (VMAC_NHBYTES/sizeof(u64));
0395         i--;
0396     }
0397 
0398     while (i--) {
0399         nh_vmac_nhbytes(mptr, kptr, VMAC_NHBYTES/8, rh, rl);
0400         rh &= m62;
0401         poly_step(ch, cl, pkh, pkl, rh, rl);
0402         mptr += (VMAC_NHBYTES/sizeof(u64));
0403     }
0404 
0405     ctx->polytmp[0] = ch;
0406     ctx->polytmp[1] = cl;
0407 }
0408 
0409 static u64 vhash(unsigned char m[], unsigned int mbytes,
0410             u64 *tagl, struct vmac_ctx *ctx)
0411 {
0412     u64 rh, rl, *mptr;
0413     const u64 *kptr = (u64 *)ctx->nhkey;
0414     int i, remaining;
0415     u64 ch, cl;
0416     u64 pkh = ctx->polykey[0];
0417     u64 pkl = ctx->polykey[1];
0418 
0419     mptr = (u64 *)m;
0420     i = mbytes / VMAC_NHBYTES;
0421     remaining = mbytes % VMAC_NHBYTES;
0422 
0423     if (ctx->first_block_processed) {
0424         ch = ctx->polytmp[0];
0425         cl = ctx->polytmp[1];
0426     } else if (i) {
0427         nh_vmac_nhbytes(mptr, kptr, VMAC_NHBYTES/8, ch, cl);
0428         ch &= m62;
0429         ADD128(ch, cl, pkh, pkl);
0430         mptr += (VMAC_NHBYTES/sizeof(u64));
0431         i--;
0432     } else if (remaining) {
0433         nh_16(mptr, kptr, 2*((remaining+15)/16), ch, cl);
0434         ch &= m62;
0435         ADD128(ch, cl, pkh, pkl);
0436         mptr += (VMAC_NHBYTES/sizeof(u64));
0437         goto do_l3;
0438     } else {/* Empty String */
0439         ch = pkh; cl = pkl;
0440         goto do_l3;
0441     }
0442 
0443     while (i--) {
0444         nh_vmac_nhbytes(mptr, kptr, VMAC_NHBYTES/8, rh, rl);
0445         rh &= m62;
0446         poly_step(ch, cl, pkh, pkl, rh, rl);
0447         mptr += (VMAC_NHBYTES/sizeof(u64));
0448     }
0449     if (remaining) {
0450         nh_16(mptr, kptr, 2*((remaining+15)/16), rh, rl);
0451         rh &= m62;
0452         poly_step(ch, cl, pkh, pkl, rh, rl);
0453     }
0454 
0455 do_l3:
0456     vhash_abort(ctx);
0457     remaining *= 8;
0458     return l3hash(ch, cl, ctx->l3key[0], ctx->l3key[1], remaining);
0459 }
0460 
0461 static u64 vmac(unsigned char m[], unsigned int mbytes,
0462             const unsigned char n[16], u64 *tagl,
0463             struct vmac_ctx_t *ctx)
0464 {
0465     u64 *in_n, *out_p;
0466     u64 p, h;
0467     int i;
0468 
0469     in_n = ctx->__vmac_ctx.cached_nonce;
0470     out_p = ctx->__vmac_ctx.cached_aes;
0471 
0472     i = n[15] & 1;
0473     if ((*(u64 *)(n+8) != in_n[1]) || (*(u64 *)(n) != in_n[0])) {
0474         in_n[0] = *(u64 *)(n);
0475         in_n[1] = *(u64 *)(n+8);
0476         ((unsigned char *)in_n)[15] &= 0xFE;
0477         crypto_cipher_encrypt_one(ctx->child,
0478             (unsigned char *)out_p, (unsigned char *)in_n);
0479 
0480         ((unsigned char *)in_n)[15] |= (unsigned char)(1-i);
0481     }
0482     p = be64_to_cpup(out_p + i);
0483     h = vhash(m, mbytes, (u64 *)0, &ctx->__vmac_ctx);
0484     return le64_to_cpu(p + h);
0485 }
0486 
0487 static int vmac_set_key(unsigned char user_key[], struct vmac_ctx_t *ctx)
0488 {
0489     u64 in[2] = {0}, out[2];
0490     unsigned i;
0491     int err = 0;
0492 
0493     err = crypto_cipher_setkey(ctx->child, user_key, VMAC_KEY_LEN);
0494     if (err)
0495         return err;
0496 
0497     /* Fill nh key */
0498     ((unsigned char *)in)[0] = 0x80;
0499     for (i = 0; i < sizeof(ctx->__vmac_ctx.nhkey)/8; i += 2) {
0500         crypto_cipher_encrypt_one(ctx->child,
0501             (unsigned char *)out, (unsigned char *)in);
0502         ctx->__vmac_ctx.nhkey[i] = be64_to_cpup(out);
0503         ctx->__vmac_ctx.nhkey[i+1] = be64_to_cpup(out+1);
0504         ((unsigned char *)in)[15] += 1;
0505     }
0506 
0507     /* Fill poly key */
0508     ((unsigned char *)in)[0] = 0xC0;
0509     in[1] = 0;
0510     for (i = 0; i < sizeof(ctx->__vmac_ctx.polykey)/8; i += 2) {
0511         crypto_cipher_encrypt_one(ctx->child,
0512             (unsigned char *)out, (unsigned char *)in);
0513         ctx->__vmac_ctx.polytmp[i] =
0514             ctx->__vmac_ctx.polykey[i] =
0515                 be64_to_cpup(out) & mpoly;
0516         ctx->__vmac_ctx.polytmp[i+1] =
0517             ctx->__vmac_ctx.polykey[i+1] =
0518                 be64_to_cpup(out+1) & mpoly;
0519         ((unsigned char *)in)[15] += 1;
0520     }
0521 
0522     /* Fill ip key */
0523     ((unsigned char *)in)[0] = 0xE0;
0524     in[1] = 0;
0525     for (i = 0; i < sizeof(ctx->__vmac_ctx.l3key)/8; i += 2) {
0526         do {
0527             crypto_cipher_encrypt_one(ctx->child,
0528                 (unsigned char *)out, (unsigned char *)in);
0529             ctx->__vmac_ctx.l3key[i] = be64_to_cpup(out);
0530             ctx->__vmac_ctx.l3key[i+1] = be64_to_cpup(out+1);
0531             ((unsigned char *)in)[15] += 1;
0532         } while (ctx->__vmac_ctx.l3key[i] >= p64
0533             || ctx->__vmac_ctx.l3key[i+1] >= p64);
0534     }
0535 
0536     /* Invalidate nonce/aes cache and reset other elements */
0537     ctx->__vmac_ctx.cached_nonce[0] = (u64)-1; /* Ensure illegal nonce */
0538     ctx->__vmac_ctx.cached_nonce[1] = (u64)0;  /* Ensure illegal nonce */
0539     ctx->__vmac_ctx.first_block_processed = 0;
0540 
0541     return err;
0542 }
0543 
0544 static int vmac_setkey(struct crypto_shash *parent,
0545         const u8 *key, unsigned int keylen)
0546 {
0547     struct vmac_ctx_t *ctx = crypto_shash_ctx(parent);
0548 
0549     if (keylen != VMAC_KEY_LEN) {
0550         crypto_shash_set_flags(parent, CRYPTO_TFM_RES_BAD_KEY_LEN);
0551         return -EINVAL;
0552     }
0553 
0554     return vmac_set_key((u8 *)key, ctx);
0555 }
0556 
0557 static int vmac_init(struct shash_desc *pdesc)
0558 {
0559     return 0;
0560 }
0561 
0562 static int vmac_update(struct shash_desc *pdesc, const u8 *p,
0563         unsigned int len)
0564 {
0565     struct crypto_shash *parent = pdesc->tfm;
0566     struct vmac_ctx_t *ctx = crypto_shash_ctx(parent);
0567     int expand;
0568     int min;
0569 
0570     expand = VMAC_NHBYTES - ctx->partial_size > 0 ?
0571             VMAC_NHBYTES - ctx->partial_size : 0;
0572 
0573     min = len < expand ? len : expand;
0574 
0575     memcpy(ctx->partial + ctx->partial_size, p, min);
0576     ctx->partial_size += min;
0577 
0578     if (len < expand)
0579         return 0;
0580 
0581     vhash_update(ctx->partial, VMAC_NHBYTES, &ctx->__vmac_ctx);
0582     ctx->partial_size = 0;
0583 
0584     len -= expand;
0585     p += expand;
0586 
0587     if (len % VMAC_NHBYTES) {
0588         memcpy(ctx->partial, p + len - (len % VMAC_NHBYTES),
0589             len % VMAC_NHBYTES);
0590         ctx->partial_size = len % VMAC_NHBYTES;
0591     }
0592 
0593     vhash_update(p, len - len % VMAC_NHBYTES, &ctx->__vmac_ctx);
0594 
0595     return 0;
0596 }
0597 
0598 static int vmac_final(struct shash_desc *pdesc, u8 *out)
0599 {
0600     struct crypto_shash *parent = pdesc->tfm;
0601     struct vmac_ctx_t *ctx = crypto_shash_ctx(parent);
0602     vmac_t mac;
0603     u8 nonce[16] = {};
0604 
0605     /* vmac() ends up accessing outside the array bounds that
0606      * we specify.  In appears to access up to the next 2-word
0607      * boundary.  We'll just be uber cautious and zero the
0608      * unwritten bytes in the buffer.
0609      */
0610     if (ctx->partial_size) {
0611         memset(ctx->partial + ctx->partial_size, 0,
0612             VMAC_NHBYTES - ctx->partial_size);
0613     }
0614     mac = vmac(ctx->partial, ctx->partial_size, nonce, NULL, ctx);
0615     memcpy(out, &mac, sizeof(vmac_t));
0616     memzero_explicit(&mac, sizeof(vmac_t));
0617     memset(&ctx->__vmac_ctx, 0, sizeof(struct vmac_ctx));
0618     ctx->partial_size = 0;
0619     return 0;
0620 }
0621 
0622 static int vmac_init_tfm(struct crypto_tfm *tfm)
0623 {
0624     struct crypto_cipher *cipher;
0625     struct crypto_instance *inst = (void *)tfm->__crt_alg;
0626     struct crypto_spawn *spawn = crypto_instance_ctx(inst);
0627     struct vmac_ctx_t *ctx = crypto_tfm_ctx(tfm);
0628 
0629     cipher = crypto_spawn_cipher(spawn);
0630     if (IS_ERR(cipher))
0631         return PTR_ERR(cipher);
0632 
0633     ctx->child = cipher;
0634     return 0;
0635 }
0636 
0637 static void vmac_exit_tfm(struct crypto_tfm *tfm)
0638 {
0639     struct vmac_ctx_t *ctx = crypto_tfm_ctx(tfm);
0640     crypto_free_cipher(ctx->child);
0641 }
0642 
0643 static int vmac_create(struct crypto_template *tmpl, struct rtattr **tb)
0644 {
0645     struct shash_instance *inst;
0646     struct crypto_alg *alg;
0647     int err;
0648 
0649     err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_SHASH);
0650     if (err)
0651         return err;
0652 
0653     alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_CIPHER,
0654             CRYPTO_ALG_TYPE_MASK);
0655     if (IS_ERR(alg))
0656         return PTR_ERR(alg);
0657 
0658     inst = shash_alloc_instance("vmac", alg);
0659     err = PTR_ERR(inst);
0660     if (IS_ERR(inst))
0661         goto out_put_alg;
0662 
0663     err = crypto_init_spawn(shash_instance_ctx(inst), alg,
0664             shash_crypto_instance(inst),
0665             CRYPTO_ALG_TYPE_MASK);
0666     if (err)
0667         goto out_free_inst;
0668 
0669     inst->alg.base.cra_priority = alg->cra_priority;
0670     inst->alg.base.cra_blocksize = alg->cra_blocksize;
0671     inst->alg.base.cra_alignmask = alg->cra_alignmask;
0672 
0673     inst->alg.digestsize = sizeof(vmac_t);
0674     inst->alg.base.cra_ctxsize = sizeof(struct vmac_ctx_t);
0675     inst->alg.base.cra_init = vmac_init_tfm;
0676     inst->alg.base.cra_exit = vmac_exit_tfm;
0677 
0678     inst->alg.init = vmac_init;
0679     inst->alg.update = vmac_update;
0680     inst->alg.final = vmac_final;
0681     inst->alg.setkey = vmac_setkey;
0682 
0683     err = shash_register_instance(tmpl, inst);
0684     if (err) {
0685 out_free_inst:
0686         shash_free_instance(shash_crypto_instance(inst));
0687     }
0688 
0689 out_put_alg:
0690     crypto_mod_put(alg);
0691     return err;
0692 }
0693 
0694 static struct crypto_template vmac_tmpl = {
0695     .name = "vmac",
0696     .create = vmac_create,
0697     .free = shash_free_instance,
0698     .module = THIS_MODULE,
0699 };
0700 
0701 static int __init vmac_module_init(void)
0702 {
0703     return crypto_register_template(&vmac_tmpl);
0704 }
0705 
0706 static void __exit vmac_module_exit(void)
0707 {
0708     crypto_unregister_template(&vmac_tmpl);
0709 }
0710 
0711 module_init(vmac_module_init);
0712 module_exit(vmac_module_exit);
0713 
0714 MODULE_LICENSE("GPL");
0715 MODULE_DESCRIPTION("VMAC hash algorithm");
0716 MODULE_ALIAS_CRYPTO("vmac");