0001
0002
0003
0004
0005
0006
0007
0008
0009
0010
0011
0012
0013 #include <asm/unaligned.h>
0014 #include <crypto/curve25519.h>
0015 #include <linux/string.h>
0016
0017 typedef __uint128_t u128;
0018
0019 static __always_inline u64 u64_eq_mask(u64 a, u64 b)
0020 {
0021 u64 x = a ^ b;
0022 u64 minus_x = ~x + (u64)1U;
0023 u64 x_or_minus_x = x | minus_x;
0024 u64 xnx = x_or_minus_x >> (u32)63U;
0025 u64 c = xnx - (u64)1U;
0026 return c;
0027 }
0028
0029 static __always_inline u64 u64_gte_mask(u64 a, u64 b)
0030 {
0031 u64 x = a;
0032 u64 y = b;
0033 u64 x_xor_y = x ^ y;
0034 u64 x_sub_y = x - y;
0035 u64 x_sub_y_xor_y = x_sub_y ^ y;
0036 u64 q = x_xor_y | x_sub_y_xor_y;
0037 u64 x_xor_q = x ^ q;
0038 u64 x_xor_q_ = x_xor_q >> (u32)63U;
0039 u64 c = x_xor_q_ - (u64)1U;
0040 return c;
0041 }
0042
0043 static __always_inline void modulo_carry_top(u64 *b)
0044 {
0045 u64 b4 = b[4];
0046 u64 b0 = b[0];
0047 u64 b4_ = b4 & 0x7ffffffffffffLLU;
0048 u64 b0_ = b0 + 19 * (b4 >> 51);
0049 b[4] = b4_;
0050 b[0] = b0_;
0051 }
0052
0053 static __always_inline void fproduct_copy_from_wide_(u64 *output, u128 *input)
0054 {
0055 {
0056 u128 xi = input[0];
0057 output[0] = ((u64)(xi));
0058 }
0059 {
0060 u128 xi = input[1];
0061 output[1] = ((u64)(xi));
0062 }
0063 {
0064 u128 xi = input[2];
0065 output[2] = ((u64)(xi));
0066 }
0067 {
0068 u128 xi = input[3];
0069 output[3] = ((u64)(xi));
0070 }
0071 {
0072 u128 xi = input[4];
0073 output[4] = ((u64)(xi));
0074 }
0075 }
0076
0077 static __always_inline void
0078 fproduct_sum_scalar_multiplication_(u128 *output, u64 *input, u64 s)
0079 {
0080 output[0] += (u128)input[0] * s;
0081 output[1] += (u128)input[1] * s;
0082 output[2] += (u128)input[2] * s;
0083 output[3] += (u128)input[3] * s;
0084 output[4] += (u128)input[4] * s;
0085 }
0086
0087 static __always_inline void fproduct_carry_wide_(u128 *tmp)
0088 {
0089 {
0090 u32 ctr = 0;
0091 u128 tctr = tmp[ctr];
0092 u128 tctrp1 = tmp[ctr + 1];
0093 u64 r0 = ((u64)(tctr)) & 0x7ffffffffffffLLU;
0094 u128 c = ((tctr) >> (51));
0095 tmp[ctr] = ((u128)(r0));
0096 tmp[ctr + 1] = ((tctrp1) + (c));
0097 }
0098 {
0099 u32 ctr = 1;
0100 u128 tctr = tmp[ctr];
0101 u128 tctrp1 = tmp[ctr + 1];
0102 u64 r0 = ((u64)(tctr)) & 0x7ffffffffffffLLU;
0103 u128 c = ((tctr) >> (51));
0104 tmp[ctr] = ((u128)(r0));
0105 tmp[ctr + 1] = ((tctrp1) + (c));
0106 }
0107
0108 {
0109 u32 ctr = 2;
0110 u128 tctr = tmp[ctr];
0111 u128 tctrp1 = tmp[ctr + 1];
0112 u64 r0 = ((u64)(tctr)) & 0x7ffffffffffffLLU;
0113 u128 c = ((tctr) >> (51));
0114 tmp[ctr] = ((u128)(r0));
0115 tmp[ctr + 1] = ((tctrp1) + (c));
0116 }
0117 {
0118 u32 ctr = 3;
0119 u128 tctr = tmp[ctr];
0120 u128 tctrp1 = tmp[ctr + 1];
0121 u64 r0 = ((u64)(tctr)) & 0x7ffffffffffffLLU;
0122 u128 c = ((tctr) >> (51));
0123 tmp[ctr] = ((u128)(r0));
0124 tmp[ctr + 1] = ((tctrp1) + (c));
0125 }
0126 }
0127
0128 static __always_inline void fmul_shift_reduce(u64 *output)
0129 {
0130 u64 tmp = output[4];
0131 u64 b0;
0132 {
0133 u32 ctr = 5 - 0 - 1;
0134 u64 z = output[ctr - 1];
0135 output[ctr] = z;
0136 }
0137 {
0138 u32 ctr = 5 - 1 - 1;
0139 u64 z = output[ctr - 1];
0140 output[ctr] = z;
0141 }
0142 {
0143 u32 ctr = 5 - 2 - 1;
0144 u64 z = output[ctr - 1];
0145 output[ctr] = z;
0146 }
0147 {
0148 u32 ctr = 5 - 3 - 1;
0149 u64 z = output[ctr - 1];
0150 output[ctr] = z;
0151 }
0152 output[0] = tmp;
0153 b0 = output[0];
0154 output[0] = 19 * b0;
0155 }
0156
0157 static __always_inline void fmul_mul_shift_reduce_(u128 *output, u64 *input,
0158 u64 *input21)
0159 {
0160 u32 i;
0161 u64 input2i;
0162 {
0163 u64 input2i = input21[0];
0164 fproduct_sum_scalar_multiplication_(output, input, input2i);
0165 fmul_shift_reduce(input);
0166 }
0167 {
0168 u64 input2i = input21[1];
0169 fproduct_sum_scalar_multiplication_(output, input, input2i);
0170 fmul_shift_reduce(input);
0171 }
0172 {
0173 u64 input2i = input21[2];
0174 fproduct_sum_scalar_multiplication_(output, input, input2i);
0175 fmul_shift_reduce(input);
0176 }
0177 {
0178 u64 input2i = input21[3];
0179 fproduct_sum_scalar_multiplication_(output, input, input2i);
0180 fmul_shift_reduce(input);
0181 }
0182 i = 4;
0183 input2i = input21[i];
0184 fproduct_sum_scalar_multiplication_(output, input, input2i);
0185 }
0186
0187 static __always_inline void fmul_fmul(u64 *output, u64 *input, u64 *input21)
0188 {
0189 u64 tmp[5] = { input[0], input[1], input[2], input[3], input[4] };
0190 {
0191 u128 b4;
0192 u128 b0;
0193 u128 b4_;
0194 u128 b0_;
0195 u64 i0;
0196 u64 i1;
0197 u64 i0_;
0198 u64 i1_;
0199 u128 t[5] = { 0 };
0200 fmul_mul_shift_reduce_(t, tmp, input21);
0201 fproduct_carry_wide_(t);
0202 b4 = t[4];
0203 b0 = t[0];
0204 b4_ = ((b4) & (((u128)(0x7ffffffffffffLLU))));
0205 b0_ = ((b0) + (((u128)(19) * (((u64)(((b4) >> (51))))))));
0206 t[4] = b4_;
0207 t[0] = b0_;
0208 fproduct_copy_from_wide_(output, t);
0209 i0 = output[0];
0210 i1 = output[1];
0211 i0_ = i0 & 0x7ffffffffffffLLU;
0212 i1_ = i1 + (i0 >> 51);
0213 output[0] = i0_;
0214 output[1] = i1_;
0215 }
0216 }
0217
0218 static __always_inline void fsquare_fsquare__(u128 *tmp, u64 *output)
0219 {
0220 u64 r0 = output[0];
0221 u64 r1 = output[1];
0222 u64 r2 = output[2];
0223 u64 r3 = output[3];
0224 u64 r4 = output[4];
0225 u64 d0 = r0 * 2;
0226 u64 d1 = r1 * 2;
0227 u64 d2 = r2 * 2 * 19;
0228 u64 d419 = r4 * 19;
0229 u64 d4 = d419 * 2;
0230 u128 s0 = ((((((u128)(r0) * (r0))) + (((u128)(d4) * (r1))))) +
0231 (((u128)(d2) * (r3))));
0232 u128 s1 = ((((((u128)(d0) * (r1))) + (((u128)(d4) * (r2))))) +
0233 (((u128)(r3 * 19) * (r3))));
0234 u128 s2 = ((((((u128)(d0) * (r2))) + (((u128)(r1) * (r1))))) +
0235 (((u128)(d4) * (r3))));
0236 u128 s3 = ((((((u128)(d0) * (r3))) + (((u128)(d1) * (r2))))) +
0237 (((u128)(r4) * (d419))));
0238 u128 s4 = ((((((u128)(d0) * (r4))) + (((u128)(d1) * (r3))))) +
0239 (((u128)(r2) * (r2))));
0240 tmp[0] = s0;
0241 tmp[1] = s1;
0242 tmp[2] = s2;
0243 tmp[3] = s3;
0244 tmp[4] = s4;
0245 }
0246
0247 static __always_inline void fsquare_fsquare_(u128 *tmp, u64 *output)
0248 {
0249 u128 b4;
0250 u128 b0;
0251 u128 b4_;
0252 u128 b0_;
0253 u64 i0;
0254 u64 i1;
0255 u64 i0_;
0256 u64 i1_;
0257 fsquare_fsquare__(tmp, output);
0258 fproduct_carry_wide_(tmp);
0259 b4 = tmp[4];
0260 b0 = tmp[0];
0261 b4_ = ((b4) & (((u128)(0x7ffffffffffffLLU))));
0262 b0_ = ((b0) + (((u128)(19) * (((u64)(((b4) >> (51))))))));
0263 tmp[4] = b4_;
0264 tmp[0] = b0_;
0265 fproduct_copy_from_wide_(output, tmp);
0266 i0 = output[0];
0267 i1 = output[1];
0268 i0_ = i0 & 0x7ffffffffffffLLU;
0269 i1_ = i1 + (i0 >> 51);
0270 output[0] = i0_;
0271 output[1] = i1_;
0272 }
0273
0274 static __always_inline void fsquare_fsquare_times_(u64 *output, u128 *tmp,
0275 u32 count1)
0276 {
0277 u32 i;
0278 fsquare_fsquare_(tmp, output);
0279 for (i = 1; i < count1; ++i)
0280 fsquare_fsquare_(tmp, output);
0281 }
0282
0283 static __always_inline void fsquare_fsquare_times(u64 *output, u64 *input,
0284 u32 count1)
0285 {
0286 u128 t[5];
0287 memcpy(output, input, 5 * sizeof(*input));
0288 fsquare_fsquare_times_(output, t, count1);
0289 }
0290
0291 static __always_inline void fsquare_fsquare_times_inplace(u64 *output,
0292 u32 count1)
0293 {
0294 u128 t[5];
0295 fsquare_fsquare_times_(output, t, count1);
0296 }
0297
0298 static __always_inline void crecip_crecip(u64 *out, u64 *z)
0299 {
0300 u64 buf[20] = { 0 };
0301 u64 *a0 = buf;
0302 u64 *t00 = buf + 5;
0303 u64 *b0 = buf + 10;
0304 u64 *t01;
0305 u64 *b1;
0306 u64 *c0;
0307 u64 *a;
0308 u64 *t0;
0309 u64 *b;
0310 u64 *c;
0311 fsquare_fsquare_times(a0, z, 1);
0312 fsquare_fsquare_times(t00, a0, 2);
0313 fmul_fmul(b0, t00, z);
0314 fmul_fmul(a0, b0, a0);
0315 fsquare_fsquare_times(t00, a0, 1);
0316 fmul_fmul(b0, t00, b0);
0317 fsquare_fsquare_times(t00, b0, 5);
0318 t01 = buf + 5;
0319 b1 = buf + 10;
0320 c0 = buf + 15;
0321 fmul_fmul(b1, t01, b1);
0322 fsquare_fsquare_times(t01, b1, 10);
0323 fmul_fmul(c0, t01, b1);
0324 fsquare_fsquare_times(t01, c0, 20);
0325 fmul_fmul(t01, t01, c0);
0326 fsquare_fsquare_times_inplace(t01, 10);
0327 fmul_fmul(b1, t01, b1);
0328 fsquare_fsquare_times(t01, b1, 50);
0329 a = buf;
0330 t0 = buf + 5;
0331 b = buf + 10;
0332 c = buf + 15;
0333 fmul_fmul(c, t0, b);
0334 fsquare_fsquare_times(t0, c, 100);
0335 fmul_fmul(t0, t0, c);
0336 fsquare_fsquare_times_inplace(t0, 50);
0337 fmul_fmul(t0, t0, b);
0338 fsquare_fsquare_times_inplace(t0, 5);
0339 fmul_fmul(out, t0, a);
0340 }
0341
0342 static __always_inline void fsum(u64 *a, u64 *b)
0343 {
0344 a[0] += b[0];
0345 a[1] += b[1];
0346 a[2] += b[2];
0347 a[3] += b[3];
0348 a[4] += b[4];
0349 }
0350
0351 static __always_inline void fdifference(u64 *a, u64 *b)
0352 {
0353 u64 tmp[5] = { 0 };
0354 u64 b0;
0355 u64 b1;
0356 u64 b2;
0357 u64 b3;
0358 u64 b4;
0359 memcpy(tmp, b, 5 * sizeof(*b));
0360 b0 = tmp[0];
0361 b1 = tmp[1];
0362 b2 = tmp[2];
0363 b3 = tmp[3];
0364 b4 = tmp[4];
0365 tmp[0] = b0 + 0x3fffffffffff68LLU;
0366 tmp[1] = b1 + 0x3ffffffffffff8LLU;
0367 tmp[2] = b2 + 0x3ffffffffffff8LLU;
0368 tmp[3] = b3 + 0x3ffffffffffff8LLU;
0369 tmp[4] = b4 + 0x3ffffffffffff8LLU;
0370 {
0371 u64 xi = a[0];
0372 u64 yi = tmp[0];
0373 a[0] = yi - xi;
0374 }
0375 {
0376 u64 xi = a[1];
0377 u64 yi = tmp[1];
0378 a[1] = yi - xi;
0379 }
0380 {
0381 u64 xi = a[2];
0382 u64 yi = tmp[2];
0383 a[2] = yi - xi;
0384 }
0385 {
0386 u64 xi = a[3];
0387 u64 yi = tmp[3];
0388 a[3] = yi - xi;
0389 }
0390 {
0391 u64 xi = a[4];
0392 u64 yi = tmp[4];
0393 a[4] = yi - xi;
0394 }
0395 }
0396
0397 static __always_inline void fscalar(u64 *output, u64 *b, u64 s)
0398 {
0399 u128 tmp[5];
0400 u128 b4;
0401 u128 b0;
0402 u128 b4_;
0403 u128 b0_;
0404 {
0405 u64 xi = b[0];
0406 tmp[0] = ((u128)(xi) * (s));
0407 }
0408 {
0409 u64 xi = b[1];
0410 tmp[1] = ((u128)(xi) * (s));
0411 }
0412 {
0413 u64 xi = b[2];
0414 tmp[2] = ((u128)(xi) * (s));
0415 }
0416 {
0417 u64 xi = b[3];
0418 tmp[3] = ((u128)(xi) * (s));
0419 }
0420 {
0421 u64 xi = b[4];
0422 tmp[4] = ((u128)(xi) * (s));
0423 }
0424 fproduct_carry_wide_(tmp);
0425 b4 = tmp[4];
0426 b0 = tmp[0];
0427 b4_ = ((b4) & (((u128)(0x7ffffffffffffLLU))));
0428 b0_ = ((b0) + (((u128)(19) * (((u64)(((b4) >> (51))))))));
0429 tmp[4] = b4_;
0430 tmp[0] = b0_;
0431 fproduct_copy_from_wide_(output, tmp);
0432 }
0433
0434 static __always_inline void fmul(u64 *output, u64 *a, u64 *b)
0435 {
0436 fmul_fmul(output, a, b);
0437 }
0438
0439 static __always_inline void crecip(u64 *output, u64 *input)
0440 {
0441 crecip_crecip(output, input);
0442 }
0443
0444 static __always_inline void point_swap_conditional_step(u64 *a, u64 *b,
0445 u64 swap1, u32 ctr)
0446 {
0447 u32 i = ctr - 1;
0448 u64 ai = a[i];
0449 u64 bi = b[i];
0450 u64 x = swap1 & (ai ^ bi);
0451 u64 ai1 = ai ^ x;
0452 u64 bi1 = bi ^ x;
0453 a[i] = ai1;
0454 b[i] = bi1;
0455 }
0456
0457 static __always_inline void point_swap_conditional5(u64 *a, u64 *b, u64 swap1)
0458 {
0459 point_swap_conditional_step(a, b, swap1, 5);
0460 point_swap_conditional_step(a, b, swap1, 4);
0461 point_swap_conditional_step(a, b, swap1, 3);
0462 point_swap_conditional_step(a, b, swap1, 2);
0463 point_swap_conditional_step(a, b, swap1, 1);
0464 }
0465
0466 static __always_inline void point_swap_conditional(u64 *a, u64 *b, u64 iswap)
0467 {
0468 u64 swap1 = 0 - iswap;
0469 point_swap_conditional5(a, b, swap1);
0470 point_swap_conditional5(a + 5, b + 5, swap1);
0471 }
0472
0473 static __always_inline void point_copy(u64 *output, u64 *input)
0474 {
0475 memcpy(output, input, 5 * sizeof(*input));
0476 memcpy(output + 5, input + 5, 5 * sizeof(*input));
0477 }
0478
0479 static __always_inline void addanddouble_fmonty(u64 *pp, u64 *ppq, u64 *p,
0480 u64 *pq, u64 *qmqp)
0481 {
0482 u64 *qx = qmqp;
0483 u64 *x2 = pp;
0484 u64 *z2 = pp + 5;
0485 u64 *x3 = ppq;
0486 u64 *z3 = ppq + 5;
0487 u64 *x = p;
0488 u64 *z = p + 5;
0489 u64 *xprime = pq;
0490 u64 *zprime = pq + 5;
0491 u64 buf[40] = { 0 };
0492 u64 *origx = buf;
0493 u64 *origxprime0 = buf + 5;
0494 u64 *xxprime0;
0495 u64 *zzprime0;
0496 u64 *origxprime;
0497 xxprime0 = buf + 25;
0498 zzprime0 = buf + 30;
0499 memcpy(origx, x, 5 * sizeof(*x));
0500 fsum(x, z);
0501 fdifference(z, origx);
0502 memcpy(origxprime0, xprime, 5 * sizeof(*xprime));
0503 fsum(xprime, zprime);
0504 fdifference(zprime, origxprime0);
0505 fmul(xxprime0, xprime, z);
0506 fmul(zzprime0, x, zprime);
0507 origxprime = buf + 5;
0508 {
0509 u64 *xx0;
0510 u64 *zz0;
0511 u64 *xxprime;
0512 u64 *zzprime;
0513 u64 *zzzprime;
0514 xx0 = buf + 15;
0515 zz0 = buf + 20;
0516 xxprime = buf + 25;
0517 zzprime = buf + 30;
0518 zzzprime = buf + 35;
0519 memcpy(origxprime, xxprime, 5 * sizeof(*xxprime));
0520 fsum(xxprime, zzprime);
0521 fdifference(zzprime, origxprime);
0522 fsquare_fsquare_times(x3, xxprime, 1);
0523 fsquare_fsquare_times(zzzprime, zzprime, 1);
0524 fmul(z3, zzzprime, qx);
0525 fsquare_fsquare_times(xx0, x, 1);
0526 fsquare_fsquare_times(zz0, z, 1);
0527 {
0528 u64 *zzz;
0529 u64 *xx;
0530 u64 *zz;
0531 u64 scalar;
0532 zzz = buf + 10;
0533 xx = buf + 15;
0534 zz = buf + 20;
0535 fmul(x2, xx, zz);
0536 fdifference(zz, xx);
0537 scalar = 121665;
0538 fscalar(zzz, zz, scalar);
0539 fsum(zzz, xx);
0540 fmul(z2, zzz, zz);
0541 }
0542 }
0543 }
0544
0545 static __always_inline void
0546 ladder_smallloop_cmult_small_loop_step(u64 *nq, u64 *nqpq, u64 *nq2, u64 *nqpq2,
0547 u64 *q, u8 byt)
0548 {
0549 u64 bit0 = (u64)(byt >> 7);
0550 u64 bit;
0551 point_swap_conditional(nq, nqpq, bit0);
0552 addanddouble_fmonty(nq2, nqpq2, nq, nqpq, q);
0553 bit = (u64)(byt >> 7);
0554 point_swap_conditional(nq2, nqpq2, bit);
0555 }
0556
0557 static __always_inline void
0558 ladder_smallloop_cmult_small_loop_double_step(u64 *nq, u64 *nqpq, u64 *nq2,
0559 u64 *nqpq2, u64 *q, u8 byt)
0560 {
0561 u8 byt1;
0562 ladder_smallloop_cmult_small_loop_step(nq, nqpq, nq2, nqpq2, q, byt);
0563 byt1 = byt << 1;
0564 ladder_smallloop_cmult_small_loop_step(nq2, nqpq2, nq, nqpq, q, byt1);
0565 }
0566
0567 static __always_inline void
0568 ladder_smallloop_cmult_small_loop(u64 *nq, u64 *nqpq, u64 *nq2, u64 *nqpq2,
0569 u64 *q, u8 byt, u32 i)
0570 {
0571 while (i--) {
0572 ladder_smallloop_cmult_small_loop_double_step(nq, nqpq, nq2,
0573 nqpq2, q, byt);
0574 byt <<= 2;
0575 }
0576 }
0577
0578 static __always_inline void ladder_bigloop_cmult_big_loop(u8 *n1, u64 *nq,
0579 u64 *nqpq, u64 *nq2,
0580 u64 *nqpq2, u64 *q,
0581 u32 i)
0582 {
0583 while (i--) {
0584 u8 byte = n1[i];
0585 ladder_smallloop_cmult_small_loop(nq, nqpq, nq2, nqpq2, q,
0586 byte, 4);
0587 }
0588 }
0589
0590 static void ladder_cmult(u64 *result, u8 *n1, u64 *q)
0591 {
0592 u64 point_buf[40] = { 0 };
0593 u64 *nq = point_buf;
0594 u64 *nqpq = point_buf + 10;
0595 u64 *nq2 = point_buf + 20;
0596 u64 *nqpq2 = point_buf + 30;
0597 point_copy(nqpq, q);
0598 nq[0] = 1;
0599 ladder_bigloop_cmult_big_loop(n1, nq, nqpq, nq2, nqpq2, q, 32);
0600 point_copy(result, nq);
0601 }
0602
0603 static __always_inline void format_fexpand(u64 *output, const u8 *input)
0604 {
0605 const u8 *x00 = input + 6;
0606 const u8 *x01 = input + 12;
0607 const u8 *x02 = input + 19;
0608 const u8 *x0 = input + 24;
0609 u64 i0, i1, i2, i3, i4, output0, output1, output2, output3, output4;
0610 i0 = get_unaligned_le64(input);
0611 i1 = get_unaligned_le64(x00);
0612 i2 = get_unaligned_le64(x01);
0613 i3 = get_unaligned_le64(x02);
0614 i4 = get_unaligned_le64(x0);
0615 output0 = i0 & 0x7ffffffffffffLLU;
0616 output1 = i1 >> 3 & 0x7ffffffffffffLLU;
0617 output2 = i2 >> 6 & 0x7ffffffffffffLLU;
0618 output3 = i3 >> 1 & 0x7ffffffffffffLLU;
0619 output4 = i4 >> 12 & 0x7ffffffffffffLLU;
0620 output[0] = output0;
0621 output[1] = output1;
0622 output[2] = output2;
0623 output[3] = output3;
0624 output[4] = output4;
0625 }
0626
0627 static __always_inline void format_fcontract_first_carry_pass(u64 *input)
0628 {
0629 u64 t0 = input[0];
0630 u64 t1 = input[1];
0631 u64 t2 = input[2];
0632 u64 t3 = input[3];
0633 u64 t4 = input[4];
0634 u64 t1_ = t1 + (t0 >> 51);
0635 u64 t0_ = t0 & 0x7ffffffffffffLLU;
0636 u64 t2_ = t2 + (t1_ >> 51);
0637 u64 t1__ = t1_ & 0x7ffffffffffffLLU;
0638 u64 t3_ = t3 + (t2_ >> 51);
0639 u64 t2__ = t2_ & 0x7ffffffffffffLLU;
0640 u64 t4_ = t4 + (t3_ >> 51);
0641 u64 t3__ = t3_ & 0x7ffffffffffffLLU;
0642 input[0] = t0_;
0643 input[1] = t1__;
0644 input[2] = t2__;
0645 input[3] = t3__;
0646 input[4] = t4_;
0647 }
0648
0649 static __always_inline void format_fcontract_first_carry_full(u64 *input)
0650 {
0651 format_fcontract_first_carry_pass(input);
0652 modulo_carry_top(input);
0653 }
0654
0655 static __always_inline void format_fcontract_second_carry_pass(u64 *input)
0656 {
0657 u64 t0 = input[0];
0658 u64 t1 = input[1];
0659 u64 t2 = input[2];
0660 u64 t3 = input[3];
0661 u64 t4 = input[4];
0662 u64 t1_ = t1 + (t0 >> 51);
0663 u64 t0_ = t0 & 0x7ffffffffffffLLU;
0664 u64 t2_ = t2 + (t1_ >> 51);
0665 u64 t1__ = t1_ & 0x7ffffffffffffLLU;
0666 u64 t3_ = t3 + (t2_ >> 51);
0667 u64 t2__ = t2_ & 0x7ffffffffffffLLU;
0668 u64 t4_ = t4 + (t3_ >> 51);
0669 u64 t3__ = t3_ & 0x7ffffffffffffLLU;
0670 input[0] = t0_;
0671 input[1] = t1__;
0672 input[2] = t2__;
0673 input[3] = t3__;
0674 input[4] = t4_;
0675 }
0676
0677 static __always_inline void format_fcontract_second_carry_full(u64 *input)
0678 {
0679 u64 i0;
0680 u64 i1;
0681 u64 i0_;
0682 u64 i1_;
0683 format_fcontract_second_carry_pass(input);
0684 modulo_carry_top(input);
0685 i0 = input[0];
0686 i1 = input[1];
0687 i0_ = i0 & 0x7ffffffffffffLLU;
0688 i1_ = i1 + (i0 >> 51);
0689 input[0] = i0_;
0690 input[1] = i1_;
0691 }
0692
0693 static __always_inline void format_fcontract_trim(u64 *input)
0694 {
0695 u64 a0 = input[0];
0696 u64 a1 = input[1];
0697 u64 a2 = input[2];
0698 u64 a3 = input[3];
0699 u64 a4 = input[4];
0700 u64 mask0 = u64_gte_mask(a0, 0x7ffffffffffedLLU);
0701 u64 mask1 = u64_eq_mask(a1, 0x7ffffffffffffLLU);
0702 u64 mask2 = u64_eq_mask(a2, 0x7ffffffffffffLLU);
0703 u64 mask3 = u64_eq_mask(a3, 0x7ffffffffffffLLU);
0704 u64 mask4 = u64_eq_mask(a4, 0x7ffffffffffffLLU);
0705 u64 mask = (((mask0 & mask1) & mask2) & mask3) & mask4;
0706 u64 a0_ = a0 - (0x7ffffffffffedLLU & mask);
0707 u64 a1_ = a1 - (0x7ffffffffffffLLU & mask);
0708 u64 a2_ = a2 - (0x7ffffffffffffLLU & mask);
0709 u64 a3_ = a3 - (0x7ffffffffffffLLU & mask);
0710 u64 a4_ = a4 - (0x7ffffffffffffLLU & mask);
0711 input[0] = a0_;
0712 input[1] = a1_;
0713 input[2] = a2_;
0714 input[3] = a3_;
0715 input[4] = a4_;
0716 }
0717
0718 static __always_inline void format_fcontract_store(u8 *output, u64 *input)
0719 {
0720 u64 t0 = input[0];
0721 u64 t1 = input[1];
0722 u64 t2 = input[2];
0723 u64 t3 = input[3];
0724 u64 t4 = input[4];
0725 u64 o0 = t1 << 51 | t0;
0726 u64 o1 = t2 << 38 | t1 >> 13;
0727 u64 o2 = t3 << 25 | t2 >> 26;
0728 u64 o3 = t4 << 12 | t3 >> 39;
0729 u8 *b0 = output;
0730 u8 *b1 = output + 8;
0731 u8 *b2 = output + 16;
0732 u8 *b3 = output + 24;
0733 put_unaligned_le64(o0, b0);
0734 put_unaligned_le64(o1, b1);
0735 put_unaligned_le64(o2, b2);
0736 put_unaligned_le64(o3, b3);
0737 }
0738
0739 static __always_inline void format_fcontract(u8 *output, u64 *input)
0740 {
0741 format_fcontract_first_carry_full(input);
0742 format_fcontract_second_carry_full(input);
0743 format_fcontract_trim(input);
0744 format_fcontract_store(output, input);
0745 }
0746
0747 static __always_inline void format_scalar_of_point(u8 *scalar, u64 *point)
0748 {
0749 u64 *x = point;
0750 u64 *z = point + 5;
0751 u64 buf[10] __aligned(32) = { 0 };
0752 u64 *zmone = buf;
0753 u64 *sc = buf + 5;
0754 crecip(zmone, z);
0755 fmul(sc, x, zmone);
0756 format_fcontract(scalar, sc);
0757 }
0758
0759 void curve25519_generic(u8 mypublic[CURVE25519_KEY_SIZE],
0760 const u8 secret[CURVE25519_KEY_SIZE],
0761 const u8 basepoint[CURVE25519_KEY_SIZE])
0762 {
0763 u64 buf0[10] __aligned(32) = { 0 };
0764 u64 *x0 = buf0;
0765 u64 *z = buf0 + 5;
0766 u64 *q;
0767 format_fexpand(x0, basepoint);
0768 z[0] = 1;
0769 q = buf0;
0770 {
0771 u8 e[32] __aligned(32) = { 0 };
0772 u8 *scalar;
0773 memcpy(e, secret, 32);
0774 curve25519_clamp_secret(e);
0775 scalar = e;
0776 {
0777 u64 buf[15] = { 0 };
0778 u64 *nq = buf;
0779 u64 *x = nq;
0780 x[0] = 1;
0781 ladder_cmult(nq, scalar, q);
0782 format_scalar_of_point(mypublic, nq);
0783 memzero_explicit(buf, sizeof(buf));
0784 }
0785 memzero_explicit(e, sizeof(e));
0786 }
0787 memzero_explicit(buf0, sizeof(buf0));
0788 }
