x86/lib/csum-partial_64.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * arch/x86_64/lib/csum-partial.c
0004  *
0005  * This file contains network checksum routines that are better done
0006  * in an architecture-specific manner due to speed.
0007  */
0008
0009 #include <linux/compiler.h>
0010 #include <linux/export.h>
0011 #include <asm/checksum.h>
0012 #include <asm/word-at-a-time.h>
0013
0014 static inline unsigned short from32to16(unsigned a)
0015 {
0016     unsigned short b = a >> 16;
0017     asm("addw %w2,%w0\n\t"
0018         "adcw $0,%w0\n"
0019         : "=r" (b)
0020         : "0" (b), "r" (a));
0021     return b;
0022 }
0023
0024 /*
0025  * Do a checksum on an arbitrary memory area.
0026  * Returns a 32bit checksum.
0027  *
0028  * This isn't as time critical as it used to be because many NICs
0029  * do hardware checksumming these days.
0030  *
0031  * Still, with CHECKSUM_COMPLETE this is called to compute
0032  * checksums on IPv6 headers (40 bytes) and other small parts.
0033  * it's best to have buff aligned on a 64-bit boundary
0034  */
0035 __wsum csum_partial(const void *buff, int len, __wsum sum)
0036 {
0037     u64 temp64 = (__force u64)sum;
0038     unsigned odd, result;
0039
0040     odd = 1 & (unsigned long) buff;
0041     if (unlikely(odd)) {
0042         if (unlikely(len == 0))
0043             return sum;
0044         temp64 = ror32((__force u32)sum, 8);
0045         temp64 += (*(unsigned char *)buff << 8);
0046         len--;
0047         buff++;
0048     }
0049
0050     while (unlikely(len >= 64)) {
0051         asm("addq 0*8(%[src]),%[res]\n\t"
0052             "adcq 1*8(%[src]),%[res]\n\t"
0053             "adcq 2*8(%[src]),%[res]\n\t"
0054             "adcq 3*8(%[src]),%[res]\n\t"
0055             "adcq 4*8(%[src]),%[res]\n\t"
0056             "adcq 5*8(%[src]),%[res]\n\t"
0057             "adcq 6*8(%[src]),%[res]\n\t"
0058             "adcq 7*8(%[src]),%[res]\n\t"
0059             "adcq $0,%[res]"
0060             : [res] "+r" (temp64)
0061             : [src] "r" (buff)
0062             : "memory");
0063         buff += 64;
0064         len -= 64;
0065     }
0066
0067     if (len & 32) {
0068         asm("addq 0*8(%[src]),%[res]\n\t"
0069             "adcq 1*8(%[src]),%[res]\n\t"
0070             "adcq 2*8(%[src]),%[res]\n\t"
0071             "adcq 3*8(%[src]),%[res]\n\t"
0072             "adcq $0,%[res]"
0073             : [res] "+r" (temp64)
0074             : [src] "r" (buff)
0075             : "memory");
0076         buff += 32;
0077     }
0078     if (len & 16) {
0079         asm("addq 0*8(%[src]),%[res]\n\t"
0080             "adcq 1*8(%[src]),%[res]\n\t"
0081             "adcq $0,%[res]"
0082             : [res] "+r" (temp64)
0083             : [src] "r" (buff)
0084             : "memory");
0085         buff += 16;
0086     }
0087     if (len & 8) {
0088         asm("addq 0*8(%[src]),%[res]\n\t"
0089             "adcq $0,%[res]"
0090             : [res] "+r" (temp64)
0091             : [src] "r" (buff)
0092             : "memory");
0093         buff += 8;
0094     }
0095     if (len & 7) {
0096         unsigned int shift = (8 - (len & 7)) * 8;
0097         unsigned long trail;
0098
0099         trail = (load_unaligned_zeropad(buff) << shift) >> shift;
0100
0101         asm("addq %[trail],%[res]\n\t"
0102             "adcq $0,%[res]"
0103             : [res] "+r" (temp64)
0104             : [trail] "r" (trail));
0105     }
0106     result = add32_with_carry(temp64 >> 32, temp64 & 0xffffffff);
0107     if (unlikely(odd)) {
0108         result = from32to16(result);
0109         result = ((result >> 8) & 0xff) | ((result & 0xff) << 8);
0110     }
0111     return (__force __wsum)result;
0112 }
0113 EXPORT_SYMBOL(csum_partial);
0114
0115 /*
0116  * this routine is used for miscellaneous IP-like checksums, mainly
0117  * in icmp.c
0118  */
0119 __sum16 ip_compute_csum(const void *buff, int len)
0120 {
0121     return csum_fold(csum_partial(buff,len,0));
0122 }
0123 EXPORT_SYMBOL(ip_compute_csum);