alpha/lib/csum_partial_copy.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * csum_partial_copy - do IP checksumming and copy
0004  *
0005  * (C) Copyright 1996 Linus Torvalds
0006  * accelerated versions (and 21264 assembly versions ) contributed by
0007  *  Rick Gorton <rick.gorton@alpha-processor.com>
0008  *
0009  * Don't look at this too closely - you'll go mad. The things
0010  * we do for performance..
0011  */
0012
0013 #include <linux/types.h>
0014 #include <linux/string.h>
0015 #include <linux/uaccess.h>
0016 #include <net/checksum.h>
0017
0018
0019 #define ldq_u(x,y) \
0020 __asm__ __volatile__("ldq_u %0,%1":"=r" (x):"m" (*(const unsigned long *)(y)))
0021
0022 #define stq_u(x,y) \
0023 __asm__ __volatile__("stq_u %1,%0":"=m" (*(unsigned long *)(y)):"r" (x))
0024
0025 #define extql(x,y,z) \
0026 __asm__ __volatile__("extql %1,%2,%0":"=r" (z):"r" (x),"r" (y))
0027
0028 #define extqh(x,y,z) \
0029 __asm__ __volatile__("extqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))
0030
0031 #define mskql(x,y,z) \
0032 __asm__ __volatile__("mskql %1,%2,%0":"=r" (z):"r" (x),"r" (y))
0033
0034 #define mskqh(x,y,z) \
0035 __asm__ __volatile__("mskqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))
0036
0037 #define insql(x,y,z) \
0038 __asm__ __volatile__("insql %1,%2,%0":"=r" (z):"r" (x),"r" (y))
0039
0040 #define insqh(x,y,z) \
0041 __asm__ __volatile__("insqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))
0042
0043 #define __get_word(insn,x,ptr)              \
0044 ({                          \
0045     long __guu_err;                 \
0046     __asm__ __volatile__(               \
0047     "1: "#insn" %0,%2\n"            \
0048     "2:\n"                      \
0049     EXC(1b,2b,%0,%1)                \
0050         : "=r"(x), "=r"(__guu_err)      \
0051         : "m"(__m(ptr)), "1"(0));       \
0052     __guu_err;                  \
0053 })
0054
0055 static inline unsigned short from64to16(unsigned long x)
0056 {
0057     /* Using extract instructions is a bit more efficient
0058        than the original shift/bitmask version.  */
0059
0060     union {
0061         unsigned long   ul;
0062         unsigned int    ui[2];
0063         unsigned short  us[4];
0064     } in_v, tmp_v, out_v;
0065
0066     in_v.ul = x;
0067     tmp_v.ul = (unsigned long) in_v.ui[0] + (unsigned long) in_v.ui[1];
0068
0069     /* Since the bits of tmp_v.sh[3] are going to always be zero,
0070        we don't have to bother to add that in.  */
0071     out_v.ul = (unsigned long) tmp_v.us[0] + (unsigned long) tmp_v.us[1]
0072             + (unsigned long) tmp_v.us[2];
0073
0074     /* Similarly, out_v.us[2] is always zero for the final add.  */
0075     return out_v.us[0] + out_v.us[1];
0076 }
0077
0078
0079
0080 /*
0081  * Ok. This isn't fun, but this is the EASY case.
0082  */
0083 static inline unsigned long
0084 csum_partial_cfu_aligned(const unsigned long __user *src, unsigned long *dst,
0085              long len)
0086 {
0087     unsigned long checksum = ~0U;
0088     unsigned long carry = 0;
0089
0090     while (len >= 0) {
0091         unsigned long word;
0092         if (__get_word(ldq, word, src))
0093             return 0;
0094         checksum += carry;
0095         src++;
0096         checksum += word;
0097         len -= 8;
0098         carry = checksum < word;
0099         *dst = word;
0100         dst++;
0101     }
0102     len += 8;
0103     checksum += carry;
0104     if (len) {
0105         unsigned long word, tmp;
0106         if (__get_word(ldq, word, src))
0107             return 0;
0108         tmp = *dst;
0109         mskql(word, len, word);
0110         checksum += word;
0111         mskqh(tmp, len, tmp);
0112         carry = checksum < word;
0113         *dst = word | tmp;
0114         checksum += carry;
0115     }
0116     return checksum;
0117 }
0118
0119 /*
0120  * This is even less fun, but this is still reasonably
0121  * easy.
0122  */
0123 static inline unsigned long
0124 csum_partial_cfu_dest_aligned(const unsigned long __user *src,
0125                   unsigned long *dst,
0126                   unsigned long soff,
0127                   long len)
0128 {
0129     unsigned long first;
0130     unsigned long word, carry;
0131     unsigned long lastsrc = 7+len+(unsigned long)src;
0132     unsigned long checksum = ~0U;
0133
0134     if (__get_word(ldq_u, first,src))
0135         return 0;
0136     carry = 0;
0137     while (len >= 0) {
0138         unsigned long second;
0139
0140         if (__get_word(ldq_u, second, src+1))
0141             return 0;
0142         extql(first, soff, word);
0143         len -= 8;
0144         src++;
0145         extqh(second, soff, first);
0146         checksum += carry;
0147         word |= first;
0148         first = second;
0149         checksum += word;
0150         *dst = word;
0151         dst++;
0152         carry = checksum < word;
0153     }
0154     len += 8;
0155     checksum += carry;
0156     if (len) {
0157         unsigned long tmp;
0158         unsigned long second;
0159         if (__get_word(ldq_u, second, lastsrc))
0160             return 0;
0161         tmp = *dst;
0162         extql(first, soff, word);
0163         extqh(second, soff, first);
0164         word |= first;
0165         mskql(word, len, word);
0166         checksum += word;
0167         mskqh(tmp, len, tmp);
0168         carry = checksum < word;
0169         *dst = word | tmp;
0170         checksum += carry;
0171     }
0172     return checksum;
0173 }
0174
0175 /*
0176  * This is slightly less fun than the above..
0177  */
0178 static inline unsigned long
0179 csum_partial_cfu_src_aligned(const unsigned long __user *src,
0180                  unsigned long *dst,
0181                  unsigned long doff,
0182                  long len,
0183                  unsigned long partial_dest)
0184 {
0185     unsigned long carry = 0;
0186     unsigned long word;
0187     unsigned long second_dest;
0188     unsigned long checksum = ~0U;
0189
0190     mskql(partial_dest, doff, partial_dest);
0191     while (len >= 0) {
0192         if (__get_word(ldq, word, src))
0193             return 0;
0194         len -= 8;
0195         insql(word, doff, second_dest);
0196         checksum += carry;
0197         stq_u(partial_dest | second_dest, dst);
0198         src++;
0199         checksum += word;
0200         insqh(word, doff, partial_dest);
0201         carry = checksum < word;
0202         dst++;
0203     }
0204     len += 8;
0205     if (len) {
0206         checksum += carry;
0207         if (__get_word(ldq, word, src))
0208             return 0;
0209         mskql(word, len, word);
0210         len -= 8;
0211         checksum += word;
0212         insql(word, doff, second_dest);
0213         len += doff;
0214         carry = checksum < word;
0215         partial_dest |= second_dest;
0216         if (len >= 0) {
0217             stq_u(partial_dest, dst);
0218             if (!len) goto out;
0219             dst++;
0220             insqh(word, doff, partial_dest);
0221         }
0222         doff = len;
0223     }
0224     ldq_u(second_dest, dst);
0225     mskqh(second_dest, doff, second_dest);
0226     stq_u(partial_dest | second_dest, dst);
0227 out:
0228     checksum += carry;
0229     return checksum;
0230 }
0231
0232 /*
0233  * This is so totally un-fun that it's frightening. Don't
0234  * look at this too closely, you'll go blind.
0235  */
0236 static inline unsigned long
0237 csum_partial_cfu_unaligned(const unsigned long __user * src,
0238                unsigned long * dst,
0239                unsigned long soff, unsigned long doff,
0240                long len, unsigned long partial_dest)
0241 {
0242     unsigned long carry = 0;
0243     unsigned long first;
0244     unsigned long lastsrc;
0245     unsigned long checksum = ~0U;
0246
0247     if (__get_word(ldq_u, first, src))
0248         return 0;
0249     lastsrc = 7+len+(unsigned long)src;
0250     mskql(partial_dest, doff, partial_dest);
0251     while (len >= 0) {
0252         unsigned long second, word;
0253         unsigned long second_dest;
0254
0255         if (__get_word(ldq_u, second, src+1))
0256             return 0;
0257         extql(first, soff, word);
0258         checksum += carry;
0259         len -= 8;
0260         extqh(second, soff, first);
0261         src++;
0262         word |= first;
0263         first = second;
0264         insql(word, doff, second_dest);
0265         checksum += word;
0266         stq_u(partial_dest | second_dest, dst);
0267         carry = checksum < word;
0268         insqh(word, doff, partial_dest);
0269         dst++;
0270     }
0271     len += doff;
0272     checksum += carry;
0273     if (len >= 0) {
0274         unsigned long second, word;
0275         unsigned long second_dest;
0276
0277         if (__get_word(ldq_u, second, lastsrc))
0278             return 0;
0279         extql(first, soff, word);
0280         extqh(second, soff, first);
0281         word |= first;
0282         first = second;
0283         mskql(word, len-doff, word);
0284         checksum += word;
0285         insql(word, doff, second_dest);
0286         carry = checksum < word;
0287         stq_u(partial_dest | second_dest, dst);
0288         if (len) {
0289             ldq_u(second_dest, dst+1);
0290             insqh(word, doff, partial_dest);
0291             mskqh(second_dest, len, second_dest);
0292             stq_u(partial_dest | second_dest, dst+1);
0293         }
0294         checksum += carry;
0295     } else {
0296         unsigned long second, word;
0297         unsigned long second_dest;
0298
0299         if (__get_word(ldq_u, second, lastsrc))
0300             return 0;
0301         extql(first, soff, word);
0302         extqh(second, soff, first);
0303         word |= first;
0304         ldq_u(second_dest, dst);
0305         mskql(word, len-doff, word);
0306         checksum += word;
0307         mskqh(second_dest, len, second_dest);
0308         carry = checksum < word;
0309         insql(word, doff, word);
0310         stq_u(partial_dest | word | second_dest, dst);
0311         checksum += carry;
0312     }
0313     return checksum;
0314 }
0315
0316 static __wsum __csum_and_copy(const void __user *src, void *dst, int len)
0317 {
0318     unsigned long soff = 7 & (unsigned long) src;
0319     unsigned long doff = 7 & (unsigned long) dst;
0320     unsigned long checksum;
0321
0322     if (!doff) {
0323         if (!soff)
0324             checksum = csum_partial_cfu_aligned(
0325                 (const unsigned long __user *) src,
0326                 (unsigned long *) dst, len-8);
0327         else
0328             checksum = csum_partial_cfu_dest_aligned(
0329                 (const unsigned long __user *) src,
0330                 (unsigned long *) dst,
0331                 soff, len-8);
0332     } else {
0333         unsigned long partial_dest;
0334         ldq_u(partial_dest, dst);
0335         if (!soff)
0336             checksum = csum_partial_cfu_src_aligned(
0337                 (const unsigned long __user *) src,
0338                 (unsigned long *) dst,
0339                 doff, len-8, partial_dest);
0340         else
0341             checksum = csum_partial_cfu_unaligned(
0342                 (const unsigned long __user *) src,
0343                 (unsigned long *) dst,
0344                 soff, doff, len-8, partial_dest);
0345     }
0346     return (__force __wsum)from64to16 (checksum);
0347 }
0348
0349 __wsum
0350 csum_and_copy_from_user(const void __user *src, void *dst, int len)
0351 {
0352     if (!access_ok(src, len))
0353         return 0;
0354     return __csum_and_copy(src, dst, len);
0355 }
0356
0357 __wsum
0358 csum_partial_copy_nocheck(const void *src, void *dst, int len)
0359 {
0360     return __csum_and_copy((__force const void __user *)src,
0361                         dst, len);
0362 }
0363 EXPORT_SYMBOL(csum_partial_copy_nocheck);