x86/lib/memcpy_32.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 #include <linux/string.h>
0003 #include <linux/export.h>
0004
0005 #undef memcpy
0006 #undef memset
0007 #undef memmove
0008
0009 __visible void *memcpy(void *to, const void *from, size_t n)
0010 {
0011     return __memcpy(to, from, n);
0012 }
0013 EXPORT_SYMBOL(memcpy);
0014
0015 __visible void *memset(void *s, int c, size_t count)
0016 {
0017     return __memset(s, c, count);
0018 }
0019 EXPORT_SYMBOL(memset);
0020
0021 __visible void *memmove(void *dest, const void *src, size_t n)
0022 {
0023     int d0,d1,d2,d3,d4,d5;
0024     char *ret = dest;
0025
0026     __asm__ __volatile__(
0027         /* Handle more 16 bytes in loop */
0028         "cmp $0x10, %0\n\t"
0029         "jb 1f\n\t"
0030
0031         /* Decide forward/backward copy mode */
0032         "cmp %2, %1\n\t"
0033         "jb 2f\n\t"
0034
0035         /*
0036          * movs instruction have many startup latency
0037          * so we handle small size by general register.
0038          */
0039         "cmp  $680, %0\n\t"
0040         "jb 3f\n\t"
0041         /*
0042          * movs instruction is only good for aligned case.
0043          */
0044         "mov %1, %3\n\t"
0045         "xor %2, %3\n\t"
0046         "and $0xff, %3\n\t"
0047         "jz 4f\n\t"
0048         "3:\n\t"
0049         "sub $0x10, %0\n\t"
0050
0051         /*
0052          * We gobble 16 bytes forward in each loop.
0053          */
0054         "3:\n\t"
0055         "sub $0x10, %0\n\t"
0056         "mov 0*4(%1), %3\n\t"
0057         "mov 1*4(%1), %4\n\t"
0058         "mov  %3, 0*4(%2)\n\t"
0059         "mov  %4, 1*4(%2)\n\t"
0060         "mov 2*4(%1), %3\n\t"
0061         "mov 3*4(%1), %4\n\t"
0062         "mov  %3, 2*4(%2)\n\t"
0063         "mov  %4, 3*4(%2)\n\t"
0064         "lea  0x10(%1), %1\n\t"
0065         "lea  0x10(%2), %2\n\t"
0066         "jae 3b\n\t"
0067         "add $0x10, %0\n\t"
0068         "jmp 1f\n\t"
0069
0070         /*
0071          * Handle data forward by movs.
0072          */
0073         ".p2align 4\n\t"
0074         "4:\n\t"
0075         "mov -4(%1, %0), %3\n\t"
0076         "lea -4(%2, %0), %4\n\t"
0077         "shr $2, %0\n\t"
0078         "rep movsl\n\t"
0079         "mov %3, (%4)\n\t"
0080         "jmp 11f\n\t"
0081         /*
0082          * Handle data backward by movs.
0083          */
0084         ".p2align 4\n\t"
0085         "6:\n\t"
0086         "mov (%1), %3\n\t"
0087         "mov %2, %4\n\t"
0088         "lea -4(%1, %0), %1\n\t"
0089         "lea -4(%2, %0), %2\n\t"
0090         "shr $2, %0\n\t"
0091         "std\n\t"
0092         "rep movsl\n\t"
0093         "mov %3,(%4)\n\t"
0094         "cld\n\t"
0095         "jmp 11f\n\t"
0096
0097         /*
0098          * Start to prepare for backward copy.
0099          */
0100         ".p2align 4\n\t"
0101         "2:\n\t"
0102         "cmp  $680, %0\n\t"
0103         "jb 5f\n\t"
0104         "mov %1, %3\n\t"
0105         "xor %2, %3\n\t"
0106         "and $0xff, %3\n\t"
0107         "jz 6b\n\t"
0108
0109         /*
0110          * Calculate copy position to tail.
0111          */
0112         "5:\n\t"
0113         "add %0, %1\n\t"
0114         "add %0, %2\n\t"
0115         "sub $0x10, %0\n\t"
0116
0117         /*
0118          * We gobble 16 bytes backward in each loop.
0119          */
0120         "7:\n\t"
0121         "sub $0x10, %0\n\t"
0122
0123         "mov -1*4(%1), %3\n\t"
0124         "mov -2*4(%1), %4\n\t"
0125         "mov  %3, -1*4(%2)\n\t"
0126         "mov  %4, -2*4(%2)\n\t"
0127         "mov -3*4(%1), %3\n\t"
0128         "mov -4*4(%1), %4\n\t"
0129         "mov  %3, -3*4(%2)\n\t"
0130         "mov  %4, -4*4(%2)\n\t"
0131         "lea  -0x10(%1), %1\n\t"
0132         "lea  -0x10(%2), %2\n\t"
0133         "jae 7b\n\t"
0134         /*
0135          * Calculate copy position to head.
0136          */
0137         "add $0x10, %0\n\t"
0138         "sub %0, %1\n\t"
0139         "sub %0, %2\n\t"
0140
0141         /*
0142          * Move data from 8 bytes to 15 bytes.
0143          */
0144         ".p2align 4\n\t"
0145         "1:\n\t"
0146         "cmp $8, %0\n\t"
0147         "jb 8f\n\t"
0148         "mov 0*4(%1), %3\n\t"
0149         "mov 1*4(%1), %4\n\t"
0150         "mov -2*4(%1, %0), %5\n\t"
0151         "mov -1*4(%1, %0), %1\n\t"
0152
0153         "mov  %3, 0*4(%2)\n\t"
0154         "mov  %4, 1*4(%2)\n\t"
0155         "mov  %5, -2*4(%2, %0)\n\t"
0156         "mov  %1, -1*4(%2, %0)\n\t"
0157         "jmp 11f\n\t"
0158
0159         /*
0160          * Move data from 4 bytes to 7 bytes.
0161          */
0162         ".p2align 4\n\t"
0163         "8:\n\t"
0164         "cmp $4, %0\n\t"
0165         "jb 9f\n\t"
0166         "mov 0*4(%1), %3\n\t"
0167         "mov -1*4(%1, %0), %4\n\t"
0168         "mov  %3, 0*4(%2)\n\t"
0169         "mov  %4, -1*4(%2, %0)\n\t"
0170         "jmp 11f\n\t"
0171
0172         /*
0173          * Move data from 2 bytes to 3 bytes.
0174          */
0175         ".p2align 4\n\t"
0176         "9:\n\t"
0177         "cmp $2, %0\n\t"
0178         "jb 10f\n\t"
0179         "movw 0*2(%1), %%dx\n\t"
0180         "movw -1*2(%1, %0), %%bx\n\t"
0181         "movw %%dx, 0*2(%2)\n\t"
0182         "movw %%bx, -1*2(%2, %0)\n\t"
0183         "jmp 11f\n\t"
0184
0185         /*
0186          * Move data for 1 byte.
0187          */
0188         ".p2align 4\n\t"
0189         "10:\n\t"
0190         "cmp $1, %0\n\t"
0191         "jb 11f\n\t"
0192         "movb (%1), %%cl\n\t"
0193         "movb %%cl, (%2)\n\t"
0194         ".p2align 4\n\t"
0195         "11:"
0196         : "=&c" (d0), "=&S" (d1), "=&D" (d2),
0197           "=r" (d3),"=r" (d4), "=r"(d5)
0198         :"0" (n),
0199          "1" (src),
0200          "2" (dest)
0201         :"memory");
0202
0203     return ret;
0204
0205 }
0206 EXPORT_SYMBOL(memmove);