powerpc/lib/copypage_power7.S

0001 /* SPDX-License-Identifier: GPL-2.0-or-later */
0002 /*
0003  *
0004  * Copyright (C) IBM Corporation, 2012
0005  *
0006  * Author: Anton Blanchard <anton@au.ibm.com>
0007  */
0008 #include <asm/page.h>
0009 #include <asm/ppc_asm.h>
0010
0011 _GLOBAL(copypage_power7)
0012     /*
0013      * We prefetch both the source and destination using enhanced touch
0014      * instructions. We use a stream ID of 0 for the load side and
0015      * 1 for the store side. Since source and destination are page
0016      * aligned we don't need to clear the bottom 7 bits of either
0017      * address.
0018      */
0019     ori r9,r3,1     /* stream=1 => to */
0020
0021 #ifdef CONFIG_PPC_64K_PAGES
0022     lis r7,0x0E01   /* depth=7
0023                  * units/cachelines=512 */
0024 #else
0025     lis r7,0x0E00   /* depth=7 */
0026     ori r7,r7,0x1000    /* units/cachelines=32 */
0027 #endif
0028     ori r10,r7,1    /* stream=1 */
0029
0030     lis r8,0x8000   /* GO=1 */
0031     clrldi  r8,r8,32
0032
0033     /* setup read stream 0  */
0034     dcbt    0,r4,0b01000    /* addr from */
0035     dcbt    0,r7,0b01010   /* length and depth from */
0036     /* setup write stream 1 */
0037     dcbtst  0,r9,0b01000   /* addr to */
0038     dcbtst  0,r10,0b01010  /* length and depth to */
0039     eieio
0040     dcbt    0,r8,0b01010    /* all streams GO */
0041
0042 #ifdef CONFIG_ALTIVEC
0043     mflr    r0
0044     std r3,-STACKFRAMESIZE+STK_REG(R31)(r1)
0045     std r4,-STACKFRAMESIZE+STK_REG(R30)(r1)
0046     std r0,16(r1)
0047     stdu    r1,-STACKFRAMESIZE(r1)
0048     bl  enter_vmx_ops
0049     cmpwi   r3,0
0050     ld  r0,STACKFRAMESIZE+16(r1)
0051     ld  r3,STK_REG(R31)(r1)
0052     ld  r4,STK_REG(R30)(r1)
0053     mtlr    r0
0054
0055     li  r0,(PAGE_SIZE/128)
0056     mtctr   r0
0057
0058     beq .Lnonvmx_copy
0059
0060     addi    r1,r1,STACKFRAMESIZE
0061
0062     li  r6,16
0063     li  r7,32
0064     li  r8,48
0065     li  r9,64
0066     li  r10,80
0067     li  r11,96
0068     li  r12,112
0069
0070     .align  5
0071 1:  lvx v7,0,r4
0072     lvx v6,r4,r6
0073     lvx v5,r4,r7
0074     lvx v4,r4,r8
0075     lvx v3,r4,r9
0076     lvx v2,r4,r10
0077     lvx v1,r4,r11
0078     lvx v0,r4,r12
0079     addi    r4,r4,128
0080     stvx    v7,0,r3
0081     stvx    v6,r3,r6
0082     stvx    v5,r3,r7
0083     stvx    v4,r3,r8
0084     stvx    v3,r3,r9
0085     stvx    v2,r3,r10
0086     stvx    v1,r3,r11
0087     stvx    v0,r3,r12
0088     addi    r3,r3,128
0089     bdnz    1b
0090
0091     b   exit_vmx_ops        /* tail call optimise */
0092
0093 #else
0094     li  r0,(PAGE_SIZE/128)
0095     mtctr   r0
0096
0097     stdu    r1,-STACKFRAMESIZE(r1)
0098 #endif
0099
0100 .Lnonvmx_copy:
0101     std r14,STK_REG(R14)(r1)
0102     std r15,STK_REG(R15)(r1)
0103     std r16,STK_REG(R16)(r1)
0104     std r17,STK_REG(R17)(r1)
0105     std r18,STK_REG(R18)(r1)
0106     std r19,STK_REG(R19)(r1)
0107     std r20,STK_REG(R20)(r1)
0108
0109 1:  ld  r0,0(r4)
0110     ld  r5,8(r4)
0111     ld  r6,16(r4)
0112     ld  r7,24(r4)
0113     ld  r8,32(r4)
0114     ld  r9,40(r4)
0115     ld  r10,48(r4)
0116     ld  r11,56(r4)
0117     ld  r12,64(r4)
0118     ld  r14,72(r4)
0119     ld  r15,80(r4)
0120     ld  r16,88(r4)
0121     ld  r17,96(r4)
0122     ld  r18,104(r4)
0123     ld  r19,112(r4)
0124     ld  r20,120(r4)
0125     addi    r4,r4,128
0126     std r0,0(r3)
0127     std r5,8(r3)
0128     std r6,16(r3)
0129     std r7,24(r3)
0130     std r8,32(r3)
0131     std r9,40(r3)
0132     std r10,48(r3)
0133     std r11,56(r3)
0134     std r12,64(r3)
0135     std r14,72(r3)
0136     std r15,80(r3)
0137     std r16,88(r3)
0138     std r17,96(r3)
0139     std r18,104(r3)
0140     std r19,112(r3)
0141     std r20,120(r3)
0142     addi    r3,r3,128
0143     bdnz    1b
0144
0145     ld  r14,STK_REG(R14)(r1)
0146     ld  r15,STK_REG(R15)(r1)
0147     ld  r16,STK_REG(R16)(r1)
0148     ld  r17,STK_REG(R17)(r1)
0149     ld  r18,STK_REG(R18)(r1)
0150     ld  r19,STK_REG(R19)(r1)
0151     ld  r20,STK_REG(R20)(r1)
0152     addi    r1,r1,STACKFRAMESIZE
0153     blr