arm64/lib/copy_page.S

0001 /* SPDX-License-Identifier: GPL-2.0-only */
0002 /*
0003  * Copyright (C) 2012 ARM Ltd.
0004  */
0005
0006 #include <linux/linkage.h>
0007 #include <linux/const.h>
0008 #include <asm/assembler.h>
0009 #include <asm/page.h>
0010 #include <asm/cpufeature.h>
0011 #include <asm/alternative.h>
0012
0013 /*
0014  * Copy a page from src to dest (both are page aligned)
0015  *
0016  * Parameters:
0017  *  x0 - dest
0018  *  x1 - src
0019  */
0020 SYM_FUNC_START(__pi_copy_page)
0021 alternative_if ARM64_HAS_NO_HW_PREFETCH
0022     // Prefetch three cache lines ahead.
0023     prfm    pldl1strm, [x1, #128]
0024     prfm    pldl1strm, [x1, #256]
0025     prfm    pldl1strm, [x1, #384]
0026 alternative_else_nop_endif
0027
0028     ldp x2, x3, [x1]
0029     ldp x4, x5, [x1, #16]
0030     ldp x6, x7, [x1, #32]
0031     ldp x8, x9, [x1, #48]
0032     ldp x10, x11, [x1, #64]
0033     ldp x12, x13, [x1, #80]
0034     ldp x14, x15, [x1, #96]
0035     ldp x16, x17, [x1, #112]
0036
0037     add x0, x0, #256
0038     add x1, x1, #128
0039 1:
0040     tst x0, #(PAGE_SIZE - 1)
0041
0042 alternative_if ARM64_HAS_NO_HW_PREFETCH
0043     prfm    pldl1strm, [x1, #384]
0044 alternative_else_nop_endif
0045
0046     stnp    x2, x3, [x0, #-256]
0047     ldp x2, x3, [x1]
0048     stnp    x4, x5, [x0, #16 - 256]
0049     ldp x4, x5, [x1, #16]
0050     stnp    x6, x7, [x0, #32 - 256]
0051     ldp x6, x7, [x1, #32]
0052     stnp    x8, x9, [x0, #48 - 256]
0053     ldp x8, x9, [x1, #48]
0054     stnp    x10, x11, [x0, #64 - 256]
0055     ldp x10, x11, [x1, #64]
0056     stnp    x12, x13, [x0, #80 - 256]
0057     ldp x12, x13, [x1, #80]
0058     stnp    x14, x15, [x0, #96 - 256]
0059     ldp x14, x15, [x1, #96]
0060     stnp    x16, x17, [x0, #112 - 256]
0061     ldp x16, x17, [x1, #112]
0062
0063     add x0, x0, #128
0064     add x1, x1, #128
0065
0066     b.ne    1b
0067
0068     stnp    x2, x3, [x0, #-256]
0069     stnp    x4, x5, [x0, #16 - 256]
0070     stnp    x6, x7, [x0, #32 - 256]
0071     stnp    x8, x9, [x0, #48 - 256]
0072     stnp    x10, x11, [x0, #64 - 256]
0073     stnp    x12, x13, [x0, #80 - 256]
0074     stnp    x14, x15, [x0, #96 - 256]
0075     stnp    x16, x17, [x0, #112 - 256]
0076
0077     ret
0078 SYM_FUNC_END(__pi_copy_page)
0079 SYM_FUNC_ALIAS(copy_page, __pi_copy_page)
0080 EXPORT_SYMBOL(copy_page)