arm64/lib/mte.S

0001 /* SPDX-License-Identifier: GPL-2.0-only */
0002 /*
0003  * Copyright (C) 2020 ARM Ltd.
0004  */
0005 #include <linux/linkage.h>
0006
0007 #include <asm/asm-uaccess.h>
0008 #include <asm/assembler.h>
0009 #include <asm/mte.h>
0010 #include <asm/page.h>
0011 #include <asm/sysreg.h>
0012
0013     .arch   armv8.5-a+memtag
0014
0015 /*
0016  * multitag_transfer_size - set \reg to the block size that is accessed by the
0017  * LDGM/STGM instructions.
0018  */
0019     .macro  multitag_transfer_size, reg, tmp
0020     mrs_s   \reg, SYS_GMID_EL1
0021     ubfx    \reg, \reg, #GMID_EL1_BS_SHIFT, #GMID_EL1_BS_SIZE
0022     mov \tmp, #4
0023     lsl \reg, \tmp, \reg
0024     .endm
0025
0026 /*
0027  * Clear the tags in a page
0028  *   x0 - address of the page to be cleared
0029  */
0030 SYM_FUNC_START(mte_clear_page_tags)
0031     multitag_transfer_size x1, x2
0032 1:  stgm    xzr, [x0]
0033     add x0, x0, x1
0034     tst x0, #(PAGE_SIZE - 1)
0035     b.ne    1b
0036     ret
0037 SYM_FUNC_END(mte_clear_page_tags)
0038
0039 /*
0040  * Zero the page and tags at the same time
0041  *
0042  * Parameters:
0043  *  x0 - address to the beginning of the page
0044  */
0045 SYM_FUNC_START(mte_zero_clear_page_tags)
0046     and x0, x0, #(1 << MTE_TAG_SHIFT) - 1   // clear the tag
0047     mrs x1, dczid_el0
0048     tbnz    x1, #4, 2f  // Branch if DC GZVA is prohibited
0049     and w1, w1, #0xf
0050     mov x2, #4
0051     lsl x1, x2, x1
0052
0053 1:  dc  gzva, x0
0054     add x0, x0, x1
0055     tst x0, #(PAGE_SIZE - 1)
0056     b.ne    1b
0057     ret
0058
0059 2:  stz2g   x0, [x0], #(MTE_GRANULE_SIZE * 2)
0060     tst x0, #(PAGE_SIZE - 1)
0061     b.ne    2b
0062     ret
0063 SYM_FUNC_END(mte_zero_clear_page_tags)
0064
0065 /*
0066  * Copy the tags from the source page to the destination one
0067  *   x0 - address of the destination page
0068  *   x1 - address of the source page
0069  */
0070 SYM_FUNC_START(mte_copy_page_tags)
0071     mov x2, x0
0072     mov x3, x1
0073     multitag_transfer_size x5, x6
0074 1:  ldgm    x4, [x3]
0075     stgm    x4, [x2]
0076     add x2, x2, x5
0077     add x3, x3, x5
0078     tst x2, #(PAGE_SIZE - 1)
0079     b.ne    1b
0080     ret
0081 SYM_FUNC_END(mte_copy_page_tags)
0082
0083 /*
0084  * Read tags from a user buffer (one tag per byte) and set the corresponding
0085  * tags at the given kernel address. Used by PTRACE_POKEMTETAGS.
0086  *   x0 - kernel address (to)
0087  *   x1 - user buffer (from)
0088  *   x2 - number of tags/bytes (n)
0089  * Returns:
0090  *   x0 - number of tags read/set
0091  */
0092 SYM_FUNC_START(mte_copy_tags_from_user)
0093     mov x3, x1
0094     cbz x2, 2f
0095 1:
0096 USER(2f, ldtrb  w4, [x1])
0097     lsl x4, x4, #MTE_TAG_SHIFT
0098     stg x4, [x0], #MTE_GRANULE_SIZE
0099     add x1, x1, #1
0100     subs    x2, x2, #1
0101     b.ne    1b
0102
0103     // exception handling and function return
0104 2:  sub x0, x1, x3      // update the number of tags set
0105     ret
0106 SYM_FUNC_END(mte_copy_tags_from_user)
0107
0108 /*
0109  * Get the tags from a kernel address range and write the tag values to the
0110  * given user buffer (one tag per byte). Used by PTRACE_PEEKMTETAGS.
0111  *   x0 - user buffer (to)
0112  *   x1 - kernel address (from)
0113  *   x2 - number of tags/bytes (n)
0114  * Returns:
0115  *   x0 - number of tags read/set
0116  */
0117 SYM_FUNC_START(mte_copy_tags_to_user)
0118     mov x3, x0
0119     cbz x2, 2f
0120 1:
0121     ldg x4, [x1]
0122     ubfx    x4, x4, #MTE_TAG_SHIFT, #MTE_TAG_SIZE
0123 USER(2f, sttrb  w4, [x0])
0124     add x0, x0, #1
0125     add x1, x1, #MTE_GRANULE_SIZE
0126     subs    x2, x2, #1
0127     b.ne    1b
0128
0129     // exception handling and function return
0130 2:  sub x0, x0, x3      // update the number of tags copied
0131     ret
0132 SYM_FUNC_END(mte_copy_tags_to_user)
0133
0134 /*
0135  * Save the tags in a page
0136  *   x0 - page address
0137  *   x1 - tag storage, MTE_PAGE_TAG_STORAGE bytes
0138  */
0139 SYM_FUNC_START(mte_save_page_tags)
0140     multitag_transfer_size x7, x5
0141 1:
0142     mov x2, #0
0143 2:
0144     ldgm    x5, [x0]
0145     orr x2, x2, x5
0146     add x0, x0, x7
0147     tst x0, #0xFF       // 16 tag values fit in a register,
0148     b.ne    2b          // which is 16*16=256 bytes
0149
0150     str x2, [x1], #8
0151
0152     tst x0, #(PAGE_SIZE - 1)
0153     b.ne    1b
0154
0155     ret
0156 SYM_FUNC_END(mte_save_page_tags)
0157
0158 /*
0159  * Restore the tags in a page
0160  *   x0 - page address
0161  *   x1 - tag storage, MTE_PAGE_TAG_STORAGE bytes
0162  */
0163 SYM_FUNC_START(mte_restore_page_tags)
0164     multitag_transfer_size x7, x5
0165 1:
0166     ldr x2, [x1], #8
0167 2:
0168     stgm    x2, [x0]
0169     add x0, x0, x7
0170     tst x0, #0xFF
0171     b.ne    2b
0172
0173     tst x0, #(PAGE_SIZE - 1)
0174     b.ne    1b
0175
0176     ret
0177 SYM_FUNC_END(mte_restore_page_tags)