arm/kernel/phys2virt.S

0001 /* SPDX-License-Identifier: GPL-2.0-only */
0002 /*
0003  *  Copyright (C) 1994-2002 Russell King
0004  *  Copyright (c) 2003, 2020 ARM Limited
0005  *  All Rights Reserved
0006  */
0007
0008 #include <linux/init.h>
0009 #include <linux/linkage.h>
0010 #include <asm/assembler.h>
0011 #include <asm/page.h>
0012
0013 #ifdef __ARMEB__
0014 #define LOW_OFFSET  0x4
0015 #define HIGH_OFFSET 0x0
0016 #else
0017 #define LOW_OFFSET  0x0
0018 #define HIGH_OFFSET 0x4
0019 #endif
0020
0021 /*
0022  * __fixup_pv_table - patch the stub instructions with the delta between
0023  *                    PHYS_OFFSET and PAGE_OFFSET, which is assumed to be
0024  *                    2 MiB aligned.
0025  *
0026  * Called from head.S, which expects the following registers to be preserved:
0027  *   r1 = machine no, r2 = atags or dtb,
0028  *   r8 = phys_offset, r9 = cpuid, r10 = procinfo
0029  */
0030     __HEAD
0031 ENTRY(__fixup_pv_table)
0032     mov r0, r8, lsr #PAGE_SHIFT @ convert to PFN
0033     str_l   r0, __pv_phys_pfn_offset, r3
0034
0035     adr_l   r0, __pv_offset
0036     subs    r3, r8, #PAGE_OFFSET    @ PHYS_OFFSET - PAGE_OFFSET
0037     mvn ip, #0
0038     strcc   ip, [r0, #HIGH_OFFSET]  @ save to __pv_offset high bits
0039     str r3, [r0, #LOW_OFFSET]   @ save to __pv_offset low bits
0040
0041     mov r0, r3, lsr #21     @ constant for add/sub instructions
0042     teq r3, r0, lsl #21     @ must be 2 MiB aligned
0043     bne 0f
0044
0045     adr_l   r4, __pv_table_begin
0046     adr_l   r5, __pv_table_end
0047     b   __fixup_a_pv_table
0048
0049 0:  mov r0, r0          @ deadloop on error
0050     b   0b
0051 ENDPROC(__fixup_pv_table)
0052
0053     .text
0054 __fixup_a_pv_table:
0055     adr_l   r6, __pv_offset
0056     ldr r0, [r6, #HIGH_OFFSET]  @ pv_offset high word
0057     ldr r6, [r6, #LOW_OFFSET]   @ pv_offset low word
0058     cmn r0, #1
0059 #ifdef CONFIG_THUMB2_KERNEL
0060     @
0061     @ The Thumb-2 versions of the patchable sequences are
0062     @
0063     @ phys-to-virt:         movw    <reg>, #offset<31:21>
0064     @               lsl <reg>, #21
0065     @               sub <VA>, <PA>, <reg>
0066     @
0067     @ virt-to-phys (non-LPAE):  movw    <reg>, #offset<31:21>
0068     @               lsl <reg>, #21
0069     @               add <PA>, <VA>, <reg>
0070     @
0071     @ virt-to-phys (LPAE):      movw    <reg>, #offset<31:21>
0072     @               lsl <reg>, #21
0073     @               adds    <PAlo>, <VA>, <reg>
0074     @               mov <PAhi>, #offset<39:32>
0075     @               adc <PAhi>, <PAhi>, #0
0076     @
0077     @ In the non-LPAE case, all patchable instructions are MOVW
0078     @ instructions, where we need to patch in the offset into the
0079     @ second halfword of the opcode (the 16-bit immediate is encoded
0080     @ as imm4:i:imm3:imm8)
0081     @
0082     @       15       11 10  9           4 3    0  15  14  12 11 8 7    0
0083     @      +-----------+---+-------------+------++---+------+----+------+
0084     @ MOVW | 1 1 1 1 0 | i | 1 0 0 1 0 0 | imm4 || 0 | imm3 | Rd | imm8 |
0085     @      +-----------+---+-------------+------++---+------+----+------+
0086     @
0087     @ In the LPAE case, we also need to patch in the high word of the
0088     @ offset into the immediate field of the MOV instruction, or patch it
0089     @ to a MVN instruction if the offset is negative. In this case, we
0090     @ need to inspect the first halfword of the opcode, to check whether
0091     @ it is MOVW or MOV/MVN, and to perform the MOV to MVN patching if
0092     @ needed. The encoding of the immediate is rather complex for values
0093     @ of i:imm3 != 0b0000, but fortunately, we never need more than 8 lower
0094     @ order bits, which can be patched into imm8 directly (and i:imm3
0095     @ cleared)
0096     @
0097     @      15       11 10  9        5         0  15  14  12 11 8 7    0
0098     @     +-----------+---+---------------------++---+------+----+------+
0099     @ MOV | 1 1 1 1 0 | i | 0 0 0 1 0 0 1 1 1 1 || 0 | imm3 | Rd | imm8 |
0100     @ MVN | 1 1 1 1 0 | i | 0 0 0 1 1 0 1 1 1 1 || 0 | imm3 | Rd | imm8 |
0101     @     +-----------+---+---------------------++---+------+----+------+
0102     @
0103     moveq   r0, #0x200000       @ set bit 21, mov to mvn instruction
0104     lsrs    r3, r6, #29     @ isolate top 3 bits of displacement
0105     ubfx    r6, r6, #21, #8     @ put bits 28:21 into the MOVW imm8 field
0106     bfi r6, r3, #12, #3     @ put bits 31:29 into the MOVW imm3 field
0107     b   .Lnext
0108 .Lloop: add r7, r4
0109     adds    r4, #4          @ clears Z flag
0110 #ifdef CONFIG_ARM_LPAE
0111     ldrh    ip, [r7]
0112 ARM_BE8(rev16   ip, ip)
0113     tst ip, #0x200      @ MOVW has bit 9 set, MVN has it clear
0114     bne 0f          @ skip to MOVW handling (Z flag is clear)
0115     bic ip, #0x20       @ clear bit 5 (MVN -> MOV)
0116     orr ip, ip, r0, lsr #16 @ MOV -> MVN if offset < 0
0117 ARM_BE8(rev16   ip, ip)
0118     strh    ip, [r7]
0119     @ Z flag is set
0120 0:
0121 #endif
0122     ldrh    ip, [r7, #2]
0123 ARM_BE8(rev16   ip, ip)
0124     and ip, #0xf00      @ clear everything except Rd field
0125     orreq   ip, r0          @ Z flag set -> MOV/MVN -> patch in high bits
0126     orrne   ip, r6          @ Z flag clear -> MOVW -> patch in low bits
0127 ARM_BE8(rev16   ip, ip)
0128     strh    ip, [r7, #2]
0129 #else
0130 #ifdef CONFIG_CPU_ENDIAN_BE8
0131 @ in BE8, we load data in BE, but instructions still in LE
0132 #define PV_BIT24    0x00000001
0133 #define PV_IMM8_MASK    0xff000000
0134 #define PV_IMMR_MSB 0x00080000
0135 #else
0136 #define PV_BIT24    0x01000000
0137 #define PV_IMM8_MASK    0x000000ff
0138 #define PV_IMMR_MSB 0x00000800
0139 #endif
0140
0141     @
0142     @ The ARM versions of the patchable sequences are
0143     @
0144     @ phys-to-virt:         sub <VA>, <PA>, #offset<31:24>, lsl #24
0145     @               sub <VA>, <PA>, #offset<23:16>, lsl #16
0146     @
0147     @ virt-to-phys (non-LPAE):  add <PA>, <VA>, #offset<31:24>, lsl #24
0148     @               add <PA>, <VA>, #offset<23:16>, lsl #16
0149     @
0150     @ virt-to-phys (LPAE):      movw    <reg>, #offset<31:20>
0151     @               adds    <PAlo>, <VA>, <reg>, lsl #20
0152     @               mov <PAhi>, #offset<39:32>
0153     @               adc <PAhi>, <PAhi>, #0
0154     @
0155     @ In the non-LPAE case, all patchable instructions are ADD or SUB
0156     @ instructions, where we need to patch in the offset into the
0157     @ immediate field of the opcode, which is emitted with the correct
0158     @ rotation value. (The effective value of the immediate is imm12<7:0>
0159     @ rotated right by [2 * imm12<11:8>] bits)
0160     @
0161     @      31   28 27      23 22  20 19  16 15  12 11    0
0162     @      +------+-----------------+------+------+-------+
0163     @  ADD | cond | 0 0 1 0 1 0 0 0 |  Rn  |  Rd  | imm12 |
0164     @  SUB | cond | 0 0 1 0 0 1 0 0 |  Rn  |  Rd  | imm12 |
0165     @  MOV | cond | 0 0 1 1 1 0 1 0 |  Rn  |  Rd  | imm12 |
0166     @  MVN | cond | 0 0 1 1 1 1 1 0 |  Rn  |  Rd  | imm12 |
0167     @      +------+-----------------+------+------+-------+
0168     @
0169     @ In the LPAE case, we use a MOVW instruction to carry the low offset
0170     @ word, and patch in the high word of the offset into the immediate
0171     @ field of the subsequent MOV instruction, or patch it to a MVN
0172     @ instruction if the offset is negative. We can distinguish MOVW
0173     @ instructions based on bits 23:22 of the opcode, and ADD/SUB can be
0174     @ distinguished from MOV/MVN (all using the encodings above) using
0175     @ bit 24.
0176     @
0177     @      31   28 27      23 22  20 19  16 15  12 11    0
0178     @      +------+-----------------+------+------+-------+
0179     @ MOVW | cond | 0 0 1 1 0 0 0 0 | imm4 |  Rd  | imm12 |
0180     @      +------+-----------------+------+------+-------+
0181     @
0182     moveq   r0, #0x400000       @ set bit 22, mov to mvn instruction
0183     mov r3, r6, lsr #16     @ put offset bits 31-16 into r3
0184     mov r6, r6, lsr #24     @ put offset bits 31-24 into r6
0185     and r3, r3, #0xf0       @ only keep offset bits 23-20 in r3
0186     b   .Lnext
0187 .Lloop: ldr ip, [r7, r4]
0188 #ifdef CONFIG_ARM_LPAE
0189     tst ip, #PV_BIT24       @ ADD/SUB have bit 24 clear
0190     beq 1f
0191 ARM_BE8(rev ip, ip)
0192     tst ip, #0xc00000       @ MOVW has bits 23:22 clear
0193     bic ip, ip, #0x400000   @ clear bit 22
0194     bfc ip, #0, #12     @ clear imm12 field of MOV[W] instruction
0195     orreq   ip, ip, r6, lsl #4  @ MOVW -> mask in offset bits 31-24
0196     orreq   ip, ip, r3, lsr #4  @ MOVW -> mask in offset bits 23-20
0197     orrne   ip, ip, r0      @ MOV  -> mask in offset bits 7-0 (or bit 22)
0198 ARM_BE8(rev ip, ip)
0199     b   2f
0200 1:
0201 #endif
0202     tst ip, #PV_IMMR_MSB        @ rotation value >= 16 ?
0203     bic ip, ip, #PV_IMM8_MASK
0204     orreq   ip, ip, r6 ARM_BE8(, lsl #24)   @ mask in offset bits 31-24
0205     orrne   ip, ip, r3 ARM_BE8(, lsl #24)   @ mask in offset bits 23-20
0206 2:
0207     str ip, [r7, r4]
0208     add r4, r4, #4
0209 #endif
0210
0211 .Lnext:
0212     cmp r4, r5
0213     ldrcc   r7, [r4]        @ use branch for delay slot
0214     bcc .Lloop
0215     ret lr
0216 ENDPROC(__fixup_a_pv_table)
0217
0218 ENTRY(fixup_pv_table)
0219     stmfd   sp!, {r4 - r7, lr}
0220     mov r4, r0          @ r0 = table start
0221     add r5, r0, r1      @ r1 = table size
0222     bl  __fixup_a_pv_table
0223     ldmfd   sp!, {r4 - r7, pc}
0224 ENDPROC(fixup_pv_table)
0225
0226     .data
0227     .align  2
0228     .globl  __pv_phys_pfn_offset
0229     .type   __pv_phys_pfn_offset, %object
0230 __pv_phys_pfn_offset:
0231     .word   0
0232     .size   __pv_phys_pfn_offset, . -__pv_phys_pfn_offset
0233
0234     .globl  __pv_offset
0235     .type   __pv_offset, %object
0236 __pv_offset:
0237     .quad   0
0238     .size   __pv_offset, . -__pv_offset