Back to home page

OSCL-LXR
 
 

     

0001 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
0002 /*
0003  * rseq-arm.h
0004  *
0005  * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
0006  */
0007 
0008 /*
0009  * - ARM little endian
0010  *
0011  * RSEQ_SIG uses the udf A32 instruction with an uncommon immediate operand
0012  * value 0x5de3. This traps if user-space reaches this instruction by mistake,
0013  * and the uncommon operand ensures the kernel does not move the instruction
0014  * pointer to attacker-controlled code on rseq abort.
0015  *
0016  * The instruction pattern in the A32 instruction set is:
0017  *
0018  * e7f5def3    udf    #24035    ; 0x5de3
0019  *
0020  * This translates to the following instruction pattern in the T16 instruction
0021  * set:
0022  *
0023  * little endian:
0024  * def3        udf    #243      ; 0xf3
0025  * e7f5        b.n    <7f5>
0026  *
0027  * - ARMv6+ big endian (BE8):
0028  *
0029  * ARMv6+ -mbig-endian generates mixed endianness code vs data: little-endian
0030  * code and big-endian data. The data value of the signature needs to have its
0031  * byte order reversed to generate the trap instruction:
0032  *
0033  * Data: 0xf3def5e7
0034  *
0035  * Translates to this A32 instruction pattern:
0036  *
0037  * e7f5def3    udf    #24035    ; 0x5de3
0038  *
0039  * Translates to this T16 instruction pattern:
0040  *
0041  * def3        udf    #243      ; 0xf3
0042  * e7f5        b.n    <7f5>
0043  *
0044  * - Prior to ARMv6 big endian (BE32):
0045  *
0046  * Prior to ARMv6, -mbig-endian generates big-endian code and data
0047  * (which match), so the endianness of the data representation of the
0048  * signature should not be reversed. However, the choice between BE32
0049  * and BE8 is done by the linker, so we cannot know whether code and
0050  * data endianness will be mixed before the linker is invoked. So rather
0051  * than try to play tricks with the linker, the rseq signature is simply
0052  * data (not a trap instruction) prior to ARMv6 on big endian. This is
0053  * why the signature is expressed as data (.word) rather than as
0054  * instruction (.inst) in assembler.
0055  */
0056 
0057 #ifdef __ARMEB__
0058 #define RSEQ_SIG    0xf3def5e7      /* udf    #24035    ; 0x5de3 (ARMv6+) */
0059 #else
0060 #define RSEQ_SIG    0xe7f5def3      /* udf    #24035    ; 0x5de3 */
0061 #endif
0062 
0063 #define rseq_smp_mb()   __asm__ __volatile__ ("dmb" ::: "memory", "cc")
0064 #define rseq_smp_rmb()  __asm__ __volatile__ ("dmb" ::: "memory", "cc")
0065 #define rseq_smp_wmb()  __asm__ __volatile__ ("dmb" ::: "memory", "cc")
0066 
0067 #define rseq_smp_load_acquire(p)                    \
0068 __extension__ ({                            \
0069     __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p);           \
0070     rseq_smp_mb();                          \
0071     ____p1;                             \
0072 })
0073 
0074 #define rseq_smp_acquire__after_ctrl_dep()  rseq_smp_rmb()
0075 
0076 #define rseq_smp_store_release(p, v)                    \
0077 do {                                    \
0078     rseq_smp_mb();                          \
0079     RSEQ_WRITE_ONCE(*p, v);                     \
0080 } while (0)
0081 
0082 #ifdef RSEQ_SKIP_FASTPATH
0083 #include "rseq-skip.h"
0084 #else /* !RSEQ_SKIP_FASTPATH */
0085 
0086 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip,    \
0087                 post_commit_offset, abort_ip)       \
0088         ".pushsection __rseq_cs, \"aw\"\n\t"            \
0089         ".balign 32\n\t"                    \
0090         __rseq_str(label) ":\n\t"                   \
0091         ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
0092         ".word " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
0093         ".popsection\n\t"                   \
0094         ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"      \
0095         ".word " __rseq_str(label) "b, 0x0\n\t"         \
0096         ".popsection\n\t"
0097 
0098 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
0099     __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,      \
0100                 (post_commit_ip - start_ip), abort_ip)
0101 
0102 /*
0103  * Exit points of a rseq critical section consist of all instructions outside
0104  * of the critical section where a critical section can either branch to or
0105  * reach through the normal course of its execution. The abort IP and the
0106  * post-commit IP are already part of the __rseq_cs section and should not be
0107  * explicitly defined as additional exit points. Knowing all exit points is
0108  * useful to assist debuggers stepping over the critical section.
0109  */
0110 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)           \
0111         ".pushsection __rseq_exit_point_array, \"aw\"\n\t"  \
0112         ".word " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) ", 0x0\n\t" \
0113         ".popsection\n\t"
0114 
0115 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)        \
0116         RSEQ_INJECT_ASM(1)                  \
0117         "adr r0, " __rseq_str(cs_label) "\n\t"          \
0118         "str r0, %[" __rseq_str(rseq_cs) "]\n\t"        \
0119         __rseq_str(label) ":\n\t"
0120 
0121 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label)      \
0122         RSEQ_INJECT_ASM(2)                  \
0123         "ldr r0, %[" __rseq_str(current_cpu_id) "]\n\t" \
0124         "cmp %[" __rseq_str(cpu_id) "], r0\n\t"     \
0125         "bne " __rseq_str(label) "\n\t"
0126 
0127 #define __RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown,       \
0128                 abort_label, version, flags,        \
0129                 start_ip, post_commit_offset, abort_ip) \
0130         ".balign 32\n\t"                    \
0131         __rseq_str(table_label) ":\n\t"             \
0132         ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
0133         ".word " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
0134         ".word " __rseq_str(RSEQ_SIG) "\n\t"            \
0135         __rseq_str(label) ":\n\t"               \
0136         teardown                        \
0137         "b %l[" __rseq_str(abort_label) "]\n\t"
0138 
0139 #define RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, abort_label, \
0140                   start_ip, post_commit_ip, abort_ip)   \
0141     __RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown,       \
0142                 abort_label, 0x0, 0x0, start_ip,    \
0143                 (post_commit_ip - start_ip), abort_ip)
0144 
0145 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label)     \
0146         __rseq_str(label) ":\n\t"               \
0147         teardown                        \
0148         "b %l[" __rseq_str(cmpfail_label) "]\n\t"
0149 
0150 static inline __attribute__((always_inline))
0151 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
0152 {
0153     RSEQ_INJECT_C(9)
0154 
0155     __asm__ __volatile__ goto (
0156         RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
0157         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
0158 #ifdef RSEQ_COMPARE_TWICE
0159         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
0160         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
0161 #endif
0162         /* Start rseq by storing table entry pointer into rseq_cs. */
0163         RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
0164         RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
0165         RSEQ_INJECT_ASM(3)
0166         "ldr r0, %[v]\n\t"
0167         "cmp %[expect], r0\n\t"
0168         "bne %l[cmpfail]\n\t"
0169         RSEQ_INJECT_ASM(4)
0170 #ifdef RSEQ_COMPARE_TWICE
0171         RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
0172         "ldr r0, %[v]\n\t"
0173         "cmp %[expect], r0\n\t"
0174         "bne %l[error2]\n\t"
0175 #endif
0176         /* final store */
0177         "str %[newv], %[v]\n\t"
0178         "2:\n\t"
0179         RSEQ_INJECT_ASM(5)
0180         "b 5f\n\t"
0181         RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
0182         "5:\n\t"
0183         : /* gcc asm goto does not allow outputs */
0184         : [cpu_id]      "r" (cpu),
0185           [current_cpu_id]  "m" (rseq_get_abi()->cpu_id),
0186           [rseq_cs]     "m" (rseq_get_abi()->rseq_cs.arch.ptr),
0187           [v]           "m" (*v),
0188           [expect]      "r" (expect),
0189           [newv]        "r" (newv)
0190           RSEQ_INJECT_INPUT
0191         : "r0", "memory", "cc"
0192           RSEQ_INJECT_CLOBBER
0193         : abort, cmpfail
0194 #ifdef RSEQ_COMPARE_TWICE
0195           , error1, error2
0196 #endif
0197     );
0198     rseq_after_asm_goto();
0199     return 0;
0200 abort:
0201     rseq_after_asm_goto();
0202     RSEQ_INJECT_FAILED
0203     return -1;
0204 cmpfail:
0205     rseq_after_asm_goto();
0206     return 1;
0207 #ifdef RSEQ_COMPARE_TWICE
0208 error1:
0209     rseq_after_asm_goto();
0210     rseq_bug("cpu_id comparison failed");
0211 error2:
0212     rseq_after_asm_goto();
0213     rseq_bug("expected value comparison failed");
0214 #endif
0215 }
0216 
0217 static inline __attribute__((always_inline))
0218 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
0219                    long voffp, intptr_t *load, int cpu)
0220 {
0221     RSEQ_INJECT_C(9)
0222 
0223     __asm__ __volatile__ goto (
0224         RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
0225         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
0226 #ifdef RSEQ_COMPARE_TWICE
0227         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
0228         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
0229 #endif
0230         /* Start rseq by storing table entry pointer into rseq_cs. */
0231         RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
0232         RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
0233         RSEQ_INJECT_ASM(3)
0234         "ldr r0, %[v]\n\t"
0235         "cmp %[expectnot], r0\n\t"
0236         "beq %l[cmpfail]\n\t"
0237         RSEQ_INJECT_ASM(4)
0238 #ifdef RSEQ_COMPARE_TWICE
0239         RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
0240         "ldr r0, %[v]\n\t"
0241         "cmp %[expectnot], r0\n\t"
0242         "beq %l[error2]\n\t"
0243 #endif
0244         "str r0, %[load]\n\t"
0245         "add r0, %[voffp]\n\t"
0246         "ldr r0, [r0]\n\t"
0247         /* final store */
0248         "str r0, %[v]\n\t"
0249         "2:\n\t"
0250         RSEQ_INJECT_ASM(5)
0251         "b 5f\n\t"
0252         RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
0253         "5:\n\t"
0254         : /* gcc asm goto does not allow outputs */
0255         : [cpu_id]      "r" (cpu),
0256           [current_cpu_id]  "m" (rseq_get_abi()->cpu_id),
0257           [rseq_cs]     "m" (rseq_get_abi()->rseq_cs.arch.ptr),
0258           /* final store input */
0259           [v]           "m" (*v),
0260           [expectnot]       "r" (expectnot),
0261           [voffp]       "Ir" (voffp),
0262           [load]        "m" (*load)
0263           RSEQ_INJECT_INPUT
0264         : "r0", "memory", "cc"
0265           RSEQ_INJECT_CLOBBER
0266         : abort, cmpfail
0267 #ifdef RSEQ_COMPARE_TWICE
0268           , error1, error2
0269 #endif
0270     );
0271     rseq_after_asm_goto();
0272     return 0;
0273 abort:
0274     rseq_after_asm_goto();
0275     RSEQ_INJECT_FAILED
0276     return -1;
0277 cmpfail:
0278     rseq_after_asm_goto();
0279     return 1;
0280 #ifdef RSEQ_COMPARE_TWICE
0281 error1:
0282     rseq_after_asm_goto();
0283     rseq_bug("cpu_id comparison failed");
0284 error2:
0285     rseq_after_asm_goto();
0286     rseq_bug("expected value comparison failed");
0287 #endif
0288 }
0289 
0290 static inline __attribute__((always_inline))
0291 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
0292 {
0293     RSEQ_INJECT_C(9)
0294 
0295     __asm__ __volatile__ goto (
0296         RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
0297 #ifdef RSEQ_COMPARE_TWICE
0298         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
0299 #endif
0300         /* Start rseq by storing table entry pointer into rseq_cs. */
0301         RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
0302         RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
0303         RSEQ_INJECT_ASM(3)
0304 #ifdef RSEQ_COMPARE_TWICE
0305         RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
0306 #endif
0307         "ldr r0, %[v]\n\t"
0308         "add r0, %[count]\n\t"
0309         /* final store */
0310         "str r0, %[v]\n\t"
0311         "2:\n\t"
0312         RSEQ_INJECT_ASM(4)
0313         "b 5f\n\t"
0314         RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
0315         "5:\n\t"
0316         : /* gcc asm goto does not allow outputs */
0317         : [cpu_id]      "r" (cpu),
0318           [current_cpu_id]  "m" (rseq_get_abi()->cpu_id),
0319           [rseq_cs]     "m" (rseq_get_abi()->rseq_cs.arch.ptr),
0320           [v]           "m" (*v),
0321           [count]       "Ir" (count)
0322           RSEQ_INJECT_INPUT
0323         : "r0", "memory", "cc"
0324           RSEQ_INJECT_CLOBBER
0325         : abort
0326 #ifdef RSEQ_COMPARE_TWICE
0327           , error1
0328 #endif
0329     );
0330     rseq_after_asm_goto();
0331     return 0;
0332 abort:
0333     rseq_after_asm_goto();
0334     RSEQ_INJECT_FAILED
0335     return -1;
0336 #ifdef RSEQ_COMPARE_TWICE
0337 error1:
0338     rseq_after_asm_goto();
0339     rseq_bug("cpu_id comparison failed");
0340 #endif
0341 }
0342 
0343 static inline __attribute__((always_inline))
0344 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
0345                  intptr_t *v2, intptr_t newv2,
0346                  intptr_t newv, int cpu)
0347 {
0348     RSEQ_INJECT_C(9)
0349 
0350     __asm__ __volatile__ goto (
0351         RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
0352         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
0353 #ifdef RSEQ_COMPARE_TWICE
0354         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
0355         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
0356 #endif
0357         /* Start rseq by storing table entry pointer into rseq_cs. */
0358         RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
0359         RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
0360         RSEQ_INJECT_ASM(3)
0361         "ldr r0, %[v]\n\t"
0362         "cmp %[expect], r0\n\t"
0363         "bne %l[cmpfail]\n\t"
0364         RSEQ_INJECT_ASM(4)
0365 #ifdef RSEQ_COMPARE_TWICE
0366         RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
0367         "ldr r0, %[v]\n\t"
0368         "cmp %[expect], r0\n\t"
0369         "bne %l[error2]\n\t"
0370 #endif
0371         /* try store */
0372         "str %[newv2], %[v2]\n\t"
0373         RSEQ_INJECT_ASM(5)
0374         /* final store */
0375         "str %[newv], %[v]\n\t"
0376         "2:\n\t"
0377         RSEQ_INJECT_ASM(6)
0378         "b 5f\n\t"
0379         RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
0380         "5:\n\t"
0381         : /* gcc asm goto does not allow outputs */
0382         : [cpu_id]      "r" (cpu),
0383           [current_cpu_id]  "m" (rseq_get_abi()->cpu_id),
0384           [rseq_cs]     "m" (rseq_get_abi()->rseq_cs.arch.ptr),
0385           /* try store input */
0386           [v2]          "m" (*v2),
0387           [newv2]       "r" (newv2),
0388           /* final store input */
0389           [v]           "m" (*v),
0390           [expect]      "r" (expect),
0391           [newv]        "r" (newv)
0392           RSEQ_INJECT_INPUT
0393         : "r0", "memory", "cc"
0394           RSEQ_INJECT_CLOBBER
0395         : abort, cmpfail
0396 #ifdef RSEQ_COMPARE_TWICE
0397           , error1, error2
0398 #endif
0399     );
0400     rseq_after_asm_goto();
0401     return 0;
0402 abort:
0403     rseq_after_asm_goto();
0404     RSEQ_INJECT_FAILED
0405     return -1;
0406 cmpfail:
0407     rseq_after_asm_goto();
0408     return 1;
0409 #ifdef RSEQ_COMPARE_TWICE
0410 error1:
0411     rseq_after_asm_goto();
0412     rseq_bug("cpu_id comparison failed");
0413 error2:
0414     rseq_after_asm_goto();
0415     rseq_bug("expected value comparison failed");
0416 #endif
0417 }
0418 
0419 static inline __attribute__((always_inline))
0420 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
0421                      intptr_t *v2, intptr_t newv2,
0422                      intptr_t newv, int cpu)
0423 {
0424     RSEQ_INJECT_C(9)
0425 
0426     __asm__ __volatile__ goto (
0427         RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
0428         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
0429 #ifdef RSEQ_COMPARE_TWICE
0430         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
0431         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
0432 #endif
0433         /* Start rseq by storing table entry pointer into rseq_cs. */
0434         RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
0435         RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
0436         RSEQ_INJECT_ASM(3)
0437         "ldr r0, %[v]\n\t"
0438         "cmp %[expect], r0\n\t"
0439         "bne %l[cmpfail]\n\t"
0440         RSEQ_INJECT_ASM(4)
0441 #ifdef RSEQ_COMPARE_TWICE
0442         RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
0443         "ldr r0, %[v]\n\t"
0444         "cmp %[expect], r0\n\t"
0445         "bne %l[error2]\n\t"
0446 #endif
0447         /* try store */
0448         "str %[newv2], %[v2]\n\t"
0449         RSEQ_INJECT_ASM(5)
0450         "dmb\n\t"   /* full mb provides store-release */
0451         /* final store */
0452         "str %[newv], %[v]\n\t"
0453         "2:\n\t"
0454         RSEQ_INJECT_ASM(6)
0455         "b 5f\n\t"
0456         RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
0457         "5:\n\t"
0458         : /* gcc asm goto does not allow outputs */
0459         : [cpu_id]      "r" (cpu),
0460           [current_cpu_id]  "m" (rseq_get_abi()->cpu_id),
0461           [rseq_cs]     "m" (rseq_get_abi()->rseq_cs.arch.ptr),
0462           /* try store input */
0463           [v2]          "m" (*v2),
0464           [newv2]       "r" (newv2),
0465           /* final store input */
0466           [v]           "m" (*v),
0467           [expect]      "r" (expect),
0468           [newv]        "r" (newv)
0469           RSEQ_INJECT_INPUT
0470         : "r0", "memory", "cc"
0471           RSEQ_INJECT_CLOBBER
0472         : abort, cmpfail
0473 #ifdef RSEQ_COMPARE_TWICE
0474           , error1, error2
0475 #endif
0476     );
0477     rseq_after_asm_goto();
0478     return 0;
0479 abort:
0480     rseq_after_asm_goto();
0481     RSEQ_INJECT_FAILED
0482     return -1;
0483 cmpfail:
0484     rseq_after_asm_goto();
0485     return 1;
0486 #ifdef RSEQ_COMPARE_TWICE
0487 error1:
0488     rseq_after_asm_goto();
0489     rseq_bug("cpu_id comparison failed");
0490 error2:
0491     rseq_after_asm_goto();
0492     rseq_bug("expected value comparison failed");
0493 #endif
0494 }
0495 
0496 static inline __attribute__((always_inline))
0497 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
0498                   intptr_t *v2, intptr_t expect2,
0499                   intptr_t newv, int cpu)
0500 {
0501     RSEQ_INJECT_C(9)
0502 
0503     __asm__ __volatile__ goto (
0504         RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
0505         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
0506 #ifdef RSEQ_COMPARE_TWICE
0507         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
0508         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
0509         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
0510 #endif
0511         /* Start rseq by storing table entry pointer into rseq_cs. */
0512         RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
0513         RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
0514         RSEQ_INJECT_ASM(3)
0515         "ldr r0, %[v]\n\t"
0516         "cmp %[expect], r0\n\t"
0517         "bne %l[cmpfail]\n\t"
0518         RSEQ_INJECT_ASM(4)
0519         "ldr r0, %[v2]\n\t"
0520         "cmp %[expect2], r0\n\t"
0521         "bne %l[cmpfail]\n\t"
0522         RSEQ_INJECT_ASM(5)
0523 #ifdef RSEQ_COMPARE_TWICE
0524         RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
0525         "ldr r0, %[v]\n\t"
0526         "cmp %[expect], r0\n\t"
0527         "bne %l[error2]\n\t"
0528         "ldr r0, %[v2]\n\t"
0529         "cmp %[expect2], r0\n\t"
0530         "bne %l[error3]\n\t"
0531 #endif
0532         /* final store */
0533         "str %[newv], %[v]\n\t"
0534         "2:\n\t"
0535         RSEQ_INJECT_ASM(6)
0536         "b 5f\n\t"
0537         RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
0538         "5:\n\t"
0539         : /* gcc asm goto does not allow outputs */
0540         : [cpu_id]      "r" (cpu),
0541           [current_cpu_id]  "m" (rseq_get_abi()->cpu_id),
0542           [rseq_cs]     "m" (rseq_get_abi()->rseq_cs.arch.ptr),
0543           /* cmp2 input */
0544           [v2]          "m" (*v2),
0545           [expect2]     "r" (expect2),
0546           /* final store input */
0547           [v]           "m" (*v),
0548           [expect]      "r" (expect),
0549           [newv]        "r" (newv)
0550           RSEQ_INJECT_INPUT
0551         : "r0", "memory", "cc"
0552           RSEQ_INJECT_CLOBBER
0553         : abort, cmpfail
0554 #ifdef RSEQ_COMPARE_TWICE
0555           , error1, error2, error3
0556 #endif
0557     );
0558     rseq_after_asm_goto();
0559     return 0;
0560 abort:
0561     rseq_after_asm_goto();
0562     RSEQ_INJECT_FAILED
0563     return -1;
0564 cmpfail:
0565     rseq_after_asm_goto();
0566     return 1;
0567 #ifdef RSEQ_COMPARE_TWICE
0568 error1:
0569     rseq_after_asm_goto();
0570     rseq_bug("cpu_id comparison failed");
0571 error2:
0572     rseq_after_asm_goto();
0573     rseq_bug("1st expected value comparison failed");
0574 error3:
0575     rseq_after_asm_goto();
0576     rseq_bug("2nd expected value comparison failed");
0577 #endif
0578 }
0579 
0580 static inline __attribute__((always_inline))
0581 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
0582                  void *dst, void *src, size_t len,
0583                  intptr_t newv, int cpu)
0584 {
0585     uint32_t rseq_scratch[3];
0586 
0587     RSEQ_INJECT_C(9)
0588 
0589     __asm__ __volatile__ goto (
0590         RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
0591         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
0592 #ifdef RSEQ_COMPARE_TWICE
0593         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
0594         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
0595 #endif
0596         "str %[src], %[rseq_scratch0]\n\t"
0597         "str %[dst], %[rseq_scratch1]\n\t"
0598         "str %[len], %[rseq_scratch2]\n\t"
0599         /* Start rseq by storing table entry pointer into rseq_cs. */
0600         RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
0601         RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
0602         RSEQ_INJECT_ASM(3)
0603         "ldr r0, %[v]\n\t"
0604         "cmp %[expect], r0\n\t"
0605         "bne 5f\n\t"
0606         RSEQ_INJECT_ASM(4)
0607 #ifdef RSEQ_COMPARE_TWICE
0608         RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
0609         "ldr r0, %[v]\n\t"
0610         "cmp %[expect], r0\n\t"
0611         "bne 7f\n\t"
0612 #endif
0613         /* try memcpy */
0614         "cmp %[len], #0\n\t" \
0615         "beq 333f\n\t" \
0616         "222:\n\t" \
0617         "ldrb %%r0, [%[src]]\n\t" \
0618         "strb %%r0, [%[dst]]\n\t" \
0619         "adds %[src], #1\n\t" \
0620         "adds %[dst], #1\n\t" \
0621         "subs %[len], #1\n\t" \
0622         "bne 222b\n\t" \
0623         "333:\n\t" \
0624         RSEQ_INJECT_ASM(5)
0625         /* final store */
0626         "str %[newv], %[v]\n\t"
0627         "2:\n\t"
0628         RSEQ_INJECT_ASM(6)
0629         /* teardown */
0630         "ldr %[len], %[rseq_scratch2]\n\t"
0631         "ldr %[dst], %[rseq_scratch1]\n\t"
0632         "ldr %[src], %[rseq_scratch0]\n\t"
0633         "b 8f\n\t"
0634         RSEQ_ASM_DEFINE_ABORT(3, 4,
0635                       /* teardown */
0636                       "ldr %[len], %[rseq_scratch2]\n\t"
0637                       "ldr %[dst], %[rseq_scratch1]\n\t"
0638                       "ldr %[src], %[rseq_scratch0]\n\t",
0639                       abort, 1b, 2b, 4f)
0640         RSEQ_ASM_DEFINE_CMPFAIL(5,
0641                     /* teardown */
0642                     "ldr %[len], %[rseq_scratch2]\n\t"
0643                     "ldr %[dst], %[rseq_scratch1]\n\t"
0644                     "ldr %[src], %[rseq_scratch0]\n\t",
0645                     cmpfail)
0646 #ifdef RSEQ_COMPARE_TWICE
0647         RSEQ_ASM_DEFINE_CMPFAIL(6,
0648                     /* teardown */
0649                     "ldr %[len], %[rseq_scratch2]\n\t"
0650                     "ldr %[dst], %[rseq_scratch1]\n\t"
0651                     "ldr %[src], %[rseq_scratch0]\n\t",
0652                     error1)
0653         RSEQ_ASM_DEFINE_CMPFAIL(7,
0654                     /* teardown */
0655                     "ldr %[len], %[rseq_scratch2]\n\t"
0656                     "ldr %[dst], %[rseq_scratch1]\n\t"
0657                     "ldr %[src], %[rseq_scratch0]\n\t",
0658                     error2)
0659 #endif
0660         "8:\n\t"
0661         : /* gcc asm goto does not allow outputs */
0662         : [cpu_id]      "r" (cpu),
0663           [current_cpu_id]  "m" (rseq_get_abi()->cpu_id),
0664           [rseq_cs]     "m" (rseq_get_abi()->rseq_cs.arch.ptr),
0665           /* final store input */
0666           [v]           "m" (*v),
0667           [expect]      "r" (expect),
0668           [newv]        "r" (newv),
0669           /* try memcpy input */
0670           [dst]         "r" (dst),
0671           [src]         "r" (src),
0672           [len]         "r" (len),
0673           [rseq_scratch0]   "m" (rseq_scratch[0]),
0674           [rseq_scratch1]   "m" (rseq_scratch[1]),
0675           [rseq_scratch2]   "m" (rseq_scratch[2])
0676           RSEQ_INJECT_INPUT
0677         : "r0", "memory", "cc"
0678           RSEQ_INJECT_CLOBBER
0679         : abort, cmpfail
0680 #ifdef RSEQ_COMPARE_TWICE
0681           , error1, error2
0682 #endif
0683     );
0684     rseq_after_asm_goto();
0685     return 0;
0686 abort:
0687     rseq_after_asm_goto();
0688     RSEQ_INJECT_FAILED
0689     return -1;
0690 cmpfail:
0691     rseq_after_asm_goto();
0692     return 1;
0693 #ifdef RSEQ_COMPARE_TWICE
0694 error1:
0695     rseq_after_asm_goto();
0696     rseq_bug("cpu_id comparison failed");
0697 error2:
0698     rseq_after_asm_goto();
0699     rseq_bug("expected value comparison failed");
0700 #endif
0701 }
0702 
0703 static inline __attribute__((always_inline))
0704 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
0705                      void *dst, void *src, size_t len,
0706                      intptr_t newv, int cpu)
0707 {
0708     uint32_t rseq_scratch[3];
0709 
0710     RSEQ_INJECT_C(9)
0711 
0712     __asm__ __volatile__ goto (
0713         RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
0714         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
0715 #ifdef RSEQ_COMPARE_TWICE
0716         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
0717         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
0718 #endif
0719         "str %[src], %[rseq_scratch0]\n\t"
0720         "str %[dst], %[rseq_scratch1]\n\t"
0721         "str %[len], %[rseq_scratch2]\n\t"
0722         /* Start rseq by storing table entry pointer into rseq_cs. */
0723         RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
0724         RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
0725         RSEQ_INJECT_ASM(3)
0726         "ldr r0, %[v]\n\t"
0727         "cmp %[expect], r0\n\t"
0728         "bne 5f\n\t"
0729         RSEQ_INJECT_ASM(4)
0730 #ifdef RSEQ_COMPARE_TWICE
0731         RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
0732         "ldr r0, %[v]\n\t"
0733         "cmp %[expect], r0\n\t"
0734         "bne 7f\n\t"
0735 #endif
0736         /* try memcpy */
0737         "cmp %[len], #0\n\t" \
0738         "beq 333f\n\t" \
0739         "222:\n\t" \
0740         "ldrb %%r0, [%[src]]\n\t" \
0741         "strb %%r0, [%[dst]]\n\t" \
0742         "adds %[src], #1\n\t" \
0743         "adds %[dst], #1\n\t" \
0744         "subs %[len], #1\n\t" \
0745         "bne 222b\n\t" \
0746         "333:\n\t" \
0747         RSEQ_INJECT_ASM(5)
0748         "dmb\n\t"   /* full mb provides store-release */
0749         /* final store */
0750         "str %[newv], %[v]\n\t"
0751         "2:\n\t"
0752         RSEQ_INJECT_ASM(6)
0753         /* teardown */
0754         "ldr %[len], %[rseq_scratch2]\n\t"
0755         "ldr %[dst], %[rseq_scratch1]\n\t"
0756         "ldr %[src], %[rseq_scratch0]\n\t"
0757         "b 8f\n\t"
0758         RSEQ_ASM_DEFINE_ABORT(3, 4,
0759                       /* teardown */
0760                       "ldr %[len], %[rseq_scratch2]\n\t"
0761                       "ldr %[dst], %[rseq_scratch1]\n\t"
0762                       "ldr %[src], %[rseq_scratch0]\n\t",
0763                       abort, 1b, 2b, 4f)
0764         RSEQ_ASM_DEFINE_CMPFAIL(5,
0765                     /* teardown */
0766                     "ldr %[len], %[rseq_scratch2]\n\t"
0767                     "ldr %[dst], %[rseq_scratch1]\n\t"
0768                     "ldr %[src], %[rseq_scratch0]\n\t",
0769                     cmpfail)
0770 #ifdef RSEQ_COMPARE_TWICE
0771         RSEQ_ASM_DEFINE_CMPFAIL(6,
0772                     /* teardown */
0773                     "ldr %[len], %[rseq_scratch2]\n\t"
0774                     "ldr %[dst], %[rseq_scratch1]\n\t"
0775                     "ldr %[src], %[rseq_scratch0]\n\t",
0776                     error1)
0777         RSEQ_ASM_DEFINE_CMPFAIL(7,
0778                     /* teardown */
0779                     "ldr %[len], %[rseq_scratch2]\n\t"
0780                     "ldr %[dst], %[rseq_scratch1]\n\t"
0781                     "ldr %[src], %[rseq_scratch0]\n\t",
0782                     error2)
0783 #endif
0784         "8:\n\t"
0785         : /* gcc asm goto does not allow outputs */
0786         : [cpu_id]      "r" (cpu),
0787           [current_cpu_id]  "m" (rseq_get_abi()->cpu_id),
0788           [rseq_cs]     "m" (rseq_get_abi()->rseq_cs.arch.ptr),
0789           /* final store input */
0790           [v]           "m" (*v),
0791           [expect]      "r" (expect),
0792           [newv]        "r" (newv),
0793           /* try memcpy input */
0794           [dst]         "r" (dst),
0795           [src]         "r" (src),
0796           [len]         "r" (len),
0797           [rseq_scratch0]   "m" (rseq_scratch[0]),
0798           [rseq_scratch1]   "m" (rseq_scratch[1]),
0799           [rseq_scratch2]   "m" (rseq_scratch[2])
0800           RSEQ_INJECT_INPUT
0801         : "r0", "memory", "cc"
0802           RSEQ_INJECT_CLOBBER
0803         : abort, cmpfail
0804 #ifdef RSEQ_COMPARE_TWICE
0805           , error1, error2
0806 #endif
0807     );
0808     rseq_after_asm_goto();
0809     return 0;
0810 abort:
0811     rseq_after_asm_goto();
0812     RSEQ_INJECT_FAILED
0813     return -1;
0814 cmpfail:
0815     rseq_after_asm_goto();
0816     return 1;
0817 #ifdef RSEQ_COMPARE_TWICE
0818 error1:
0819     rseq_after_asm_goto();
0820     rseq_bug("cpu_id comparison failed");
0821 error2:
0822     rseq_after_asm_goto();
0823     rseq_bug("expected value comparison failed");
0824 #endif
0825 }
0826 
0827 #endif /* !RSEQ_SKIP_FASTPATH */
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!