Back to home page

OSCL-LXR
 
 

     

0001 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
0002 /*
0003  * rseq-ppc.h
0004  *
0005  * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
0006  * (C) Copyright 2016-2018 - Boqun Feng <boqun.feng@gmail.com>
0007  */
0008 
0009 /*
0010  * RSEQ_SIG is used with the following trap instruction:
0011  *
0012  * powerpc-be:    0f e5 00 0b           twui   r5,11
0013  * powerpc64-le:  0b 00 e5 0f           twui   r5,11
0014  * powerpc64-be:  0f e5 00 0b           twui   r5,11
0015  */
0016 
0017 #define RSEQ_SIG    0x0fe5000b
0018 
0019 #define rseq_smp_mb()       __asm__ __volatile__ ("sync"    ::: "memory", "cc")
0020 #define rseq_smp_lwsync()   __asm__ __volatile__ ("lwsync"  ::: "memory", "cc")
0021 #define rseq_smp_rmb()      rseq_smp_lwsync()
0022 #define rseq_smp_wmb()      rseq_smp_lwsync()
0023 
0024 #define rseq_smp_load_acquire(p)                    \
0025 __extension__ ({                            \
0026     __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p);           \
0027     rseq_smp_lwsync();                      \
0028     ____p1;                             \
0029 })
0030 
0031 #define rseq_smp_acquire__after_ctrl_dep()  rseq_smp_lwsync()
0032 
0033 #define rseq_smp_store_release(p, v)                    \
0034 do {                                    \
0035     rseq_smp_lwsync();                      \
0036     RSEQ_WRITE_ONCE(*p, v);                     \
0037 } while (0)
0038 
0039 #ifdef RSEQ_SKIP_FASTPATH
0040 #include "rseq-skip.h"
0041 #else /* !RSEQ_SKIP_FASTPATH */
0042 
0043 /*
0044  * The __rseq_cs_ptr_array and __rseq_cs sections can be used by debuggers to
0045  * better handle single-stepping through the restartable critical sections.
0046  */
0047 
0048 #ifdef __PPC64__
0049 
0050 #define RSEQ_STORE_LONG(arg)    "std%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] "    /* To memory ("m" constraint) */
0051 #define RSEQ_STORE_INT(arg) "stw%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] "    /* To memory ("m" constraint) */
0052 #define RSEQ_LOAD_LONG(arg) "ld%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* From memory ("m" constraint) */
0053 #define RSEQ_LOAD_INT(arg)  "lwz%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] "    /* From memory ("m" constraint) */
0054 #define RSEQ_LOADX_LONG     "ldx "                          /* From base register ("b" constraint) */
0055 #define RSEQ_CMP_LONG       "cmpd "
0056 #define RSEQ_CMP_LONG_INT   "cmpdi "
0057 
0058 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,              \
0059             start_ip, post_commit_offset, abort_ip)         \
0060         ".pushsection __rseq_cs, \"aw\"\n\t"                \
0061         ".balign 32\n\t"                        \
0062         __rseq_str(label) ":\n\t"                   \
0063         ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t"  \
0064         ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
0065         ".popsection\n\t"                       \
0066         ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"          \
0067         ".quad " __rseq_str(label) "b\n\t"              \
0068         ".popsection\n\t"
0069 
0070 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)            \
0071         RSEQ_INJECT_ASM(1)                      \
0072         "lis %%r17, (" __rseq_str(cs_label) ")@highest\n\t"     \
0073         "ori %%r17, %%r17, (" __rseq_str(cs_label) ")@higher\n\t"   \
0074         "rldicr %%r17, %%r17, 32, 31\n\t"               \
0075         "oris %%r17, %%r17, (" __rseq_str(cs_label) ")@high\n\t"    \
0076         "ori %%r17, %%r17, (" __rseq_str(cs_label) ")@l\n\t"        \
0077         "std %%r17, %[" __rseq_str(rseq_cs) "]\n\t"         \
0078         __rseq_str(label) ":\n\t"
0079 
0080 /*
0081  * Exit points of a rseq critical section consist of all instructions outside
0082  * of the critical section where a critical section can either branch to or
0083  * reach through the normal course of its execution. The abort IP and the
0084  * post-commit IP are already part of the __rseq_cs section and should not be
0085  * explicitly defined as additional exit points. Knowing all exit points is
0086  * useful to assist debuggers stepping over the critical section.
0087  */
0088 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)           \
0089         ".pushsection __rseq_exit_point_array, \"aw\"\n\t"  \
0090         ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
0091         ".popsection\n\t"
0092 
0093 #else /* #ifdef __PPC64__ */
0094 
0095 #define RSEQ_STORE_LONG(arg)    "stw%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] "    /* To memory ("m" constraint) */
0096 #define RSEQ_STORE_INT(arg) RSEQ_STORE_LONG(arg)                    /* To memory ("m" constraint) */
0097 #define RSEQ_LOAD_LONG(arg) "lwz%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] "    /* From memory ("m" constraint) */
0098 #define RSEQ_LOAD_INT(arg)  RSEQ_LOAD_LONG(arg)                 /* From memory ("m" constraint) */
0099 #define RSEQ_LOADX_LONG     "lwzx "                         /* From base register ("b" constraint) */
0100 #define RSEQ_CMP_LONG       "cmpw "
0101 #define RSEQ_CMP_LONG_INT   "cmpwi "
0102 
0103 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,              \
0104             start_ip, post_commit_offset, abort_ip)         \
0105         ".pushsection __rseq_cs, \"aw\"\n\t"                \
0106         ".balign 32\n\t"                        \
0107         __rseq_str(label) ":\n\t"                   \
0108         ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t"  \
0109         /* 32-bit only supported on BE */               \
0110         ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) "\n\t" \
0111         ".popsection\n\t"                   \
0112         ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"      \
0113         ".long 0x0, " __rseq_str(label) "b\n\t"         \
0114         ".popsection\n\t"
0115 
0116 /*
0117  * Exit points of a rseq critical section consist of all instructions outside
0118  * of the critical section where a critical section can either branch to or
0119  * reach through the normal course of its execution. The abort IP and the
0120  * post-commit IP are already part of the __rseq_cs section and should not be
0121  * explicitly defined as additional exit points. Knowing all exit points is
0122  * useful to assist debuggers stepping over the critical section.
0123  */
0124 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)               \
0125         ".pushsection __rseq_exit_point_array, \"aw\"\n\t"      \
0126         /* 32-bit only supported on BE */               \
0127         ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) "\n\t" \
0128         ".popsection\n\t"
0129 
0130 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)            \
0131         RSEQ_INJECT_ASM(1)                      \
0132         "lis %%r17, (" __rseq_str(cs_label) ")@ha\n\t"          \
0133         "addi %%r17, %%r17, (" __rseq_str(cs_label) ")@l\n\t"       \
0134         RSEQ_STORE_INT(rseq_cs) "%%r17, %[" __rseq_str(rseq_cs) "]\n\t" \
0135         __rseq_str(label) ":\n\t"
0136 
0137 #endif /* #ifdef __PPC64__ */
0138 
0139 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip)    \
0140         __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,      \
0141                     (post_commit_ip - start_ip), abort_ip)
0142 
0143 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label)          \
0144         RSEQ_INJECT_ASM(2)                      \
0145         RSEQ_LOAD_INT(current_cpu_id) "%%r17, %[" __rseq_str(current_cpu_id) "]\n\t" \
0146         "cmpw cr7, %[" __rseq_str(cpu_id) "], %%r17\n\t"        \
0147         "bne- cr7, " __rseq_str(label) "\n\t"
0148 
0149 #define RSEQ_ASM_DEFINE_ABORT(label, abort_label)               \
0150         ".pushsection __rseq_failure, \"ax\"\n\t"           \
0151         ".long " __rseq_str(RSEQ_SIG) "\n\t"                \
0152         __rseq_str(label) ":\n\t"                   \
0153         "b %l[" __rseq_str(abort_label) "]\n\t"             \
0154         ".popsection\n\t"
0155 
0156 /*
0157  * RSEQ_ASM_OPs: asm operations for rseq
0158  *  RSEQ_ASM_OP_R_*: has hard-code registers in it
0159  *  RSEQ_ASM_OP_* (else): doesn't have hard-code registers(unless cr7)
0160  */
0161 #define RSEQ_ASM_OP_CMPEQ(var, expect, label)                   \
0162         RSEQ_LOAD_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t"     \
0163         RSEQ_CMP_LONG "cr7, %%r17, %[" __rseq_str(expect) "]\n\t"       \
0164         "bne- cr7, " __rseq_str(label) "\n\t"
0165 
0166 #define RSEQ_ASM_OP_CMPNE(var, expectnot, label)                \
0167         RSEQ_LOAD_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t"     \
0168         RSEQ_CMP_LONG "cr7, %%r17, %[" __rseq_str(expectnot) "]\n\t"        \
0169         "beq- cr7, " __rseq_str(label) "\n\t"
0170 
0171 #define RSEQ_ASM_OP_STORE(value, var)                       \
0172         RSEQ_STORE_LONG(var) "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t"
0173 
0174 /* Load @var to r17 */
0175 #define RSEQ_ASM_OP_R_LOAD(var)                         \
0176         RSEQ_LOAD_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t"
0177 
0178 /* Store r17 to @var */
0179 #define RSEQ_ASM_OP_R_STORE(var)                        \
0180         RSEQ_STORE_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t"
0181 
0182 /* Add @count to r17 */
0183 #define RSEQ_ASM_OP_R_ADD(count)                        \
0184         "add %%r17, %[" __rseq_str(count) "], %%r17\n\t"
0185 
0186 /* Load (r17 + voffp) to r17 */
0187 #define RSEQ_ASM_OP_R_LOADX(voffp)                      \
0188         RSEQ_LOADX_LONG "%%r17, %[" __rseq_str(voffp) "], %%r17\n\t"
0189 
0190 /* TODO: implement a faster memcpy. */
0191 #define RSEQ_ASM_OP_R_MEMCPY() \
0192         RSEQ_CMP_LONG_INT "%%r19, 0\n\t" \
0193         "beq 333f\n\t" \
0194         "addi %%r20, %%r20, -1\n\t" \
0195         "addi %%r21, %%r21, -1\n\t" \
0196         "222:\n\t" \
0197         "lbzu %%r18, 1(%%r20)\n\t" \
0198         "stbu %%r18, 1(%%r21)\n\t" \
0199         "addi %%r19, %%r19, -1\n\t" \
0200         RSEQ_CMP_LONG_INT "%%r19, 0\n\t" \
0201         "bne 222b\n\t" \
0202         "333:\n\t" \
0203 
0204 #define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label)           \
0205         RSEQ_STORE_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t"            \
0206         __rseq_str(post_commit_label) ":\n\t"
0207 
0208 #define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label)          \
0209         RSEQ_STORE_LONG(var) "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t" \
0210         __rseq_str(post_commit_label) ":\n\t"
0211 
0212 static inline __attribute__((always_inline))
0213 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
0214 {
0215     RSEQ_INJECT_C(9)
0216 
0217     __asm__ __volatile__ goto (
0218         RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
0219         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
0220 #ifdef RSEQ_COMPARE_TWICE
0221         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
0222         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
0223 #endif
0224         /* Start rseq by storing table entry pointer into rseq_cs. */
0225         RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
0226         /* cmp cpuid */
0227         RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
0228         RSEQ_INJECT_ASM(3)
0229         /* cmp @v equal to @expect */
0230         RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
0231         RSEQ_INJECT_ASM(4)
0232 #ifdef RSEQ_COMPARE_TWICE
0233         /* cmp cpuid */
0234         RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
0235         /* cmp @v equal to @expect */
0236         RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
0237 #endif
0238         /* final store */
0239         RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
0240         RSEQ_INJECT_ASM(5)
0241         RSEQ_ASM_DEFINE_ABORT(4, abort)
0242         : /* gcc asm goto does not allow outputs */
0243         : [cpu_id]      "r" (cpu),
0244           [current_cpu_id]  "m" (rseq_get_abi()->cpu_id),
0245           [rseq_cs]     "m" (rseq_get_abi()->rseq_cs.arch.ptr),
0246           [v]           "m" (*v),
0247           [expect]      "r" (expect),
0248           [newv]        "r" (newv)
0249           RSEQ_INJECT_INPUT
0250         : "memory", "cc", "r17"
0251           RSEQ_INJECT_CLOBBER
0252         : abort, cmpfail
0253 #ifdef RSEQ_COMPARE_TWICE
0254           , error1, error2
0255 #endif
0256     );
0257     rseq_after_asm_goto();
0258     return 0;
0259 abort:
0260     rseq_after_asm_goto();
0261     RSEQ_INJECT_FAILED
0262     return -1;
0263 cmpfail:
0264     rseq_after_asm_goto();
0265     return 1;
0266 #ifdef RSEQ_COMPARE_TWICE
0267 error1:
0268     rseq_after_asm_goto();
0269     rseq_bug("cpu_id comparison failed");
0270 error2:
0271     rseq_after_asm_goto();
0272     rseq_bug("expected value comparison failed");
0273 #endif
0274 }
0275 
0276 static inline __attribute__((always_inline))
0277 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
0278                    long voffp, intptr_t *load, int cpu)
0279 {
0280     RSEQ_INJECT_C(9)
0281 
0282     __asm__ __volatile__ goto (
0283         RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
0284         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
0285 #ifdef RSEQ_COMPARE_TWICE
0286         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
0287         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
0288 #endif
0289         /* Start rseq by storing table entry pointer into rseq_cs. */
0290         RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
0291         /* cmp cpuid */
0292         RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
0293         RSEQ_INJECT_ASM(3)
0294         /* cmp @v not equal to @expectnot */
0295         RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail])
0296         RSEQ_INJECT_ASM(4)
0297 #ifdef RSEQ_COMPARE_TWICE
0298         /* cmp cpuid */
0299         RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
0300         /* cmp @v not equal to @expectnot */
0301         RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2])
0302 #endif
0303         /* load the value of @v */
0304         RSEQ_ASM_OP_R_LOAD(v)
0305         /* store it in @load */
0306         RSEQ_ASM_OP_R_STORE(load)
0307         /* dereference voffp(v) */
0308         RSEQ_ASM_OP_R_LOADX(voffp)
0309         /* final store the value at voffp(v) */
0310         RSEQ_ASM_OP_R_FINAL_STORE(v, 2)
0311         RSEQ_INJECT_ASM(5)
0312         RSEQ_ASM_DEFINE_ABORT(4, abort)
0313         : /* gcc asm goto does not allow outputs */
0314         : [cpu_id]      "r" (cpu),
0315           [current_cpu_id]  "m" (rseq_get_abi()->cpu_id),
0316           [rseq_cs]     "m" (rseq_get_abi()->rseq_cs.arch.ptr),
0317           /* final store input */
0318           [v]           "m" (*v),
0319           [expectnot]       "r" (expectnot),
0320           [voffp]       "b" (voffp),
0321           [load]        "m" (*load)
0322           RSEQ_INJECT_INPUT
0323         : "memory", "cc", "r17"
0324           RSEQ_INJECT_CLOBBER
0325         : abort, cmpfail
0326 #ifdef RSEQ_COMPARE_TWICE
0327           , error1, error2
0328 #endif
0329     );
0330     rseq_after_asm_goto();
0331     return 0;
0332 abort:
0333     rseq_after_asm_goto();
0334     RSEQ_INJECT_FAILED
0335     return -1;
0336 cmpfail:
0337     rseq_after_asm_goto();
0338     return 1;
0339 #ifdef RSEQ_COMPARE_TWICE
0340 error1:
0341     rseq_after_asm_goto();
0342     rseq_bug("cpu_id comparison failed");
0343 error2:
0344     rseq_after_asm_goto();
0345     rseq_bug("expected value comparison failed");
0346 #endif
0347 }
0348 
0349 static inline __attribute__((always_inline))
0350 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
0351 {
0352     RSEQ_INJECT_C(9)
0353 
0354     __asm__ __volatile__ goto (
0355         RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
0356 #ifdef RSEQ_COMPARE_TWICE
0357         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
0358 #endif
0359         /* Start rseq by storing table entry pointer into rseq_cs. */
0360         RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
0361         /* cmp cpuid */
0362         RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
0363         RSEQ_INJECT_ASM(3)
0364 #ifdef RSEQ_COMPARE_TWICE
0365         /* cmp cpuid */
0366         RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
0367 #endif
0368         /* load the value of @v */
0369         RSEQ_ASM_OP_R_LOAD(v)
0370         /* add @count to it */
0371         RSEQ_ASM_OP_R_ADD(count)
0372         /* final store */
0373         RSEQ_ASM_OP_R_FINAL_STORE(v, 2)
0374         RSEQ_INJECT_ASM(4)
0375         RSEQ_ASM_DEFINE_ABORT(4, abort)
0376         : /* gcc asm goto does not allow outputs */
0377         : [cpu_id]      "r" (cpu),
0378           [current_cpu_id]  "m" (rseq_get_abi()->cpu_id),
0379           [rseq_cs]     "m" (rseq_get_abi()->rseq_cs.arch.ptr),
0380           /* final store input */
0381           [v]           "m" (*v),
0382           [count]       "r" (count)
0383           RSEQ_INJECT_INPUT
0384         : "memory", "cc", "r17"
0385           RSEQ_INJECT_CLOBBER
0386         : abort
0387 #ifdef RSEQ_COMPARE_TWICE
0388           , error1
0389 #endif
0390     );
0391     rseq_after_asm_goto();
0392     return 0;
0393 abort:
0394     rseq_after_asm_goto();
0395     RSEQ_INJECT_FAILED
0396     return -1;
0397 #ifdef RSEQ_COMPARE_TWICE
0398 error1:
0399     rseq_after_asm_goto();
0400     rseq_bug("cpu_id comparison failed");
0401 #endif
0402 }
0403 
0404 static inline __attribute__((always_inline))
0405 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
0406                  intptr_t *v2, intptr_t newv2,
0407                  intptr_t newv, int cpu)
0408 {
0409     RSEQ_INJECT_C(9)
0410 
0411     __asm__ __volatile__ goto (
0412         RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
0413         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
0414 #ifdef RSEQ_COMPARE_TWICE
0415         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
0416         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
0417 #endif
0418         /* Start rseq by storing table entry pointer into rseq_cs. */
0419         RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
0420         /* cmp cpuid */
0421         RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
0422         RSEQ_INJECT_ASM(3)
0423         /* cmp @v equal to @expect */
0424         RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
0425         RSEQ_INJECT_ASM(4)
0426 #ifdef RSEQ_COMPARE_TWICE
0427         /* cmp cpuid */
0428         RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
0429         /* cmp @v equal to @expect */
0430         RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
0431 #endif
0432         /* try store */
0433         RSEQ_ASM_OP_STORE(newv2, v2)
0434         RSEQ_INJECT_ASM(5)
0435         /* final store */
0436         RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
0437         RSEQ_INJECT_ASM(6)
0438         RSEQ_ASM_DEFINE_ABORT(4, abort)
0439         : /* gcc asm goto does not allow outputs */
0440         : [cpu_id]      "r" (cpu),
0441           [current_cpu_id]  "m" (rseq_get_abi()->cpu_id),
0442           [rseq_cs]     "m" (rseq_get_abi()->rseq_cs.arch.ptr),
0443           /* try store input */
0444           [v2]          "m" (*v2),
0445           [newv2]       "r" (newv2),
0446           /* final store input */
0447           [v]           "m" (*v),
0448           [expect]      "r" (expect),
0449           [newv]        "r" (newv)
0450           RSEQ_INJECT_INPUT
0451         : "memory", "cc", "r17"
0452           RSEQ_INJECT_CLOBBER
0453         : abort, cmpfail
0454 #ifdef RSEQ_COMPARE_TWICE
0455           , error1, error2
0456 #endif
0457     );
0458     rseq_after_asm_goto();
0459     return 0;
0460 abort:
0461     rseq_after_asm_goto();
0462     RSEQ_INJECT_FAILED
0463     return -1;
0464 cmpfail:
0465     rseq_after_asm_goto();
0466     return 1;
0467 #ifdef RSEQ_COMPARE_TWICE
0468 error1:
0469     rseq_after_asm_goto();
0470     rseq_bug("cpu_id comparison failed");
0471 error2:
0472     rseq_after_asm_goto();
0473     rseq_bug("expected value comparison failed");
0474 #endif
0475 }
0476 
0477 static inline __attribute__((always_inline))
0478 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
0479                      intptr_t *v2, intptr_t newv2,
0480                      intptr_t newv, int cpu)
0481 {
0482     RSEQ_INJECT_C(9)
0483 
0484     __asm__ __volatile__ goto (
0485         RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
0486         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
0487 #ifdef RSEQ_COMPARE_TWICE
0488         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
0489         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
0490 #endif
0491         /* Start rseq by storing table entry pointer into rseq_cs. */
0492         RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
0493         /* cmp cpuid */
0494         RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
0495         RSEQ_INJECT_ASM(3)
0496         /* cmp @v equal to @expect */
0497         RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
0498         RSEQ_INJECT_ASM(4)
0499 #ifdef RSEQ_COMPARE_TWICE
0500         /* cmp cpuid */
0501         RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
0502         /* cmp @v equal to @expect */
0503         RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
0504 #endif
0505         /* try store */
0506         RSEQ_ASM_OP_STORE(newv2, v2)
0507         RSEQ_INJECT_ASM(5)
0508         /* for 'release' */
0509         "lwsync\n\t"
0510         /* final store */
0511         RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
0512         RSEQ_INJECT_ASM(6)
0513         RSEQ_ASM_DEFINE_ABORT(4, abort)
0514         : /* gcc asm goto does not allow outputs */
0515         : [cpu_id]      "r" (cpu),
0516           [current_cpu_id]  "m" (rseq_get_abi()->cpu_id),
0517           [rseq_cs]     "m" (rseq_get_abi()->rseq_cs.arch.ptr),
0518           /* try store input */
0519           [v2]          "m" (*v2),
0520           [newv2]       "r" (newv2),
0521           /* final store input */
0522           [v]           "m" (*v),
0523           [expect]      "r" (expect),
0524           [newv]        "r" (newv)
0525           RSEQ_INJECT_INPUT
0526         : "memory", "cc", "r17"
0527           RSEQ_INJECT_CLOBBER
0528         : abort, cmpfail
0529 #ifdef RSEQ_COMPARE_TWICE
0530           , error1, error2
0531 #endif
0532     );
0533     rseq_after_asm_goto();
0534     return 0;
0535 abort:
0536     rseq_after_asm_goto();
0537     RSEQ_INJECT_FAILED
0538     return -1;
0539 cmpfail:
0540     rseq_after_asm_goto();
0541     return 1;
0542 #ifdef RSEQ_COMPARE_TWICE
0543 error1:
0544     rseq_after_asm_goto();
0545     rseq_bug("cpu_id comparison failed");
0546 error2:
0547     rseq_after_asm_goto();
0548     rseq_bug("expected value comparison failed");
0549 #endif
0550 }
0551 
0552 static inline __attribute__((always_inline))
0553 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
0554                   intptr_t *v2, intptr_t expect2,
0555                   intptr_t newv, int cpu)
0556 {
0557     RSEQ_INJECT_C(9)
0558 
0559     __asm__ __volatile__ goto (
0560         RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
0561         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
0562 #ifdef RSEQ_COMPARE_TWICE
0563         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
0564         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
0565         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
0566 #endif
0567         /* Start rseq by storing table entry pointer into rseq_cs. */
0568         RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
0569         /* cmp cpuid */
0570         RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
0571         RSEQ_INJECT_ASM(3)
0572         /* cmp @v equal to @expect */
0573         RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
0574         RSEQ_INJECT_ASM(4)
0575         /* cmp @v2 equal to @expct2 */
0576         RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail])
0577         RSEQ_INJECT_ASM(5)
0578 #ifdef RSEQ_COMPARE_TWICE
0579         /* cmp cpuid */
0580         RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
0581         /* cmp @v equal to @expect */
0582         RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
0583         /* cmp @v2 equal to @expct2 */
0584         RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3])
0585 #endif
0586         /* final store */
0587         RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
0588         RSEQ_INJECT_ASM(6)
0589         RSEQ_ASM_DEFINE_ABORT(4, abort)
0590         : /* gcc asm goto does not allow outputs */
0591         : [cpu_id]      "r" (cpu),
0592           [current_cpu_id]  "m" (rseq_get_abi()->cpu_id),
0593           [rseq_cs]     "m" (rseq_get_abi()->rseq_cs.arch.ptr),
0594           /* cmp2 input */
0595           [v2]          "m" (*v2),
0596           [expect2]     "r" (expect2),
0597           /* final store input */
0598           [v]           "m" (*v),
0599           [expect]      "r" (expect),
0600           [newv]        "r" (newv)
0601           RSEQ_INJECT_INPUT
0602         : "memory", "cc", "r17"
0603           RSEQ_INJECT_CLOBBER
0604         : abort, cmpfail
0605 #ifdef RSEQ_COMPARE_TWICE
0606           , error1, error2, error3
0607 #endif
0608     );
0609     rseq_after_asm_goto();
0610     return 0;
0611 abort:
0612     rseq_after_asm_goto();
0613     RSEQ_INJECT_FAILED
0614     return -1;
0615 cmpfail:
0616     rseq_after_asm_goto();
0617     return 1;
0618 #ifdef RSEQ_COMPARE_TWICE
0619 error1:
0620     rseq_after_asm_goto();
0621     rseq_bug("cpu_id comparison failed");
0622 error2:
0623     rseq_after_asm_goto();
0624     rseq_bug("1st expected value comparison failed");
0625 error3:
0626     rseq_after_asm_goto();
0627     rseq_bug("2nd expected value comparison failed");
0628 #endif
0629 }
0630 
0631 static inline __attribute__((always_inline))
0632 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
0633                  void *dst, void *src, size_t len,
0634                  intptr_t newv, int cpu)
0635 {
0636     RSEQ_INJECT_C(9)
0637 
0638     __asm__ __volatile__ goto (
0639         RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
0640         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
0641 #ifdef RSEQ_COMPARE_TWICE
0642         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
0643         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
0644 #endif
0645         /* setup for mempcy */
0646         "mr %%r19, %[len]\n\t"
0647         "mr %%r20, %[src]\n\t"
0648         "mr %%r21, %[dst]\n\t"
0649         /* Start rseq by storing table entry pointer into rseq_cs. */
0650         RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
0651         /* cmp cpuid */
0652         RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
0653         RSEQ_INJECT_ASM(3)
0654         /* cmp @v equal to @expect */
0655         RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
0656         RSEQ_INJECT_ASM(4)
0657 #ifdef RSEQ_COMPARE_TWICE
0658         /* cmp cpuid */
0659         RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
0660         /* cmp @v equal to @expect */
0661         RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
0662 #endif
0663         /* try memcpy */
0664         RSEQ_ASM_OP_R_MEMCPY()
0665         RSEQ_INJECT_ASM(5)
0666         /* final store */
0667         RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
0668         RSEQ_INJECT_ASM(6)
0669         /* teardown */
0670         RSEQ_ASM_DEFINE_ABORT(4, abort)
0671         : /* gcc asm goto does not allow outputs */
0672         : [cpu_id]      "r" (cpu),
0673           [current_cpu_id]  "m" (rseq_get_abi()->cpu_id),
0674           [rseq_cs]     "m" (rseq_get_abi()->rseq_cs.arch.ptr),
0675           /* final store input */
0676           [v]           "m" (*v),
0677           [expect]      "r" (expect),
0678           [newv]        "r" (newv),
0679           /* try memcpy input */
0680           [dst]         "r" (dst),
0681           [src]         "r" (src),
0682           [len]         "r" (len)
0683           RSEQ_INJECT_INPUT
0684         : "memory", "cc", "r17", "r18", "r19", "r20", "r21"
0685           RSEQ_INJECT_CLOBBER
0686         : abort, cmpfail
0687 #ifdef RSEQ_COMPARE_TWICE
0688           , error1, error2
0689 #endif
0690     );
0691     rseq_after_asm_goto();
0692     return 0;
0693 abort:
0694     rseq_after_asm_goto();
0695     RSEQ_INJECT_FAILED
0696     return -1;
0697 cmpfail:
0698     rseq_after_asm_goto();
0699     return 1;
0700 #ifdef RSEQ_COMPARE_TWICE
0701 error1:
0702     rseq_after_asm_goto();
0703     rseq_bug("cpu_id comparison failed");
0704 error2:
0705     rseq_after_asm_goto();
0706     rseq_bug("expected value comparison failed");
0707 #endif
0708 }
0709 
0710 static inline __attribute__((always_inline))
0711 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
0712                      void *dst, void *src, size_t len,
0713                      intptr_t newv, int cpu)
0714 {
0715     RSEQ_INJECT_C(9)
0716 
0717     __asm__ __volatile__ goto (
0718         RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
0719         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
0720 #ifdef RSEQ_COMPARE_TWICE
0721         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
0722         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
0723 #endif
0724         /* setup for mempcy */
0725         "mr %%r19, %[len]\n\t"
0726         "mr %%r20, %[src]\n\t"
0727         "mr %%r21, %[dst]\n\t"
0728         /* Start rseq by storing table entry pointer into rseq_cs. */
0729         RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
0730         /* cmp cpuid */
0731         RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
0732         RSEQ_INJECT_ASM(3)
0733         /* cmp @v equal to @expect */
0734         RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
0735         RSEQ_INJECT_ASM(4)
0736 #ifdef RSEQ_COMPARE_TWICE
0737         /* cmp cpuid */
0738         RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
0739         /* cmp @v equal to @expect */
0740         RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
0741 #endif
0742         /* try memcpy */
0743         RSEQ_ASM_OP_R_MEMCPY()
0744         RSEQ_INJECT_ASM(5)
0745         /* for 'release' */
0746         "lwsync\n\t"
0747         /* final store */
0748         RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
0749         RSEQ_INJECT_ASM(6)
0750         /* teardown */
0751         RSEQ_ASM_DEFINE_ABORT(4, abort)
0752         : /* gcc asm goto does not allow outputs */
0753         : [cpu_id]      "r" (cpu),
0754           [current_cpu_id]  "m" (rseq_get_abi()->cpu_id),
0755           [rseq_cs]     "m" (rseq_get_abi()->rseq_cs.arch.ptr),
0756           /* final store input */
0757           [v]           "m" (*v),
0758           [expect]      "r" (expect),
0759           [newv]        "r" (newv),
0760           /* try memcpy input */
0761           [dst]         "r" (dst),
0762           [src]         "r" (src),
0763           [len]         "r" (len)
0764           RSEQ_INJECT_INPUT
0765         : "memory", "cc", "r17", "r18", "r19", "r20", "r21"
0766           RSEQ_INJECT_CLOBBER
0767         : abort, cmpfail
0768 #ifdef RSEQ_COMPARE_TWICE
0769           , error1, error2
0770 #endif
0771     );
0772     rseq_after_asm_goto();
0773     return 0;
0774 abort:
0775     rseq_after_asm_goto();
0776     RSEQ_INJECT_FAILED
0777     return -1;
0778 cmpfail:
0779     rseq_after_asm_goto();
0780     return 1;
0781 #ifdef RSEQ_COMPARE_TWICE
0782 error1:
0783     rseq_after_asm_goto();
0784     rseq_bug("cpu_id comparison failed");
0785 error2:
0786     rseq_after_asm_goto();
0787     rseq_bug("expected value comparison failed");
0788 #endif
0789 }
0790 
0791 #endif /* !RSEQ_SKIP_FASTPATH */
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!