Back to home page

OSCL-LXR
 
 

     

0001 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
0002 
0003 /*
0004  * RSEQ_SIG uses the trap4 instruction. As Linux does not make use of the
0005  * access-register mode nor the linkage stack this instruction will always
0006  * cause a special-operation exception (the trap-enabled bit in the DUCT
0007  * is and will stay 0). The instruction pattern is
0008  *  b2 ff 0f ff trap4   4095(%r0)
0009  */
0010 #define RSEQ_SIG    0xB2FF0FFF
0011 
0012 #define rseq_smp_mb()   __asm__ __volatile__ ("bcr 15,0" ::: "memory")
0013 #define rseq_smp_rmb()  rseq_smp_mb()
0014 #define rseq_smp_wmb()  rseq_smp_mb()
0015 
0016 #define rseq_smp_load_acquire(p)                    \
0017 __extension__ ({                            \
0018     __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p);           \
0019     rseq_barrier();                         \
0020     ____p1;                             \
0021 })
0022 
0023 #define rseq_smp_acquire__after_ctrl_dep()  rseq_smp_rmb()
0024 
0025 #define rseq_smp_store_release(p, v)                    \
0026 do {                                    \
0027     rseq_barrier();                         \
0028     RSEQ_WRITE_ONCE(*p, v);                     \
0029 } while (0)
0030 
0031 #ifdef RSEQ_SKIP_FASTPATH
0032 #include "rseq-skip.h"
0033 #else /* !RSEQ_SKIP_FASTPATH */
0034 
0035 #ifdef __s390x__
0036 
0037 #define LONG_L          "lg"
0038 #define LONG_S          "stg"
0039 #define LONG_LT_R       "ltgr"
0040 #define LONG_CMP        "cg"
0041 #define LONG_CMP_R      "cgr"
0042 #define LONG_ADDI       "aghi"
0043 #define LONG_ADD_R      "agr"
0044 
0045 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,          \
0046                 start_ip, post_commit_offset, abort_ip) \
0047         ".pushsection __rseq_cs, \"aw\"\n\t"            \
0048         ".balign 32\n\t"                    \
0049         __rseq_str(label) ":\n\t"               \
0050         ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
0051         ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
0052         ".popsection\n\t"                   \
0053         ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"      \
0054         ".quad " __rseq_str(label) "b\n\t"          \
0055         ".popsection\n\t"
0056 
0057 /*
0058  * Exit points of a rseq critical section consist of all instructions outside
0059  * of the critical section where a critical section can either branch to or
0060  * reach through the normal course of its execution. The abort IP and the
0061  * post-commit IP are already part of the __rseq_cs section and should not be
0062  * explicitly defined as additional exit points. Knowing all exit points is
0063  * useful to assist debuggers stepping over the critical section.
0064  */
0065 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)           \
0066         ".pushsection __rseq_exit_point_array, \"aw\"\n\t"  \
0067         ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
0068         ".popsection\n\t"
0069 
0070 #elif __s390__
0071 
0072 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,          \
0073                 start_ip, post_commit_offset, abort_ip) \
0074         ".pushsection __rseq_cs, \"aw\"\n\t"            \
0075         ".balign 32\n\t"                    \
0076         __rseq_str(label) ":\n\t"               \
0077         ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
0078         ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) "\n\t" \
0079         ".popsection\n\t"                   \
0080         ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"      \
0081         ".long 0x0, " __rseq_str(label) "b\n\t"         \
0082         ".popsection\n\t"
0083 
0084 /*
0085  * Exit points of a rseq critical section consist of all instructions outside
0086  * of the critical section where a critical section can either branch to or
0087  * reach through the normal course of its execution. The abort IP and the
0088  * post-commit IP are already part of the __rseq_cs section and should not be
0089  * explicitly defined as additional exit points. Knowing all exit points is
0090  * useful to assist debuggers stepping over the critical section.
0091  */
0092 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)           \
0093         ".pushsection __rseq_exit_point_array, \"aw\"\n\t"  \
0094         ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) "\n\t" \
0095         ".popsection\n\t"
0096 
0097 #define LONG_L          "l"
0098 #define LONG_S          "st"
0099 #define LONG_LT_R       "ltr"
0100 #define LONG_CMP        "c"
0101 #define LONG_CMP_R      "cr"
0102 #define LONG_ADDI       "ahi"
0103 #define LONG_ADD_R      "ar"
0104 
0105 #endif
0106 
0107 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
0108     __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,      \
0109                 (post_commit_ip - start_ip), abort_ip)
0110 
0111 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)        \
0112         RSEQ_INJECT_ASM(1)                  \
0113         "larl %%r0, " __rseq_str(cs_label) "\n\t"       \
0114         LONG_S " %%r0, %[" __rseq_str(rseq_cs) "]\n\t"      \
0115         __rseq_str(label) ":\n\t"
0116 
0117 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label)      \
0118         RSEQ_INJECT_ASM(2)                  \
0119         "c %[" __rseq_str(cpu_id) "], %[" __rseq_str(current_cpu_id) "]\n\t" \
0120         "jnz " __rseq_str(label) "\n\t"
0121 
0122 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label)     \
0123         ".pushsection __rseq_failure, \"ax\"\n\t"       \
0124         ".long " __rseq_str(RSEQ_SIG) "\n\t"            \
0125         __rseq_str(label) ":\n\t"               \
0126         teardown                        \
0127         "jg %l[" __rseq_str(abort_label) "]\n\t"        \
0128         ".popsection\n\t"
0129 
0130 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label)     \
0131         ".pushsection __rseq_failure, \"ax\"\n\t"       \
0132         __rseq_str(label) ":\n\t"               \
0133         teardown                        \
0134         "jg %l[" __rseq_str(cmpfail_label) "]\n\t"      \
0135         ".popsection\n\t"
0136 
0137 static inline __attribute__((always_inline))
0138 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
0139 {
0140     RSEQ_INJECT_C(9)
0141 
0142     __asm__ __volatile__ goto (
0143         RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
0144         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
0145 #ifdef RSEQ_COMPARE_TWICE
0146         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
0147         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
0148 #endif
0149         /* Start rseq by storing table entry pointer into rseq_cs. */
0150         RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
0151         RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
0152         RSEQ_INJECT_ASM(3)
0153         LONG_CMP " %[expect], %[v]\n\t"
0154         "jnz %l[cmpfail]\n\t"
0155         RSEQ_INJECT_ASM(4)
0156 #ifdef RSEQ_COMPARE_TWICE
0157         RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
0158         LONG_CMP " %[expect], %[v]\n\t"
0159         "jnz %l[error2]\n\t"
0160 #endif
0161         /* final store */
0162         LONG_S " %[newv], %[v]\n\t"
0163         "2:\n\t"
0164         RSEQ_INJECT_ASM(5)
0165         RSEQ_ASM_DEFINE_ABORT(4, "", abort)
0166         : /* gcc asm goto does not allow outputs */
0167         : [cpu_id]      "r" (cpu),
0168           [current_cpu_id]  "m" (rseq_get_abi()->cpu_id),
0169           [rseq_cs]     "m" (rseq_get_abi()->rseq_cs.arch.ptr),
0170           [v]           "m" (*v),
0171           [expect]      "r" (expect),
0172           [newv]        "r" (newv)
0173           RSEQ_INJECT_INPUT
0174         : "memory", "cc", "r0"
0175           RSEQ_INJECT_CLOBBER
0176         : abort, cmpfail
0177 #ifdef RSEQ_COMPARE_TWICE
0178           , error1, error2
0179 #endif
0180     );
0181     rseq_after_asm_goto();
0182     return 0;
0183 abort:
0184     rseq_after_asm_goto();
0185     RSEQ_INJECT_FAILED
0186     return -1;
0187 cmpfail:
0188     rseq_after_asm_goto();
0189     return 1;
0190 #ifdef RSEQ_COMPARE_TWICE
0191 error1:
0192     rseq_after_asm_goto();
0193     rseq_bug("cpu_id comparison failed");
0194 error2:
0195     rseq_after_asm_goto();
0196     rseq_bug("expected value comparison failed");
0197 #endif
0198 }
0199 
0200 /*
0201  * Compare @v against @expectnot. When it does _not_ match, load @v
0202  * into @load, and store the content of *@v + voffp into @v.
0203  */
0204 static inline __attribute__((always_inline))
0205 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
0206                    long voffp, intptr_t *load, int cpu)
0207 {
0208     RSEQ_INJECT_C(9)
0209 
0210     __asm__ __volatile__ goto (
0211         RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
0212         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
0213 #ifdef RSEQ_COMPARE_TWICE
0214         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
0215         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
0216 #endif
0217         /* Start rseq by storing table entry pointer into rseq_cs. */
0218         RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
0219         RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
0220         RSEQ_INJECT_ASM(3)
0221         LONG_L " %%r1, %[v]\n\t"
0222         LONG_CMP_R " %%r1, %[expectnot]\n\t"
0223         "je %l[cmpfail]\n\t"
0224         RSEQ_INJECT_ASM(4)
0225 #ifdef RSEQ_COMPARE_TWICE
0226         RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
0227         LONG_L " %%r1, %[v]\n\t"
0228         LONG_CMP_R " %%r1, %[expectnot]\n\t"
0229         "je %l[error2]\n\t"
0230 #endif
0231         LONG_S " %%r1, %[load]\n\t"
0232         LONG_ADD_R " %%r1, %[voffp]\n\t"
0233         LONG_L " %%r1, 0(%%r1)\n\t"
0234         /* final store */
0235         LONG_S " %%r1, %[v]\n\t"
0236         "2:\n\t"
0237         RSEQ_INJECT_ASM(5)
0238         RSEQ_ASM_DEFINE_ABORT(4, "", abort)
0239         : /* gcc asm goto does not allow outputs */
0240         : [cpu_id]      "r" (cpu),
0241           [current_cpu_id]  "m" (rseq_get_abi()->cpu_id),
0242           [rseq_cs]     "m" (rseq_get_abi()->rseq_cs.arch.ptr),
0243           /* final store input */
0244           [v]           "m" (*v),
0245           [expectnot]       "r" (expectnot),
0246           [voffp]       "r" (voffp),
0247           [load]        "m" (*load)
0248           RSEQ_INJECT_INPUT
0249         : "memory", "cc", "r0", "r1"
0250           RSEQ_INJECT_CLOBBER
0251         : abort, cmpfail
0252 #ifdef RSEQ_COMPARE_TWICE
0253           , error1, error2
0254 #endif
0255     );
0256     rseq_after_asm_goto();
0257     return 0;
0258 abort:
0259     rseq_after_asm_goto();
0260     RSEQ_INJECT_FAILED
0261     return -1;
0262 cmpfail:
0263     rseq_after_asm_goto();
0264     return 1;
0265 #ifdef RSEQ_COMPARE_TWICE
0266 error1:
0267     rseq_after_asm_goto();
0268     rseq_bug("cpu_id comparison failed");
0269 error2:
0270     rseq_after_asm_goto();
0271     rseq_bug("expected value comparison failed");
0272 #endif
0273 }
0274 
0275 static inline __attribute__((always_inline))
0276 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
0277 {
0278     RSEQ_INJECT_C(9)
0279 
0280     __asm__ __volatile__ goto (
0281         RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
0282 #ifdef RSEQ_COMPARE_TWICE
0283         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
0284 #endif
0285         /* Start rseq by storing table entry pointer into rseq_cs. */
0286         RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
0287         RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
0288         RSEQ_INJECT_ASM(3)
0289 #ifdef RSEQ_COMPARE_TWICE
0290         RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
0291 #endif
0292         LONG_L " %%r0, %[v]\n\t"
0293         LONG_ADD_R " %%r0, %[count]\n\t"
0294         /* final store */
0295         LONG_S " %%r0, %[v]\n\t"
0296         "2:\n\t"
0297         RSEQ_INJECT_ASM(4)
0298         RSEQ_ASM_DEFINE_ABORT(4, "", abort)
0299         : /* gcc asm goto does not allow outputs */
0300         : [cpu_id]      "r" (cpu),
0301           [current_cpu_id]  "m" (rseq_get_abi()->cpu_id),
0302           [rseq_cs]     "m" (rseq_get_abi()->rseq_cs.arch.ptr),
0303           /* final store input */
0304           [v]           "m" (*v),
0305           [count]       "r" (count)
0306           RSEQ_INJECT_INPUT
0307         : "memory", "cc", "r0"
0308           RSEQ_INJECT_CLOBBER
0309         : abort
0310 #ifdef RSEQ_COMPARE_TWICE
0311           , error1
0312 #endif
0313     );
0314     rseq_after_asm_goto();
0315     return 0;
0316 abort:
0317     rseq_after_asm_goto();
0318     RSEQ_INJECT_FAILED
0319     return -1;
0320 #ifdef RSEQ_COMPARE_TWICE
0321 error1:
0322     rseq_after_asm_goto();
0323     rseq_bug("cpu_id comparison failed");
0324 #endif
0325 }
0326 
0327 static inline __attribute__((always_inline))
0328 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
0329                  intptr_t *v2, intptr_t newv2,
0330                  intptr_t newv, int cpu)
0331 {
0332     RSEQ_INJECT_C(9)
0333 
0334     __asm__ __volatile__ goto (
0335         RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
0336         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
0337 #ifdef RSEQ_COMPARE_TWICE
0338         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
0339         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
0340 #endif
0341         /* Start rseq by storing table entry pointer into rseq_cs. */
0342         RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
0343         RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
0344         RSEQ_INJECT_ASM(3)
0345         LONG_CMP " %[expect], %[v]\n\t"
0346         "jnz %l[cmpfail]\n\t"
0347         RSEQ_INJECT_ASM(4)
0348 #ifdef RSEQ_COMPARE_TWICE
0349         RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
0350         LONG_CMP " %[expect], %[v]\n\t"
0351         "jnz %l[error2]\n\t"
0352 #endif
0353         /* try store */
0354         LONG_S " %[newv2], %[v2]\n\t"
0355         RSEQ_INJECT_ASM(5)
0356         /* final store */
0357         LONG_S " %[newv], %[v]\n\t"
0358         "2:\n\t"
0359         RSEQ_INJECT_ASM(6)
0360         RSEQ_ASM_DEFINE_ABORT(4, "", abort)
0361         : /* gcc asm goto does not allow outputs */
0362         : [cpu_id]      "r" (cpu),
0363           [current_cpu_id]  "m" (rseq_get_abi()->cpu_id),
0364           [rseq_cs]     "m" (rseq_get_abi()->rseq_cs.arch.ptr),
0365           /* try store input */
0366           [v2]          "m" (*v2),
0367           [newv2]       "r" (newv2),
0368           /* final store input */
0369           [v]           "m" (*v),
0370           [expect]      "r" (expect),
0371           [newv]        "r" (newv)
0372           RSEQ_INJECT_INPUT
0373         : "memory", "cc", "r0"
0374           RSEQ_INJECT_CLOBBER
0375         : abort, cmpfail
0376 #ifdef RSEQ_COMPARE_TWICE
0377           , error1, error2
0378 #endif
0379     );
0380     rseq_after_asm_goto();
0381     return 0;
0382 abort:
0383     rseq_after_asm_goto();
0384     RSEQ_INJECT_FAILED
0385     return -1;
0386 cmpfail:
0387     rseq_after_asm_goto();
0388     return 1;
0389 #ifdef RSEQ_COMPARE_TWICE
0390 error1:
0391     rseq_after_asm_goto();
0392     rseq_bug("cpu_id comparison failed");
0393 error2:
0394     rseq_after_asm_goto();
0395     rseq_bug("expected value comparison failed");
0396 #endif
0397 }
0398 
0399 /* s390 is TSO. */
0400 static inline __attribute__((always_inline))
0401 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
0402                      intptr_t *v2, intptr_t newv2,
0403                      intptr_t newv, int cpu)
0404 {
0405     return rseq_cmpeqv_trystorev_storev(v, expect, v2, newv2, newv, cpu);
0406 }
0407 
0408 static inline __attribute__((always_inline))
0409 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
0410                   intptr_t *v2, intptr_t expect2,
0411                   intptr_t newv, int cpu)
0412 {
0413     RSEQ_INJECT_C(9)
0414 
0415     __asm__ __volatile__ goto (
0416         RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
0417         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
0418 #ifdef RSEQ_COMPARE_TWICE
0419         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
0420         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
0421         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
0422 #endif
0423         /* Start rseq by storing table entry pointer into rseq_cs. */
0424         RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
0425         RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
0426         RSEQ_INJECT_ASM(3)
0427         LONG_CMP " %[expect], %[v]\n\t"
0428         "jnz %l[cmpfail]\n\t"
0429         RSEQ_INJECT_ASM(4)
0430         LONG_CMP " %[expect2], %[v2]\n\t"
0431         "jnz %l[cmpfail]\n\t"
0432         RSEQ_INJECT_ASM(5)
0433 #ifdef RSEQ_COMPARE_TWICE
0434         RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
0435         LONG_CMP " %[expect], %[v]\n\t"
0436         "jnz %l[error2]\n\t"
0437         LONG_CMP " %[expect2], %[v2]\n\t"
0438         "jnz %l[error3]\n\t"
0439 #endif
0440         /* final store */
0441         LONG_S " %[newv], %[v]\n\t"
0442         "2:\n\t"
0443         RSEQ_INJECT_ASM(6)
0444         RSEQ_ASM_DEFINE_ABORT(4, "", abort)
0445         : /* gcc asm goto does not allow outputs */
0446         : [cpu_id]      "r" (cpu),
0447           [current_cpu_id]  "m" (rseq_get_abi()->cpu_id),
0448           [rseq_cs]     "m" (rseq_get_abi()->rseq_cs.arch.ptr),
0449           /* cmp2 input */
0450           [v2]          "m" (*v2),
0451           [expect2]     "r" (expect2),
0452           /* final store input */
0453           [v]           "m" (*v),
0454           [expect]      "r" (expect),
0455           [newv]        "r" (newv)
0456           RSEQ_INJECT_INPUT
0457         : "memory", "cc", "r0"
0458           RSEQ_INJECT_CLOBBER
0459         : abort, cmpfail
0460 #ifdef RSEQ_COMPARE_TWICE
0461           , error1, error2, error3
0462 #endif
0463     );
0464     rseq_after_asm_goto();
0465     return 0;
0466 abort:
0467     rseq_after_asm_goto();
0468     RSEQ_INJECT_FAILED
0469     return -1;
0470 cmpfail:
0471     rseq_after_asm_goto();
0472     return 1;
0473 #ifdef RSEQ_COMPARE_TWICE
0474 error1:
0475     rseq_after_asm_goto();
0476     rseq_bug("cpu_id comparison failed");
0477 error2:
0478     rseq_after_asm_goto();
0479     rseq_bug("1st expected value comparison failed");
0480 error3:
0481     rseq_after_asm_goto();
0482     rseq_bug("2nd expected value comparison failed");
0483 #endif
0484 }
0485 
0486 static inline __attribute__((always_inline))
0487 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
0488                  void *dst, void *src, size_t len,
0489                  intptr_t newv, int cpu)
0490 {
0491     uint64_t rseq_scratch[3];
0492 
0493     RSEQ_INJECT_C(9)
0494 
0495     __asm__ __volatile__ goto (
0496         RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
0497         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
0498 #ifdef RSEQ_COMPARE_TWICE
0499         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
0500         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
0501 #endif
0502         LONG_S " %[src], %[rseq_scratch0]\n\t"
0503         LONG_S " %[dst], %[rseq_scratch1]\n\t"
0504         LONG_S " %[len], %[rseq_scratch2]\n\t"
0505         /* Start rseq by storing table entry pointer into rseq_cs. */
0506         RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
0507         RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
0508         RSEQ_INJECT_ASM(3)
0509         LONG_CMP " %[expect], %[v]\n\t"
0510         "jnz 5f\n\t"
0511         RSEQ_INJECT_ASM(4)
0512 #ifdef RSEQ_COMPARE_TWICE
0513         RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
0514         LONG_CMP " %[expect], %[v]\n\t"
0515         "jnz 7f\n\t"
0516 #endif
0517         /* try memcpy */
0518         LONG_LT_R " %[len], %[len]\n\t"
0519         "jz 333f\n\t"
0520         "222:\n\t"
0521         "ic %%r0,0(%[src])\n\t"
0522         "stc %%r0,0(%[dst])\n\t"
0523         LONG_ADDI " %[src], 1\n\t"
0524         LONG_ADDI " %[dst], 1\n\t"
0525         LONG_ADDI " %[len], -1\n\t"
0526         "jnz 222b\n\t"
0527         "333:\n\t"
0528         RSEQ_INJECT_ASM(5)
0529         /* final store */
0530         LONG_S " %[newv], %[v]\n\t"
0531         "2:\n\t"
0532         RSEQ_INJECT_ASM(6)
0533         /* teardown */
0534         LONG_L " %[len], %[rseq_scratch2]\n\t"
0535         LONG_L " %[dst], %[rseq_scratch1]\n\t"
0536         LONG_L " %[src], %[rseq_scratch0]\n\t"
0537         RSEQ_ASM_DEFINE_ABORT(4,
0538             LONG_L " %[len], %[rseq_scratch2]\n\t"
0539             LONG_L " %[dst], %[rseq_scratch1]\n\t"
0540             LONG_L " %[src], %[rseq_scratch0]\n\t",
0541             abort)
0542         RSEQ_ASM_DEFINE_CMPFAIL(5,
0543             LONG_L " %[len], %[rseq_scratch2]\n\t"
0544             LONG_L " %[dst], %[rseq_scratch1]\n\t"
0545             LONG_L " %[src], %[rseq_scratch0]\n\t",
0546             cmpfail)
0547 #ifdef RSEQ_COMPARE_TWICE
0548         RSEQ_ASM_DEFINE_CMPFAIL(6,
0549             LONG_L " %[len], %[rseq_scratch2]\n\t"
0550             LONG_L " %[dst], %[rseq_scratch1]\n\t"
0551             LONG_L " %[src], %[rseq_scratch0]\n\t",
0552             error1)
0553         RSEQ_ASM_DEFINE_CMPFAIL(7,
0554             LONG_L " %[len], %[rseq_scratch2]\n\t"
0555             LONG_L " %[dst], %[rseq_scratch1]\n\t"
0556             LONG_L " %[src], %[rseq_scratch0]\n\t",
0557             error2)
0558 #endif
0559         : /* gcc asm goto does not allow outputs */
0560         : [cpu_id]      "r" (cpu),
0561           [current_cpu_id]  "m" (rseq_get_abi()->cpu_id),
0562           [rseq_cs]     "m" (rseq_get_abi()->rseq_cs.arch.ptr),
0563           /* final store input */
0564           [v]           "m" (*v),
0565           [expect]      "r" (expect),
0566           [newv]        "r" (newv),
0567           /* try memcpy input */
0568           [dst]         "r" (dst),
0569           [src]         "r" (src),
0570           [len]         "r" (len),
0571           [rseq_scratch0]   "m" (rseq_scratch[0]),
0572           [rseq_scratch1]   "m" (rseq_scratch[1]),
0573           [rseq_scratch2]   "m" (rseq_scratch[2])
0574           RSEQ_INJECT_INPUT
0575         : "memory", "cc", "r0"
0576           RSEQ_INJECT_CLOBBER
0577         : abort, cmpfail
0578 #ifdef RSEQ_COMPARE_TWICE
0579           , error1, error2
0580 #endif
0581     );
0582     rseq_after_asm_goto();
0583     return 0;
0584 abort:
0585     rseq_after_asm_goto();
0586     RSEQ_INJECT_FAILED
0587     return -1;
0588 cmpfail:
0589     rseq_after_asm_goto();
0590     return 1;
0591 #ifdef RSEQ_COMPARE_TWICE
0592 error1:
0593     rseq_after_asm_goto();
0594     rseq_bug("cpu_id comparison failed");
0595 error2:
0596     rseq_after_asm_goto();
0597     rseq_bug("expected value comparison failed");
0598 #endif
0599 }
0600 
0601 /* s390 is TSO. */
0602 static inline __attribute__((always_inline))
0603 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
0604                      void *dst, void *src, size_t len,
0605                      intptr_t newv, int cpu)
0606 {
0607     return rseq_cmpeqv_trymemcpy_storev(v, expect, dst, src, len,
0608                         newv, cpu);
0609 }
0610 #endif /* !RSEQ_SKIP_FASTPATH */
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!