Back to home page

OSCL-LXR
 
 

     

0001 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
0002 /*
0003  * rseq-x86.h
0004  *
0005  * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
0006  */
0007 
0008 #include <stdint.h>
0009 
0010 /*
0011  * RSEQ_SIG is used with the following reserved undefined instructions, which
0012  * trap in user-space:
0013  *
0014  * x86-32:    0f b9 3d 53 30 05 53      ud1    0x53053053,%edi
0015  * x86-64:    0f b9 3d 53 30 05 53      ud1    0x53053053(%rip),%edi
0016  */
0017 #define RSEQ_SIG    0x53053053
0018 
0019 /*
0020  * Due to a compiler optimization bug in gcc-8 with asm goto and TLS asm input
0021  * operands, we cannot use "m" input operands, and rather pass the __rseq_abi
0022  * address through a "r" input operand.
0023  */
0024 
0025 /* Offset of cpu_id and rseq_cs fields in struct rseq. */
0026 #define RSEQ_CPU_ID_OFFSET  4
0027 #define RSEQ_CS_OFFSET      8
0028 
0029 #ifdef __x86_64__
0030 
0031 #define RSEQ_ASM_TP_SEGMENT %%fs
0032 
0033 #define rseq_smp_mb()   \
0034     __asm__ __volatile__ ("lock; addl $0,-128(%%rsp)" ::: "memory", "cc")
0035 #define rseq_smp_rmb()  rseq_barrier()
0036 #define rseq_smp_wmb()  rseq_barrier()
0037 
0038 #define rseq_smp_load_acquire(p)                    \
0039 __extension__ ({                            \
0040     __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p);           \
0041     rseq_barrier();                         \
0042     ____p1;                             \
0043 })
0044 
0045 #define rseq_smp_acquire__after_ctrl_dep()  rseq_smp_rmb()
0046 
0047 #define rseq_smp_store_release(p, v)                    \
0048 do {                                    \
0049     rseq_barrier();                         \
0050     RSEQ_WRITE_ONCE(*p, v);                     \
0051 } while (0)
0052 
0053 #ifdef RSEQ_SKIP_FASTPATH
0054 #include "rseq-skip.h"
0055 #else /* !RSEQ_SKIP_FASTPATH */
0056 
0057 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,          \
0058                 start_ip, post_commit_offset, abort_ip) \
0059         ".pushsection __rseq_cs, \"aw\"\n\t"            \
0060         ".balign 32\n\t"                    \
0061         __rseq_str(label) ":\n\t"               \
0062         ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
0063         ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
0064         ".popsection\n\t"                   \
0065         ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"      \
0066         ".quad " __rseq_str(label) "b\n\t"          \
0067         ".popsection\n\t"
0068 
0069 
0070 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
0071     __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,      \
0072                 (post_commit_ip - start_ip), abort_ip)
0073 
0074 /*
0075  * Exit points of a rseq critical section consist of all instructions outside
0076  * of the critical section where a critical section can either branch to or
0077  * reach through the normal course of its execution. The abort IP and the
0078  * post-commit IP are already part of the __rseq_cs section and should not be
0079  * explicitly defined as additional exit points. Knowing all exit points is
0080  * useful to assist debuggers stepping over the critical section.
0081  */
0082 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)           \
0083         ".pushsection __rseq_exit_point_array, \"aw\"\n\t"  \
0084         ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
0085         ".popsection\n\t"
0086 
0087 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)        \
0088         RSEQ_INJECT_ASM(1)                  \
0089         "leaq " __rseq_str(cs_label) "(%%rip), %%rax\n\t"   \
0090         "movq %%rax, " __rseq_str(rseq_cs) "\n\t"       \
0091         __rseq_str(label) ":\n\t"
0092 
0093 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label)      \
0094         RSEQ_INJECT_ASM(2)                  \
0095         "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \
0096         "jnz " __rseq_str(label) "\n\t"
0097 
0098 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label)     \
0099         ".pushsection __rseq_failure, \"ax\"\n\t"       \
0100         /* Disassembler-friendly signature: ud1 <sig>(%rip),%edi. */ \
0101         ".byte 0x0f, 0xb9, 0x3d\n\t"                \
0102         ".long " __rseq_str(RSEQ_SIG) "\n\t"            \
0103         __rseq_str(label) ":\n\t"               \
0104         teardown                        \
0105         "jmp %l[" __rseq_str(abort_label) "]\n\t"       \
0106         ".popsection\n\t"
0107 
0108 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label)     \
0109         ".pushsection __rseq_failure, \"ax\"\n\t"       \
0110         __rseq_str(label) ":\n\t"               \
0111         teardown                        \
0112         "jmp %l[" __rseq_str(cmpfail_label) "]\n\t"     \
0113         ".popsection\n\t"
0114 
0115 static inline __attribute__((always_inline))
0116 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
0117 {
0118     RSEQ_INJECT_C(9)
0119 
0120     __asm__ __volatile__ goto (
0121         RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
0122         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
0123 #ifdef RSEQ_COMPARE_TWICE
0124         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
0125         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
0126 #endif
0127         /* Start rseq by storing table entry pointer into rseq_cs. */
0128         RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
0129         RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
0130         RSEQ_INJECT_ASM(3)
0131         "cmpq %[v], %[expect]\n\t"
0132         "jnz %l[cmpfail]\n\t"
0133         RSEQ_INJECT_ASM(4)
0134 #ifdef RSEQ_COMPARE_TWICE
0135         RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
0136         "cmpq %[v], %[expect]\n\t"
0137         "jnz %l[error2]\n\t"
0138 #endif
0139         /* final store */
0140         "movq %[newv], %[v]\n\t"
0141         "2:\n\t"
0142         RSEQ_INJECT_ASM(5)
0143         RSEQ_ASM_DEFINE_ABORT(4, "", abort)
0144         : /* gcc asm goto does not allow outputs */
0145         : [cpu_id]      "r" (cpu),
0146           [rseq_offset]     "r" (rseq_offset),
0147           [v]           "m" (*v),
0148           [expect]      "r" (expect),
0149           [newv]        "r" (newv)
0150         : "memory", "cc", "rax"
0151           RSEQ_INJECT_CLOBBER
0152         : abort, cmpfail
0153 #ifdef RSEQ_COMPARE_TWICE
0154           , error1, error2
0155 #endif
0156     );
0157     rseq_after_asm_goto();
0158     return 0;
0159 abort:
0160     rseq_after_asm_goto();
0161     RSEQ_INJECT_FAILED
0162     return -1;
0163 cmpfail:
0164     rseq_after_asm_goto();
0165     return 1;
0166 #ifdef RSEQ_COMPARE_TWICE
0167 error1:
0168     rseq_after_asm_goto();
0169     rseq_bug("cpu_id comparison failed");
0170 error2:
0171     rseq_after_asm_goto();
0172     rseq_bug("expected value comparison failed");
0173 #endif
0174 }
0175 
0176 /*
0177  * Compare @v against @expectnot. When it does _not_ match, load @v
0178  * into @load, and store the content of *@v + voffp into @v.
0179  */
0180 static inline __attribute__((always_inline))
0181 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
0182                    long voffp, intptr_t *load, int cpu)
0183 {
0184     RSEQ_INJECT_C(9)
0185 
0186     __asm__ __volatile__ goto (
0187         RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
0188         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
0189 #ifdef RSEQ_COMPARE_TWICE
0190         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
0191         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
0192 #endif
0193         /* Start rseq by storing table entry pointer into rseq_cs. */
0194         RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
0195         RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
0196         RSEQ_INJECT_ASM(3)
0197         "movq %[v], %%rbx\n\t"
0198         "cmpq %%rbx, %[expectnot]\n\t"
0199         "je %l[cmpfail]\n\t"
0200         RSEQ_INJECT_ASM(4)
0201 #ifdef RSEQ_COMPARE_TWICE
0202         RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
0203         "movq %[v], %%rbx\n\t"
0204         "cmpq %%rbx, %[expectnot]\n\t"
0205         "je %l[error2]\n\t"
0206 #endif
0207         "movq %%rbx, %[load]\n\t"
0208         "addq %[voffp], %%rbx\n\t"
0209         "movq (%%rbx), %%rbx\n\t"
0210         /* final store */
0211         "movq %%rbx, %[v]\n\t"
0212         "2:\n\t"
0213         RSEQ_INJECT_ASM(5)
0214         RSEQ_ASM_DEFINE_ABORT(4, "", abort)
0215         : /* gcc asm goto does not allow outputs */
0216         : [cpu_id]      "r" (cpu),
0217           [rseq_offset]     "r" (rseq_offset),
0218           /* final store input */
0219           [v]           "m" (*v),
0220           [expectnot]       "r" (expectnot),
0221           [voffp]       "er" (voffp),
0222           [load]        "m" (*load)
0223         : "memory", "cc", "rax", "rbx"
0224           RSEQ_INJECT_CLOBBER
0225         : abort, cmpfail
0226 #ifdef RSEQ_COMPARE_TWICE
0227           , error1, error2
0228 #endif
0229     );
0230     rseq_after_asm_goto();
0231     return 0;
0232 abort:
0233     rseq_after_asm_goto();
0234     RSEQ_INJECT_FAILED
0235     return -1;
0236 cmpfail:
0237     rseq_after_asm_goto();
0238     return 1;
0239 #ifdef RSEQ_COMPARE_TWICE
0240 error1:
0241     rseq_after_asm_goto();
0242     rseq_bug("cpu_id comparison failed");
0243 error2:
0244     rseq_after_asm_goto();
0245     rseq_bug("expected value comparison failed");
0246 #endif
0247 }
0248 
0249 static inline __attribute__((always_inline))
0250 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
0251 {
0252     RSEQ_INJECT_C(9)
0253 
0254     __asm__ __volatile__ goto (
0255         RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
0256 #ifdef RSEQ_COMPARE_TWICE
0257         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
0258 #endif
0259         /* Start rseq by storing table entry pointer into rseq_cs. */
0260         RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
0261         RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
0262         RSEQ_INJECT_ASM(3)
0263 #ifdef RSEQ_COMPARE_TWICE
0264         RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
0265 #endif
0266         /* final store */
0267         "addq %[count], %[v]\n\t"
0268         "2:\n\t"
0269         RSEQ_INJECT_ASM(4)
0270         RSEQ_ASM_DEFINE_ABORT(4, "", abort)
0271         : /* gcc asm goto does not allow outputs */
0272         : [cpu_id]      "r" (cpu),
0273           [rseq_offset]     "r" (rseq_offset),
0274           /* final store input */
0275           [v]           "m" (*v),
0276           [count]       "er" (count)
0277         : "memory", "cc", "rax"
0278           RSEQ_INJECT_CLOBBER
0279         : abort
0280 #ifdef RSEQ_COMPARE_TWICE
0281           , error1
0282 #endif
0283     );
0284     rseq_after_asm_goto();
0285     return 0;
0286 abort:
0287     rseq_after_asm_goto();
0288     RSEQ_INJECT_FAILED
0289     return -1;
0290 #ifdef RSEQ_COMPARE_TWICE
0291 error1:
0292     rseq_after_asm_goto();
0293     rseq_bug("cpu_id comparison failed");
0294 #endif
0295 }
0296 
0297 #define RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV
0298 
0299 /*
0300  *   pval = *(ptr+off)
0301  *  *pval += inc;
0302  */
0303 static inline __attribute__((always_inline))
0304 int rseq_offset_deref_addv(intptr_t *ptr, long off, intptr_t inc, int cpu)
0305 {
0306     RSEQ_INJECT_C(9)
0307 
0308     __asm__ __volatile__ goto (
0309         RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
0310 #ifdef RSEQ_COMPARE_TWICE
0311         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
0312 #endif
0313         /* Start rseq by storing table entry pointer into rseq_cs. */
0314         RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
0315         RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
0316         RSEQ_INJECT_ASM(3)
0317 #ifdef RSEQ_COMPARE_TWICE
0318         RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
0319 #endif
0320         /* get p+v */
0321         "movq %[ptr], %%rbx\n\t"
0322         "addq %[off], %%rbx\n\t"
0323         /* get pv */
0324         "movq (%%rbx), %%rcx\n\t"
0325         /* *pv += inc */
0326         "addq %[inc], (%%rcx)\n\t"
0327         "2:\n\t"
0328         RSEQ_INJECT_ASM(4)
0329         RSEQ_ASM_DEFINE_ABORT(4, "", abort)
0330         : /* gcc asm goto does not allow outputs */
0331         : [cpu_id]      "r" (cpu),
0332           [rseq_offset]     "r" (rseq_offset),
0333           /* final store input */
0334           [ptr]         "m" (*ptr),
0335           [off]         "er" (off),
0336           [inc]         "er" (inc)
0337         : "memory", "cc", "rax", "rbx", "rcx"
0338           RSEQ_INJECT_CLOBBER
0339         : abort
0340 #ifdef RSEQ_COMPARE_TWICE
0341           , error1
0342 #endif
0343     );
0344     return 0;
0345 abort:
0346     RSEQ_INJECT_FAILED
0347     return -1;
0348 #ifdef RSEQ_COMPARE_TWICE
0349 error1:
0350     rseq_bug("cpu_id comparison failed");
0351 #endif
0352 }
0353 
0354 static inline __attribute__((always_inline))
0355 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
0356                  intptr_t *v2, intptr_t newv2,
0357                  intptr_t newv, int cpu)
0358 {
0359     RSEQ_INJECT_C(9)
0360 
0361     __asm__ __volatile__ goto (
0362         RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
0363         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
0364 #ifdef RSEQ_COMPARE_TWICE
0365         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
0366         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
0367 #endif
0368         /* Start rseq by storing table entry pointer into rseq_cs. */
0369         RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
0370         RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
0371         RSEQ_INJECT_ASM(3)
0372         "cmpq %[v], %[expect]\n\t"
0373         "jnz %l[cmpfail]\n\t"
0374         RSEQ_INJECT_ASM(4)
0375 #ifdef RSEQ_COMPARE_TWICE
0376         RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
0377         "cmpq %[v], %[expect]\n\t"
0378         "jnz %l[error2]\n\t"
0379 #endif
0380         /* try store */
0381         "movq %[newv2], %[v2]\n\t"
0382         RSEQ_INJECT_ASM(5)
0383         /* final store */
0384         "movq %[newv], %[v]\n\t"
0385         "2:\n\t"
0386         RSEQ_INJECT_ASM(6)
0387         RSEQ_ASM_DEFINE_ABORT(4, "", abort)
0388         : /* gcc asm goto does not allow outputs */
0389         : [cpu_id]      "r" (cpu),
0390           [rseq_offset]     "r" (rseq_offset),
0391           /* try store input */
0392           [v2]          "m" (*v2),
0393           [newv2]       "r" (newv2),
0394           /* final store input */
0395           [v]           "m" (*v),
0396           [expect]      "r" (expect),
0397           [newv]        "r" (newv)
0398         : "memory", "cc", "rax"
0399           RSEQ_INJECT_CLOBBER
0400         : abort, cmpfail
0401 #ifdef RSEQ_COMPARE_TWICE
0402           , error1, error2
0403 #endif
0404     );
0405     rseq_after_asm_goto();
0406     return 0;
0407 abort:
0408     rseq_after_asm_goto();
0409     RSEQ_INJECT_FAILED
0410     return -1;
0411 cmpfail:
0412     rseq_after_asm_goto();
0413     return 1;
0414 #ifdef RSEQ_COMPARE_TWICE
0415 error1:
0416     rseq_after_asm_goto();
0417     rseq_bug("cpu_id comparison failed");
0418 error2:
0419     rseq_after_asm_goto();
0420     rseq_bug("expected value comparison failed");
0421 #endif
0422 }
0423 
0424 /* x86-64 is TSO. */
0425 static inline __attribute__((always_inline))
0426 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
0427                      intptr_t *v2, intptr_t newv2,
0428                      intptr_t newv, int cpu)
0429 {
0430     return rseq_cmpeqv_trystorev_storev(v, expect, v2, newv2, newv, cpu);
0431 }
0432 
0433 static inline __attribute__((always_inline))
0434 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
0435                   intptr_t *v2, intptr_t expect2,
0436                   intptr_t newv, int cpu)
0437 {
0438     RSEQ_INJECT_C(9)
0439 
0440     __asm__ __volatile__ goto (
0441         RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
0442         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
0443 #ifdef RSEQ_COMPARE_TWICE
0444         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
0445         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
0446         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
0447 #endif
0448         /* Start rseq by storing table entry pointer into rseq_cs. */
0449         RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
0450         RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
0451         RSEQ_INJECT_ASM(3)
0452         "cmpq %[v], %[expect]\n\t"
0453         "jnz %l[cmpfail]\n\t"
0454         RSEQ_INJECT_ASM(4)
0455         "cmpq %[v2], %[expect2]\n\t"
0456         "jnz %l[cmpfail]\n\t"
0457         RSEQ_INJECT_ASM(5)
0458 #ifdef RSEQ_COMPARE_TWICE
0459         RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
0460         "cmpq %[v], %[expect]\n\t"
0461         "jnz %l[error2]\n\t"
0462         "cmpq %[v2], %[expect2]\n\t"
0463         "jnz %l[error3]\n\t"
0464 #endif
0465         /* final store */
0466         "movq %[newv], %[v]\n\t"
0467         "2:\n\t"
0468         RSEQ_INJECT_ASM(6)
0469         RSEQ_ASM_DEFINE_ABORT(4, "", abort)
0470         : /* gcc asm goto does not allow outputs */
0471         : [cpu_id]      "r" (cpu),
0472           [rseq_offset]     "r" (rseq_offset),
0473           /* cmp2 input */
0474           [v2]          "m" (*v2),
0475           [expect2]     "r" (expect2),
0476           /* final store input */
0477           [v]           "m" (*v),
0478           [expect]      "r" (expect),
0479           [newv]        "r" (newv)
0480         : "memory", "cc", "rax"
0481           RSEQ_INJECT_CLOBBER
0482         : abort, cmpfail
0483 #ifdef RSEQ_COMPARE_TWICE
0484           , error1, error2, error3
0485 #endif
0486     );
0487     rseq_after_asm_goto();
0488     return 0;
0489 abort:
0490     rseq_after_asm_goto();
0491     RSEQ_INJECT_FAILED
0492     return -1;
0493 cmpfail:
0494     rseq_after_asm_goto();
0495     return 1;
0496 #ifdef RSEQ_COMPARE_TWICE
0497 error1:
0498     rseq_after_asm_goto();
0499     rseq_bug("cpu_id comparison failed");
0500 error2:
0501     rseq_after_asm_goto();
0502     rseq_bug("1st expected value comparison failed");
0503 error3:
0504     rseq_after_asm_goto();
0505     rseq_bug("2nd expected value comparison failed");
0506 #endif
0507 }
0508 
0509 static inline __attribute__((always_inline))
0510 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
0511                  void *dst, void *src, size_t len,
0512                  intptr_t newv, int cpu)
0513 {
0514     uint64_t rseq_scratch[3];
0515 
0516     RSEQ_INJECT_C(9)
0517 
0518     __asm__ __volatile__ goto (
0519         RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
0520         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
0521 #ifdef RSEQ_COMPARE_TWICE
0522         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
0523         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
0524 #endif
0525         "movq %[src], %[rseq_scratch0]\n\t"
0526         "movq %[dst], %[rseq_scratch1]\n\t"
0527         "movq %[len], %[rseq_scratch2]\n\t"
0528         /* Start rseq by storing table entry pointer into rseq_cs. */
0529         RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
0530         RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
0531         RSEQ_INJECT_ASM(3)
0532         "cmpq %[v], %[expect]\n\t"
0533         "jnz 5f\n\t"
0534         RSEQ_INJECT_ASM(4)
0535 #ifdef RSEQ_COMPARE_TWICE
0536         RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 6f)
0537         "cmpq %[v], %[expect]\n\t"
0538         "jnz 7f\n\t"
0539 #endif
0540         /* try memcpy */
0541         "test %[len], %[len]\n\t" \
0542         "jz 333f\n\t" \
0543         "222:\n\t" \
0544         "movb (%[src]), %%al\n\t" \
0545         "movb %%al, (%[dst])\n\t" \
0546         "inc %[src]\n\t" \
0547         "inc %[dst]\n\t" \
0548         "dec %[len]\n\t" \
0549         "jnz 222b\n\t" \
0550         "333:\n\t" \
0551         RSEQ_INJECT_ASM(5)
0552         /* final store */
0553         "movq %[newv], %[v]\n\t"
0554         "2:\n\t"
0555         RSEQ_INJECT_ASM(6)
0556         /* teardown */
0557         "movq %[rseq_scratch2], %[len]\n\t"
0558         "movq %[rseq_scratch1], %[dst]\n\t"
0559         "movq %[rseq_scratch0], %[src]\n\t"
0560         RSEQ_ASM_DEFINE_ABORT(4,
0561             "movq %[rseq_scratch2], %[len]\n\t"
0562             "movq %[rseq_scratch1], %[dst]\n\t"
0563             "movq %[rseq_scratch0], %[src]\n\t",
0564             abort)
0565         RSEQ_ASM_DEFINE_CMPFAIL(5,
0566             "movq %[rseq_scratch2], %[len]\n\t"
0567             "movq %[rseq_scratch1], %[dst]\n\t"
0568             "movq %[rseq_scratch0], %[src]\n\t",
0569             cmpfail)
0570 #ifdef RSEQ_COMPARE_TWICE
0571         RSEQ_ASM_DEFINE_CMPFAIL(6,
0572             "movq %[rseq_scratch2], %[len]\n\t"
0573             "movq %[rseq_scratch1], %[dst]\n\t"
0574             "movq %[rseq_scratch0], %[src]\n\t",
0575             error1)
0576         RSEQ_ASM_DEFINE_CMPFAIL(7,
0577             "movq %[rseq_scratch2], %[len]\n\t"
0578             "movq %[rseq_scratch1], %[dst]\n\t"
0579             "movq %[rseq_scratch0], %[src]\n\t",
0580             error2)
0581 #endif
0582         : /* gcc asm goto does not allow outputs */
0583         : [cpu_id]      "r" (cpu),
0584           [rseq_offset]     "r" (rseq_offset),
0585           /* final store input */
0586           [v]           "m" (*v),
0587           [expect]      "r" (expect),
0588           [newv]        "r" (newv),
0589           /* try memcpy input */
0590           [dst]         "r" (dst),
0591           [src]         "r" (src),
0592           [len]         "r" (len),
0593           [rseq_scratch0]   "m" (rseq_scratch[0]),
0594           [rseq_scratch1]   "m" (rseq_scratch[1]),
0595           [rseq_scratch2]   "m" (rseq_scratch[2])
0596         : "memory", "cc", "rax"
0597           RSEQ_INJECT_CLOBBER
0598         : abort, cmpfail
0599 #ifdef RSEQ_COMPARE_TWICE
0600           , error1, error2
0601 #endif
0602     );
0603     rseq_after_asm_goto();
0604     return 0;
0605 abort:
0606     rseq_after_asm_goto();
0607     RSEQ_INJECT_FAILED
0608     return -1;
0609 cmpfail:
0610     rseq_after_asm_goto();
0611     return 1;
0612 #ifdef RSEQ_COMPARE_TWICE
0613 error1:
0614     rseq_after_asm_goto();
0615     rseq_bug("cpu_id comparison failed");
0616 error2:
0617     rseq_after_asm_goto();
0618     rseq_bug("expected value comparison failed");
0619 #endif
0620 }
0621 
0622 /* x86-64 is TSO. */
0623 static inline __attribute__((always_inline))
0624 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
0625                      void *dst, void *src, size_t len,
0626                      intptr_t newv, int cpu)
0627 {
0628     return rseq_cmpeqv_trymemcpy_storev(v, expect, dst, src, len,
0629                         newv, cpu);
0630 }
0631 
0632 #endif /* !RSEQ_SKIP_FASTPATH */
0633 
0634 #elif defined(__i386__)
0635 
0636 #define RSEQ_ASM_TP_SEGMENT %%gs
0637 
0638 #define rseq_smp_mb()   \
0639     __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
0640 #define rseq_smp_rmb()  \
0641     __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
0642 #define rseq_smp_wmb()  \
0643     __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
0644 
0645 #define rseq_smp_load_acquire(p)                    \
0646 __extension__ ({                            \
0647     __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p);           \
0648     rseq_smp_mb();                          \
0649     ____p1;                             \
0650 })
0651 
0652 #define rseq_smp_acquire__after_ctrl_dep()  rseq_smp_rmb()
0653 
0654 #define rseq_smp_store_release(p, v)                    \
0655 do {                                    \
0656     rseq_smp_mb();                          \
0657     RSEQ_WRITE_ONCE(*p, v);                     \
0658 } while (0)
0659 
0660 #ifdef RSEQ_SKIP_FASTPATH
0661 #include "rseq-skip.h"
0662 #else /* !RSEQ_SKIP_FASTPATH */
0663 
0664 /*
0665  * Use eax as scratch register and take memory operands as input to
0666  * lessen register pressure. Especially needed when compiling in O0.
0667  */
0668 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,          \
0669                 start_ip, post_commit_offset, abort_ip) \
0670         ".pushsection __rseq_cs, \"aw\"\n\t"            \
0671         ".balign 32\n\t"                    \
0672         __rseq_str(label) ":\n\t"               \
0673         ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
0674         ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
0675         ".popsection\n\t"                   \
0676         ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"      \
0677         ".long " __rseq_str(label) "b, 0x0\n\t"         \
0678         ".popsection\n\t"
0679 
0680 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
0681     __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,      \
0682                 (post_commit_ip - start_ip), abort_ip)
0683 
0684 /*
0685  * Exit points of a rseq critical section consist of all instructions outside
0686  * of the critical section where a critical section can either branch to or
0687  * reach through the normal course of its execution. The abort IP and the
0688  * post-commit IP are already part of the __rseq_cs section and should not be
0689  * explicitly defined as additional exit points. Knowing all exit points is
0690  * useful to assist debuggers stepping over the critical section.
0691  */
0692 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)           \
0693         ".pushsection __rseq_exit_point_array, \"aw\"\n\t"  \
0694         ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) ", 0x0\n\t" \
0695         ".popsection\n\t"
0696 
0697 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)        \
0698         RSEQ_INJECT_ASM(1)                  \
0699         "movl $" __rseq_str(cs_label) ", " __rseq_str(rseq_cs) "\n\t"   \
0700         __rseq_str(label) ":\n\t"
0701 
0702 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label)      \
0703         RSEQ_INJECT_ASM(2)                  \
0704         "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \
0705         "jnz " __rseq_str(label) "\n\t"
0706 
0707 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label)     \
0708         ".pushsection __rseq_failure, \"ax\"\n\t"       \
0709         /* Disassembler-friendly signature: ud1 <sig>,%edi. */  \
0710         ".byte 0x0f, 0xb9, 0x3d\n\t"                \
0711         ".long " __rseq_str(RSEQ_SIG) "\n\t"            \
0712         __rseq_str(label) ":\n\t"               \
0713         teardown                        \
0714         "jmp %l[" __rseq_str(abort_label) "]\n\t"       \
0715         ".popsection\n\t"
0716 
0717 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label)     \
0718         ".pushsection __rseq_failure, \"ax\"\n\t"       \
0719         __rseq_str(label) ":\n\t"               \
0720         teardown                        \
0721         "jmp %l[" __rseq_str(cmpfail_label) "]\n\t"     \
0722         ".popsection\n\t"
0723 
0724 static inline __attribute__((always_inline))
0725 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
0726 {
0727     RSEQ_INJECT_C(9)
0728 
0729     __asm__ __volatile__ goto (
0730         RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
0731         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
0732 #ifdef RSEQ_COMPARE_TWICE
0733         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
0734         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
0735 #endif
0736         /* Start rseq by storing table entry pointer into rseq_cs. */
0737         RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
0738         RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
0739         RSEQ_INJECT_ASM(3)
0740         "cmpl %[v], %[expect]\n\t"
0741         "jnz %l[cmpfail]\n\t"
0742         RSEQ_INJECT_ASM(4)
0743 #ifdef RSEQ_COMPARE_TWICE
0744         RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
0745         "cmpl %[v], %[expect]\n\t"
0746         "jnz %l[error2]\n\t"
0747 #endif
0748         /* final store */
0749         "movl %[newv], %[v]\n\t"
0750         "2:\n\t"
0751         RSEQ_INJECT_ASM(5)
0752         RSEQ_ASM_DEFINE_ABORT(4, "", abort)
0753         : /* gcc asm goto does not allow outputs */
0754         : [cpu_id]      "r" (cpu),
0755           [rseq_offset]     "r" (rseq_offset),
0756           [v]           "m" (*v),
0757           [expect]      "r" (expect),
0758           [newv]        "r" (newv)
0759         : "memory", "cc", "eax"
0760           RSEQ_INJECT_CLOBBER
0761         : abort, cmpfail
0762 #ifdef RSEQ_COMPARE_TWICE
0763           , error1, error2
0764 #endif
0765     );
0766     rseq_after_asm_goto();
0767     return 0;
0768 abort:
0769     rseq_after_asm_goto();
0770     RSEQ_INJECT_FAILED
0771     return -1;
0772 cmpfail:
0773     rseq_after_asm_goto();
0774     return 1;
0775 #ifdef RSEQ_COMPARE_TWICE
0776 error1:
0777     rseq_after_asm_goto();
0778     rseq_bug("cpu_id comparison failed");
0779 error2:
0780     rseq_after_asm_goto();
0781     rseq_bug("expected value comparison failed");
0782 #endif
0783 }
0784 
0785 /*
0786  * Compare @v against @expectnot. When it does _not_ match, load @v
0787  * into @load, and store the content of *@v + voffp into @v.
0788  */
0789 static inline __attribute__((always_inline))
0790 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
0791                    long voffp, intptr_t *load, int cpu)
0792 {
0793     RSEQ_INJECT_C(9)
0794 
0795     __asm__ __volatile__ goto (
0796         RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
0797         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
0798 #ifdef RSEQ_COMPARE_TWICE
0799         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
0800         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
0801 #endif
0802         /* Start rseq by storing table entry pointer into rseq_cs. */
0803         RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
0804         RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
0805         RSEQ_INJECT_ASM(3)
0806         "movl %[v], %%ebx\n\t"
0807         "cmpl %%ebx, %[expectnot]\n\t"
0808         "je %l[cmpfail]\n\t"
0809         RSEQ_INJECT_ASM(4)
0810 #ifdef RSEQ_COMPARE_TWICE
0811         RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
0812         "movl %[v], %%ebx\n\t"
0813         "cmpl %%ebx, %[expectnot]\n\t"
0814         "je %l[error2]\n\t"
0815 #endif
0816         "movl %%ebx, %[load]\n\t"
0817         "addl %[voffp], %%ebx\n\t"
0818         "movl (%%ebx), %%ebx\n\t"
0819         /* final store */
0820         "movl %%ebx, %[v]\n\t"
0821         "2:\n\t"
0822         RSEQ_INJECT_ASM(5)
0823         RSEQ_ASM_DEFINE_ABORT(4, "", abort)
0824         : /* gcc asm goto does not allow outputs */
0825         : [cpu_id]      "r" (cpu),
0826           [rseq_offset]     "r" (rseq_offset),
0827           /* final store input */
0828           [v]           "m" (*v),
0829           [expectnot]       "r" (expectnot),
0830           [voffp]       "ir" (voffp),
0831           [load]        "m" (*load)
0832         : "memory", "cc", "eax", "ebx"
0833           RSEQ_INJECT_CLOBBER
0834         : abort, cmpfail
0835 #ifdef RSEQ_COMPARE_TWICE
0836           , error1, error2
0837 #endif
0838     );
0839     rseq_after_asm_goto();
0840     return 0;
0841 abort:
0842     rseq_after_asm_goto();
0843     RSEQ_INJECT_FAILED
0844     return -1;
0845 cmpfail:
0846     rseq_after_asm_goto();
0847     return 1;
0848 #ifdef RSEQ_COMPARE_TWICE
0849 error1:
0850     rseq_after_asm_goto();
0851     rseq_bug("cpu_id comparison failed");
0852 error2:
0853     rseq_after_asm_goto();
0854     rseq_bug("expected value comparison failed");
0855 #endif
0856 }
0857 
0858 static inline __attribute__((always_inline))
0859 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
0860 {
0861     RSEQ_INJECT_C(9)
0862 
0863     __asm__ __volatile__ goto (
0864         RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
0865 #ifdef RSEQ_COMPARE_TWICE
0866         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
0867 #endif
0868         /* Start rseq by storing table entry pointer into rseq_cs. */
0869         RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
0870         RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
0871         RSEQ_INJECT_ASM(3)
0872 #ifdef RSEQ_COMPARE_TWICE
0873         RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
0874 #endif
0875         /* final store */
0876         "addl %[count], %[v]\n\t"
0877         "2:\n\t"
0878         RSEQ_INJECT_ASM(4)
0879         RSEQ_ASM_DEFINE_ABORT(4, "", abort)
0880         : /* gcc asm goto does not allow outputs */
0881         : [cpu_id]      "r" (cpu),
0882           [rseq_offset]     "r" (rseq_offset),
0883           /* final store input */
0884           [v]           "m" (*v),
0885           [count]       "ir" (count)
0886         : "memory", "cc", "eax"
0887           RSEQ_INJECT_CLOBBER
0888         : abort
0889 #ifdef RSEQ_COMPARE_TWICE
0890           , error1
0891 #endif
0892     );
0893     rseq_after_asm_goto();
0894     return 0;
0895 abort:
0896     rseq_after_asm_goto();
0897     RSEQ_INJECT_FAILED
0898     return -1;
0899 #ifdef RSEQ_COMPARE_TWICE
0900 error1:
0901     rseq_after_asm_goto();
0902     rseq_bug("cpu_id comparison failed");
0903 #endif
0904 }
0905 
0906 static inline __attribute__((always_inline))
0907 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
0908                  intptr_t *v2, intptr_t newv2,
0909                  intptr_t newv, int cpu)
0910 {
0911     RSEQ_INJECT_C(9)
0912 
0913     __asm__ __volatile__ goto (
0914         RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
0915         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
0916 #ifdef RSEQ_COMPARE_TWICE
0917         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
0918         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
0919 #endif
0920         /* Start rseq by storing table entry pointer into rseq_cs. */
0921         RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
0922         RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
0923         RSEQ_INJECT_ASM(3)
0924         "cmpl %[v], %[expect]\n\t"
0925         "jnz %l[cmpfail]\n\t"
0926         RSEQ_INJECT_ASM(4)
0927 #ifdef RSEQ_COMPARE_TWICE
0928         RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
0929         "cmpl %[v], %[expect]\n\t"
0930         "jnz %l[error2]\n\t"
0931 #endif
0932         /* try store */
0933         "movl %[newv2], %%eax\n\t"
0934         "movl %%eax, %[v2]\n\t"
0935         RSEQ_INJECT_ASM(5)
0936         /* final store */
0937         "movl %[newv], %[v]\n\t"
0938         "2:\n\t"
0939         RSEQ_INJECT_ASM(6)
0940         RSEQ_ASM_DEFINE_ABORT(4, "", abort)
0941         : /* gcc asm goto does not allow outputs */
0942         : [cpu_id]      "r" (cpu),
0943           [rseq_offset]     "r" (rseq_offset),
0944           /* try store input */
0945           [v2]          "m" (*v2),
0946           [newv2]       "m" (newv2),
0947           /* final store input */
0948           [v]           "m" (*v),
0949           [expect]      "r" (expect),
0950           [newv]        "r" (newv)
0951         : "memory", "cc", "eax"
0952           RSEQ_INJECT_CLOBBER
0953         : abort, cmpfail
0954 #ifdef RSEQ_COMPARE_TWICE
0955           , error1, error2
0956 #endif
0957     );
0958     rseq_after_asm_goto();
0959     return 0;
0960 abort:
0961     rseq_after_asm_goto();
0962     RSEQ_INJECT_FAILED
0963     return -1;
0964 cmpfail:
0965     rseq_after_asm_goto();
0966     return 1;
0967 #ifdef RSEQ_COMPARE_TWICE
0968 error1:
0969     rseq_after_asm_goto();
0970     rseq_bug("cpu_id comparison failed");
0971 error2:
0972     rseq_after_asm_goto();
0973     rseq_bug("expected value comparison failed");
0974 #endif
0975 }
0976 
0977 static inline __attribute__((always_inline))
0978 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
0979                      intptr_t *v2, intptr_t newv2,
0980                      intptr_t newv, int cpu)
0981 {
0982     RSEQ_INJECT_C(9)
0983 
0984     __asm__ __volatile__ goto (
0985         RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
0986         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
0987 #ifdef RSEQ_COMPARE_TWICE
0988         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
0989         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
0990 #endif
0991         /* Start rseq by storing table entry pointer into rseq_cs. */
0992         RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
0993         RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
0994         RSEQ_INJECT_ASM(3)
0995         "movl %[expect], %%eax\n\t"
0996         "cmpl %[v], %%eax\n\t"
0997         "jnz %l[cmpfail]\n\t"
0998         RSEQ_INJECT_ASM(4)
0999 #ifdef RSEQ_COMPARE_TWICE
1000         RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
1001         "movl %[expect], %%eax\n\t"
1002         "cmpl %[v], %%eax\n\t"
1003         "jnz %l[error2]\n\t"
1004 #endif
1005         /* try store */
1006         "movl %[newv2], %[v2]\n\t"
1007         RSEQ_INJECT_ASM(5)
1008         "lock; addl $0,-128(%%esp)\n\t"
1009         /* final store */
1010         "movl %[newv], %[v]\n\t"
1011         "2:\n\t"
1012         RSEQ_INJECT_ASM(6)
1013         RSEQ_ASM_DEFINE_ABORT(4, "", abort)
1014         : /* gcc asm goto does not allow outputs */
1015         : [cpu_id]      "r" (cpu),
1016           [rseq_offset]     "r" (rseq_offset),
1017           /* try store input */
1018           [v2]          "m" (*v2),
1019           [newv2]       "r" (newv2),
1020           /* final store input */
1021           [v]           "m" (*v),
1022           [expect]      "m" (expect),
1023           [newv]        "r" (newv)
1024         : "memory", "cc", "eax"
1025           RSEQ_INJECT_CLOBBER
1026         : abort, cmpfail
1027 #ifdef RSEQ_COMPARE_TWICE
1028           , error1, error2
1029 #endif
1030     );
1031     rseq_after_asm_goto();
1032     return 0;
1033 abort:
1034     rseq_after_asm_goto();
1035     RSEQ_INJECT_FAILED
1036     return -1;
1037 cmpfail:
1038     rseq_after_asm_goto();
1039     return 1;
1040 #ifdef RSEQ_COMPARE_TWICE
1041 error1:
1042     rseq_after_asm_goto();
1043     rseq_bug("cpu_id comparison failed");
1044 error2:
1045     rseq_after_asm_goto();
1046     rseq_bug("expected value comparison failed");
1047 #endif
1048 
1049 }
1050 
1051 static inline __attribute__((always_inline))
1052 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
1053                   intptr_t *v2, intptr_t expect2,
1054                   intptr_t newv, int cpu)
1055 {
1056     RSEQ_INJECT_C(9)
1057 
1058     __asm__ __volatile__ goto (
1059         RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
1060         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
1061 #ifdef RSEQ_COMPARE_TWICE
1062         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
1063         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
1064         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
1065 #endif
1066         /* Start rseq by storing table entry pointer into rseq_cs. */
1067         RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
1068         RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
1069         RSEQ_INJECT_ASM(3)
1070         "cmpl %[v], %[expect]\n\t"
1071         "jnz %l[cmpfail]\n\t"
1072         RSEQ_INJECT_ASM(4)
1073         "cmpl %[expect2], %[v2]\n\t"
1074         "jnz %l[cmpfail]\n\t"
1075         RSEQ_INJECT_ASM(5)
1076 #ifdef RSEQ_COMPARE_TWICE
1077         RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
1078         "cmpl %[v], %[expect]\n\t"
1079         "jnz %l[error2]\n\t"
1080         "cmpl %[expect2], %[v2]\n\t"
1081         "jnz %l[error3]\n\t"
1082 #endif
1083         "movl %[newv], %%eax\n\t"
1084         /* final store */
1085         "movl %%eax, %[v]\n\t"
1086         "2:\n\t"
1087         RSEQ_INJECT_ASM(6)
1088         RSEQ_ASM_DEFINE_ABORT(4, "", abort)
1089         : /* gcc asm goto does not allow outputs */
1090         : [cpu_id]      "r" (cpu),
1091           [rseq_offset]     "r" (rseq_offset),
1092           /* cmp2 input */
1093           [v2]          "m" (*v2),
1094           [expect2]     "r" (expect2),
1095           /* final store input */
1096           [v]           "m" (*v),
1097           [expect]      "r" (expect),
1098           [newv]        "m" (newv)
1099         : "memory", "cc", "eax"
1100           RSEQ_INJECT_CLOBBER
1101         : abort, cmpfail
1102 #ifdef RSEQ_COMPARE_TWICE
1103           , error1, error2, error3
1104 #endif
1105     );
1106     rseq_after_asm_goto();
1107     return 0;
1108 abort:
1109     rseq_after_asm_goto();
1110     RSEQ_INJECT_FAILED
1111     return -1;
1112 cmpfail:
1113     rseq_after_asm_goto();
1114     return 1;
1115 #ifdef RSEQ_COMPARE_TWICE
1116 error1:
1117     rseq_after_asm_goto();
1118     rseq_bug("cpu_id comparison failed");
1119 error2:
1120     rseq_after_asm_goto();
1121     rseq_bug("1st expected value comparison failed");
1122 error3:
1123     rseq_after_asm_goto();
1124     rseq_bug("2nd expected value comparison failed");
1125 #endif
1126 }
1127 
1128 /* TODO: implement a faster memcpy. */
1129 static inline __attribute__((always_inline))
1130 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
1131                  void *dst, void *src, size_t len,
1132                  intptr_t newv, int cpu)
1133 {
1134     uint32_t rseq_scratch[3];
1135 
1136     RSEQ_INJECT_C(9)
1137 
1138     __asm__ __volatile__ goto (
1139         RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
1140         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
1141 #ifdef RSEQ_COMPARE_TWICE
1142         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
1143         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
1144 #endif
1145         "movl %[src], %[rseq_scratch0]\n\t"
1146         "movl %[dst], %[rseq_scratch1]\n\t"
1147         "movl %[len], %[rseq_scratch2]\n\t"
1148         /* Start rseq by storing table entry pointer into rseq_cs. */
1149         RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
1150         RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
1151         RSEQ_INJECT_ASM(3)
1152         "movl %[expect], %%eax\n\t"
1153         "cmpl %%eax, %[v]\n\t"
1154         "jnz 5f\n\t"
1155         RSEQ_INJECT_ASM(4)
1156 #ifdef RSEQ_COMPARE_TWICE
1157         RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 6f)
1158         "movl %[expect], %%eax\n\t"
1159         "cmpl %%eax, %[v]\n\t"
1160         "jnz 7f\n\t"
1161 #endif
1162         /* try memcpy */
1163         "test %[len], %[len]\n\t" \
1164         "jz 333f\n\t" \
1165         "222:\n\t" \
1166         "movb (%[src]), %%al\n\t" \
1167         "movb %%al, (%[dst])\n\t" \
1168         "inc %[src]\n\t" \
1169         "inc %[dst]\n\t" \
1170         "dec %[len]\n\t" \
1171         "jnz 222b\n\t" \
1172         "333:\n\t" \
1173         RSEQ_INJECT_ASM(5)
1174         "movl %[newv], %%eax\n\t"
1175         /* final store */
1176         "movl %%eax, %[v]\n\t"
1177         "2:\n\t"
1178         RSEQ_INJECT_ASM(6)
1179         /* teardown */
1180         "movl %[rseq_scratch2], %[len]\n\t"
1181         "movl %[rseq_scratch1], %[dst]\n\t"
1182         "movl %[rseq_scratch0], %[src]\n\t"
1183         RSEQ_ASM_DEFINE_ABORT(4,
1184             "movl %[rseq_scratch2], %[len]\n\t"
1185             "movl %[rseq_scratch1], %[dst]\n\t"
1186             "movl %[rseq_scratch0], %[src]\n\t",
1187             abort)
1188         RSEQ_ASM_DEFINE_CMPFAIL(5,
1189             "movl %[rseq_scratch2], %[len]\n\t"
1190             "movl %[rseq_scratch1], %[dst]\n\t"
1191             "movl %[rseq_scratch0], %[src]\n\t",
1192             cmpfail)
1193 #ifdef RSEQ_COMPARE_TWICE
1194         RSEQ_ASM_DEFINE_CMPFAIL(6,
1195             "movl %[rseq_scratch2], %[len]\n\t"
1196             "movl %[rseq_scratch1], %[dst]\n\t"
1197             "movl %[rseq_scratch0], %[src]\n\t",
1198             error1)
1199         RSEQ_ASM_DEFINE_CMPFAIL(7,
1200             "movl %[rseq_scratch2], %[len]\n\t"
1201             "movl %[rseq_scratch1], %[dst]\n\t"
1202             "movl %[rseq_scratch0], %[src]\n\t",
1203             error2)
1204 #endif
1205         : /* gcc asm goto does not allow outputs */
1206         : [cpu_id]      "r" (cpu),
1207           [rseq_offset]     "r" (rseq_offset),
1208           /* final store input */
1209           [v]           "m" (*v),
1210           [expect]      "m" (expect),
1211           [newv]        "m" (newv),
1212           /* try memcpy input */
1213           [dst]         "r" (dst),
1214           [src]         "r" (src),
1215           [len]         "r" (len),
1216           [rseq_scratch0]   "m" (rseq_scratch[0]),
1217           [rseq_scratch1]   "m" (rseq_scratch[1]),
1218           [rseq_scratch2]   "m" (rseq_scratch[2])
1219         : "memory", "cc", "eax"
1220           RSEQ_INJECT_CLOBBER
1221         : abort, cmpfail
1222 #ifdef RSEQ_COMPARE_TWICE
1223           , error1, error2
1224 #endif
1225     );
1226     rseq_after_asm_goto();
1227     return 0;
1228 abort:
1229     rseq_after_asm_goto();
1230     RSEQ_INJECT_FAILED
1231     return -1;
1232 cmpfail:
1233     rseq_after_asm_goto();
1234     return 1;
1235 #ifdef RSEQ_COMPARE_TWICE
1236 error1:
1237     rseq_after_asm_goto();
1238     rseq_bug("cpu_id comparison failed");
1239 error2:
1240     rseq_after_asm_goto();
1241     rseq_bug("expected value comparison failed");
1242 #endif
1243 }
1244 
1245 /* TODO: implement a faster memcpy. */
1246 static inline __attribute__((always_inline))
1247 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
1248                      void *dst, void *src, size_t len,
1249                      intptr_t newv, int cpu)
1250 {
1251     uint32_t rseq_scratch[3];
1252 
1253     RSEQ_INJECT_C(9)
1254 
1255     __asm__ __volatile__ goto (
1256         RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
1257         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
1258 #ifdef RSEQ_COMPARE_TWICE
1259         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
1260         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
1261 #endif
1262         "movl %[src], %[rseq_scratch0]\n\t"
1263         "movl %[dst], %[rseq_scratch1]\n\t"
1264         "movl %[len], %[rseq_scratch2]\n\t"
1265         /* Start rseq by storing table entry pointer into rseq_cs. */
1266         RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
1267         RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
1268         RSEQ_INJECT_ASM(3)
1269         "movl %[expect], %%eax\n\t"
1270         "cmpl %%eax, %[v]\n\t"
1271         "jnz 5f\n\t"
1272         RSEQ_INJECT_ASM(4)
1273 #ifdef RSEQ_COMPARE_TWICE
1274         RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 6f)
1275         "movl %[expect], %%eax\n\t"
1276         "cmpl %%eax, %[v]\n\t"
1277         "jnz 7f\n\t"
1278 #endif
1279         /* try memcpy */
1280         "test %[len], %[len]\n\t" \
1281         "jz 333f\n\t" \
1282         "222:\n\t" \
1283         "movb (%[src]), %%al\n\t" \
1284         "movb %%al, (%[dst])\n\t" \
1285         "inc %[src]\n\t" \
1286         "inc %[dst]\n\t" \
1287         "dec %[len]\n\t" \
1288         "jnz 222b\n\t" \
1289         "333:\n\t" \
1290         RSEQ_INJECT_ASM(5)
1291         "lock; addl $0,-128(%%esp)\n\t"
1292         "movl %[newv], %%eax\n\t"
1293         /* final store */
1294         "movl %%eax, %[v]\n\t"
1295         "2:\n\t"
1296         RSEQ_INJECT_ASM(6)
1297         /* teardown */
1298         "movl %[rseq_scratch2], %[len]\n\t"
1299         "movl %[rseq_scratch1], %[dst]\n\t"
1300         "movl %[rseq_scratch0], %[src]\n\t"
1301         RSEQ_ASM_DEFINE_ABORT(4,
1302             "movl %[rseq_scratch2], %[len]\n\t"
1303             "movl %[rseq_scratch1], %[dst]\n\t"
1304             "movl %[rseq_scratch0], %[src]\n\t",
1305             abort)
1306         RSEQ_ASM_DEFINE_CMPFAIL(5,
1307             "movl %[rseq_scratch2], %[len]\n\t"
1308             "movl %[rseq_scratch1], %[dst]\n\t"
1309             "movl %[rseq_scratch0], %[src]\n\t",
1310             cmpfail)
1311 #ifdef RSEQ_COMPARE_TWICE
1312         RSEQ_ASM_DEFINE_CMPFAIL(6,
1313             "movl %[rseq_scratch2], %[len]\n\t"
1314             "movl %[rseq_scratch1], %[dst]\n\t"
1315             "movl %[rseq_scratch0], %[src]\n\t",
1316             error1)
1317         RSEQ_ASM_DEFINE_CMPFAIL(7,
1318             "movl %[rseq_scratch2], %[len]\n\t"
1319             "movl %[rseq_scratch1], %[dst]\n\t"
1320             "movl %[rseq_scratch0], %[src]\n\t",
1321             error2)
1322 #endif
1323         : /* gcc asm goto does not allow outputs */
1324         : [cpu_id]      "r" (cpu),
1325           [rseq_offset]     "r" (rseq_offset),
1326           /* final store input */
1327           [v]           "m" (*v),
1328           [expect]      "m" (expect),
1329           [newv]        "m" (newv),
1330           /* try memcpy input */
1331           [dst]         "r" (dst),
1332           [src]         "r" (src),
1333           [len]         "r" (len),
1334           [rseq_scratch0]   "m" (rseq_scratch[0]),
1335           [rseq_scratch1]   "m" (rseq_scratch[1]),
1336           [rseq_scratch2]   "m" (rseq_scratch[2])
1337         : "memory", "cc", "eax"
1338           RSEQ_INJECT_CLOBBER
1339         : abort, cmpfail
1340 #ifdef RSEQ_COMPARE_TWICE
1341           , error1, error2
1342 #endif
1343     );
1344     rseq_after_asm_goto();
1345     return 0;
1346 abort:
1347     rseq_after_asm_goto();
1348     RSEQ_INJECT_FAILED
1349     return -1;
1350 cmpfail:
1351     rseq_after_asm_goto();
1352     return 1;
1353 #ifdef RSEQ_COMPARE_TWICE
1354 error1:
1355     rseq_after_asm_goto();
1356     rseq_bug("cpu_id comparison failed");
1357 error2:
1358     rseq_after_asm_goto();
1359     rseq_bug("expected value comparison failed");
1360 #endif
1361 }
1362 
1363 #endif /* !RSEQ_SKIP_FASTPATH */
1364 
1365 #endif
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!