Back to home page

OSCL-LXR
 
 

     

0001 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
0002 /*
0003  * Select the instruction "csrw mhartid, x0" as the RSEQ_SIG. Unlike
0004  * other architectures, the ebreak instruction has no immediate field for
0005  * distinguishing purposes. Hence, ebreak is not suitable as RSEQ_SIG.
0006  * "csrw mhartid, x0" can also satisfy the RSEQ requirement because it
0007  * is an uncommon instruction and will raise an illegal instruction
0008  * exception when executed in all modes.
0009  */
0010 #include <endian.h>
0011 
0012 #if defined(__BYTE_ORDER) ? (__BYTE_ORDER == __LITTLE_ENDIAN) : defined(__LITTLE_ENDIAN)
0013 #define RSEQ_SIG   0xf1401073  /* csrr mhartid, x0 */
0014 #else
0015 #error "Currently, RSEQ only supports Little-Endian version"
0016 #endif
0017 
0018 #if __riscv_xlen == 64
0019 #define __REG_SEL(a, b) a
0020 #elif __riscv_xlen == 32
0021 #define __REG_SEL(a, b) b
0022 #endif
0023 
0024 #define REG_L   __REG_SEL("ld ", "lw ")
0025 #define REG_S   __REG_SEL("sd ", "sw ")
0026 
0027 #define RISCV_FENCE(p, s) \
0028     __asm__ __volatile__ ("fence " #p "," #s : : : "memory")
0029 #define rseq_smp_mb()   RISCV_FENCE(rw, rw)
0030 #define rseq_smp_rmb()  RISCV_FENCE(r, r)
0031 #define rseq_smp_wmb()  RISCV_FENCE(w, w)
0032 #define RSEQ_ASM_TMP_REG_1  "t6"
0033 #define RSEQ_ASM_TMP_REG_2  "t5"
0034 #define RSEQ_ASM_TMP_REG_3  "t4"
0035 #define RSEQ_ASM_TMP_REG_4  "t3"
0036 
0037 #define rseq_smp_load_acquire(p)                    \
0038 __extension__ ({                            \
0039     __typeof(*(p)) ____p1 = RSEQ_READ_ONCE(*(p));           \
0040     RISCV_FENCE(r, rw)                      \
0041     ____p1;                             \
0042 })
0043 
0044 #define rseq_smp_acquire__after_ctrl_dep()  rseq_smp_rmb()
0045 
0046 #define rseq_smp_store_release(p, v)                    \
0047 do {                                    \
0048     RISCV_FENCE(rw, w);                     \
0049     RSEQ_WRITE_ONCE(*(p), v);                       \
0050 } while (0)
0051 
0052 #ifdef RSEQ_SKIP_FASTPATH
0053 #include "rseq-skip.h"
0054 #else /* !RSEQ_SKIP_FASTPATH */
0055 
0056 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip,    \
0057                 post_commit_offset, abort_ip)       \
0058     ".pushsection   __rseq_cs, \"aw\"\n"                \
0059     ".balign    32\n"                       \
0060     __rseq_str(label) ":\n"                     \
0061     ".long  " __rseq_str(version) ", " __rseq_str(flags) "\n"   \
0062     ".quad  " __rseq_str(start_ip) ", "             \
0063               __rseq_str(post_commit_offset) ", "       \
0064               __rseq_str(abort_ip) "\n"         \
0065     ".popsection\n\t"                       \
0066     ".pushsection __rseq_cs_ptr_array, \"aw\"\n"            \
0067     ".quad " __rseq_str(label) "b\n"                \
0068     ".popsection\n"
0069 
0070 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
0071     __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,       \
0072                 ((post_commit_ip) - (start_ip)), abort_ip)
0073 
0074 /*
0075  * Exit points of a rseq critical section consist of all instructions outside
0076  * of the critical section where a critical section can either branch to or
0077  * reach through the normal course of its execution. The abort IP and the
0078  * post-commit IP are already part of the __rseq_cs section and should not be
0079  * explicitly defined as additional exit points. Knowing all exit points is
0080  * useful to assist debuggers stepping over the critical section.
0081  */
0082 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)           \
0083     ".pushsection __rseq_exit_point_array, \"aw\"\n"        \
0084     ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n" \
0085     ".popsection\n"
0086 
0087 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)        \
0088     RSEQ_INJECT_ASM(1)                      \
0089     "la " RSEQ_ASM_TMP_REG_1 ", " __rseq_str(cs_label) "\n" \
0090     REG_S   RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(rseq_cs) "]\n" \
0091     __rseq_str(label) ":\n"
0092 
0093 #define RSEQ_ASM_DEFINE_ABORT(label, abort_label)           \
0094     "j  222f\n"                         \
0095     ".balign    4\n"                        \
0096     ".long "    __rseq_str(RSEQ_SIG) "\n"           \
0097     __rseq_str(label) ":\n"                     \
0098     "j  %l[" __rseq_str(abort_label) "]\n"          \
0099     "222:\n"
0100 
0101 #define RSEQ_ASM_OP_STORE(value, var)                   \
0102     REG_S   "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n"
0103 
0104 #define RSEQ_ASM_OP_CMPEQ(var, expect, label)               \
0105     REG_L   RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n"     \
0106     "bne    " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(expect) "] ,"    \
0107           __rseq_str(label) "\n"
0108 
0109 #define RSEQ_ASM_OP_CMPEQ32(var, expect, label)             \
0110     "lw " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n"   \
0111     "bne    " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(expect) "] ,"    \
0112           __rseq_str(label) "\n"
0113 
0114 #define RSEQ_ASM_OP_CMPNE(var, expect, label)               \
0115     REG_L   RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n"     \
0116     "beq    " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(expect) "] ,"    \
0117           __rseq_str(label) "\n"
0118 
0119 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label)      \
0120     RSEQ_INJECT_ASM(2)                      \
0121     RSEQ_ASM_OP_CMPEQ32(current_cpu_id, cpu_id, label)
0122 
0123 #define RSEQ_ASM_OP_R_LOAD(var)                     \
0124     REG_L   RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n"
0125 
0126 #define RSEQ_ASM_OP_R_STORE(var)                    \
0127     REG_S   RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n"
0128 
0129 #define RSEQ_ASM_OP_R_LOAD_OFF(offset)                  \
0130     "add    " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(offset) "], "    \
0131          RSEQ_ASM_TMP_REG_1 "\n"                \
0132     REG_L   RSEQ_ASM_TMP_REG_1 ", (" RSEQ_ASM_TMP_REG_1 ")\n"
0133 
0134 #define RSEQ_ASM_OP_R_ADD(count)                    \
0135     "add    " RSEQ_ASM_TMP_REG_1 ", " RSEQ_ASM_TMP_REG_1        \
0136         ", %[" __rseq_str(count) "]\n"
0137 
0138 #define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label)      \
0139     RSEQ_ASM_OP_STORE(value, var)                   \
0140     __rseq_str(post_commit_label) ":\n"
0141 
0142 #define RSEQ_ASM_OP_FINAL_STORE_RELEASE(value, var, post_commit_label)  \
0143     "fence  rw, w\n"                        \
0144     RSEQ_ASM_OP_STORE(value, var)                   \
0145     __rseq_str(post_commit_label) ":\n"
0146 
0147 #define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label)       \
0148     REG_S   RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n"     \
0149     __rseq_str(post_commit_label) ":\n"
0150 
0151 #define RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)             \
0152     "beqz   %[" __rseq_str(len) "], 333f\n"             \
0153     "mv " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(len) "]\n"   \
0154     "mv " RSEQ_ASM_TMP_REG_2 ", %[" __rseq_str(src) "]\n"   \
0155     "mv " RSEQ_ASM_TMP_REG_3 ", %[" __rseq_str(dst) "]\n"   \
0156     "222:\n"                            \
0157     "lb " RSEQ_ASM_TMP_REG_4 ", 0(" RSEQ_ASM_TMP_REG_2 ")\n"    \
0158     "sb " RSEQ_ASM_TMP_REG_4 ", 0(" RSEQ_ASM_TMP_REG_3 ")\n"    \
0159     "addi   " RSEQ_ASM_TMP_REG_1 ", " RSEQ_ASM_TMP_REG_1 ", -1\n"   \
0160     "addi   " RSEQ_ASM_TMP_REG_2 ", " RSEQ_ASM_TMP_REG_2 ", 1\n"    \
0161     "addi   " RSEQ_ASM_TMP_REG_3 ", " RSEQ_ASM_TMP_REG_3 ", 1\n"    \
0162     "bnez   " RSEQ_ASM_TMP_REG_1 ", 222b\n"             \
0163     "333:\n"
0164 
0165 #define RSEQ_ASM_OP_R_DEREF_ADDV(ptr, off, post_commit_label)       \
0166     "mv " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(ptr) "]\n"   \
0167     RSEQ_ASM_OP_R_ADD(off)                      \
0168     REG_L     RSEQ_ASM_TMP_REG_1 ", 0(" RSEQ_ASM_TMP_REG_1 ")\n"    \
0169     RSEQ_ASM_OP_R_ADD(inc)                      \
0170     __rseq_str(post_commit_label) ":\n"
0171 
0172 static inline __always_inline
0173 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
0174 {
0175     RSEQ_INJECT_C(9)
0176 
0177     __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
0178                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
0179 #ifdef RSEQ_COMPARE_TWICE
0180                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
0181                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
0182 #endif
0183                   RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
0184                   RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
0185                   RSEQ_INJECT_ASM(3)
0186                   RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]")
0187                   RSEQ_INJECT_ASM(4)
0188 #ifdef RSEQ_COMPARE_TWICE
0189                   RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
0190                   RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]")
0191 #endif
0192                   RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
0193                   RSEQ_INJECT_ASM(5)
0194                   RSEQ_ASM_DEFINE_ABORT(4, abort)
0195                   : /* gcc asm goto does not allow outputs */
0196                   : [cpu_id]        "r" (cpu),
0197                     [current_cpu_id]    "m" (rseq_get_abi()->cpu_id),
0198                     [rseq_cs]       "m" (rseq_get_abi()->rseq_cs.arch.ptr),
0199                     [v]         "m" (*v),
0200                     [expect]        "r" (expect),
0201                     [newv]      "r" (newv)
0202                     RSEQ_INJECT_INPUT
0203                   : "memory", RSEQ_ASM_TMP_REG_1
0204                     RSEQ_INJECT_CLOBBER
0205                   : abort, cmpfail
0206 #ifdef RSEQ_COMPARE_TWICE
0207                     , error1, error2
0208 #endif
0209     );
0210 
0211     return 0;
0212 abort:
0213     RSEQ_INJECT_FAILED
0214     return -1;
0215 cmpfail:
0216     return 1;
0217 #ifdef RSEQ_COMPARE_TWICE
0218 error1:
0219     rseq_bug("cpu_id comparison failed");
0220 error2:
0221     rseq_bug("expected value comparison failed");
0222 #endif
0223 }
0224 
0225 static inline __always_inline
0226 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
0227                    off_t voffp, intptr_t *load, int cpu)
0228 {
0229     RSEQ_INJECT_C(9)
0230 
0231     __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
0232                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
0233 #ifdef RSEQ_COMPARE_TWICE
0234                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
0235                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
0236 #endif
0237                   RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
0238                   RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
0239                   RSEQ_INJECT_ASM(3)
0240                   RSEQ_ASM_OP_CMPNE(v, expectnot, "%l[cmpfail]")
0241                   RSEQ_INJECT_ASM(4)
0242 #ifdef RSEQ_COMPARE_TWICE
0243                   RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
0244                   RSEQ_ASM_OP_CMPNE(v, expectnot, "%l[error2]")
0245 #endif
0246                   RSEQ_ASM_OP_R_LOAD(v)
0247                   RSEQ_ASM_OP_R_STORE(load)
0248                   RSEQ_ASM_OP_R_LOAD_OFF(voffp)
0249                   RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
0250                   RSEQ_INJECT_ASM(5)
0251                   RSEQ_ASM_DEFINE_ABORT(4, abort)
0252                   : /* gcc asm goto does not allow outputs */
0253                   : [cpu_id]        "r" (cpu),
0254                     [current_cpu_id]    "m" (rseq_get_abi()->cpu_id),
0255                     [rseq_cs]       "m" (rseq_get_abi()->rseq_cs.arch.ptr),
0256                     [v]         "m" (*v),
0257                     [expectnot]     "r" (expectnot),
0258                     [load]      "m" (*load),
0259                     [voffp]     "r" (voffp)
0260                     RSEQ_INJECT_INPUT
0261                   : "memory", RSEQ_ASM_TMP_REG_1
0262                     RSEQ_INJECT_CLOBBER
0263                   : abort, cmpfail
0264 #ifdef RSEQ_COMPARE_TWICE
0265                     , error1, error2
0266 #endif
0267     );
0268     return 0;
0269 abort:
0270     RSEQ_INJECT_FAILED
0271     return -1;
0272 cmpfail:
0273     return 1;
0274 #ifdef RSEQ_COMPARE_TWICE
0275 error1:
0276     rseq_bug("cpu_id comparison failed");
0277 error2:
0278     rseq_bug("expected value comparison failed");
0279 #endif
0280 }
0281 
0282 static inline __always_inline
0283 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
0284 {
0285     RSEQ_INJECT_C(9)
0286 
0287     __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
0288 #ifdef RSEQ_COMPARE_TWICE
0289                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
0290 #endif
0291                   RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
0292                   RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
0293                   RSEQ_INJECT_ASM(3)
0294 #ifdef RSEQ_COMPARE_TWICE
0295                   RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
0296 #endif
0297                   RSEQ_ASM_OP_R_LOAD(v)
0298                   RSEQ_ASM_OP_R_ADD(count)
0299                   RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
0300                   RSEQ_INJECT_ASM(4)
0301                   RSEQ_ASM_DEFINE_ABORT(4, abort)
0302                   : /* gcc asm goto does not allow outputs */
0303                   : [cpu_id]        "r" (cpu),
0304                     [current_cpu_id]    "m" (rseq_get_abi()->cpu_id),
0305                     [rseq_cs]       "m" (rseq_get_abi()->rseq_cs.arch.ptr),
0306                     [v]         "m" (*v),
0307                     [count]     "r" (count)
0308                     RSEQ_INJECT_INPUT
0309                   : "memory", RSEQ_ASM_TMP_REG_1
0310                     RSEQ_INJECT_CLOBBER
0311                   : abort
0312 #ifdef RSEQ_COMPARE_TWICE
0313                     , error1
0314 #endif
0315     );
0316     return 0;
0317 abort:
0318     RSEQ_INJECT_FAILED
0319     return -1;
0320 #ifdef RSEQ_COMPARE_TWICE
0321 error1:
0322     rseq_bug("cpu_id comparison failed");
0323 #endif
0324 }
0325 
0326 static inline __always_inline
0327 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
0328                  intptr_t *v2, intptr_t newv2,
0329                  intptr_t newv, int cpu)
0330 {
0331     RSEQ_INJECT_C(9)
0332 
0333     __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
0334                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
0335 #ifdef RSEQ_COMPARE_TWICE
0336                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
0337                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
0338 #endif
0339                   RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
0340                   RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
0341                   RSEQ_INJECT_ASM(3)
0342                   RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]")
0343                   RSEQ_INJECT_ASM(4)
0344 #ifdef RSEQ_COMPARE_TWICE
0345                   RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
0346                   RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]")
0347 #endif
0348                   RSEQ_ASM_OP_STORE(newv2, v2)
0349                   RSEQ_INJECT_ASM(5)
0350                   RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
0351                   RSEQ_INJECT_ASM(6)
0352                   RSEQ_ASM_DEFINE_ABORT(4, abort)
0353                   : /* gcc asm goto does not allow outputs */
0354                   : [cpu_id]        "r" (cpu),
0355                     [current_cpu_id]    "m" (rseq_get_abi()->cpu_id),
0356                     [rseq_cs]       "m" (rseq_get_abi()->rseq_cs.arch.ptr),
0357                     [expect]        "r" (expect),
0358                     [v]         "m" (*v),
0359                     [newv]      "r" (newv),
0360                     [v2]            "m" (*v2),
0361                     [newv2]     "r" (newv2)
0362                     RSEQ_INJECT_INPUT
0363                   : "memory", RSEQ_ASM_TMP_REG_1
0364                     RSEQ_INJECT_CLOBBER
0365                   : abort, cmpfail
0366 #ifdef RSEQ_COMPARE_TWICE
0367                     , error1, error2
0368 #endif
0369     );
0370 
0371     return 0;
0372 abort:
0373     RSEQ_INJECT_FAILED
0374     return -1;
0375 cmpfail:
0376     return 1;
0377 #ifdef RSEQ_COMPARE_TWICE
0378 error1:
0379     rseq_bug("cpu_id comparison failed");
0380 error2:
0381     rseq_bug("expected value comparison failed");
0382 #endif
0383 }
0384 
0385 static inline __always_inline
0386 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
0387                      intptr_t *v2, intptr_t newv2,
0388                      intptr_t newv, int cpu)
0389 {
0390     RSEQ_INJECT_C(9)
0391 
0392     __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
0393                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
0394 #ifdef RSEQ_COMPARE_TWICE
0395                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
0396                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
0397 #endif
0398                   RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
0399                   RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
0400                   RSEQ_INJECT_ASM(3)
0401                   RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]")
0402                   RSEQ_INJECT_ASM(4)
0403 #ifdef RSEQ_COMPARE_TWICE
0404                   RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
0405                   RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]")
0406 #endif
0407                   RSEQ_ASM_OP_STORE(newv2, v2)
0408                   RSEQ_INJECT_ASM(5)
0409                   RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
0410                   RSEQ_INJECT_ASM(6)
0411                   RSEQ_ASM_DEFINE_ABORT(4, abort)
0412                   : /* gcc asm goto does not allow outputs */
0413                   : [cpu_id]        "r" (cpu),
0414                     [current_cpu_id]    "m" (rseq_get_abi()->cpu_id),
0415                     [rseq_cs]       "m" (rseq_get_abi()->rseq_cs.arch.ptr),
0416                     [expect]        "r" (expect),
0417                     [v]         "m" (*v),
0418                     [newv]      "r" (newv),
0419                     [v2]            "m" (*v2),
0420                     [newv2]     "r" (newv2)
0421                     RSEQ_INJECT_INPUT
0422                   : "memory", RSEQ_ASM_TMP_REG_1
0423                     RSEQ_INJECT_CLOBBER
0424                   : abort, cmpfail
0425 #ifdef RSEQ_COMPARE_TWICE
0426                     , error1, error2
0427 #endif
0428     );
0429 
0430     return 0;
0431 abort:
0432     RSEQ_INJECT_FAILED
0433     return -1;
0434 cmpfail:
0435     return 1;
0436 #ifdef RSEQ_COMPARE_TWICE
0437 error1:
0438     rseq_bug("cpu_id comparison failed");
0439 error2:
0440     rseq_bug("expected value comparison failed");
0441 #endif
0442 }
0443 
0444 static inline __always_inline
0445 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
0446                   intptr_t *v2, intptr_t expect2,
0447                   intptr_t newv, int cpu)
0448 {
0449     RSEQ_INJECT_C(9)
0450 
0451     __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
0452                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
0453 #ifdef RSEQ_COMPARE_TWICE
0454                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
0455                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
0456                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error3]")
0457 #endif
0458                   RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
0459                   RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
0460                   RSEQ_INJECT_ASM(3)
0461                   RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]")
0462                   RSEQ_INJECT_ASM(4)
0463                   RSEQ_ASM_OP_CMPEQ(v2, expect2, "%l[cmpfail]")
0464                   RSEQ_INJECT_ASM(5)
0465 #ifdef RSEQ_COMPARE_TWICE
0466                   RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
0467                   RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]")
0468                   RSEQ_ASM_OP_CMPEQ(v2, expect2, "%l[error3]")
0469 #endif
0470                   RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
0471                   RSEQ_INJECT_ASM(6)
0472                   RSEQ_ASM_DEFINE_ABORT(4, abort)
0473                   : /* gcc asm goto does not allow outputs */
0474                   : [cpu_id]        "r" (cpu),
0475                     [current_cpu_id]    "m" (rseq_get_abi()->cpu_id),
0476                     [rseq_cs]       "m" (rseq_get_abi()->rseq_cs.arch.ptr),
0477                     [v]         "m" (*v),
0478                     [expect]        "r" (expect),
0479                     [v2]            "m" (*v2),
0480                     [expect2]       "r" (expect2),
0481                     [newv]      "r" (newv)
0482                     RSEQ_INJECT_INPUT
0483                   : "memory", RSEQ_ASM_TMP_REG_1
0484                     RSEQ_INJECT_CLOBBER
0485                   : abort, cmpfail
0486 #ifdef RSEQ_COMPARE_TWICE
0487                     , error1, error2, error3
0488 #endif
0489     );
0490 
0491     return 0;
0492 abort:
0493     RSEQ_INJECT_FAILED
0494     return -1;
0495 cmpfail:
0496     return 1;
0497 #ifdef RSEQ_COMPARE_TWICE
0498 error1:
0499     rseq_bug("cpu_id comparison failed");
0500 error2:
0501     rseq_bug("expected value comparison failed");
0502 error3:
0503     rseq_bug("2nd expected value comparison failed");
0504 #endif
0505 }
0506 
0507 static inline __always_inline
0508 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
0509                  void *dst, void *src, size_t len,
0510                  intptr_t newv, int cpu)
0511 {
0512     RSEQ_INJECT_C(9)
0513     __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
0514                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
0515 #ifdef RSEQ_COMPARE_TWICE
0516                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
0517                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
0518 #endif
0519                   RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
0520                   RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
0521                   RSEQ_INJECT_ASM(3)
0522                   RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]")
0523                   RSEQ_INJECT_ASM(4)
0524 #ifdef RSEQ_COMPARE_TWICE
0525                   RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
0526                   RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]")
0527 #endif
0528                   RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)
0529                   RSEQ_INJECT_ASM(5)
0530                   RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
0531                   RSEQ_INJECT_ASM(6)
0532                   RSEQ_ASM_DEFINE_ABORT(4, abort)
0533                   : /* gcc asm goto does not allow outputs */
0534                   : [cpu_id]        "r" (cpu),
0535                     [current_cpu_id]    "m" (rseq_get_abi()->cpu_id),
0536                     [rseq_cs]       "m" (rseq_get_abi()->rseq_cs.arch.ptr),
0537                     [expect]        "r" (expect),
0538                     [v]         "m" (*v),
0539                     [newv]      "r" (newv),
0540                     [dst]           "r" (dst),
0541                     [src]           "r" (src),
0542                     [len]           "r" (len)
0543                     RSEQ_INJECT_INPUT
0544                   : "memory", RSEQ_ASM_TMP_REG_1, RSEQ_ASM_TMP_REG_2,
0545                     RSEQ_ASM_TMP_REG_3, RSEQ_ASM_TMP_REG_4
0546                     RSEQ_INJECT_CLOBBER
0547                   : abort, cmpfail
0548 #ifdef RSEQ_COMPARE_TWICE
0549                     , error1, error2
0550 #endif
0551     );
0552 
0553     return 0;
0554 abort:
0555     RSEQ_INJECT_FAILED
0556     return -1;
0557 cmpfail:
0558     return 1;
0559 #ifdef RSEQ_COMPARE_TWICE
0560 error1:
0561     rseq_bug("cpu_id comparison failed");
0562 error2:
0563     rseq_bug("expected value comparison failed");
0564 #endif
0565 }
0566 
0567 static inline __always_inline
0568 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
0569                      void *dst, void *src, size_t len,
0570                      intptr_t newv, int cpu)
0571 {
0572     RSEQ_INJECT_C(9)
0573 
0574     __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
0575                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
0576 #ifdef RSEQ_COMPARE_TWICE
0577                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
0578                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
0579 #endif
0580                   RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
0581                   RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
0582                   RSEQ_INJECT_ASM(3)
0583                   RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]")
0584                   RSEQ_INJECT_ASM(4)
0585 #ifdef RSEQ_COMPARE_TWICE
0586                   RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
0587                   RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]")
0588 #endif
0589                   RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)
0590                   RSEQ_INJECT_ASM(5)
0591                   RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
0592                   RSEQ_INJECT_ASM(6)
0593                   RSEQ_ASM_DEFINE_ABORT(4, abort)
0594                   : /* gcc asm goto does not allow outputs */
0595                   : [cpu_id]        "r" (cpu),
0596                     [current_cpu_id]    "m" (rseq_get_abi()->cpu_id),
0597                     [rseq_cs]       "m" (rseq_get_abi()->rseq_cs.arch.ptr),
0598                     [expect]        "r" (expect),
0599                     [v]         "m" (*v),
0600                     [newv]      "r" (newv),
0601                     [dst]           "r" (dst),
0602                     [src]           "r" (src),
0603                     [len]           "r" (len)
0604                     RSEQ_INJECT_INPUT
0605                   : "memory", RSEQ_ASM_TMP_REG_1, RSEQ_ASM_TMP_REG_2,
0606                     RSEQ_ASM_TMP_REG_3, RSEQ_ASM_TMP_REG_4
0607                     RSEQ_INJECT_CLOBBER
0608                   : abort, cmpfail
0609 #ifdef RSEQ_COMPARE_TWICE
0610                     , error1, error2
0611 #endif
0612     );
0613 
0614     return 0;
0615 abort:
0616     RSEQ_INJECT_FAILED
0617     return -1;
0618 cmpfail:
0619     return 1;
0620 #ifdef RSEQ_COMPARE_TWICE
0621 error1:
0622     rseq_bug("cpu_id comparison failed");
0623 error2:
0624     rseq_bug("expected value comparison failed");
0625 #endif
0626 }
0627 
0628 #define RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV
0629 
0630 /*
0631  *   pval = *(ptr+off)
0632  *  *pval += inc;
0633  */
0634 static inline __always_inline
0635 int rseq_offset_deref_addv(intptr_t *ptr, off_t off, intptr_t inc, int cpu)
0636 {
0637     RSEQ_INJECT_C(9)
0638 
0639     __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
0640 #ifdef RSEQ_COMPARE_TWICE
0641                   RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
0642 #endif
0643                   RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
0644                   RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
0645                   RSEQ_INJECT_ASM(3)
0646 #ifdef RSEQ_COMPARE_TWICE
0647                   RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
0648 #endif
0649                   RSEQ_ASM_OP_R_DEREF_ADDV(ptr, off, 3)
0650                   RSEQ_INJECT_ASM(4)
0651                   RSEQ_ASM_DEFINE_ABORT(4, abort)
0652                   : /* gcc asm goto does not allow outputs */
0653                   : [cpu_id]        "r" (cpu),
0654                     [current_cpu_id]      "m" (rseq_get_abi()->cpu_id),
0655                     [rseq_cs]       "m" (rseq_get_abi()->rseq_cs.arch.ptr),
0656                     [ptr]           "r" (ptr),
0657                     [off]           "er" (off),
0658                     [inc]           "er" (inc)
0659                     RSEQ_INJECT_INPUT
0660                   : "memory", RSEQ_ASM_TMP_REG_1
0661                     RSEQ_INJECT_CLOBBER
0662                   : abort
0663 #ifdef RSEQ_COMPARE_TWICE
0664                     , error1
0665 #endif
0666     );
0667     return 0;
0668 abort:
0669     RSEQ_INJECT_FAILED
0670     return -1;
0671 #ifdef RSEQ_COMPARE_TWICE
0672 error1:
0673     rseq_bug("cpu_id comparison failed");
0674 #endif
0675 }
0676 
0677 #endif /* !RSEQ_SKIP_FASTPATH */
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!