Back to home page

OSCL-LXR
 
 

     

0001 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
0002 /*
0003  * Author: Paul Burton <paul.burton@mips.com>
0004  * (C) Copyright 2018 MIPS Tech LLC
0005  *
0006  * Based on rseq-arm.h:
0007  * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
0008  */
0009 
0010 /*
0011  * RSEQ_SIG uses the break instruction. The instruction pattern is:
0012  *
0013  * On MIPS:
0014  *  0350000d        break     0x350
0015  *
0016  * On nanoMIPS:
0017  *      00100350        break     0x350
0018  *
0019  * On microMIPS:
0020  *      0000d407        break     0x350
0021  *
0022  * For nanoMIPS32 and microMIPS, the instruction stream is encoded as 16-bit
0023  * halfwords, so the signature halfwords need to be swapped accordingly for
0024  * little-endian.
0025  */
0026 #if defined(__nanomips__)
0027 # ifdef __MIPSEL__
0028 #  define RSEQ_SIG  0x03500010
0029 # else
0030 #  define RSEQ_SIG  0x00100350
0031 # endif
0032 #elif defined(__mips_micromips)
0033 # ifdef __MIPSEL__
0034 #  define RSEQ_SIG  0xd4070000
0035 # else
0036 #  define RSEQ_SIG  0x0000d407
0037 # endif
0038 #elif defined(__mips__)
0039 # define RSEQ_SIG   0x0350000d
0040 #else
0041 /* Unknown MIPS architecture. */
0042 #endif
0043 
0044 #define rseq_smp_mb()   __asm__ __volatile__ ("sync" ::: "memory")
0045 #define rseq_smp_rmb()  rseq_smp_mb()
0046 #define rseq_smp_wmb()  rseq_smp_mb()
0047 
0048 #define rseq_smp_load_acquire(p)                    \
0049 __extension__ ({                            \
0050     __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p);           \
0051     rseq_smp_mb();                          \
0052     ____p1;                             \
0053 })
0054 
0055 #define rseq_smp_acquire__after_ctrl_dep()  rseq_smp_rmb()
0056 
0057 #define rseq_smp_store_release(p, v)                    \
0058 do {                                    \
0059     rseq_smp_mb();                          \
0060     RSEQ_WRITE_ONCE(*p, v);                     \
0061 } while (0)
0062 
0063 #ifdef RSEQ_SKIP_FASTPATH
0064 #include "rseq-skip.h"
0065 #else /* !RSEQ_SKIP_FASTPATH */
0066 
0067 #if _MIPS_SZLONG == 64
0068 # define LONG           ".dword"
0069 # define LONG_LA        "dla"
0070 # define LONG_L         "ld"
0071 # define LONG_S         "sd"
0072 # define LONG_ADDI      "daddiu"
0073 # define U32_U64_PAD(x)     x
0074 #elif _MIPS_SZLONG == 32
0075 # define LONG           ".word"
0076 # define LONG_LA        "la"
0077 # define LONG_L         "lw"
0078 # define LONG_S         "sw"
0079 # define LONG_ADDI      "addiu"
0080 # ifdef __BIG_ENDIAN
0081 #  define U32_U64_PAD(x)    "0x0, " x
0082 # else
0083 #  define U32_U64_PAD(x)    x ", 0x0"
0084 # endif
0085 #else
0086 # error unsupported _MIPS_SZLONG
0087 #endif
0088 
0089 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip, \
0090                 post_commit_offset, abort_ip) \
0091         ".pushsection __rseq_cs, \"aw\"\n\t" \
0092         ".balign 32\n\t" \
0093         __rseq_str(label) ":\n\t"                   \
0094         ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
0095         LONG " " U32_U64_PAD(__rseq_str(start_ip)) "\n\t" \
0096         LONG " " U32_U64_PAD(__rseq_str(post_commit_offset)) "\n\t" \
0097         LONG " " U32_U64_PAD(__rseq_str(abort_ip)) "\n\t" \
0098         ".popsection\n\t" \
0099         ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
0100         LONG " " U32_U64_PAD(__rseq_str(label) "b") "\n\t" \
0101         ".popsection\n\t"
0102 
0103 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
0104     __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
0105                 (post_commit_ip - start_ip), abort_ip)
0106 
0107 /*
0108  * Exit points of a rseq critical section consist of all instructions outside
0109  * of the critical section where a critical section can either branch to or
0110  * reach through the normal course of its execution. The abort IP and the
0111  * post-commit IP are already part of the __rseq_cs section and should not be
0112  * explicitly defined as additional exit points. Knowing all exit points is
0113  * useful to assist debuggers stepping over the critical section.
0114  */
0115 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
0116         ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
0117         LONG " " U32_U64_PAD(__rseq_str(start_ip)) "\n\t" \
0118         LONG " " U32_U64_PAD(__rseq_str(exit_ip)) "\n\t" \
0119         ".popsection\n\t"
0120 
0121 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
0122         RSEQ_INJECT_ASM(1) \
0123         LONG_LA " $4, " __rseq_str(cs_label) "\n\t" \
0124         LONG_S  " $4, %[" __rseq_str(rseq_cs) "]\n\t" \
0125         __rseq_str(label) ":\n\t"
0126 
0127 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
0128         RSEQ_INJECT_ASM(2) \
0129         "lw  $4, %[" __rseq_str(current_cpu_id) "]\n\t" \
0130         "bne $4, %[" __rseq_str(cpu_id) "], " __rseq_str(label) "\n\t"
0131 
0132 #define __RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, \
0133                 abort_label, version, flags, \
0134                 start_ip, post_commit_offset, abort_ip) \
0135         ".balign 32\n\t" \
0136         __rseq_str(table_label) ":\n\t" \
0137         ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
0138         LONG " " U32_U64_PAD(__rseq_str(start_ip)) "\n\t" \
0139         LONG " " U32_U64_PAD(__rseq_str(post_commit_offset)) "\n\t" \
0140         LONG " " U32_U64_PAD(__rseq_str(abort_ip)) "\n\t" \
0141         ".word " __rseq_str(RSEQ_SIG) "\n\t" \
0142         __rseq_str(label) ":\n\t" \
0143         teardown \
0144         "b %l[" __rseq_str(abort_label) "]\n\t"
0145 
0146 #define RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, abort_label, \
0147                   start_ip, post_commit_ip, abort_ip) \
0148     __RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, \
0149                 abort_label, 0x0, 0x0, start_ip, \
0150                 (post_commit_ip - start_ip), abort_ip)
0151 
0152 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \
0153         __rseq_str(label) ":\n\t" \
0154         teardown \
0155         "b %l[" __rseq_str(cmpfail_label) "]\n\t"
0156 
0157 static inline __attribute__((always_inline))
0158 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
0159 {
0160     RSEQ_INJECT_C(9)
0161 
0162     __asm__ __volatile__ goto (
0163         RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
0164         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
0165 #ifdef RSEQ_COMPARE_TWICE
0166         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
0167         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
0168 #endif
0169         /* Start rseq by storing table entry pointer into rseq_cs. */
0170         RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
0171         RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
0172         RSEQ_INJECT_ASM(3)
0173         LONG_L " $4, %[v]\n\t"
0174         "bne $4, %[expect], %l[cmpfail]\n\t"
0175         RSEQ_INJECT_ASM(4)
0176 #ifdef RSEQ_COMPARE_TWICE
0177         RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
0178         LONG_L " $4, %[v]\n\t"
0179         "bne $4, %[expect], %l[error2]\n\t"
0180 #endif
0181         /* final store */
0182         LONG_S " %[newv], %[v]\n\t"
0183         "2:\n\t"
0184         RSEQ_INJECT_ASM(5)
0185         "b 5f\n\t"
0186         RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
0187         "5:\n\t"
0188         : /* gcc asm goto does not allow outputs */
0189         : [cpu_id]      "r" (cpu),
0190           [current_cpu_id]  "m" (rseq_get_abi()->cpu_id),
0191           [rseq_cs]     "m" (rseq_get_abi()->rseq_cs.arch.ptr),
0192           [v]           "m" (*v),
0193           [expect]      "r" (expect),
0194           [newv]        "r" (newv)
0195           RSEQ_INJECT_INPUT
0196         : "$4", "memory"
0197           RSEQ_INJECT_CLOBBER
0198         : abort, cmpfail
0199 #ifdef RSEQ_COMPARE_TWICE
0200           , error1, error2
0201 #endif
0202     );
0203     return 0;
0204 abort:
0205     RSEQ_INJECT_FAILED
0206     return -1;
0207 cmpfail:
0208     return 1;
0209 #ifdef RSEQ_COMPARE_TWICE
0210 error1:
0211     rseq_bug("cpu_id comparison failed");
0212 error2:
0213     rseq_bug("expected value comparison failed");
0214 #endif
0215 }
0216 
0217 static inline __attribute__((always_inline))
0218 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
0219                    long voffp, intptr_t *load, int cpu)
0220 {
0221     RSEQ_INJECT_C(9)
0222 
0223     __asm__ __volatile__ goto (
0224         RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
0225         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
0226 #ifdef RSEQ_COMPARE_TWICE
0227         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
0228         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
0229 #endif
0230         /* Start rseq by storing table entry pointer into rseq_cs. */
0231         RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
0232         RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
0233         RSEQ_INJECT_ASM(3)
0234         LONG_L " $4, %[v]\n\t"
0235         "beq $4, %[expectnot], %l[cmpfail]\n\t"
0236         RSEQ_INJECT_ASM(4)
0237 #ifdef RSEQ_COMPARE_TWICE
0238         RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
0239         LONG_L " $4, %[v]\n\t"
0240         "beq $4, %[expectnot], %l[error2]\n\t"
0241 #endif
0242         LONG_S " $4, %[load]\n\t"
0243         LONG_ADDI " $4, %[voffp]\n\t"
0244         LONG_L " $4, 0($4)\n\t"
0245         /* final store */
0246         LONG_S " $4, %[v]\n\t"
0247         "2:\n\t"
0248         RSEQ_INJECT_ASM(5)
0249         "b 5f\n\t"
0250         RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
0251         "5:\n\t"
0252         : /* gcc asm goto does not allow outputs */
0253         : [cpu_id]      "r" (cpu),
0254           [current_cpu_id]  "m" (rseq_get_abi()->cpu_id),
0255           [rseq_cs]     "m" (rseq_get_abi()->rseq_cs.arch.ptr),
0256           /* final store input */
0257           [v]           "m" (*v),
0258           [expectnot]       "r" (expectnot),
0259           [voffp]       "Ir" (voffp),
0260           [load]        "m" (*load)
0261           RSEQ_INJECT_INPUT
0262         : "$4", "memory"
0263           RSEQ_INJECT_CLOBBER
0264         : abort, cmpfail
0265 #ifdef RSEQ_COMPARE_TWICE
0266           , error1, error2
0267 #endif
0268     );
0269     return 0;
0270 abort:
0271     RSEQ_INJECT_FAILED
0272     return -1;
0273 cmpfail:
0274     return 1;
0275 #ifdef RSEQ_COMPARE_TWICE
0276 error1:
0277     rseq_bug("cpu_id comparison failed");
0278 error2:
0279     rseq_bug("expected value comparison failed");
0280 #endif
0281 }
0282 
0283 static inline __attribute__((always_inline))
0284 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
0285 {
0286     RSEQ_INJECT_C(9)
0287 
0288     __asm__ __volatile__ goto (
0289         RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
0290 #ifdef RSEQ_COMPARE_TWICE
0291         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
0292 #endif
0293         /* Start rseq by storing table entry pointer into rseq_cs. */
0294         RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
0295         RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
0296         RSEQ_INJECT_ASM(3)
0297 #ifdef RSEQ_COMPARE_TWICE
0298         RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
0299 #endif
0300         LONG_L " $4, %[v]\n\t"
0301         LONG_ADDI " $4, %[count]\n\t"
0302         /* final store */
0303         LONG_S " $4, %[v]\n\t"
0304         "2:\n\t"
0305         RSEQ_INJECT_ASM(4)
0306         "b 5f\n\t"
0307         RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
0308         "5:\n\t"
0309         : /* gcc asm goto does not allow outputs */
0310         : [cpu_id]      "r" (cpu),
0311           [current_cpu_id]  "m" (rseq_get_abi()->cpu_id),
0312           [rseq_cs]     "m" (rseq_get_abi()->rseq_cs.arch.ptr),
0313           [v]           "m" (*v),
0314           [count]       "Ir" (count)
0315           RSEQ_INJECT_INPUT
0316         : "$4", "memory"
0317           RSEQ_INJECT_CLOBBER
0318         : abort
0319 #ifdef RSEQ_COMPARE_TWICE
0320           , error1
0321 #endif
0322     );
0323     return 0;
0324 abort:
0325     RSEQ_INJECT_FAILED
0326     return -1;
0327 #ifdef RSEQ_COMPARE_TWICE
0328 error1:
0329     rseq_bug("cpu_id comparison failed");
0330 #endif
0331 }
0332 
0333 static inline __attribute__((always_inline))
0334 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
0335                  intptr_t *v2, intptr_t newv2,
0336                  intptr_t newv, int cpu)
0337 {
0338     RSEQ_INJECT_C(9)
0339 
0340     __asm__ __volatile__ goto (
0341         RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
0342         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
0343 #ifdef RSEQ_COMPARE_TWICE
0344         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
0345         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
0346 #endif
0347         /* Start rseq by storing table entry pointer into rseq_cs. */
0348         RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
0349         RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
0350         RSEQ_INJECT_ASM(3)
0351         LONG_L " $4, %[v]\n\t"
0352         "bne $4, %[expect], %l[cmpfail]\n\t"
0353         RSEQ_INJECT_ASM(4)
0354 #ifdef RSEQ_COMPARE_TWICE
0355         RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
0356         LONG_L " $4, %[v]\n\t"
0357         "bne $4, %[expect], %l[error2]\n\t"
0358 #endif
0359         /* try store */
0360         LONG_S " %[newv2], %[v2]\n\t"
0361         RSEQ_INJECT_ASM(5)
0362         /* final store */
0363         LONG_S " %[newv], %[v]\n\t"
0364         "2:\n\t"
0365         RSEQ_INJECT_ASM(6)
0366         "b 5f\n\t"
0367         RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
0368         "5:\n\t"
0369         : /* gcc asm goto does not allow outputs */
0370         : [cpu_id]      "r" (cpu),
0371           [current_cpu_id]  "m" (rseq_get_abi()->cpu_id),
0372           [rseq_cs]     "m" (rseq_get_abi()->rseq_cs.arch.ptr),
0373           /* try store input */
0374           [v2]          "m" (*v2),
0375           [newv2]       "r" (newv2),
0376           /* final store input */
0377           [v]           "m" (*v),
0378           [expect]      "r" (expect),
0379           [newv]        "r" (newv)
0380           RSEQ_INJECT_INPUT
0381         : "$4", "memory"
0382           RSEQ_INJECT_CLOBBER
0383         : abort, cmpfail
0384 #ifdef RSEQ_COMPARE_TWICE
0385           , error1, error2
0386 #endif
0387     );
0388     return 0;
0389 abort:
0390     RSEQ_INJECT_FAILED
0391     return -1;
0392 cmpfail:
0393     return 1;
0394 #ifdef RSEQ_COMPARE_TWICE
0395 error1:
0396     rseq_bug("cpu_id comparison failed");
0397 error2:
0398     rseq_bug("expected value comparison failed");
0399 #endif
0400 }
0401 
0402 static inline __attribute__((always_inline))
0403 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
0404                      intptr_t *v2, intptr_t newv2,
0405                      intptr_t newv, int cpu)
0406 {
0407     RSEQ_INJECT_C(9)
0408 
0409     __asm__ __volatile__ goto (
0410         RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
0411         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
0412 #ifdef RSEQ_COMPARE_TWICE
0413         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
0414         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
0415 #endif
0416         /* Start rseq by storing table entry pointer into rseq_cs. */
0417         RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
0418         RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
0419         RSEQ_INJECT_ASM(3)
0420         LONG_L " $4, %[v]\n\t"
0421         "bne $4, %[expect], %l[cmpfail]\n\t"
0422         RSEQ_INJECT_ASM(4)
0423 #ifdef RSEQ_COMPARE_TWICE
0424         RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
0425         LONG_L " $4, %[v]\n\t"
0426         "bne $4, %[expect], %l[error2]\n\t"
0427 #endif
0428         /* try store */
0429         LONG_S " %[newv2], %[v2]\n\t"
0430         RSEQ_INJECT_ASM(5)
0431         "sync\n\t"  /* full sync provides store-release */
0432         /* final store */
0433         LONG_S " %[newv], %[v]\n\t"
0434         "2:\n\t"
0435         RSEQ_INJECT_ASM(6)
0436         "b 5f\n\t"
0437         RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
0438         "5:\n\t"
0439         : /* gcc asm goto does not allow outputs */
0440         : [cpu_id]      "r" (cpu),
0441           [current_cpu_id]  "m" (rseq_get_abi()->cpu_id),
0442           [rseq_cs]     "m" (rseq_get_abi()->rseq_cs.arch.ptr),
0443           /* try store input */
0444           [v2]          "m" (*v2),
0445           [newv2]       "r" (newv2),
0446           /* final store input */
0447           [v]           "m" (*v),
0448           [expect]      "r" (expect),
0449           [newv]        "r" (newv)
0450           RSEQ_INJECT_INPUT
0451         : "$4", "memory"
0452           RSEQ_INJECT_CLOBBER
0453         : abort, cmpfail
0454 #ifdef RSEQ_COMPARE_TWICE
0455           , error1, error2
0456 #endif
0457     );
0458     return 0;
0459 abort:
0460     RSEQ_INJECT_FAILED
0461     return -1;
0462 cmpfail:
0463     return 1;
0464 #ifdef RSEQ_COMPARE_TWICE
0465 error1:
0466     rseq_bug("cpu_id comparison failed");
0467 error2:
0468     rseq_bug("expected value comparison failed");
0469 #endif
0470 }
0471 
0472 static inline __attribute__((always_inline))
0473 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
0474                   intptr_t *v2, intptr_t expect2,
0475                   intptr_t newv, int cpu)
0476 {
0477     RSEQ_INJECT_C(9)
0478 
0479     __asm__ __volatile__ goto (
0480         RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
0481         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
0482 #ifdef RSEQ_COMPARE_TWICE
0483         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
0484         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
0485         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
0486 #endif
0487         /* Start rseq by storing table entry pointer into rseq_cs. */
0488         RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
0489         RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
0490         RSEQ_INJECT_ASM(3)
0491         LONG_L " $4, %[v]\n\t"
0492         "bne $4, %[expect], %l[cmpfail]\n\t"
0493         RSEQ_INJECT_ASM(4)
0494         LONG_L " $4, %[v2]\n\t"
0495         "bne $4, %[expect2], %l[cmpfail]\n\t"
0496         RSEQ_INJECT_ASM(5)
0497 #ifdef RSEQ_COMPARE_TWICE
0498         RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
0499         LONG_L " $4, %[v]\n\t"
0500         "bne $4, %[expect], %l[error2]\n\t"
0501         LONG_L " $4, %[v2]\n\t"
0502         "bne $4, %[expect2], %l[error3]\n\t"
0503 #endif
0504         /* final store */
0505         LONG_S " %[newv], %[v]\n\t"
0506         "2:\n\t"
0507         RSEQ_INJECT_ASM(6)
0508         "b 5f\n\t"
0509         RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
0510         "5:\n\t"
0511         : /* gcc asm goto does not allow outputs */
0512         : [cpu_id]      "r" (cpu),
0513           [current_cpu_id]  "m" (rseq_get_abi()->cpu_id),
0514           [rseq_cs]     "m" (rseq_get_abi()->rseq_cs.arch.ptr),
0515           /* cmp2 input */
0516           [v2]          "m" (*v2),
0517           [expect2]     "r" (expect2),
0518           /* final store input */
0519           [v]           "m" (*v),
0520           [expect]      "r" (expect),
0521           [newv]        "r" (newv)
0522           RSEQ_INJECT_INPUT
0523         : "$4", "memory"
0524           RSEQ_INJECT_CLOBBER
0525         : abort, cmpfail
0526 #ifdef RSEQ_COMPARE_TWICE
0527           , error1, error2, error3
0528 #endif
0529     );
0530     return 0;
0531 abort:
0532     RSEQ_INJECT_FAILED
0533     return -1;
0534 cmpfail:
0535     return 1;
0536 #ifdef RSEQ_COMPARE_TWICE
0537 error1:
0538     rseq_bug("cpu_id comparison failed");
0539 error2:
0540     rseq_bug("1st expected value comparison failed");
0541 error3:
0542     rseq_bug("2nd expected value comparison failed");
0543 #endif
0544 }
0545 
0546 static inline __attribute__((always_inline))
0547 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
0548                  void *dst, void *src, size_t len,
0549                  intptr_t newv, int cpu)
0550 {
0551     uintptr_t rseq_scratch[3];
0552 
0553     RSEQ_INJECT_C(9)
0554 
0555     __asm__ __volatile__ goto (
0556         RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
0557         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
0558 #ifdef RSEQ_COMPARE_TWICE
0559         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
0560         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
0561 #endif
0562         LONG_S " %[src], %[rseq_scratch0]\n\t"
0563         LONG_S "  %[dst], %[rseq_scratch1]\n\t"
0564         LONG_S " %[len], %[rseq_scratch2]\n\t"
0565         /* Start rseq by storing table entry pointer into rseq_cs. */
0566         RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
0567         RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
0568         RSEQ_INJECT_ASM(3)
0569         LONG_L " $4, %[v]\n\t"
0570         "bne $4, %[expect], 5f\n\t"
0571         RSEQ_INJECT_ASM(4)
0572 #ifdef RSEQ_COMPARE_TWICE
0573         RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
0574         LONG_L " $4, %[v]\n\t"
0575         "bne $4, %[expect], 7f\n\t"
0576 #endif
0577         /* try memcpy */
0578         "beqz %[len], 333f\n\t" \
0579         "222:\n\t" \
0580         "lb   $4, 0(%[src])\n\t" \
0581         "sb   $4, 0(%[dst])\n\t" \
0582         LONG_ADDI " %[src], 1\n\t" \
0583         LONG_ADDI " %[dst], 1\n\t" \
0584         LONG_ADDI " %[len], -1\n\t" \
0585         "bnez %[len], 222b\n\t" \
0586         "333:\n\t" \
0587         RSEQ_INJECT_ASM(5)
0588         /* final store */
0589         LONG_S " %[newv], %[v]\n\t"
0590         "2:\n\t"
0591         RSEQ_INJECT_ASM(6)
0592         /* teardown */
0593         LONG_L " %[len], %[rseq_scratch2]\n\t"
0594         LONG_L " %[dst], %[rseq_scratch1]\n\t"
0595         LONG_L " %[src], %[rseq_scratch0]\n\t"
0596         "b 8f\n\t"
0597         RSEQ_ASM_DEFINE_ABORT(3, 4,
0598                       /* teardown */
0599                       LONG_L " %[len], %[rseq_scratch2]\n\t"
0600                       LONG_L " %[dst], %[rseq_scratch1]\n\t"
0601                       LONG_L " %[src], %[rseq_scratch0]\n\t",
0602                       abort, 1b, 2b, 4f)
0603         RSEQ_ASM_DEFINE_CMPFAIL(5,
0604                     /* teardown */
0605                     LONG_L " %[len], %[rseq_scratch2]\n\t"
0606                     LONG_L " %[dst], %[rseq_scratch1]\n\t"
0607                     LONG_L " %[src], %[rseq_scratch0]\n\t",
0608                     cmpfail)
0609 #ifdef RSEQ_COMPARE_TWICE
0610         RSEQ_ASM_DEFINE_CMPFAIL(6,
0611                     /* teardown */
0612                     LONG_L " %[len], %[rseq_scratch2]\n\t"
0613                     LONG_L " %[dst], %[rseq_scratch1]\n\t"
0614                     LONG_L " %[src], %[rseq_scratch0]\n\t",
0615                     error1)
0616         RSEQ_ASM_DEFINE_CMPFAIL(7,
0617                     /* teardown */
0618                     LONG_L " %[len], %[rseq_scratch2]\n\t"
0619                     LONG_L " %[dst], %[rseq_scratch1]\n\t"
0620                     LONG_L " %[src], %[rseq_scratch0]\n\t",
0621                     error2)
0622 #endif
0623         "8:\n\t"
0624         : /* gcc asm goto does not allow outputs */
0625         : [cpu_id]      "r" (cpu),
0626           [current_cpu_id]  "m" (rseq_get_abi()->cpu_id),
0627           [rseq_cs]     "m" (rseq_get_abi()->rseq_cs.arch.ptr),
0628           /* final store input */
0629           [v]           "m" (*v),
0630           [expect]      "r" (expect),
0631           [newv]        "r" (newv),
0632           /* try memcpy input */
0633           [dst]         "r" (dst),
0634           [src]         "r" (src),
0635           [len]         "r" (len),
0636           [rseq_scratch0]   "m" (rseq_scratch[0]),
0637           [rseq_scratch1]   "m" (rseq_scratch[1]),
0638           [rseq_scratch2]   "m" (rseq_scratch[2])
0639           RSEQ_INJECT_INPUT
0640         : "$4", "memory"
0641           RSEQ_INJECT_CLOBBER
0642         : abort, cmpfail
0643 #ifdef RSEQ_COMPARE_TWICE
0644           , error1, error2
0645 #endif
0646     );
0647     return 0;
0648 abort:
0649     RSEQ_INJECT_FAILED
0650     return -1;
0651 cmpfail:
0652     return 1;
0653 #ifdef RSEQ_COMPARE_TWICE
0654 error1:
0655     rseq_bug("cpu_id comparison failed");
0656 error2:
0657     rseq_bug("expected value comparison failed");
0658 #endif
0659 }
0660 
0661 static inline __attribute__((always_inline))
0662 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
0663                      void *dst, void *src, size_t len,
0664                      intptr_t newv, int cpu)
0665 {
0666     uintptr_t rseq_scratch[3];
0667 
0668     RSEQ_INJECT_C(9)
0669 
0670     __asm__ __volatile__ goto (
0671         RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
0672         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
0673 #ifdef RSEQ_COMPARE_TWICE
0674         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
0675         RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
0676 #endif
0677         LONG_S " %[src], %[rseq_scratch0]\n\t"
0678         LONG_S " %[dst], %[rseq_scratch1]\n\t"
0679         LONG_S " %[len], %[rseq_scratch2]\n\t"
0680         /* Start rseq by storing table entry pointer into rseq_cs. */
0681         RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
0682         RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
0683         RSEQ_INJECT_ASM(3)
0684         LONG_L " $4, %[v]\n\t"
0685         "bne $4, %[expect], 5f\n\t"
0686         RSEQ_INJECT_ASM(4)
0687 #ifdef RSEQ_COMPARE_TWICE
0688         RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
0689         LONG_L " $4, %[v]\n\t"
0690         "bne $4, %[expect], 7f\n\t"
0691 #endif
0692         /* try memcpy */
0693         "beqz %[len], 333f\n\t" \
0694         "222:\n\t" \
0695         "lb   $4, 0(%[src])\n\t" \
0696         "sb   $4, 0(%[dst])\n\t" \
0697         LONG_ADDI " %[src], 1\n\t" \
0698         LONG_ADDI " %[dst], 1\n\t" \
0699         LONG_ADDI " %[len], -1\n\t" \
0700         "bnez %[len], 222b\n\t" \
0701         "333:\n\t" \
0702         RSEQ_INJECT_ASM(5)
0703         "sync\n\t"  /* full sync provides store-release */
0704         /* final store */
0705         LONG_S " %[newv], %[v]\n\t"
0706         "2:\n\t"
0707         RSEQ_INJECT_ASM(6)
0708         /* teardown */
0709         LONG_L " %[len], %[rseq_scratch2]\n\t"
0710         LONG_L " %[dst], %[rseq_scratch1]\n\t"
0711         LONG_L " %[src], %[rseq_scratch0]\n\t"
0712         "b 8f\n\t"
0713         RSEQ_ASM_DEFINE_ABORT(3, 4,
0714                       /* teardown */
0715                       LONG_L " %[len], %[rseq_scratch2]\n\t"
0716                       LONG_L " %[dst], %[rseq_scratch1]\n\t"
0717                       LONG_L " %[src], %[rseq_scratch0]\n\t",
0718                       abort, 1b, 2b, 4f)
0719         RSEQ_ASM_DEFINE_CMPFAIL(5,
0720                     /* teardown */
0721                     LONG_L " %[len], %[rseq_scratch2]\n\t"
0722                     LONG_L " %[dst], %[rseq_scratch1]\n\t"
0723                     LONG_L " %[src], %[rseq_scratch0]\n\t",
0724                     cmpfail)
0725 #ifdef RSEQ_COMPARE_TWICE
0726         RSEQ_ASM_DEFINE_CMPFAIL(6,
0727                     /* teardown */
0728                     LONG_L " %[len], %[rseq_scratch2]\n\t"
0729                     LONG_L " %[dst], %[rseq_scratch1]\n\t"
0730                     LONG_L " %[src], %[rseq_scratch0]\n\t",
0731                     error1)
0732         RSEQ_ASM_DEFINE_CMPFAIL(7,
0733                     /* teardown */
0734                     LONG_L " %[len], %[rseq_scratch2]\n\t"
0735                     LONG_L " %[dst], %[rseq_scratch1]\n\t"
0736                     LONG_L " %[src], %[rseq_scratch0]\n\t",
0737                     error2)
0738 #endif
0739         "8:\n\t"
0740         : /* gcc asm goto does not allow outputs */
0741         : [cpu_id]      "r" (cpu),
0742           [current_cpu_id]  "m" (rseq_get_abi()->cpu_id),
0743           [rseq_cs]     "m" (rseq_get_abi()->rseq_cs.arch.ptr),
0744           /* final store input */
0745           [v]           "m" (*v),
0746           [expect]      "r" (expect),
0747           [newv]        "r" (newv),
0748           /* try memcpy input */
0749           [dst]         "r" (dst),
0750           [src]         "r" (src),
0751           [len]         "r" (len),
0752           [rseq_scratch0]   "m" (rseq_scratch[0]),
0753           [rseq_scratch1]   "m" (rseq_scratch[1]),
0754           [rseq_scratch2]   "m" (rseq_scratch[2])
0755           RSEQ_INJECT_INPUT
0756         : "$4", "memory"
0757           RSEQ_INJECT_CLOBBER
0758         : abort, cmpfail
0759 #ifdef RSEQ_COMPARE_TWICE
0760           , error1, error2
0761 #endif
0762     );
0763     return 0;
0764 abort:
0765     RSEQ_INJECT_FAILED
0766     return -1;
0767 cmpfail:
0768     return 1;
0769 #ifdef RSEQ_COMPARE_TWICE
0770 error1:
0771     rseq_bug("cpu_id comparison failed");
0772 error2:
0773     rseq_bug("expected value comparison failed");
0774 #endif
0775 }
0776 
0777 #endif /* !RSEQ_SKIP_FASTPATH */
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!