linux/atomic/atomic-instrumented.h

0001 // SPDX-License-Identifier: GPL-2.0
0002
0003 // Generated by scripts/atomic/gen-atomic-instrumented.sh
0004 // DO NOT MODIFY THIS FILE DIRECTLY
0005
0006 /*
0007  * This file provides wrappers with KASAN instrumentation for atomic operations.
0008  * To use this functionality an arch's atomic.h file needs to define all
0009  * atomic operations with arch_ prefix (e.g. arch_atomic_read()) and include
0010  * this file at the end. This file provides atomic_read() that forwards to
0011  * arch_atomic_read() for actual atomic operation.
0012  * Note: if an arch atomic operation is implemented by means of other atomic
0013  * operations (e.g. atomic_read()/atomic_cmpxchg() loop), then it needs to use
0014  * arch_ variants (i.e. arch_atomic_read()/arch_atomic_cmpxchg()) to avoid
0015  * double instrumentation.
0016  */
0017 #ifndef _LINUX_ATOMIC_INSTRUMENTED_H
0018 #define _LINUX_ATOMIC_INSTRUMENTED_H
0019
0020 #include <linux/build_bug.h>
0021 #include <linux/compiler.h>
0022 #include <linux/instrumented.h>
0023
0024 static __always_inline int
0025 atomic_read(const atomic_t *v)
0026 {
0027     instrument_atomic_read(v, sizeof(*v));
0028     return arch_atomic_read(v);
0029 }
0030
0031 static __always_inline int
0032 atomic_read_acquire(const atomic_t *v)
0033 {
0034     instrument_atomic_read(v, sizeof(*v));
0035     return arch_atomic_read_acquire(v);
0036 }
0037
0038 static __always_inline void
0039 atomic_set(atomic_t *v, int i)
0040 {
0041     instrument_atomic_write(v, sizeof(*v));
0042     arch_atomic_set(v, i);
0043 }
0044
0045 static __always_inline void
0046 atomic_set_release(atomic_t *v, int i)
0047 {
0048     kcsan_release();
0049     instrument_atomic_write(v, sizeof(*v));
0050     arch_atomic_set_release(v, i);
0051 }
0052
0053 static __always_inline void
0054 atomic_add(int i, atomic_t *v)
0055 {
0056     instrument_atomic_read_write(v, sizeof(*v));
0057     arch_atomic_add(i, v);
0058 }
0059
0060 static __always_inline int
0061 atomic_add_return(int i, atomic_t *v)
0062 {
0063     kcsan_mb();
0064     instrument_atomic_read_write(v, sizeof(*v));
0065     return arch_atomic_add_return(i, v);
0066 }
0067
0068 static __always_inline int
0069 atomic_add_return_acquire(int i, atomic_t *v)
0070 {
0071     instrument_atomic_read_write(v, sizeof(*v));
0072     return arch_atomic_add_return_acquire(i, v);
0073 }
0074
0075 static __always_inline int
0076 atomic_add_return_release(int i, atomic_t *v)
0077 {
0078     kcsan_release();
0079     instrument_atomic_read_write(v, sizeof(*v));
0080     return arch_atomic_add_return_release(i, v);
0081 }
0082
0083 static __always_inline int
0084 atomic_add_return_relaxed(int i, atomic_t *v)
0085 {
0086     instrument_atomic_read_write(v, sizeof(*v));
0087     return arch_atomic_add_return_relaxed(i, v);
0088 }
0089
0090 static __always_inline int
0091 atomic_fetch_add(int i, atomic_t *v)
0092 {
0093     kcsan_mb();
0094     instrument_atomic_read_write(v, sizeof(*v));
0095     return arch_atomic_fetch_add(i, v);
0096 }
0097
0098 static __always_inline int
0099 atomic_fetch_add_acquire(int i, atomic_t *v)
0100 {
0101     instrument_atomic_read_write(v, sizeof(*v));
0102     return arch_atomic_fetch_add_acquire(i, v);
0103 }
0104
0105 static __always_inline int
0106 atomic_fetch_add_release(int i, atomic_t *v)
0107 {
0108     kcsan_release();
0109     instrument_atomic_read_write(v, sizeof(*v));
0110     return arch_atomic_fetch_add_release(i, v);
0111 }
0112
0113 static __always_inline int
0114 atomic_fetch_add_relaxed(int i, atomic_t *v)
0115 {
0116     instrument_atomic_read_write(v, sizeof(*v));
0117     return arch_atomic_fetch_add_relaxed(i, v);
0118 }
0119
0120 static __always_inline void
0121 atomic_sub(int i, atomic_t *v)
0122 {
0123     instrument_atomic_read_write(v, sizeof(*v));
0124     arch_atomic_sub(i, v);
0125 }
0126
0127 static __always_inline int
0128 atomic_sub_return(int i, atomic_t *v)
0129 {
0130     kcsan_mb();
0131     instrument_atomic_read_write(v, sizeof(*v));
0132     return arch_atomic_sub_return(i, v);
0133 }
0134
0135 static __always_inline int
0136 atomic_sub_return_acquire(int i, atomic_t *v)
0137 {
0138     instrument_atomic_read_write(v, sizeof(*v));
0139     return arch_atomic_sub_return_acquire(i, v);
0140 }
0141
0142 static __always_inline int
0143 atomic_sub_return_release(int i, atomic_t *v)
0144 {
0145     kcsan_release();
0146     instrument_atomic_read_write(v, sizeof(*v));
0147     return arch_atomic_sub_return_release(i, v);
0148 }
0149
0150 static __always_inline int
0151 atomic_sub_return_relaxed(int i, atomic_t *v)
0152 {
0153     instrument_atomic_read_write(v, sizeof(*v));
0154     return arch_atomic_sub_return_relaxed(i, v);
0155 }
0156
0157 static __always_inline int
0158 atomic_fetch_sub(int i, atomic_t *v)
0159 {
0160     kcsan_mb();
0161     instrument_atomic_read_write(v, sizeof(*v));
0162     return arch_atomic_fetch_sub(i, v);
0163 }
0164
0165 static __always_inline int
0166 atomic_fetch_sub_acquire(int i, atomic_t *v)
0167 {
0168     instrument_atomic_read_write(v, sizeof(*v));
0169     return arch_atomic_fetch_sub_acquire(i, v);
0170 }
0171
0172 static __always_inline int
0173 atomic_fetch_sub_release(int i, atomic_t *v)
0174 {
0175     kcsan_release();
0176     instrument_atomic_read_write(v, sizeof(*v));
0177     return arch_atomic_fetch_sub_release(i, v);
0178 }
0179
0180 static __always_inline int
0181 atomic_fetch_sub_relaxed(int i, atomic_t *v)
0182 {
0183     instrument_atomic_read_write(v, sizeof(*v));
0184     return arch_atomic_fetch_sub_relaxed(i, v);
0185 }
0186
0187 static __always_inline void
0188 atomic_inc(atomic_t *v)
0189 {
0190     instrument_atomic_read_write(v, sizeof(*v));
0191     arch_atomic_inc(v);
0192 }
0193
0194 static __always_inline int
0195 atomic_inc_return(atomic_t *v)
0196 {
0197     kcsan_mb();
0198     instrument_atomic_read_write(v, sizeof(*v));
0199     return arch_atomic_inc_return(v);
0200 }
0201
0202 static __always_inline int
0203 atomic_inc_return_acquire(atomic_t *v)
0204 {
0205     instrument_atomic_read_write(v, sizeof(*v));
0206     return arch_atomic_inc_return_acquire(v);
0207 }
0208
0209 static __always_inline int
0210 atomic_inc_return_release(atomic_t *v)
0211 {
0212     kcsan_release();
0213     instrument_atomic_read_write(v, sizeof(*v));
0214     return arch_atomic_inc_return_release(v);
0215 }
0216
0217 static __always_inline int
0218 atomic_inc_return_relaxed(atomic_t *v)
0219 {
0220     instrument_atomic_read_write(v, sizeof(*v));
0221     return arch_atomic_inc_return_relaxed(v);
0222 }
0223
0224 static __always_inline int
0225 atomic_fetch_inc(atomic_t *v)
0226 {
0227     kcsan_mb();
0228     instrument_atomic_read_write(v, sizeof(*v));
0229     return arch_atomic_fetch_inc(v);
0230 }
0231
0232 static __always_inline int
0233 atomic_fetch_inc_acquire(atomic_t *v)
0234 {
0235     instrument_atomic_read_write(v, sizeof(*v));
0236     return arch_atomic_fetch_inc_acquire(v);
0237 }
0238
0239 static __always_inline int
0240 atomic_fetch_inc_release(atomic_t *v)
0241 {
0242     kcsan_release();
0243     instrument_atomic_read_write(v, sizeof(*v));
0244     return arch_atomic_fetch_inc_release(v);
0245 }
0246
0247 static __always_inline int
0248 atomic_fetch_inc_relaxed(atomic_t *v)
0249 {
0250     instrument_atomic_read_write(v, sizeof(*v));
0251     return arch_atomic_fetch_inc_relaxed(v);
0252 }
0253
0254 static __always_inline void
0255 atomic_dec(atomic_t *v)
0256 {
0257     instrument_atomic_read_write(v, sizeof(*v));
0258     arch_atomic_dec(v);
0259 }
0260
0261 static __always_inline int
0262 atomic_dec_return(atomic_t *v)
0263 {
0264     kcsan_mb();
0265     instrument_atomic_read_write(v, sizeof(*v));
0266     return arch_atomic_dec_return(v);
0267 }
0268
0269 static __always_inline int
0270 atomic_dec_return_acquire(atomic_t *v)
0271 {
0272     instrument_atomic_read_write(v, sizeof(*v));
0273     return arch_atomic_dec_return_acquire(v);
0274 }
0275
0276 static __always_inline int
0277 atomic_dec_return_release(atomic_t *v)
0278 {
0279     kcsan_release();
0280     instrument_atomic_read_write(v, sizeof(*v));
0281     return arch_atomic_dec_return_release(v);
0282 }
0283
0284 static __always_inline int
0285 atomic_dec_return_relaxed(atomic_t *v)
0286 {
0287     instrument_atomic_read_write(v, sizeof(*v));
0288     return arch_atomic_dec_return_relaxed(v);
0289 }
0290
0291 static __always_inline int
0292 atomic_fetch_dec(atomic_t *v)
0293 {
0294     kcsan_mb();
0295     instrument_atomic_read_write(v, sizeof(*v));
0296     return arch_atomic_fetch_dec(v);
0297 }
0298
0299 static __always_inline int
0300 atomic_fetch_dec_acquire(atomic_t *v)
0301 {
0302     instrument_atomic_read_write(v, sizeof(*v));
0303     return arch_atomic_fetch_dec_acquire(v);
0304 }
0305
0306 static __always_inline int
0307 atomic_fetch_dec_release(atomic_t *v)
0308 {
0309     kcsan_release();
0310     instrument_atomic_read_write(v, sizeof(*v));
0311     return arch_atomic_fetch_dec_release(v);
0312 }
0313
0314 static __always_inline int
0315 atomic_fetch_dec_relaxed(atomic_t *v)
0316 {
0317     instrument_atomic_read_write(v, sizeof(*v));
0318     return arch_atomic_fetch_dec_relaxed(v);
0319 }
0320
0321 static __always_inline void
0322 atomic_and(int i, atomic_t *v)
0323 {
0324     instrument_atomic_read_write(v, sizeof(*v));
0325     arch_atomic_and(i, v);
0326 }
0327
0328 static __always_inline int
0329 atomic_fetch_and(int i, atomic_t *v)
0330 {
0331     kcsan_mb();
0332     instrument_atomic_read_write(v, sizeof(*v));
0333     return arch_atomic_fetch_and(i, v);
0334 }
0335
0336 static __always_inline int
0337 atomic_fetch_and_acquire(int i, atomic_t *v)
0338 {
0339     instrument_atomic_read_write(v, sizeof(*v));
0340     return arch_atomic_fetch_and_acquire(i, v);
0341 }
0342
0343 static __always_inline int
0344 atomic_fetch_and_release(int i, atomic_t *v)
0345 {
0346     kcsan_release();
0347     instrument_atomic_read_write(v, sizeof(*v));
0348     return arch_atomic_fetch_and_release(i, v);
0349 }
0350
0351 static __always_inline int
0352 atomic_fetch_and_relaxed(int i, atomic_t *v)
0353 {
0354     instrument_atomic_read_write(v, sizeof(*v));
0355     return arch_atomic_fetch_and_relaxed(i, v);
0356 }
0357
0358 static __always_inline void
0359 atomic_andnot(int i, atomic_t *v)
0360 {
0361     instrument_atomic_read_write(v, sizeof(*v));
0362     arch_atomic_andnot(i, v);
0363 }
0364
0365 static __always_inline int
0366 atomic_fetch_andnot(int i, atomic_t *v)
0367 {
0368     kcsan_mb();
0369     instrument_atomic_read_write(v, sizeof(*v));
0370     return arch_atomic_fetch_andnot(i, v);
0371 }
0372
0373 static __always_inline int
0374 atomic_fetch_andnot_acquire(int i, atomic_t *v)
0375 {
0376     instrument_atomic_read_write(v, sizeof(*v));
0377     return arch_atomic_fetch_andnot_acquire(i, v);
0378 }
0379
0380 static __always_inline int
0381 atomic_fetch_andnot_release(int i, atomic_t *v)
0382 {
0383     kcsan_release();
0384     instrument_atomic_read_write(v, sizeof(*v));
0385     return arch_atomic_fetch_andnot_release(i, v);
0386 }
0387
0388 static __always_inline int
0389 atomic_fetch_andnot_relaxed(int i, atomic_t *v)
0390 {
0391     instrument_atomic_read_write(v, sizeof(*v));
0392     return arch_atomic_fetch_andnot_relaxed(i, v);
0393 }
0394
0395 static __always_inline void
0396 atomic_or(int i, atomic_t *v)
0397 {
0398     instrument_atomic_read_write(v, sizeof(*v));
0399     arch_atomic_or(i, v);
0400 }
0401
0402 static __always_inline int
0403 atomic_fetch_or(int i, atomic_t *v)
0404 {
0405     kcsan_mb();
0406     instrument_atomic_read_write(v, sizeof(*v));
0407     return arch_atomic_fetch_or(i, v);
0408 }
0409
0410 static __always_inline int
0411 atomic_fetch_or_acquire(int i, atomic_t *v)
0412 {
0413     instrument_atomic_read_write(v, sizeof(*v));
0414     return arch_atomic_fetch_or_acquire(i, v);
0415 }
0416
0417 static __always_inline int
0418 atomic_fetch_or_release(int i, atomic_t *v)
0419 {
0420     kcsan_release();
0421     instrument_atomic_read_write(v, sizeof(*v));
0422     return arch_atomic_fetch_or_release(i, v);
0423 }
0424
0425 static __always_inline int
0426 atomic_fetch_or_relaxed(int i, atomic_t *v)
0427 {
0428     instrument_atomic_read_write(v, sizeof(*v));
0429     return arch_atomic_fetch_or_relaxed(i, v);
0430 }
0431
0432 static __always_inline void
0433 atomic_xor(int i, atomic_t *v)
0434 {
0435     instrument_atomic_read_write(v, sizeof(*v));
0436     arch_atomic_xor(i, v);
0437 }
0438
0439 static __always_inline int
0440 atomic_fetch_xor(int i, atomic_t *v)
0441 {
0442     kcsan_mb();
0443     instrument_atomic_read_write(v, sizeof(*v));
0444     return arch_atomic_fetch_xor(i, v);
0445 }
0446
0447 static __always_inline int
0448 atomic_fetch_xor_acquire(int i, atomic_t *v)
0449 {
0450     instrument_atomic_read_write(v, sizeof(*v));
0451     return arch_atomic_fetch_xor_acquire(i, v);
0452 }
0453
0454 static __always_inline int
0455 atomic_fetch_xor_release(int i, atomic_t *v)
0456 {
0457     kcsan_release();
0458     instrument_atomic_read_write(v, sizeof(*v));
0459     return arch_atomic_fetch_xor_release(i, v);
0460 }
0461
0462 static __always_inline int
0463 atomic_fetch_xor_relaxed(int i, atomic_t *v)
0464 {
0465     instrument_atomic_read_write(v, sizeof(*v));
0466     return arch_atomic_fetch_xor_relaxed(i, v);
0467 }
0468
0469 static __always_inline int
0470 atomic_xchg(atomic_t *v, int i)
0471 {
0472     kcsan_mb();
0473     instrument_atomic_read_write(v, sizeof(*v));
0474     return arch_atomic_xchg(v, i);
0475 }
0476
0477 static __always_inline int
0478 atomic_xchg_acquire(atomic_t *v, int i)
0479 {
0480     instrument_atomic_read_write(v, sizeof(*v));
0481     return arch_atomic_xchg_acquire(v, i);
0482 }
0483
0484 static __always_inline int
0485 atomic_xchg_release(atomic_t *v, int i)
0486 {
0487     kcsan_release();
0488     instrument_atomic_read_write(v, sizeof(*v));
0489     return arch_atomic_xchg_release(v, i);
0490 }
0491
0492 static __always_inline int
0493 atomic_xchg_relaxed(atomic_t *v, int i)
0494 {
0495     instrument_atomic_read_write(v, sizeof(*v));
0496     return arch_atomic_xchg_relaxed(v, i);
0497 }
0498
0499 static __always_inline int
0500 atomic_cmpxchg(atomic_t *v, int old, int new)
0501 {
0502     kcsan_mb();
0503     instrument_atomic_read_write(v, sizeof(*v));
0504     return arch_atomic_cmpxchg(v, old, new);
0505 }
0506
0507 static __always_inline int
0508 atomic_cmpxchg_acquire(atomic_t *v, int old, int new)
0509 {
0510     instrument_atomic_read_write(v, sizeof(*v));
0511     return arch_atomic_cmpxchg_acquire(v, old, new);
0512 }
0513
0514 static __always_inline int
0515 atomic_cmpxchg_release(atomic_t *v, int old, int new)
0516 {
0517     kcsan_release();
0518     instrument_atomic_read_write(v, sizeof(*v));
0519     return arch_atomic_cmpxchg_release(v, old, new);
0520 }
0521
0522 static __always_inline int
0523 atomic_cmpxchg_relaxed(atomic_t *v, int old, int new)
0524 {
0525     instrument_atomic_read_write(v, sizeof(*v));
0526     return arch_atomic_cmpxchg_relaxed(v, old, new);
0527 }
0528
0529 static __always_inline bool
0530 atomic_try_cmpxchg(atomic_t *v, int *old, int new)
0531 {
0532     kcsan_mb();
0533     instrument_atomic_read_write(v, sizeof(*v));
0534     instrument_atomic_read_write(old, sizeof(*old));
0535     return arch_atomic_try_cmpxchg(v, old, new);
0536 }
0537
0538 static __always_inline bool
0539 atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
0540 {
0541     instrument_atomic_read_write(v, sizeof(*v));
0542     instrument_atomic_read_write(old, sizeof(*old));
0543     return arch_atomic_try_cmpxchg_acquire(v, old, new);
0544 }
0545
0546 static __always_inline bool
0547 atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
0548 {
0549     kcsan_release();
0550     instrument_atomic_read_write(v, sizeof(*v));
0551     instrument_atomic_read_write(old, sizeof(*old));
0552     return arch_atomic_try_cmpxchg_release(v, old, new);
0553 }
0554
0555 static __always_inline bool
0556 atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new)
0557 {
0558     instrument_atomic_read_write(v, sizeof(*v));
0559     instrument_atomic_read_write(old, sizeof(*old));
0560     return arch_atomic_try_cmpxchg_relaxed(v, old, new);
0561 }
0562
0563 static __always_inline bool
0564 atomic_sub_and_test(int i, atomic_t *v)
0565 {
0566     kcsan_mb();
0567     instrument_atomic_read_write(v, sizeof(*v));
0568     return arch_atomic_sub_and_test(i, v);
0569 }
0570
0571 static __always_inline bool
0572 atomic_dec_and_test(atomic_t *v)
0573 {
0574     kcsan_mb();
0575     instrument_atomic_read_write(v, sizeof(*v));
0576     return arch_atomic_dec_and_test(v);
0577 }
0578
0579 static __always_inline bool
0580 atomic_inc_and_test(atomic_t *v)
0581 {
0582     kcsan_mb();
0583     instrument_atomic_read_write(v, sizeof(*v));
0584     return arch_atomic_inc_and_test(v);
0585 }
0586
0587 static __always_inline bool
0588 atomic_add_negative(int i, atomic_t *v)
0589 {
0590     kcsan_mb();
0591     instrument_atomic_read_write(v, sizeof(*v));
0592     return arch_atomic_add_negative(i, v);
0593 }
0594
0595 static __always_inline int
0596 atomic_fetch_add_unless(atomic_t *v, int a, int u)
0597 {
0598     kcsan_mb();
0599     instrument_atomic_read_write(v, sizeof(*v));
0600     return arch_atomic_fetch_add_unless(v, a, u);
0601 }
0602
0603 static __always_inline bool
0604 atomic_add_unless(atomic_t *v, int a, int u)
0605 {
0606     kcsan_mb();
0607     instrument_atomic_read_write(v, sizeof(*v));
0608     return arch_atomic_add_unless(v, a, u);
0609 }
0610
0611 static __always_inline bool
0612 atomic_inc_not_zero(atomic_t *v)
0613 {
0614     kcsan_mb();
0615     instrument_atomic_read_write(v, sizeof(*v));
0616     return arch_atomic_inc_not_zero(v);
0617 }
0618
0619 static __always_inline bool
0620 atomic_inc_unless_negative(atomic_t *v)
0621 {
0622     kcsan_mb();
0623     instrument_atomic_read_write(v, sizeof(*v));
0624     return arch_atomic_inc_unless_negative(v);
0625 }
0626
0627 static __always_inline bool
0628 atomic_dec_unless_positive(atomic_t *v)
0629 {
0630     kcsan_mb();
0631     instrument_atomic_read_write(v, sizeof(*v));
0632     return arch_atomic_dec_unless_positive(v);
0633 }
0634
0635 static __always_inline int
0636 atomic_dec_if_positive(atomic_t *v)
0637 {
0638     kcsan_mb();
0639     instrument_atomic_read_write(v, sizeof(*v));
0640     return arch_atomic_dec_if_positive(v);
0641 }
0642
0643 static __always_inline s64
0644 atomic64_read(const atomic64_t *v)
0645 {
0646     instrument_atomic_read(v, sizeof(*v));
0647     return arch_atomic64_read(v);
0648 }
0649
0650 static __always_inline s64
0651 atomic64_read_acquire(const atomic64_t *v)
0652 {
0653     instrument_atomic_read(v, sizeof(*v));
0654     return arch_atomic64_read_acquire(v);
0655 }
0656
0657 static __always_inline void
0658 atomic64_set(atomic64_t *v, s64 i)
0659 {
0660     instrument_atomic_write(v, sizeof(*v));
0661     arch_atomic64_set(v, i);
0662 }
0663
0664 static __always_inline void
0665 atomic64_set_release(atomic64_t *v, s64 i)
0666 {
0667     kcsan_release();
0668     instrument_atomic_write(v, sizeof(*v));
0669     arch_atomic64_set_release(v, i);
0670 }
0671
0672 static __always_inline void
0673 atomic64_add(s64 i, atomic64_t *v)
0674 {
0675     instrument_atomic_read_write(v, sizeof(*v));
0676     arch_atomic64_add(i, v);
0677 }
0678
0679 static __always_inline s64
0680 atomic64_add_return(s64 i, atomic64_t *v)
0681 {
0682     kcsan_mb();
0683     instrument_atomic_read_write(v, sizeof(*v));
0684     return arch_atomic64_add_return(i, v);
0685 }
0686
0687 static __always_inline s64
0688 atomic64_add_return_acquire(s64 i, atomic64_t *v)
0689 {
0690     instrument_atomic_read_write(v, sizeof(*v));
0691     return arch_atomic64_add_return_acquire(i, v);
0692 }
0693
0694 static __always_inline s64
0695 atomic64_add_return_release(s64 i, atomic64_t *v)
0696 {
0697     kcsan_release();
0698     instrument_atomic_read_write(v, sizeof(*v));
0699     return arch_atomic64_add_return_release(i, v);
0700 }
0701
0702 static __always_inline s64
0703 atomic64_add_return_relaxed(s64 i, atomic64_t *v)
0704 {
0705     instrument_atomic_read_write(v, sizeof(*v));
0706     return arch_atomic64_add_return_relaxed(i, v);
0707 }
0708
0709 static __always_inline s64
0710 atomic64_fetch_add(s64 i, atomic64_t *v)
0711 {
0712     kcsan_mb();
0713     instrument_atomic_read_write(v, sizeof(*v));
0714     return arch_atomic64_fetch_add(i, v);
0715 }
0716
0717 static __always_inline s64
0718 atomic64_fetch_add_acquire(s64 i, atomic64_t *v)
0719 {
0720     instrument_atomic_read_write(v, sizeof(*v));
0721     return arch_atomic64_fetch_add_acquire(i, v);
0722 }
0723
0724 static __always_inline s64
0725 atomic64_fetch_add_release(s64 i, atomic64_t *v)
0726 {
0727     kcsan_release();
0728     instrument_atomic_read_write(v, sizeof(*v));
0729     return arch_atomic64_fetch_add_release(i, v);
0730 }
0731
0732 static __always_inline s64
0733 atomic64_fetch_add_relaxed(s64 i, atomic64_t *v)
0734 {
0735     instrument_atomic_read_write(v, sizeof(*v));
0736     return arch_atomic64_fetch_add_relaxed(i, v);
0737 }
0738
0739 static __always_inline void
0740 atomic64_sub(s64 i, atomic64_t *v)
0741 {
0742     instrument_atomic_read_write(v, sizeof(*v));
0743     arch_atomic64_sub(i, v);
0744 }
0745
0746 static __always_inline s64
0747 atomic64_sub_return(s64 i, atomic64_t *v)
0748 {
0749     kcsan_mb();
0750     instrument_atomic_read_write(v, sizeof(*v));
0751     return arch_atomic64_sub_return(i, v);
0752 }
0753
0754 static __always_inline s64
0755 atomic64_sub_return_acquire(s64 i, atomic64_t *v)
0756 {
0757     instrument_atomic_read_write(v, sizeof(*v));
0758     return arch_atomic64_sub_return_acquire(i, v);
0759 }
0760
0761 static __always_inline s64
0762 atomic64_sub_return_release(s64 i, atomic64_t *v)
0763 {
0764     kcsan_release();
0765     instrument_atomic_read_write(v, sizeof(*v));
0766     return arch_atomic64_sub_return_release(i, v);
0767 }
0768
0769 static __always_inline s64
0770 atomic64_sub_return_relaxed(s64 i, atomic64_t *v)
0771 {
0772     instrument_atomic_read_write(v, sizeof(*v));
0773     return arch_atomic64_sub_return_relaxed(i, v);
0774 }
0775
0776 static __always_inline s64
0777 atomic64_fetch_sub(s64 i, atomic64_t *v)
0778 {
0779     kcsan_mb();
0780     instrument_atomic_read_write(v, sizeof(*v));
0781     return arch_atomic64_fetch_sub(i, v);
0782 }
0783
0784 static __always_inline s64
0785 atomic64_fetch_sub_acquire(s64 i, atomic64_t *v)
0786 {
0787     instrument_atomic_read_write(v, sizeof(*v));
0788     return arch_atomic64_fetch_sub_acquire(i, v);
0789 }
0790
0791 static __always_inline s64
0792 atomic64_fetch_sub_release(s64 i, atomic64_t *v)
0793 {
0794     kcsan_release();
0795     instrument_atomic_read_write(v, sizeof(*v));
0796     return arch_atomic64_fetch_sub_release(i, v);
0797 }
0798
0799 static __always_inline s64
0800 atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v)
0801 {
0802     instrument_atomic_read_write(v, sizeof(*v));
0803     return arch_atomic64_fetch_sub_relaxed(i, v);
0804 }
0805
0806 static __always_inline void
0807 atomic64_inc(atomic64_t *v)
0808 {
0809     instrument_atomic_read_write(v, sizeof(*v));
0810     arch_atomic64_inc(v);
0811 }
0812
0813 static __always_inline s64
0814 atomic64_inc_return(atomic64_t *v)
0815 {
0816     kcsan_mb();
0817     instrument_atomic_read_write(v, sizeof(*v));
0818     return arch_atomic64_inc_return(v);
0819 }
0820
0821 static __always_inline s64
0822 atomic64_inc_return_acquire(atomic64_t *v)
0823 {
0824     instrument_atomic_read_write(v, sizeof(*v));
0825     return arch_atomic64_inc_return_acquire(v);
0826 }
0827
0828 static __always_inline s64
0829 atomic64_inc_return_release(atomic64_t *v)
0830 {
0831     kcsan_release();
0832     instrument_atomic_read_write(v, sizeof(*v));
0833     return arch_atomic64_inc_return_release(v);
0834 }
0835
0836 static __always_inline s64
0837 atomic64_inc_return_relaxed(atomic64_t *v)
0838 {
0839     instrument_atomic_read_write(v, sizeof(*v));
0840     return arch_atomic64_inc_return_relaxed(v);
0841 }
0842
0843 static __always_inline s64
0844 atomic64_fetch_inc(atomic64_t *v)
0845 {
0846     kcsan_mb();
0847     instrument_atomic_read_write(v, sizeof(*v));
0848     return arch_atomic64_fetch_inc(v);
0849 }
0850
0851 static __always_inline s64
0852 atomic64_fetch_inc_acquire(atomic64_t *v)
0853 {
0854     instrument_atomic_read_write(v, sizeof(*v));
0855     return arch_atomic64_fetch_inc_acquire(v);
0856 }
0857
0858 static __always_inline s64
0859 atomic64_fetch_inc_release(atomic64_t *v)
0860 {
0861     kcsan_release();
0862     instrument_atomic_read_write(v, sizeof(*v));
0863     return arch_atomic64_fetch_inc_release(v);
0864 }
0865
0866 static __always_inline s64
0867 atomic64_fetch_inc_relaxed(atomic64_t *v)
0868 {
0869     instrument_atomic_read_write(v, sizeof(*v));
0870     return arch_atomic64_fetch_inc_relaxed(v);
0871 }
0872
0873 static __always_inline void
0874 atomic64_dec(atomic64_t *v)
0875 {
0876     instrument_atomic_read_write(v, sizeof(*v));
0877     arch_atomic64_dec(v);
0878 }
0879
0880 static __always_inline s64
0881 atomic64_dec_return(atomic64_t *v)
0882 {
0883     kcsan_mb();
0884     instrument_atomic_read_write(v, sizeof(*v));
0885     return arch_atomic64_dec_return(v);
0886 }
0887
0888 static __always_inline s64
0889 atomic64_dec_return_acquire(atomic64_t *v)
0890 {
0891     instrument_atomic_read_write(v, sizeof(*v));
0892     return arch_atomic64_dec_return_acquire(v);
0893 }
0894
0895 static __always_inline s64
0896 atomic64_dec_return_release(atomic64_t *v)
0897 {
0898     kcsan_release();
0899     instrument_atomic_read_write(v, sizeof(*v));
0900     return arch_atomic64_dec_return_release(v);
0901 }
0902
0903 static __always_inline s64
0904 atomic64_dec_return_relaxed(atomic64_t *v)
0905 {
0906     instrument_atomic_read_write(v, sizeof(*v));
0907     return arch_atomic64_dec_return_relaxed(v);
0908 }
0909
0910 static __always_inline s64
0911 atomic64_fetch_dec(atomic64_t *v)
0912 {
0913     kcsan_mb();
0914     instrument_atomic_read_write(v, sizeof(*v));
0915     return arch_atomic64_fetch_dec(v);
0916 }
0917
0918 static __always_inline s64
0919 atomic64_fetch_dec_acquire(atomic64_t *v)
0920 {
0921     instrument_atomic_read_write(v, sizeof(*v));
0922     return arch_atomic64_fetch_dec_acquire(v);
0923 }
0924
0925 static __always_inline s64
0926 atomic64_fetch_dec_release(atomic64_t *v)
0927 {
0928     kcsan_release();
0929     instrument_atomic_read_write(v, sizeof(*v));
0930     return arch_atomic64_fetch_dec_release(v);
0931 }
0932
0933 static __always_inline s64
0934 atomic64_fetch_dec_relaxed(atomic64_t *v)
0935 {
0936     instrument_atomic_read_write(v, sizeof(*v));
0937     return arch_atomic64_fetch_dec_relaxed(v);
0938 }
0939
0940 static __always_inline void
0941 atomic64_and(s64 i, atomic64_t *v)
0942 {
0943     instrument_atomic_read_write(v, sizeof(*v));
0944     arch_atomic64_and(i, v);
0945 }
0946
0947 static __always_inline s64
0948 atomic64_fetch_and(s64 i, atomic64_t *v)
0949 {
0950     kcsan_mb();
0951     instrument_atomic_read_write(v, sizeof(*v));
0952     return arch_atomic64_fetch_and(i, v);
0953 }
0954
0955 static __always_inline s64
0956 atomic64_fetch_and_acquire(s64 i, atomic64_t *v)
0957 {
0958     instrument_atomic_read_write(v, sizeof(*v));
0959     return arch_atomic64_fetch_and_acquire(i, v);
0960 }
0961
0962 static __always_inline s64
0963 atomic64_fetch_and_release(s64 i, atomic64_t *v)
0964 {
0965     kcsan_release();
0966     instrument_atomic_read_write(v, sizeof(*v));
0967     return arch_atomic64_fetch_and_release(i, v);
0968 }
0969
0970 static __always_inline s64
0971 atomic64_fetch_and_relaxed(s64 i, atomic64_t *v)
0972 {
0973     instrument_atomic_read_write(v, sizeof(*v));
0974     return arch_atomic64_fetch_and_relaxed(i, v);
0975 }
0976
0977 static __always_inline void
0978 atomic64_andnot(s64 i, atomic64_t *v)
0979 {
0980     instrument_atomic_read_write(v, sizeof(*v));
0981     arch_atomic64_andnot(i, v);
0982 }
0983
0984 static __always_inline s64
0985 atomic64_fetch_andnot(s64 i, atomic64_t *v)
0986 {
0987     kcsan_mb();
0988     instrument_atomic_read_write(v, sizeof(*v));
0989     return arch_atomic64_fetch_andnot(i, v);
0990 }
0991
0992 static __always_inline s64
0993 atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
0994 {
0995     instrument_atomic_read_write(v, sizeof(*v));
0996     return arch_atomic64_fetch_andnot_acquire(i, v);
0997 }
0998
0999 static __always_inline s64
1000 atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
1001 {
1002     kcsan_release();
1003     instrument_atomic_read_write(v, sizeof(*v));
1004     return arch_atomic64_fetch_andnot_release(i, v);
1005 }
1006
1007 static __always_inline s64
1008 atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v)
1009 {
1010     instrument_atomic_read_write(v, sizeof(*v));
1011     return arch_atomic64_fetch_andnot_relaxed(i, v);
1012 }
1013
1014 static __always_inline void
1015 atomic64_or(s64 i, atomic64_t *v)
1016 {
1017     instrument_atomic_read_write(v, sizeof(*v));
1018     arch_atomic64_or(i, v);
1019 }
1020
1021 static __always_inline s64
1022 atomic64_fetch_or(s64 i, atomic64_t *v)
1023 {
1024     kcsan_mb();
1025     instrument_atomic_read_write(v, sizeof(*v));
1026     return arch_atomic64_fetch_or(i, v);
1027 }
1028
1029 static __always_inline s64
1030 atomic64_fetch_or_acquire(s64 i, atomic64_t *v)
1031 {
1032     instrument_atomic_read_write(v, sizeof(*v));
1033     return arch_atomic64_fetch_or_acquire(i, v);
1034 }
1035
1036 static __always_inline s64
1037 atomic64_fetch_or_release(s64 i, atomic64_t *v)
1038 {
1039     kcsan_release();
1040     instrument_atomic_read_write(v, sizeof(*v));
1041     return arch_atomic64_fetch_or_release(i, v);
1042 }
1043
1044 static __always_inline s64
1045 atomic64_fetch_or_relaxed(s64 i, atomic64_t *v)
1046 {
1047     instrument_atomic_read_write(v, sizeof(*v));
1048     return arch_atomic64_fetch_or_relaxed(i, v);
1049 }
1050
1051 static __always_inline void
1052 atomic64_xor(s64 i, atomic64_t *v)
1053 {
1054     instrument_atomic_read_write(v, sizeof(*v));
1055     arch_atomic64_xor(i, v);
1056 }
1057
1058 static __always_inline s64
1059 atomic64_fetch_xor(s64 i, atomic64_t *v)
1060 {
1061     kcsan_mb();
1062     instrument_atomic_read_write(v, sizeof(*v));
1063     return arch_atomic64_fetch_xor(i, v);
1064 }
1065
1066 static __always_inline s64
1067 atomic64_fetch_xor_acquire(s64 i, atomic64_t *v)
1068 {
1069     instrument_atomic_read_write(v, sizeof(*v));
1070     return arch_atomic64_fetch_xor_acquire(i, v);
1071 }
1072
1073 static __always_inline s64
1074 atomic64_fetch_xor_release(s64 i, atomic64_t *v)
1075 {
1076     kcsan_release();
1077     instrument_atomic_read_write(v, sizeof(*v));
1078     return arch_atomic64_fetch_xor_release(i, v);
1079 }
1080
1081 static __always_inline s64
1082 atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v)
1083 {
1084     instrument_atomic_read_write(v, sizeof(*v));
1085     return arch_atomic64_fetch_xor_relaxed(i, v);
1086 }
1087
1088 static __always_inline s64
1089 atomic64_xchg(atomic64_t *v, s64 i)
1090 {
1091     kcsan_mb();
1092     instrument_atomic_read_write(v, sizeof(*v));
1093     return arch_atomic64_xchg(v, i);
1094 }
1095
1096 static __always_inline s64
1097 atomic64_xchg_acquire(atomic64_t *v, s64 i)
1098 {
1099     instrument_atomic_read_write(v, sizeof(*v));
1100     return arch_atomic64_xchg_acquire(v, i);
1101 }
1102
1103 static __always_inline s64
1104 atomic64_xchg_release(atomic64_t *v, s64 i)
1105 {
1106     kcsan_release();
1107     instrument_atomic_read_write(v, sizeof(*v));
1108     return arch_atomic64_xchg_release(v, i);
1109 }
1110
1111 static __always_inline s64
1112 atomic64_xchg_relaxed(atomic64_t *v, s64 i)
1113 {
1114     instrument_atomic_read_write(v, sizeof(*v));
1115     return arch_atomic64_xchg_relaxed(v, i);
1116 }
1117
1118 static __always_inline s64
1119 atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
1120 {
1121     kcsan_mb();
1122     instrument_atomic_read_write(v, sizeof(*v));
1123     return arch_atomic64_cmpxchg(v, old, new);
1124 }
1125
1126 static __always_inline s64
1127 atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new)
1128 {
1129     instrument_atomic_read_write(v, sizeof(*v));
1130     return arch_atomic64_cmpxchg_acquire(v, old, new);
1131 }
1132
1133 static __always_inline s64
1134 atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new)
1135 {
1136     kcsan_release();
1137     instrument_atomic_read_write(v, sizeof(*v));
1138     return arch_atomic64_cmpxchg_release(v, old, new);
1139 }
1140
1141 static __always_inline s64
1142 atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new)
1143 {
1144     instrument_atomic_read_write(v, sizeof(*v));
1145     return arch_atomic64_cmpxchg_relaxed(v, old, new);
1146 }
1147
1148 static __always_inline bool
1149 atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
1150 {
1151     kcsan_mb();
1152     instrument_atomic_read_write(v, sizeof(*v));
1153     instrument_atomic_read_write(old, sizeof(*old));
1154     return arch_atomic64_try_cmpxchg(v, old, new);
1155 }
1156
1157 static __always_inline bool
1158 atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
1159 {
1160     instrument_atomic_read_write(v, sizeof(*v));
1161     instrument_atomic_read_write(old, sizeof(*old));
1162     return arch_atomic64_try_cmpxchg_acquire(v, old, new);
1163 }
1164
1165 static __always_inline bool
1166 atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
1167 {
1168     kcsan_release();
1169     instrument_atomic_read_write(v, sizeof(*v));
1170     instrument_atomic_read_write(old, sizeof(*old));
1171     return arch_atomic64_try_cmpxchg_release(v, old, new);
1172 }
1173
1174 static __always_inline bool
1175 atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new)
1176 {
1177     instrument_atomic_read_write(v, sizeof(*v));
1178     instrument_atomic_read_write(old, sizeof(*old));
1179     return arch_atomic64_try_cmpxchg_relaxed(v, old, new);
1180 }
1181
1182 static __always_inline bool
1183 atomic64_sub_and_test(s64 i, atomic64_t *v)
1184 {
1185     kcsan_mb();
1186     instrument_atomic_read_write(v, sizeof(*v));
1187     return arch_atomic64_sub_and_test(i, v);
1188 }
1189
1190 static __always_inline bool
1191 atomic64_dec_and_test(atomic64_t *v)
1192 {
1193     kcsan_mb();
1194     instrument_atomic_read_write(v, sizeof(*v));
1195     return arch_atomic64_dec_and_test(v);
1196 }
1197
1198 static __always_inline bool
1199 atomic64_inc_and_test(atomic64_t *v)
1200 {
1201     kcsan_mb();
1202     instrument_atomic_read_write(v, sizeof(*v));
1203     return arch_atomic64_inc_and_test(v);
1204 }
1205
1206 static __always_inline bool
1207 atomic64_add_negative(s64 i, atomic64_t *v)
1208 {
1209     kcsan_mb();
1210     instrument_atomic_read_write(v, sizeof(*v));
1211     return arch_atomic64_add_negative(i, v);
1212 }
1213
1214 static __always_inline s64
1215 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
1216 {
1217     kcsan_mb();
1218     instrument_atomic_read_write(v, sizeof(*v));
1219     return arch_atomic64_fetch_add_unless(v, a, u);
1220 }
1221
1222 static __always_inline bool
1223 atomic64_add_unless(atomic64_t *v, s64 a, s64 u)
1224 {
1225     kcsan_mb();
1226     instrument_atomic_read_write(v, sizeof(*v));
1227     return arch_atomic64_add_unless(v, a, u);
1228 }
1229
1230 static __always_inline bool
1231 atomic64_inc_not_zero(atomic64_t *v)
1232 {
1233     kcsan_mb();
1234     instrument_atomic_read_write(v, sizeof(*v));
1235     return arch_atomic64_inc_not_zero(v);
1236 }
1237
1238 static __always_inline bool
1239 atomic64_inc_unless_negative(atomic64_t *v)
1240 {
1241     kcsan_mb();
1242     instrument_atomic_read_write(v, sizeof(*v));
1243     return arch_atomic64_inc_unless_negative(v);
1244 }
1245
1246 static __always_inline bool
1247 atomic64_dec_unless_positive(atomic64_t *v)
1248 {
1249     kcsan_mb();
1250     instrument_atomic_read_write(v, sizeof(*v));
1251     return arch_atomic64_dec_unless_positive(v);
1252 }
1253
1254 static __always_inline s64
1255 atomic64_dec_if_positive(atomic64_t *v)
1256 {
1257     kcsan_mb();
1258     instrument_atomic_read_write(v, sizeof(*v));
1259     return arch_atomic64_dec_if_positive(v);
1260 }
1261
1262 static __always_inline long
1263 atomic_long_read(const atomic_long_t *v)
1264 {
1265     instrument_atomic_read(v, sizeof(*v));
1266     return arch_atomic_long_read(v);
1267 }
1268
1269 static __always_inline long
1270 atomic_long_read_acquire(const atomic_long_t *v)
1271 {
1272     instrument_atomic_read(v, sizeof(*v));
1273     return arch_atomic_long_read_acquire(v);
1274 }
1275
1276 static __always_inline void
1277 atomic_long_set(atomic_long_t *v, long i)
1278 {
1279     instrument_atomic_write(v, sizeof(*v));
1280     arch_atomic_long_set(v, i);
1281 }
1282
1283 static __always_inline void
1284 atomic_long_set_release(atomic_long_t *v, long i)
1285 {
1286     kcsan_release();
1287     instrument_atomic_write(v, sizeof(*v));
1288     arch_atomic_long_set_release(v, i);
1289 }
1290
1291 static __always_inline void
1292 atomic_long_add(long i, atomic_long_t *v)
1293 {
1294     instrument_atomic_read_write(v, sizeof(*v));
1295     arch_atomic_long_add(i, v);
1296 }
1297
1298 static __always_inline long
1299 atomic_long_add_return(long i, atomic_long_t *v)
1300 {
1301     kcsan_mb();
1302     instrument_atomic_read_write(v, sizeof(*v));
1303     return arch_atomic_long_add_return(i, v);
1304 }
1305
1306 static __always_inline long
1307 atomic_long_add_return_acquire(long i, atomic_long_t *v)
1308 {
1309     instrument_atomic_read_write(v, sizeof(*v));
1310     return arch_atomic_long_add_return_acquire(i, v);
1311 }
1312
1313 static __always_inline long
1314 atomic_long_add_return_release(long i, atomic_long_t *v)
1315 {
1316     kcsan_release();
1317     instrument_atomic_read_write(v, sizeof(*v));
1318     return arch_atomic_long_add_return_release(i, v);
1319 }
1320
1321 static __always_inline long
1322 atomic_long_add_return_relaxed(long i, atomic_long_t *v)
1323 {
1324     instrument_atomic_read_write(v, sizeof(*v));
1325     return arch_atomic_long_add_return_relaxed(i, v);
1326 }
1327
1328 static __always_inline long
1329 atomic_long_fetch_add(long i, atomic_long_t *v)
1330 {
1331     kcsan_mb();
1332     instrument_atomic_read_write(v, sizeof(*v));
1333     return arch_atomic_long_fetch_add(i, v);
1334 }
1335
1336 static __always_inline long
1337 atomic_long_fetch_add_acquire(long i, atomic_long_t *v)
1338 {
1339     instrument_atomic_read_write(v, sizeof(*v));
1340     return arch_atomic_long_fetch_add_acquire(i, v);
1341 }
1342
1343 static __always_inline long
1344 atomic_long_fetch_add_release(long i, atomic_long_t *v)
1345 {
1346     kcsan_release();
1347     instrument_atomic_read_write(v, sizeof(*v));
1348     return arch_atomic_long_fetch_add_release(i, v);
1349 }
1350
1351 static __always_inline long
1352 atomic_long_fetch_add_relaxed(long i, atomic_long_t *v)
1353 {
1354     instrument_atomic_read_write(v, sizeof(*v));
1355     return arch_atomic_long_fetch_add_relaxed(i, v);
1356 }
1357
1358 static __always_inline void
1359 atomic_long_sub(long i, atomic_long_t *v)
1360 {
1361     instrument_atomic_read_write(v, sizeof(*v));
1362     arch_atomic_long_sub(i, v);
1363 }
1364
1365 static __always_inline long
1366 atomic_long_sub_return(long i, atomic_long_t *v)
1367 {
1368     kcsan_mb();
1369     instrument_atomic_read_write(v, sizeof(*v));
1370     return arch_atomic_long_sub_return(i, v);
1371 }
1372
1373 static __always_inline long
1374 atomic_long_sub_return_acquire(long i, atomic_long_t *v)
1375 {
1376     instrument_atomic_read_write(v, sizeof(*v));
1377     return arch_atomic_long_sub_return_acquire(i, v);
1378 }
1379
1380 static __always_inline long
1381 atomic_long_sub_return_release(long i, atomic_long_t *v)
1382 {
1383     kcsan_release();
1384     instrument_atomic_read_write(v, sizeof(*v));
1385     return arch_atomic_long_sub_return_release(i, v);
1386 }
1387
1388 static __always_inline long
1389 atomic_long_sub_return_relaxed(long i, atomic_long_t *v)
1390 {
1391     instrument_atomic_read_write(v, sizeof(*v));
1392     return arch_atomic_long_sub_return_relaxed(i, v);
1393 }
1394
1395 static __always_inline long
1396 atomic_long_fetch_sub(long i, atomic_long_t *v)
1397 {
1398     kcsan_mb();
1399     instrument_atomic_read_write(v, sizeof(*v));
1400     return arch_atomic_long_fetch_sub(i, v);
1401 }
1402
1403 static __always_inline long
1404 atomic_long_fetch_sub_acquire(long i, atomic_long_t *v)
1405 {
1406     instrument_atomic_read_write(v, sizeof(*v));
1407     return arch_atomic_long_fetch_sub_acquire(i, v);
1408 }
1409
1410 static __always_inline long
1411 atomic_long_fetch_sub_release(long i, atomic_long_t *v)
1412 {
1413     kcsan_release();
1414     instrument_atomic_read_write(v, sizeof(*v));
1415     return arch_atomic_long_fetch_sub_release(i, v);
1416 }
1417
1418 static __always_inline long
1419 atomic_long_fetch_sub_relaxed(long i, atomic_long_t *v)
1420 {
1421     instrument_atomic_read_write(v, sizeof(*v));
1422     return arch_atomic_long_fetch_sub_relaxed(i, v);
1423 }
1424
1425 static __always_inline void
1426 atomic_long_inc(atomic_long_t *v)
1427 {
1428     instrument_atomic_read_write(v, sizeof(*v));
1429     arch_atomic_long_inc(v);
1430 }
1431
1432 static __always_inline long
1433 atomic_long_inc_return(atomic_long_t *v)
1434 {
1435     kcsan_mb();
1436     instrument_atomic_read_write(v, sizeof(*v));
1437     return arch_atomic_long_inc_return(v);
1438 }
1439
1440 static __always_inline long
1441 atomic_long_inc_return_acquire(atomic_long_t *v)
1442 {
1443     instrument_atomic_read_write(v, sizeof(*v));
1444     return arch_atomic_long_inc_return_acquire(v);
1445 }
1446
1447 static __always_inline long
1448 atomic_long_inc_return_release(atomic_long_t *v)
1449 {
1450     kcsan_release();
1451     instrument_atomic_read_write(v, sizeof(*v));
1452     return arch_atomic_long_inc_return_release(v);
1453 }
1454
1455 static __always_inline long
1456 atomic_long_inc_return_relaxed(atomic_long_t *v)
1457 {
1458     instrument_atomic_read_write(v, sizeof(*v));
1459     return arch_atomic_long_inc_return_relaxed(v);
1460 }
1461
1462 static __always_inline long
1463 atomic_long_fetch_inc(atomic_long_t *v)
1464 {
1465     kcsan_mb();
1466     instrument_atomic_read_write(v, sizeof(*v));
1467     return arch_atomic_long_fetch_inc(v);
1468 }
1469
1470 static __always_inline long
1471 atomic_long_fetch_inc_acquire(atomic_long_t *v)
1472 {
1473     instrument_atomic_read_write(v, sizeof(*v));
1474     return arch_atomic_long_fetch_inc_acquire(v);
1475 }
1476
1477 static __always_inline long
1478 atomic_long_fetch_inc_release(atomic_long_t *v)
1479 {
1480     kcsan_release();
1481     instrument_atomic_read_write(v, sizeof(*v));
1482     return arch_atomic_long_fetch_inc_release(v);
1483 }
1484
1485 static __always_inline long
1486 atomic_long_fetch_inc_relaxed(atomic_long_t *v)
1487 {
1488     instrument_atomic_read_write(v, sizeof(*v));
1489     return arch_atomic_long_fetch_inc_relaxed(v);
1490 }
1491
1492 static __always_inline void
1493 atomic_long_dec(atomic_long_t *v)
1494 {
1495     instrument_atomic_read_write(v, sizeof(*v));
1496     arch_atomic_long_dec(v);
1497 }
1498
1499 static __always_inline long
1500 atomic_long_dec_return(atomic_long_t *v)
1501 {
1502     kcsan_mb();
1503     instrument_atomic_read_write(v, sizeof(*v));
1504     return arch_atomic_long_dec_return(v);
1505 }
1506
1507 static __always_inline long
1508 atomic_long_dec_return_acquire(atomic_long_t *v)
1509 {
1510     instrument_atomic_read_write(v, sizeof(*v));
1511     return arch_atomic_long_dec_return_acquire(v);
1512 }
1513
1514 static __always_inline long
1515 atomic_long_dec_return_release(atomic_long_t *v)
1516 {
1517     kcsan_release();
1518     instrument_atomic_read_write(v, sizeof(*v));
1519     return arch_atomic_long_dec_return_release(v);
1520 }
1521
1522 static __always_inline long
1523 atomic_long_dec_return_relaxed(atomic_long_t *v)
1524 {
1525     instrument_atomic_read_write(v, sizeof(*v));
1526     return arch_atomic_long_dec_return_relaxed(v);
1527 }
1528
1529 static __always_inline long
1530 atomic_long_fetch_dec(atomic_long_t *v)
1531 {
1532     kcsan_mb();
1533     instrument_atomic_read_write(v, sizeof(*v));
1534     return arch_atomic_long_fetch_dec(v);
1535 }
1536
1537 static __always_inline long
1538 atomic_long_fetch_dec_acquire(atomic_long_t *v)
1539 {
1540     instrument_atomic_read_write(v, sizeof(*v));
1541     return arch_atomic_long_fetch_dec_acquire(v);
1542 }
1543
1544 static __always_inline long
1545 atomic_long_fetch_dec_release(atomic_long_t *v)
1546 {
1547     kcsan_release();
1548     instrument_atomic_read_write(v, sizeof(*v));
1549     return arch_atomic_long_fetch_dec_release(v);
1550 }
1551
1552 static __always_inline long
1553 atomic_long_fetch_dec_relaxed(atomic_long_t *v)
1554 {
1555     instrument_atomic_read_write(v, sizeof(*v));
1556     return arch_atomic_long_fetch_dec_relaxed(v);
1557 }
1558
1559 static __always_inline void
1560 atomic_long_and(long i, atomic_long_t *v)
1561 {
1562     instrument_atomic_read_write(v, sizeof(*v));
1563     arch_atomic_long_and(i, v);
1564 }
1565
1566 static __always_inline long
1567 atomic_long_fetch_and(long i, atomic_long_t *v)
1568 {
1569     kcsan_mb();
1570     instrument_atomic_read_write(v, sizeof(*v));
1571     return arch_atomic_long_fetch_and(i, v);
1572 }
1573
1574 static __always_inline long
1575 atomic_long_fetch_and_acquire(long i, atomic_long_t *v)
1576 {
1577     instrument_atomic_read_write(v, sizeof(*v));
1578     return arch_atomic_long_fetch_and_acquire(i, v);
1579 }
1580
1581 static __always_inline long
1582 atomic_long_fetch_and_release(long i, atomic_long_t *v)
1583 {
1584     kcsan_release();
1585     instrument_atomic_read_write(v, sizeof(*v));
1586     return arch_atomic_long_fetch_and_release(i, v);
1587 }
1588
1589 static __always_inline long
1590 atomic_long_fetch_and_relaxed(long i, atomic_long_t *v)
1591 {
1592     instrument_atomic_read_write(v, sizeof(*v));
1593     return arch_atomic_long_fetch_and_relaxed(i, v);
1594 }
1595
1596 static __always_inline void
1597 atomic_long_andnot(long i, atomic_long_t *v)
1598 {
1599     instrument_atomic_read_write(v, sizeof(*v));
1600     arch_atomic_long_andnot(i, v);
1601 }
1602
1603 static __always_inline long
1604 atomic_long_fetch_andnot(long i, atomic_long_t *v)
1605 {
1606     kcsan_mb();
1607     instrument_atomic_read_write(v, sizeof(*v));
1608     return arch_atomic_long_fetch_andnot(i, v);
1609 }
1610
1611 static __always_inline long
1612 atomic_long_fetch_andnot_acquire(long i, atomic_long_t *v)
1613 {
1614     instrument_atomic_read_write(v, sizeof(*v));
1615     return arch_atomic_long_fetch_andnot_acquire(i, v);
1616 }
1617
1618 static __always_inline long
1619 atomic_long_fetch_andnot_release(long i, atomic_long_t *v)
1620 {
1621     kcsan_release();
1622     instrument_atomic_read_write(v, sizeof(*v));
1623     return arch_atomic_long_fetch_andnot_release(i, v);
1624 }
1625
1626 static __always_inline long
1627 atomic_long_fetch_andnot_relaxed(long i, atomic_long_t *v)
1628 {
1629     instrument_atomic_read_write(v, sizeof(*v));
1630     return arch_atomic_long_fetch_andnot_relaxed(i, v);
1631 }
1632
1633 static __always_inline void
1634 atomic_long_or(long i, atomic_long_t *v)
1635 {
1636     instrument_atomic_read_write(v, sizeof(*v));
1637     arch_atomic_long_or(i, v);
1638 }
1639
1640 static __always_inline long
1641 atomic_long_fetch_or(long i, atomic_long_t *v)
1642 {
1643     kcsan_mb();
1644     instrument_atomic_read_write(v, sizeof(*v));
1645     return arch_atomic_long_fetch_or(i, v);
1646 }
1647
1648 static __always_inline long
1649 atomic_long_fetch_or_acquire(long i, atomic_long_t *v)
1650 {
1651     instrument_atomic_read_write(v, sizeof(*v));
1652     return arch_atomic_long_fetch_or_acquire(i, v);
1653 }
1654
1655 static __always_inline long
1656 atomic_long_fetch_or_release(long i, atomic_long_t *v)
1657 {
1658     kcsan_release();
1659     instrument_atomic_read_write(v, sizeof(*v));
1660     return arch_atomic_long_fetch_or_release(i, v);
1661 }
1662
1663 static __always_inline long
1664 atomic_long_fetch_or_relaxed(long i, atomic_long_t *v)
1665 {
1666     instrument_atomic_read_write(v, sizeof(*v));
1667     return arch_atomic_long_fetch_or_relaxed(i, v);
1668 }
1669
1670 static __always_inline void
1671 atomic_long_xor(long i, atomic_long_t *v)
1672 {
1673     instrument_atomic_read_write(v, sizeof(*v));
1674     arch_atomic_long_xor(i, v);
1675 }
1676
1677 static __always_inline long
1678 atomic_long_fetch_xor(long i, atomic_long_t *v)
1679 {
1680     kcsan_mb();
1681     instrument_atomic_read_write(v, sizeof(*v));
1682     return arch_atomic_long_fetch_xor(i, v);
1683 }
1684
1685 static __always_inline long
1686 atomic_long_fetch_xor_acquire(long i, atomic_long_t *v)
1687 {
1688     instrument_atomic_read_write(v, sizeof(*v));
1689     return arch_atomic_long_fetch_xor_acquire(i, v);
1690 }
1691
1692 static __always_inline long
1693 atomic_long_fetch_xor_release(long i, atomic_long_t *v)
1694 {
1695     kcsan_release();
1696     instrument_atomic_read_write(v, sizeof(*v));
1697     return arch_atomic_long_fetch_xor_release(i, v);
1698 }
1699
1700 static __always_inline long
1701 atomic_long_fetch_xor_relaxed(long i, atomic_long_t *v)
1702 {
1703     instrument_atomic_read_write(v, sizeof(*v));
1704     return arch_atomic_long_fetch_xor_relaxed(i, v);
1705 }
1706
1707 static __always_inline long
1708 atomic_long_xchg(atomic_long_t *v, long i)
1709 {
1710     kcsan_mb();
1711     instrument_atomic_read_write(v, sizeof(*v));
1712     return arch_atomic_long_xchg(v, i);
1713 }
1714
1715 static __always_inline long
1716 atomic_long_xchg_acquire(atomic_long_t *v, long i)
1717 {
1718     instrument_atomic_read_write(v, sizeof(*v));
1719     return arch_atomic_long_xchg_acquire(v, i);
1720 }
1721
1722 static __always_inline long
1723 atomic_long_xchg_release(atomic_long_t *v, long i)
1724 {
1725     kcsan_release();
1726     instrument_atomic_read_write(v, sizeof(*v));
1727     return arch_atomic_long_xchg_release(v, i);
1728 }
1729
1730 static __always_inline long
1731 atomic_long_xchg_relaxed(atomic_long_t *v, long i)
1732 {
1733     instrument_atomic_read_write(v, sizeof(*v));
1734     return arch_atomic_long_xchg_relaxed(v, i);
1735 }
1736
1737 static __always_inline long
1738 atomic_long_cmpxchg(atomic_long_t *v, long old, long new)
1739 {
1740     kcsan_mb();
1741     instrument_atomic_read_write(v, sizeof(*v));
1742     return arch_atomic_long_cmpxchg(v, old, new);
1743 }
1744
1745 static __always_inline long
1746 atomic_long_cmpxchg_acquire(atomic_long_t *v, long old, long new)
1747 {
1748     instrument_atomic_read_write(v, sizeof(*v));
1749     return arch_atomic_long_cmpxchg_acquire(v, old, new);
1750 }
1751
1752 static __always_inline long
1753 atomic_long_cmpxchg_release(atomic_long_t *v, long old, long new)
1754 {
1755     kcsan_release();
1756     instrument_atomic_read_write(v, sizeof(*v));
1757     return arch_atomic_long_cmpxchg_release(v, old, new);
1758 }
1759
1760 static __always_inline long
1761 atomic_long_cmpxchg_relaxed(atomic_long_t *v, long old, long new)
1762 {
1763     instrument_atomic_read_write(v, sizeof(*v));
1764     return arch_atomic_long_cmpxchg_relaxed(v, old, new);
1765 }
1766
1767 static __always_inline bool
1768 atomic_long_try_cmpxchg(atomic_long_t *v, long *old, long new)
1769 {
1770     kcsan_mb();
1771     instrument_atomic_read_write(v, sizeof(*v));
1772     instrument_atomic_read_write(old, sizeof(*old));
1773     return arch_atomic_long_try_cmpxchg(v, old, new);
1774 }
1775
1776 static __always_inline bool
1777 atomic_long_try_cmpxchg_acquire(atomic_long_t *v, long *old, long new)
1778 {
1779     instrument_atomic_read_write(v, sizeof(*v));
1780     instrument_atomic_read_write(old, sizeof(*old));
1781     return arch_atomic_long_try_cmpxchg_acquire(v, old, new);
1782 }
1783
1784 static __always_inline bool
1785 atomic_long_try_cmpxchg_release(atomic_long_t *v, long *old, long new)
1786 {
1787     kcsan_release();
1788     instrument_atomic_read_write(v, sizeof(*v));
1789     instrument_atomic_read_write(old, sizeof(*old));
1790     return arch_atomic_long_try_cmpxchg_release(v, old, new);
1791 }
1792
1793 static __always_inline bool
1794 atomic_long_try_cmpxchg_relaxed(atomic_long_t *v, long *old, long new)
1795 {
1796     instrument_atomic_read_write(v, sizeof(*v));
1797     instrument_atomic_read_write(old, sizeof(*old));
1798     return arch_atomic_long_try_cmpxchg_relaxed(v, old, new);
1799 }
1800
1801 static __always_inline bool
1802 atomic_long_sub_and_test(long i, atomic_long_t *v)
1803 {
1804     kcsan_mb();
1805     instrument_atomic_read_write(v, sizeof(*v));
1806     return arch_atomic_long_sub_and_test(i, v);
1807 }
1808
1809 static __always_inline bool
1810 atomic_long_dec_and_test(atomic_long_t *v)
1811 {
1812     kcsan_mb();
1813     instrument_atomic_read_write(v, sizeof(*v));
1814     return arch_atomic_long_dec_and_test(v);
1815 }
1816
1817 static __always_inline bool
1818 atomic_long_inc_and_test(atomic_long_t *v)
1819 {
1820     kcsan_mb();
1821     instrument_atomic_read_write(v, sizeof(*v));
1822     return arch_atomic_long_inc_and_test(v);
1823 }
1824
1825 static __always_inline bool
1826 atomic_long_add_negative(long i, atomic_long_t *v)
1827 {
1828     kcsan_mb();
1829     instrument_atomic_read_write(v, sizeof(*v));
1830     return arch_atomic_long_add_negative(i, v);
1831 }
1832
1833 static __always_inline long
1834 atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u)
1835 {
1836     kcsan_mb();
1837     instrument_atomic_read_write(v, sizeof(*v));
1838     return arch_atomic_long_fetch_add_unless(v, a, u);
1839 }
1840
1841 static __always_inline bool
1842 atomic_long_add_unless(atomic_long_t *v, long a, long u)
1843 {
1844     kcsan_mb();
1845     instrument_atomic_read_write(v, sizeof(*v));
1846     return arch_atomic_long_add_unless(v, a, u);
1847 }
1848
1849 static __always_inline bool
1850 atomic_long_inc_not_zero(atomic_long_t *v)
1851 {
1852     kcsan_mb();
1853     instrument_atomic_read_write(v, sizeof(*v));
1854     return arch_atomic_long_inc_not_zero(v);
1855 }
1856
1857 static __always_inline bool
1858 atomic_long_inc_unless_negative(atomic_long_t *v)
1859 {
1860     kcsan_mb();
1861     instrument_atomic_read_write(v, sizeof(*v));
1862     return arch_atomic_long_inc_unless_negative(v);
1863 }
1864
1865 static __always_inline bool
1866 atomic_long_dec_unless_positive(atomic_long_t *v)
1867 {
1868     kcsan_mb();
1869     instrument_atomic_read_write(v, sizeof(*v));
1870     return arch_atomic_long_dec_unless_positive(v);
1871 }
1872
1873 static __always_inline long
1874 atomic_long_dec_if_positive(atomic_long_t *v)
1875 {
1876     kcsan_mb();
1877     instrument_atomic_read_write(v, sizeof(*v));
1878     return arch_atomic_long_dec_if_positive(v);
1879 }
1880
1881 #define xchg(ptr, ...) \
1882 ({ \
1883     typeof(ptr) __ai_ptr = (ptr); \
1884     kcsan_mb(); \
1885     instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1886     arch_xchg(__ai_ptr, __VA_ARGS__); \
1887 })
1888
1889 #define xchg_acquire(ptr, ...) \
1890 ({ \
1891     typeof(ptr) __ai_ptr = (ptr); \
1892     instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1893     arch_xchg_acquire(__ai_ptr, __VA_ARGS__); \
1894 })
1895
1896 #define xchg_release(ptr, ...) \
1897 ({ \
1898     typeof(ptr) __ai_ptr = (ptr); \
1899     kcsan_release(); \
1900     instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1901     arch_xchg_release(__ai_ptr, __VA_ARGS__); \
1902 })
1903
1904 #define xchg_relaxed(ptr, ...) \
1905 ({ \
1906     typeof(ptr) __ai_ptr = (ptr); \
1907     instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1908     arch_xchg_relaxed(__ai_ptr, __VA_ARGS__); \
1909 })
1910
1911 #define cmpxchg(ptr, ...) \
1912 ({ \
1913     typeof(ptr) __ai_ptr = (ptr); \
1914     kcsan_mb(); \
1915     instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1916     arch_cmpxchg(__ai_ptr, __VA_ARGS__); \
1917 })
1918
1919 #define cmpxchg_acquire(ptr, ...) \
1920 ({ \
1921     typeof(ptr) __ai_ptr = (ptr); \
1922     instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1923     arch_cmpxchg_acquire(__ai_ptr, __VA_ARGS__); \
1924 })
1925
1926 #define cmpxchg_release(ptr, ...) \
1927 ({ \
1928     typeof(ptr) __ai_ptr = (ptr); \
1929     kcsan_release(); \
1930     instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1931     arch_cmpxchg_release(__ai_ptr, __VA_ARGS__); \
1932 })
1933
1934 #define cmpxchg_relaxed(ptr, ...) \
1935 ({ \
1936     typeof(ptr) __ai_ptr = (ptr); \
1937     instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1938     arch_cmpxchg_relaxed(__ai_ptr, __VA_ARGS__); \
1939 })
1940
1941 #define cmpxchg64(ptr, ...) \
1942 ({ \
1943     typeof(ptr) __ai_ptr = (ptr); \
1944     kcsan_mb(); \
1945     instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1946     arch_cmpxchg64(__ai_ptr, __VA_ARGS__); \
1947 })
1948
1949 #define cmpxchg64_acquire(ptr, ...) \
1950 ({ \
1951     typeof(ptr) __ai_ptr = (ptr); \
1952     instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1953     arch_cmpxchg64_acquire(__ai_ptr, __VA_ARGS__); \
1954 })
1955
1956 #define cmpxchg64_release(ptr, ...) \
1957 ({ \
1958     typeof(ptr) __ai_ptr = (ptr); \
1959     kcsan_release(); \
1960     instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1961     arch_cmpxchg64_release(__ai_ptr, __VA_ARGS__); \
1962 })
1963
1964 #define cmpxchg64_relaxed(ptr, ...) \
1965 ({ \
1966     typeof(ptr) __ai_ptr = (ptr); \
1967     instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1968     arch_cmpxchg64_relaxed(__ai_ptr, __VA_ARGS__); \
1969 })
1970
1971 #define try_cmpxchg(ptr, oldp, ...) \
1972 ({ \
1973     typeof(ptr) __ai_ptr = (ptr); \
1974     typeof(oldp) __ai_oldp = (oldp); \
1975     kcsan_mb(); \
1976     instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1977     instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
1978     arch_try_cmpxchg(__ai_ptr, __ai_oldp, __VA_ARGS__); \
1979 })
1980
1981 #define try_cmpxchg_acquire(ptr, oldp, ...) \
1982 ({ \
1983     typeof(ptr) __ai_ptr = (ptr); \
1984     typeof(oldp) __ai_oldp = (oldp); \
1985     instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1986     instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
1987     arch_try_cmpxchg_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
1988 })
1989
1990 #define try_cmpxchg_release(ptr, oldp, ...) \
1991 ({ \
1992     typeof(ptr) __ai_ptr = (ptr); \
1993     typeof(oldp) __ai_oldp = (oldp); \
1994     kcsan_release(); \
1995     instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1996     instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
1997     arch_try_cmpxchg_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
1998 })
1999
2000 #define try_cmpxchg_relaxed(ptr, oldp, ...) \
2001 ({ \
2002     typeof(ptr) __ai_ptr = (ptr); \
2003     typeof(oldp) __ai_oldp = (oldp); \
2004     instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2005     instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
2006     arch_try_cmpxchg_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2007 })
2008
2009 #define try_cmpxchg64(ptr, oldp, ...) \
2010 ({ \
2011     typeof(ptr) __ai_ptr = (ptr); \
2012     typeof(oldp) __ai_oldp = (oldp); \
2013     kcsan_mb(); \
2014     instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2015     instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
2016     arch_try_cmpxchg64(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2017 })
2018
2019 #define try_cmpxchg64_acquire(ptr, oldp, ...) \
2020 ({ \
2021     typeof(ptr) __ai_ptr = (ptr); \
2022     typeof(oldp) __ai_oldp = (oldp); \
2023     instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2024     instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
2025     arch_try_cmpxchg64_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2026 })
2027
2028 #define try_cmpxchg64_release(ptr, oldp, ...) \
2029 ({ \
2030     typeof(ptr) __ai_ptr = (ptr); \
2031     typeof(oldp) __ai_oldp = (oldp); \
2032     kcsan_release(); \
2033     instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2034     instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
2035     arch_try_cmpxchg64_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2036 })
2037
2038 #define try_cmpxchg64_relaxed(ptr, oldp, ...) \
2039 ({ \
2040     typeof(ptr) __ai_ptr = (ptr); \
2041     typeof(oldp) __ai_oldp = (oldp); \
2042     instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2043     instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
2044     arch_try_cmpxchg64_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2045 })
2046
2047 #define cmpxchg_local(ptr, ...) \
2048 ({ \
2049     typeof(ptr) __ai_ptr = (ptr); \
2050     instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2051     arch_cmpxchg_local(__ai_ptr, __VA_ARGS__); \
2052 })
2053
2054 #define cmpxchg64_local(ptr, ...) \
2055 ({ \
2056     typeof(ptr) __ai_ptr = (ptr); \
2057     instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2058     arch_cmpxchg64_local(__ai_ptr, __VA_ARGS__); \
2059 })
2060
2061 #define sync_cmpxchg(ptr, ...) \
2062 ({ \
2063     typeof(ptr) __ai_ptr = (ptr); \
2064     kcsan_mb(); \
2065     instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2066     arch_sync_cmpxchg(__ai_ptr, __VA_ARGS__); \
2067 })
2068
2069 #define cmpxchg_double(ptr, ...) \
2070 ({ \
2071     typeof(ptr) __ai_ptr = (ptr); \
2072     kcsan_mb(); \
2073     instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
2074     arch_cmpxchg_double(__ai_ptr, __VA_ARGS__); \
2075 })
2076
2077
2078 #define cmpxchg_double_local(ptr, ...) \
2079 ({ \
2080     typeof(ptr) __ai_ptr = (ptr); \
2081     instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
2082     arch_cmpxchg_double_local(__ai_ptr, __VA_ARGS__); \
2083 })
2084
2085 #endif /* _LINUX_ATOMIC_INSTRUMENTED_H */
2086 // 764f741eb77a7ad565dc8d99ce2837d5542e8aee