asm-generic/bitops/instrumented-non-atomic.h

0001 /* SPDX-License-Identifier: GPL-2.0 */
0002
0003 /*
0004  * This file provides wrappers with sanitizer instrumentation for non-atomic
0005  * bit operations.
0006  *
0007  * To use this functionality, an arch's bitops.h file needs to define each of
0008  * the below bit operations with an arch_ prefix (e.g. arch_set_bit(),
0009  * arch___set_bit(), etc.).
0010  */
0011 #ifndef _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H
0012 #define _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H
0013
0014 #include <linux/instrumented.h>
0015
0016 /**
0017  * ___set_bit - Set a bit in memory
0018  * @nr: the bit to set
0019  * @addr: the address to start counting from
0020  *
0021  * Unlike set_bit(), this function is non-atomic. If it is called on the same
0022  * region of memory concurrently, the effect may be that only one operation
0023  * succeeds.
0024  */
0025 static __always_inline void
0026 ___set_bit(unsigned long nr, volatile unsigned long *addr)
0027 {
0028     instrument_write(addr + BIT_WORD(nr), sizeof(long));
0029     arch___set_bit(nr, addr);
0030 }
0031
0032 /**
0033  * ___clear_bit - Clears a bit in memory
0034  * @nr: the bit to clear
0035  * @addr: the address to start counting from
0036  *
0037  * Unlike clear_bit(), this function is non-atomic. If it is called on the same
0038  * region of memory concurrently, the effect may be that only one operation
0039  * succeeds.
0040  */
0041 static __always_inline void
0042 ___clear_bit(unsigned long nr, volatile unsigned long *addr)
0043 {
0044     instrument_write(addr + BIT_WORD(nr), sizeof(long));
0045     arch___clear_bit(nr, addr);
0046 }
0047
0048 /**
0049  * ___change_bit - Toggle a bit in memory
0050  * @nr: the bit to change
0051  * @addr: the address to start counting from
0052  *
0053  * Unlike change_bit(), this function is non-atomic. If it is called on the same
0054  * region of memory concurrently, the effect may be that only one operation
0055  * succeeds.
0056  */
0057 static __always_inline void
0058 ___change_bit(unsigned long nr, volatile unsigned long *addr)
0059 {
0060     instrument_write(addr + BIT_WORD(nr), sizeof(long));
0061     arch___change_bit(nr, addr);
0062 }
0063
0064 static __always_inline void __instrument_read_write_bitop(long nr, volatile unsigned long *addr)
0065 {
0066     if (IS_ENABLED(CONFIG_KCSAN_ASSUME_PLAIN_WRITES_ATOMIC)) {
0067         /*
0068          * We treat non-atomic read-write bitops a little more special.
0069          * Given the operations here only modify a single bit, assuming
0070          * non-atomicity of the writer is sufficient may be reasonable
0071          * for certain usage (and follows the permissible nature of the
0072          * assume-plain-writes-atomic rule):
0073          * 1. report read-modify-write races -> check read;
0074          * 2. do not report races with marked readers, but do report
0075          *    races with unmarked readers -> check "atomic" write.
0076          */
0077         kcsan_check_read(addr + BIT_WORD(nr), sizeof(long));
0078         /*
0079          * Use generic write instrumentation, in case other sanitizers
0080          * or tools are enabled alongside KCSAN.
0081          */
0082         instrument_write(addr + BIT_WORD(nr), sizeof(long));
0083     } else {
0084         instrument_read_write(addr + BIT_WORD(nr), sizeof(long));
0085     }
0086 }
0087
0088 /**
0089  * ___test_and_set_bit - Set a bit and return its old value
0090  * @nr: Bit to set
0091  * @addr: Address to count from
0092  *
0093  * This operation is non-atomic. If two instances of this operation race, one
0094  * can appear to succeed but actually fail.
0095  */
0096 static __always_inline bool
0097 ___test_and_set_bit(unsigned long nr, volatile unsigned long *addr)
0098 {
0099     __instrument_read_write_bitop(nr, addr);
0100     return arch___test_and_set_bit(nr, addr);
0101 }
0102
0103 /**
0104  * ___test_and_clear_bit - Clear a bit and return its old value
0105  * @nr: Bit to clear
0106  * @addr: Address to count from
0107  *
0108  * This operation is non-atomic. If two instances of this operation race, one
0109  * can appear to succeed but actually fail.
0110  */
0111 static __always_inline bool
0112 ___test_and_clear_bit(unsigned long nr, volatile unsigned long *addr)
0113 {
0114     __instrument_read_write_bitop(nr, addr);
0115     return arch___test_and_clear_bit(nr, addr);
0116 }
0117
0118 /**
0119  * ___test_and_change_bit - Change a bit and return its old value
0120  * @nr: Bit to change
0121  * @addr: Address to count from
0122  *
0123  * This operation is non-atomic. If two instances of this operation race, one
0124  * can appear to succeed but actually fail.
0125  */
0126 static __always_inline bool
0127 ___test_and_change_bit(unsigned long nr, volatile unsigned long *addr)
0128 {
0129     __instrument_read_write_bitop(nr, addr);
0130     return arch___test_and_change_bit(nr, addr);
0131 }
0132
0133 /**
0134  * _test_bit - Determine whether a bit is set
0135  * @nr: bit number to test
0136  * @addr: Address to start counting from
0137  */
0138 static __always_inline bool
0139 _test_bit(unsigned long nr, const volatile unsigned long *addr)
0140 {
0141     instrument_atomic_read(addr + BIT_WORD(nr), sizeof(long));
0142     return arch_test_bit(nr, addr);
0143 }
0144
0145 /**
0146  * _test_bit_acquire - Determine, with acquire semantics, whether a bit is set
0147  * @nr: bit number to test
0148  * @addr: Address to start counting from
0149  */
0150 static __always_inline bool
0151 _test_bit_acquire(unsigned long nr, const volatile unsigned long *addr)
0152 {
0153     instrument_atomic_read(addr + BIT_WORD(nr), sizeof(long));
0154     return arch_test_bit_acquire(nr, addr);
0155 }
0156
0157 #endif /* _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H */