include/asm-generic/percpu.h

0001 /* SPDX-License-Identifier: GPL-2.0 */
0002 #ifndef _ASM_GENERIC_PERCPU_H_
0003 #define _ASM_GENERIC_PERCPU_H_
0004
0005 #include <linux/compiler.h>
0006 #include <linux/threads.h>
0007 #include <linux/percpu-defs.h>
0008
0009 #ifdef CONFIG_SMP
0010
0011 /*
0012  * per_cpu_offset() is the offset that has to be added to a
0013  * percpu variable to get to the instance for a certain processor.
0014  *
0015  * Most arches use the __per_cpu_offset array for those offsets but
0016  * some arches have their own ways of determining the offset (x86_64, s390).
0017  */
0018 #ifndef __per_cpu_offset
0019 extern unsigned long __per_cpu_offset[NR_CPUS];
0020
0021 #define per_cpu_offset(x) (__per_cpu_offset[x])
0022 #endif
0023
0024 /*
0025  * Determine the offset for the currently active processor.
0026  * An arch may define __my_cpu_offset to provide a more effective
0027  * means of obtaining the offset to the per cpu variables of the
0028  * current processor.
0029  */
0030 #ifndef __my_cpu_offset
0031 #define __my_cpu_offset per_cpu_offset(raw_smp_processor_id())
0032 #endif
0033 #ifdef CONFIG_DEBUG_PREEMPT
0034 #define my_cpu_offset per_cpu_offset(smp_processor_id())
0035 #else
0036 #define my_cpu_offset __my_cpu_offset
0037 #endif
0038
0039 /*
0040  * Arch may define arch_raw_cpu_ptr() to provide more efficient address
0041  * translations for raw_cpu_ptr().
0042  */
0043 #ifndef arch_raw_cpu_ptr
0044 #define arch_raw_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, __my_cpu_offset)
0045 #endif
0046
0047 #ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA
0048 extern void setup_per_cpu_areas(void);
0049 #endif
0050
0051 #endif  /* SMP */
0052
0053 #ifndef PER_CPU_BASE_SECTION
0054 #ifdef CONFIG_SMP
0055 #define PER_CPU_BASE_SECTION ".data..percpu"
0056 #else
0057 #define PER_CPU_BASE_SECTION ".data"
0058 #endif
0059 #endif
0060
0061 #ifndef PER_CPU_ATTRIBUTES
0062 #define PER_CPU_ATTRIBUTES
0063 #endif
0064
0065 #define raw_cpu_generic_read(pcp)                   \
0066 ({                                  \
0067     *raw_cpu_ptr(&(pcp));                       \
0068 })
0069
0070 #define raw_cpu_generic_to_op(pcp, val, op)             \
0071 do {                                    \
0072     *raw_cpu_ptr(&(pcp)) op val;                    \
0073 } while (0)
0074
0075 #define raw_cpu_generic_add_return(pcp, val)                \
0076 ({                                  \
0077     typeof(pcp) *__p = raw_cpu_ptr(&(pcp));             \
0078                                     \
0079     *__p += val;                            \
0080     *__p;                               \
0081 })
0082
0083 #define raw_cpu_generic_xchg(pcp, nval)                 \
0084 ({                                  \
0085     typeof(pcp) *__p = raw_cpu_ptr(&(pcp));             \
0086     typeof(pcp) __ret;                      \
0087     __ret = *__p;                           \
0088     *__p = nval;                            \
0089     __ret;                              \
0090 })
0091
0092 #define raw_cpu_generic_cmpxchg(pcp, oval, nval)            \
0093 ({                                  \
0094     typeof(pcp) *__p = raw_cpu_ptr(&(pcp));             \
0095     typeof(pcp) __ret;                      \
0096     __ret = *__p;                           \
0097     if (__ret == (oval))                        \
0098         *__p = nval;                        \
0099     __ret;                              \
0100 })
0101
0102 #define raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
0103 ({                                  \
0104     typeof(pcp1) *__p1 = raw_cpu_ptr(&(pcp1));          \
0105     typeof(pcp2) *__p2 = raw_cpu_ptr(&(pcp2));          \
0106     int __ret = 0;                          \
0107     if (*__p1 == (oval1) && *__p2  == (oval2)) {            \
0108         *__p1 = nval1;                      \
0109         *__p2 = nval2;                      \
0110         __ret = 1;                      \
0111     }                               \
0112     (__ret);                            \
0113 })
0114
0115 #define __this_cpu_generic_read_nopreempt(pcp)              \
0116 ({                                  \
0117     typeof(pcp) ___ret;                     \
0118     preempt_disable_notrace();                  \
0119     ___ret = READ_ONCE(*raw_cpu_ptr(&(pcp)));           \
0120     preempt_enable_notrace();                   \
0121     ___ret;                             \
0122 })
0123
0124 #define __this_cpu_generic_read_noirq(pcp)              \
0125 ({                                  \
0126     typeof(pcp) ___ret;                     \
0127     unsigned long ___flags;                     \
0128     raw_local_irq_save(___flags);                   \
0129     ___ret = raw_cpu_generic_read(pcp);             \
0130     raw_local_irq_restore(___flags);                \
0131     ___ret;                             \
0132 })
0133
0134 #define this_cpu_generic_read(pcp)                  \
0135 ({                                  \
0136     typeof(pcp) __ret;                      \
0137     if (__native_word(pcp))                     \
0138         __ret = __this_cpu_generic_read_nopreempt(pcp);     \
0139     else                                \
0140         __ret = __this_cpu_generic_read_noirq(pcp);     \
0141     __ret;                              \
0142 })
0143
0144 #define this_cpu_generic_to_op(pcp, val, op)                \
0145 do {                                    \
0146     unsigned long __flags;                      \
0147     raw_local_irq_save(__flags);                    \
0148     raw_cpu_generic_to_op(pcp, val, op);                \
0149     raw_local_irq_restore(__flags);                 \
0150 } while (0)
0151
0152
0153 #define this_cpu_generic_add_return(pcp, val)               \
0154 ({                                  \
0155     typeof(pcp) __ret;                      \
0156     unsigned long __flags;                      \
0157     raw_local_irq_save(__flags);                    \
0158     __ret = raw_cpu_generic_add_return(pcp, val);           \
0159     raw_local_irq_restore(__flags);                 \
0160     __ret;                              \
0161 })
0162
0163 #define this_cpu_generic_xchg(pcp, nval)                \
0164 ({                                  \
0165     typeof(pcp) __ret;                      \
0166     unsigned long __flags;                      \
0167     raw_local_irq_save(__flags);                    \
0168     __ret = raw_cpu_generic_xchg(pcp, nval);            \
0169     raw_local_irq_restore(__flags);                 \
0170     __ret;                              \
0171 })
0172
0173 #define this_cpu_generic_cmpxchg(pcp, oval, nval)           \
0174 ({                                  \
0175     typeof(pcp) __ret;                      \
0176     unsigned long __flags;                      \
0177     raw_local_irq_save(__flags);                    \
0178     __ret = raw_cpu_generic_cmpxchg(pcp, oval, nval);       \
0179     raw_local_irq_restore(__flags);                 \
0180     __ret;                              \
0181 })
0182
0183 #define this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
0184 ({                                  \
0185     int __ret;                          \
0186     unsigned long __flags;                      \
0187     raw_local_irq_save(__flags);                    \
0188     __ret = raw_cpu_generic_cmpxchg_double(pcp1, pcp2,      \
0189             oval1, oval2, nval1, nval2);            \
0190     raw_local_irq_restore(__flags);                 \
0191     __ret;                              \
0192 })
0193
0194 #ifndef raw_cpu_read_1
0195 #define raw_cpu_read_1(pcp)     raw_cpu_generic_read(pcp)
0196 #endif
0197 #ifndef raw_cpu_read_2
0198 #define raw_cpu_read_2(pcp)     raw_cpu_generic_read(pcp)
0199 #endif
0200 #ifndef raw_cpu_read_4
0201 #define raw_cpu_read_4(pcp)     raw_cpu_generic_read(pcp)
0202 #endif
0203 #ifndef raw_cpu_read_8
0204 #define raw_cpu_read_8(pcp)     raw_cpu_generic_read(pcp)
0205 #endif
0206
0207 #ifndef raw_cpu_write_1
0208 #define raw_cpu_write_1(pcp, val)   raw_cpu_generic_to_op(pcp, val, =)
0209 #endif
0210 #ifndef raw_cpu_write_2
0211 #define raw_cpu_write_2(pcp, val)   raw_cpu_generic_to_op(pcp, val, =)
0212 #endif
0213 #ifndef raw_cpu_write_4
0214 #define raw_cpu_write_4(pcp, val)   raw_cpu_generic_to_op(pcp, val, =)
0215 #endif
0216 #ifndef raw_cpu_write_8
0217 #define raw_cpu_write_8(pcp, val)   raw_cpu_generic_to_op(pcp, val, =)
0218 #endif
0219
0220 #ifndef raw_cpu_add_1
0221 #define raw_cpu_add_1(pcp, val)     raw_cpu_generic_to_op(pcp, val, +=)
0222 #endif
0223 #ifndef raw_cpu_add_2
0224 #define raw_cpu_add_2(pcp, val)     raw_cpu_generic_to_op(pcp, val, +=)
0225 #endif
0226 #ifndef raw_cpu_add_4
0227 #define raw_cpu_add_4(pcp, val)     raw_cpu_generic_to_op(pcp, val, +=)
0228 #endif
0229 #ifndef raw_cpu_add_8
0230 #define raw_cpu_add_8(pcp, val)     raw_cpu_generic_to_op(pcp, val, +=)
0231 #endif
0232
0233 #ifndef raw_cpu_and_1
0234 #define raw_cpu_and_1(pcp, val)     raw_cpu_generic_to_op(pcp, val, &=)
0235 #endif
0236 #ifndef raw_cpu_and_2
0237 #define raw_cpu_and_2(pcp, val)     raw_cpu_generic_to_op(pcp, val, &=)
0238 #endif
0239 #ifndef raw_cpu_and_4
0240 #define raw_cpu_and_4(pcp, val)     raw_cpu_generic_to_op(pcp, val, &=)
0241 #endif
0242 #ifndef raw_cpu_and_8
0243 #define raw_cpu_and_8(pcp, val)     raw_cpu_generic_to_op(pcp, val, &=)
0244 #endif
0245
0246 #ifndef raw_cpu_or_1
0247 #define raw_cpu_or_1(pcp, val)      raw_cpu_generic_to_op(pcp, val, |=)
0248 #endif
0249 #ifndef raw_cpu_or_2
0250 #define raw_cpu_or_2(pcp, val)      raw_cpu_generic_to_op(pcp, val, |=)
0251 #endif
0252 #ifndef raw_cpu_or_4
0253 #define raw_cpu_or_4(pcp, val)      raw_cpu_generic_to_op(pcp, val, |=)
0254 #endif
0255 #ifndef raw_cpu_or_8
0256 #define raw_cpu_or_8(pcp, val)      raw_cpu_generic_to_op(pcp, val, |=)
0257 #endif
0258
0259 #ifndef raw_cpu_add_return_1
0260 #define raw_cpu_add_return_1(pcp, val)  raw_cpu_generic_add_return(pcp, val)
0261 #endif
0262 #ifndef raw_cpu_add_return_2
0263 #define raw_cpu_add_return_2(pcp, val)  raw_cpu_generic_add_return(pcp, val)
0264 #endif
0265 #ifndef raw_cpu_add_return_4
0266 #define raw_cpu_add_return_4(pcp, val)  raw_cpu_generic_add_return(pcp, val)
0267 #endif
0268 #ifndef raw_cpu_add_return_8
0269 #define raw_cpu_add_return_8(pcp, val)  raw_cpu_generic_add_return(pcp, val)
0270 #endif
0271
0272 #ifndef raw_cpu_xchg_1
0273 #define raw_cpu_xchg_1(pcp, nval)   raw_cpu_generic_xchg(pcp, nval)
0274 #endif
0275 #ifndef raw_cpu_xchg_2
0276 #define raw_cpu_xchg_2(pcp, nval)   raw_cpu_generic_xchg(pcp, nval)
0277 #endif
0278 #ifndef raw_cpu_xchg_4
0279 #define raw_cpu_xchg_4(pcp, nval)   raw_cpu_generic_xchg(pcp, nval)
0280 #endif
0281 #ifndef raw_cpu_xchg_8
0282 #define raw_cpu_xchg_8(pcp, nval)   raw_cpu_generic_xchg(pcp, nval)
0283 #endif
0284
0285 #ifndef raw_cpu_cmpxchg_1
0286 #define raw_cpu_cmpxchg_1(pcp, oval, nval) \
0287     raw_cpu_generic_cmpxchg(pcp, oval, nval)
0288 #endif
0289 #ifndef raw_cpu_cmpxchg_2
0290 #define raw_cpu_cmpxchg_2(pcp, oval, nval) \
0291     raw_cpu_generic_cmpxchg(pcp, oval, nval)
0292 #endif
0293 #ifndef raw_cpu_cmpxchg_4
0294 #define raw_cpu_cmpxchg_4(pcp, oval, nval) \
0295     raw_cpu_generic_cmpxchg(pcp, oval, nval)
0296 #endif
0297 #ifndef raw_cpu_cmpxchg_8
0298 #define raw_cpu_cmpxchg_8(pcp, oval, nval) \
0299     raw_cpu_generic_cmpxchg(pcp, oval, nval)
0300 #endif
0301
0302 #ifndef raw_cpu_cmpxchg_double_1
0303 #define raw_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \
0304     raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
0305 #endif
0306 #ifndef raw_cpu_cmpxchg_double_2
0307 #define raw_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \
0308     raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
0309 #endif
0310 #ifndef raw_cpu_cmpxchg_double_4
0311 #define raw_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \
0312     raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
0313 #endif
0314 #ifndef raw_cpu_cmpxchg_double_8
0315 #define raw_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \
0316     raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
0317 #endif
0318
0319 #ifndef this_cpu_read_1
0320 #define this_cpu_read_1(pcp)        this_cpu_generic_read(pcp)
0321 #endif
0322 #ifndef this_cpu_read_2
0323 #define this_cpu_read_2(pcp)        this_cpu_generic_read(pcp)
0324 #endif
0325 #ifndef this_cpu_read_4
0326 #define this_cpu_read_4(pcp)        this_cpu_generic_read(pcp)
0327 #endif
0328 #ifndef this_cpu_read_8
0329 #define this_cpu_read_8(pcp)        this_cpu_generic_read(pcp)
0330 #endif
0331
0332 #ifndef this_cpu_write_1
0333 #define this_cpu_write_1(pcp, val)  this_cpu_generic_to_op(pcp, val, =)
0334 #endif
0335 #ifndef this_cpu_write_2
0336 #define this_cpu_write_2(pcp, val)  this_cpu_generic_to_op(pcp, val, =)
0337 #endif
0338 #ifndef this_cpu_write_4
0339 #define this_cpu_write_4(pcp, val)  this_cpu_generic_to_op(pcp, val, =)
0340 #endif
0341 #ifndef this_cpu_write_8
0342 #define this_cpu_write_8(pcp, val)  this_cpu_generic_to_op(pcp, val, =)
0343 #endif
0344
0345 #ifndef this_cpu_add_1
0346 #define this_cpu_add_1(pcp, val)    this_cpu_generic_to_op(pcp, val, +=)
0347 #endif
0348 #ifndef this_cpu_add_2
0349 #define this_cpu_add_2(pcp, val)    this_cpu_generic_to_op(pcp, val, +=)
0350 #endif
0351 #ifndef this_cpu_add_4
0352 #define this_cpu_add_4(pcp, val)    this_cpu_generic_to_op(pcp, val, +=)
0353 #endif
0354 #ifndef this_cpu_add_8
0355 #define this_cpu_add_8(pcp, val)    this_cpu_generic_to_op(pcp, val, +=)
0356 #endif
0357
0358 #ifndef this_cpu_and_1
0359 #define this_cpu_and_1(pcp, val)    this_cpu_generic_to_op(pcp, val, &=)
0360 #endif
0361 #ifndef this_cpu_and_2
0362 #define this_cpu_and_2(pcp, val)    this_cpu_generic_to_op(pcp, val, &=)
0363 #endif
0364 #ifndef this_cpu_and_4
0365 #define this_cpu_and_4(pcp, val)    this_cpu_generic_to_op(pcp, val, &=)
0366 #endif
0367 #ifndef this_cpu_and_8
0368 #define this_cpu_and_8(pcp, val)    this_cpu_generic_to_op(pcp, val, &=)
0369 #endif
0370
0371 #ifndef this_cpu_or_1
0372 #define this_cpu_or_1(pcp, val)     this_cpu_generic_to_op(pcp, val, |=)
0373 #endif
0374 #ifndef this_cpu_or_2
0375 #define this_cpu_or_2(pcp, val)     this_cpu_generic_to_op(pcp, val, |=)
0376 #endif
0377 #ifndef this_cpu_or_4
0378 #define this_cpu_or_4(pcp, val)     this_cpu_generic_to_op(pcp, val, |=)
0379 #endif
0380 #ifndef this_cpu_or_8
0381 #define this_cpu_or_8(pcp, val)     this_cpu_generic_to_op(pcp, val, |=)
0382 #endif
0383
0384 #ifndef this_cpu_add_return_1
0385 #define this_cpu_add_return_1(pcp, val) this_cpu_generic_add_return(pcp, val)
0386 #endif
0387 #ifndef this_cpu_add_return_2
0388 #define this_cpu_add_return_2(pcp, val) this_cpu_generic_add_return(pcp, val)
0389 #endif
0390 #ifndef this_cpu_add_return_4
0391 #define this_cpu_add_return_4(pcp, val) this_cpu_generic_add_return(pcp, val)
0392 #endif
0393 #ifndef this_cpu_add_return_8
0394 #define this_cpu_add_return_8(pcp, val) this_cpu_generic_add_return(pcp, val)
0395 #endif
0396
0397 #ifndef this_cpu_xchg_1
0398 #define this_cpu_xchg_1(pcp, nval)  this_cpu_generic_xchg(pcp, nval)
0399 #endif
0400 #ifndef this_cpu_xchg_2
0401 #define this_cpu_xchg_2(pcp, nval)  this_cpu_generic_xchg(pcp, nval)
0402 #endif
0403 #ifndef this_cpu_xchg_4
0404 #define this_cpu_xchg_4(pcp, nval)  this_cpu_generic_xchg(pcp, nval)
0405 #endif
0406 #ifndef this_cpu_xchg_8
0407 #define this_cpu_xchg_8(pcp, nval)  this_cpu_generic_xchg(pcp, nval)
0408 #endif
0409
0410 #ifndef this_cpu_cmpxchg_1
0411 #define this_cpu_cmpxchg_1(pcp, oval, nval) \
0412     this_cpu_generic_cmpxchg(pcp, oval, nval)
0413 #endif
0414 #ifndef this_cpu_cmpxchg_2
0415 #define this_cpu_cmpxchg_2(pcp, oval, nval) \
0416     this_cpu_generic_cmpxchg(pcp, oval, nval)
0417 #endif
0418 #ifndef this_cpu_cmpxchg_4
0419 #define this_cpu_cmpxchg_4(pcp, oval, nval) \
0420     this_cpu_generic_cmpxchg(pcp, oval, nval)
0421 #endif
0422 #ifndef this_cpu_cmpxchg_8
0423 #define this_cpu_cmpxchg_8(pcp, oval, nval) \
0424     this_cpu_generic_cmpxchg(pcp, oval, nval)
0425 #endif
0426
0427 #ifndef this_cpu_cmpxchg_double_1
0428 #define this_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \
0429     this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
0430 #endif
0431 #ifndef this_cpu_cmpxchg_double_2
0432 #define this_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \
0433     this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
0434 #endif
0435 #ifndef this_cpu_cmpxchg_double_4
0436 #define this_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \
0437     this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
0438 #endif
0439 #ifndef this_cpu_cmpxchg_double_8
0440 #define this_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \
0441     this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
0442 #endif
0443
0444 #endif /* _ASM_GENERIC_PERCPU_H_ */