include/vdso/helpers.h

0001 /* SPDX-License-Identifier: GPL-2.0 */
0002 #ifndef __VDSO_HELPERS_H
0003 #define __VDSO_HELPERS_H
0004
0005 #ifndef __ASSEMBLY__
0006
0007 #include <vdso/datapage.h>
0008
0009 static __always_inline u32 vdso_read_begin(const struct vdso_data *vd)
0010 {
0011     u32 seq;
0012
0013     while (unlikely((seq = READ_ONCE(vd->seq)) & 1))
0014         cpu_relax();
0015
0016     smp_rmb();
0017     return seq;
0018 }
0019
0020 static __always_inline u32 vdso_read_retry(const struct vdso_data *vd,
0021                        u32 start)
0022 {
0023     u32 seq;
0024
0025     smp_rmb();
0026     seq = READ_ONCE(vd->seq);
0027     return seq != start;
0028 }
0029
0030 static __always_inline void vdso_write_begin(struct vdso_data *vd)
0031 {
0032     /*
0033      * WRITE_ONCE it is required otherwise the compiler can validly tear
0034      * updates to vd[x].seq and it is possible that the value seen by the
0035      * reader it is inconsistent.
0036      */
0037     WRITE_ONCE(vd[CS_HRES_COARSE].seq, vd[CS_HRES_COARSE].seq + 1);
0038     WRITE_ONCE(vd[CS_RAW].seq, vd[CS_RAW].seq + 1);
0039     smp_wmb();
0040 }
0041
0042 static __always_inline void vdso_write_end(struct vdso_data *vd)
0043 {
0044     smp_wmb();
0045     /*
0046      * WRITE_ONCE it is required otherwise the compiler can validly tear
0047      * updates to vd[x].seq and it is possible that the value seen by the
0048      * reader it is inconsistent.
0049      */
0050     WRITE_ONCE(vd[CS_HRES_COARSE].seq, vd[CS_HRES_COARSE].seq + 1);
0051     WRITE_ONCE(vd[CS_RAW].seq, vd[CS_RAW].seq + 1);
0052 }
0053
0054 #endif /* !__ASSEMBLY__ */
0055
0056 #endif /* __VDSO_HELPERS_H */