Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * Copyright 2006 Andi Kleen, SUSE Labs.
0004  *
0005  * Fast user context implementation of clock_gettime, gettimeofday, and time.
0006  *
0007  * The code should have no internal unresolved relocations.
0008  * Check with readelf after changing.
0009  * Also alternative() doesn't work.
0010  */
0011 /*
0012  * Copyright (c) 2017 Oracle and/or its affiliates. All rights reserved.
0013  */
0014 
0015 #include <linux/kernel.h>
0016 #include <linux/time.h>
0017 #include <linux/string.h>
0018 #include <asm/io.h>
0019 #include <asm/unistd.h>
0020 #include <asm/timex.h>
0021 #include <asm/clocksource.h>
0022 #include <asm/vvar.h>
0023 
0024 #ifdef  CONFIG_SPARC64
0025 #define SYSCALL_STRING                          \
0026     "ta 0x6d;"                          \
0027     "bcs,a  1f;"                            \
0028     " sub   %%g0, %%o0, %%o0;"                  \
0029     "1:"
0030 #else
0031 #define SYSCALL_STRING                          \
0032     "ta 0x10;"                          \
0033     "bcs,a  1f;"                            \
0034     " sub   %%g0, %%o0, %%o0;"                  \
0035     "1:"
0036 #endif
0037 
0038 #define SYSCALL_CLOBBERS                        \
0039     "f0", "f1", "f2", "f3", "f4", "f5", "f6", "f7",         \
0040     "f8", "f9", "f10", "f11", "f12", "f13", "f14", "f15",       \
0041     "f16", "f17", "f18", "f19", "f20", "f21", "f22", "f23",     \
0042     "f24", "f25", "f26", "f27", "f28", "f29", "f30", "f31",     \
0043     "f32", "f34", "f36", "f38", "f40", "f42", "f44", "f46",     \
0044     "f48", "f50", "f52", "f54", "f56", "f58", "f60", "f62",     \
0045     "cc", "memory"
0046 
0047 /*
0048  * Compute the vvar page's address in the process address space, and return it
0049  * as a pointer to the vvar_data.
0050  */
0051 notrace static __always_inline struct vvar_data *get_vvar_data(void)
0052 {
0053     unsigned long ret;
0054 
0055     /*
0056      * vdso data page is the first vDSO page so grab the PC
0057      * and move up a page to get to the data page.
0058      */
0059     __asm__("rd %%pc, %0" : "=r" (ret));
0060     ret &= ~(8192 - 1);
0061     ret -= 8192;
0062 
0063     return (struct vvar_data *) ret;
0064 }
0065 
0066 notrace static long vdso_fallback_gettime(long clock, struct __kernel_old_timespec *ts)
0067 {
0068     register long num __asm__("g1") = __NR_clock_gettime;
0069     register long o0 __asm__("o0") = clock;
0070     register long o1 __asm__("o1") = (long) ts;
0071 
0072     __asm__ __volatile__(SYSCALL_STRING : "=r" (o0) : "r" (num),
0073                  "0" (o0), "r" (o1) : SYSCALL_CLOBBERS);
0074     return o0;
0075 }
0076 
0077 notrace static long vdso_fallback_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz)
0078 {
0079     register long num __asm__("g1") = __NR_gettimeofday;
0080     register long o0 __asm__("o0") = (long) tv;
0081     register long o1 __asm__("o1") = (long) tz;
0082 
0083     __asm__ __volatile__(SYSCALL_STRING : "=r" (o0) : "r" (num),
0084                  "0" (o0), "r" (o1) : SYSCALL_CLOBBERS);
0085     return o0;
0086 }
0087 
0088 #ifdef  CONFIG_SPARC64
0089 notrace static __always_inline u64 vread_tick(void)
0090 {
0091     u64 ret;
0092 
0093     __asm__ __volatile__("rd %%tick, %0" : "=r" (ret));
0094     return ret;
0095 }
0096 
0097 notrace static __always_inline u64 vread_tick_stick(void)
0098 {
0099     u64 ret;
0100 
0101     __asm__ __volatile__("rd %%asr24, %0" : "=r" (ret));
0102     return ret;
0103 }
0104 #else
0105 notrace static __always_inline u64 vread_tick(void)
0106 {
0107     register unsigned long long ret asm("o4");
0108 
0109     __asm__ __volatile__("rd %%tick, %L0\n\t"
0110                  "srlx %L0, 32, %H0"
0111                  : "=r" (ret));
0112     return ret;
0113 }
0114 
0115 notrace static __always_inline u64 vread_tick_stick(void)
0116 {
0117     register unsigned long long ret asm("o4");
0118 
0119     __asm__ __volatile__("rd %%asr24, %L0\n\t"
0120                  "srlx %L0, 32, %H0"
0121                  : "=r" (ret));
0122     return ret;
0123 }
0124 #endif
0125 
0126 notrace static __always_inline u64 vgetsns(struct vvar_data *vvar)
0127 {
0128     u64 v;
0129     u64 cycles;
0130 
0131     cycles = vread_tick();
0132     v = (cycles - vvar->clock.cycle_last) & vvar->clock.mask;
0133     return v * vvar->clock.mult;
0134 }
0135 
0136 notrace static __always_inline u64 vgetsns_stick(struct vvar_data *vvar)
0137 {
0138     u64 v;
0139     u64 cycles;
0140 
0141     cycles = vread_tick_stick();
0142     v = (cycles - vvar->clock.cycle_last) & vvar->clock.mask;
0143     return v * vvar->clock.mult;
0144 }
0145 
0146 notrace static __always_inline int do_realtime(struct vvar_data *vvar,
0147                            struct __kernel_old_timespec *ts)
0148 {
0149     unsigned long seq;
0150     u64 ns;
0151 
0152     do {
0153         seq = vvar_read_begin(vvar);
0154         ts->tv_sec = vvar->wall_time_sec;
0155         ns = vvar->wall_time_snsec;
0156         ns += vgetsns(vvar);
0157         ns >>= vvar->clock.shift;
0158     } while (unlikely(vvar_read_retry(vvar, seq)));
0159 
0160     ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
0161     ts->tv_nsec = ns;
0162 
0163     return 0;
0164 }
0165 
0166 notrace static __always_inline int do_realtime_stick(struct vvar_data *vvar,
0167                              struct __kernel_old_timespec *ts)
0168 {
0169     unsigned long seq;
0170     u64 ns;
0171 
0172     do {
0173         seq = vvar_read_begin(vvar);
0174         ts->tv_sec = vvar->wall_time_sec;
0175         ns = vvar->wall_time_snsec;
0176         ns += vgetsns_stick(vvar);
0177         ns >>= vvar->clock.shift;
0178     } while (unlikely(vvar_read_retry(vvar, seq)));
0179 
0180     ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
0181     ts->tv_nsec = ns;
0182 
0183     return 0;
0184 }
0185 
0186 notrace static __always_inline int do_monotonic(struct vvar_data *vvar,
0187                         struct __kernel_old_timespec *ts)
0188 {
0189     unsigned long seq;
0190     u64 ns;
0191 
0192     do {
0193         seq = vvar_read_begin(vvar);
0194         ts->tv_sec = vvar->monotonic_time_sec;
0195         ns = vvar->monotonic_time_snsec;
0196         ns += vgetsns(vvar);
0197         ns >>= vvar->clock.shift;
0198     } while (unlikely(vvar_read_retry(vvar, seq)));
0199 
0200     ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
0201     ts->tv_nsec = ns;
0202 
0203     return 0;
0204 }
0205 
0206 notrace static __always_inline int do_monotonic_stick(struct vvar_data *vvar,
0207                               struct __kernel_old_timespec *ts)
0208 {
0209     unsigned long seq;
0210     u64 ns;
0211 
0212     do {
0213         seq = vvar_read_begin(vvar);
0214         ts->tv_sec = vvar->monotonic_time_sec;
0215         ns = vvar->monotonic_time_snsec;
0216         ns += vgetsns_stick(vvar);
0217         ns >>= vvar->clock.shift;
0218     } while (unlikely(vvar_read_retry(vvar, seq)));
0219 
0220     ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
0221     ts->tv_nsec = ns;
0222 
0223     return 0;
0224 }
0225 
0226 notrace static int do_realtime_coarse(struct vvar_data *vvar,
0227                       struct __kernel_old_timespec *ts)
0228 {
0229     unsigned long seq;
0230 
0231     do {
0232         seq = vvar_read_begin(vvar);
0233         ts->tv_sec = vvar->wall_time_coarse_sec;
0234         ts->tv_nsec = vvar->wall_time_coarse_nsec;
0235     } while (unlikely(vvar_read_retry(vvar, seq)));
0236     return 0;
0237 }
0238 
0239 notrace static int do_monotonic_coarse(struct vvar_data *vvar,
0240                        struct __kernel_old_timespec *ts)
0241 {
0242     unsigned long seq;
0243 
0244     do {
0245         seq = vvar_read_begin(vvar);
0246         ts->tv_sec = vvar->monotonic_time_coarse_sec;
0247         ts->tv_nsec = vvar->monotonic_time_coarse_nsec;
0248     } while (unlikely(vvar_read_retry(vvar, seq)));
0249 
0250     return 0;
0251 }
0252 
0253 notrace int
0254 __vdso_clock_gettime(clockid_t clock, struct __kernel_old_timespec *ts)
0255 {
0256     struct vvar_data *vvd = get_vvar_data();
0257 
0258     switch (clock) {
0259     case CLOCK_REALTIME:
0260         if (unlikely(vvd->vclock_mode == VCLOCK_NONE))
0261             break;
0262         return do_realtime(vvd, ts);
0263     case CLOCK_MONOTONIC:
0264         if (unlikely(vvd->vclock_mode == VCLOCK_NONE))
0265             break;
0266         return do_monotonic(vvd, ts);
0267     case CLOCK_REALTIME_COARSE:
0268         return do_realtime_coarse(vvd, ts);
0269     case CLOCK_MONOTONIC_COARSE:
0270         return do_monotonic_coarse(vvd, ts);
0271     }
0272     /*
0273      * Unknown clock ID ? Fall back to the syscall.
0274      */
0275     return vdso_fallback_gettime(clock, ts);
0276 }
0277 int
0278 clock_gettime(clockid_t, struct __kernel_old_timespec *)
0279     __attribute__((weak, alias("__vdso_clock_gettime")));
0280 
0281 notrace int
0282 __vdso_clock_gettime_stick(clockid_t clock, struct __kernel_old_timespec *ts)
0283 {
0284     struct vvar_data *vvd = get_vvar_data();
0285 
0286     switch (clock) {
0287     case CLOCK_REALTIME:
0288         if (unlikely(vvd->vclock_mode == VCLOCK_NONE))
0289             break;
0290         return do_realtime_stick(vvd, ts);
0291     case CLOCK_MONOTONIC:
0292         if (unlikely(vvd->vclock_mode == VCLOCK_NONE))
0293             break;
0294         return do_monotonic_stick(vvd, ts);
0295     case CLOCK_REALTIME_COARSE:
0296         return do_realtime_coarse(vvd, ts);
0297     case CLOCK_MONOTONIC_COARSE:
0298         return do_monotonic_coarse(vvd, ts);
0299     }
0300     /*
0301      * Unknown clock ID ? Fall back to the syscall.
0302      */
0303     return vdso_fallback_gettime(clock, ts);
0304 }
0305 
0306 notrace int
0307 __vdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz)
0308 {
0309     struct vvar_data *vvd = get_vvar_data();
0310 
0311     if (likely(vvd->vclock_mode != VCLOCK_NONE)) {
0312         if (likely(tv != NULL)) {
0313             union tstv_t {
0314                 struct __kernel_old_timespec ts;
0315                 struct __kernel_old_timeval tv;
0316             } *tstv = (union tstv_t *) tv;
0317             do_realtime(vvd, &tstv->ts);
0318             /*
0319              * Assign before dividing to ensure that the division is
0320              * done in the type of tv_usec, not tv_nsec.
0321              *
0322              * There cannot be > 1 billion usec in a second:
0323              * do_realtime() has already distributed such overflow
0324              * into tv_sec.  So we can assign it to an int safely.
0325              */
0326             tstv->tv.tv_usec = tstv->ts.tv_nsec;
0327             tstv->tv.tv_usec /= 1000;
0328         }
0329         if (unlikely(tz != NULL)) {
0330             /* Avoid memcpy. Some old compilers fail to inline it */
0331             tz->tz_minuteswest = vvd->tz_minuteswest;
0332             tz->tz_dsttime = vvd->tz_dsttime;
0333         }
0334         return 0;
0335     }
0336     return vdso_fallback_gettimeofday(tv, tz);
0337 }
0338 int
0339 gettimeofday(struct __kernel_old_timeval *, struct timezone *)
0340     __attribute__((weak, alias("__vdso_gettimeofday")));
0341 
0342 notrace int
0343 __vdso_gettimeofday_stick(struct __kernel_old_timeval *tv, struct timezone *tz)
0344 {
0345     struct vvar_data *vvd = get_vvar_data();
0346 
0347     if (likely(vvd->vclock_mode != VCLOCK_NONE)) {
0348         if (likely(tv != NULL)) {
0349             union tstv_t {
0350                 struct __kernel_old_timespec ts;
0351                 struct __kernel_old_timeval tv;
0352             } *tstv = (union tstv_t *) tv;
0353             do_realtime_stick(vvd, &tstv->ts);
0354             /*
0355              * Assign before dividing to ensure that the division is
0356              * done in the type of tv_usec, not tv_nsec.
0357              *
0358              * There cannot be > 1 billion usec in a second:
0359              * do_realtime() has already distributed such overflow
0360              * into tv_sec.  So we can assign it to an int safely.
0361              */
0362             tstv->tv.tv_usec = tstv->ts.tv_nsec;
0363             tstv->tv.tv_usec /= 1000;
0364         }
0365         if (unlikely(tz != NULL)) {
0366             /* Avoid memcpy. Some old compilers fail to inline it */
0367             tz->tz_minuteswest = vvd->tz_minuteswest;
0368             tz->tz_dsttime = vvd->tz_dsttime;
0369         }
0370         return 0;
0371     }
0372     return vdso_fallback_gettimeofday(tv, tz);
0373 }
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!