blob: 03c3eb77bfcebce765b838271b6d7789365ff3de [file] [log] [blame]
Andi Kleen2aae9502007-07-21 17:10:01 +02001/*
2 * Copyright 2006 Andi Kleen, SUSE Labs.
3 * Subject to the GNU Public License, v.2
4 *
Andy Lutomirskif144a6b2011-05-23 09:31:30 -04005 * Fast user context implementation of clock_gettime, gettimeofday, and time.
Andi Kleen2aae9502007-07-21 17:10:01 +02006 *
Stefani Seibold7a59ed42014-03-17 23:22:09 +01007 * 32 Bit compat layer by Stefani Seibold <stefani@seibold.net>
8 * sponsored by Rohde & Schwarz GmbH & Co. KG Munich/Germany
9 *
Andi Kleen2aae9502007-07-21 17:10:01 +020010 * The code should have no internal unresolved relocations.
11 * Check with readelf after changing.
Andi Kleen2aae9502007-07-21 17:10:01 +020012 */
13
Stefani Seibold7a59ed42014-03-17 23:22:09 +010014#include <uapi/linux/time.h>
Andi Kleen2aae9502007-07-21 17:10:01 +020015#include <asm/vgtod.h>
Andi Kleen2aae9502007-07-21 17:10:01 +020016#include <asm/hpet.h>
Stefani Seibold7c031562014-03-17 23:22:10 +010017#include <asm/vvar.h>
Andi Kleen2aae9502007-07-21 17:10:01 +020018#include <asm/unistd.h>
Stefani Seibold7c031562014-03-17 23:22:10 +010019#include <asm/msr.h>
Andy Lutomirski76480a62015-12-10 19:20:22 -080020#include <asm/pvclock.h>
Stefani Seibold7c031562014-03-17 23:22:10 +010021#include <linux/math64.h>
22#include <linux/time.h>
Andy Lutomirski76480a62015-12-10 19:20:22 -080023#include <linux/kernel.h>
Andi Kleen2aae9502007-07-21 17:10:01 +020024
Andy Lutomirski8c49d9a2011-05-23 09:31:24 -040025#define gtod (&VVAR(vsyscall_gtod_data))
Andi Kleen2aae9502007-07-21 17:10:01 +020026
Stefani Seibold7a59ed42014-03-17 23:22:09 +010027extern int __vdso_clock_gettime(clockid_t clock, struct timespec *ts);
28extern int __vdso_gettimeofday(struct timeval *tv, struct timezone *tz);
29extern time_t __vdso_time(time_t *t);
30
Stefani Seibold7c031562014-03-17 23:22:10 +010031#ifdef CONFIG_HPET_TIMER
Andy Lutomirskif40c3302014-05-05 12:19:36 -070032extern u8 hpet_page
33 __attribute__((visibility("hidden")));
34
35static notrace cycle_t vread_hpet(void)
Stefani Seibold7c031562014-03-17 23:22:10 +010036{
Andy Lutomirskif40c3302014-05-05 12:19:36 -070037 return *(const volatile u32 *)(&hpet_page + HPET_COUNTER);
Stefani Seibold7c031562014-03-17 23:22:10 +010038}
39#endif
40
Andy Lutomirskidac16fb2015-12-10 19:20:20 -080041#ifdef CONFIG_PARAVIRT_CLOCK
42extern u8 pvclock_page
43 __attribute__((visibility("hidden")));
44#endif
45
Stefani Seibold7a59ed42014-03-17 23:22:09 +010046#ifndef BUILD_VDSO32
47
Stefani Seibold411f7902014-03-17 23:22:03 +010048notrace static long vdso_fallback_gettime(long clock, struct timespec *ts)
49{
50 long ret;
51 asm("syscall" : "=a" (ret) :
52 "0" (__NR_clock_gettime), "D" (clock), "S" (ts) : "memory");
53 return ret;
54}
55
56notrace static long vdso_fallback_gtod(struct timeval *tv, struct timezone *tz)
57{
58 long ret;
59
60 asm("syscall" : "=a" (ret) :
61 "0" (__NR_gettimeofday), "D" (tv), "S" (tz) : "memory");
62 return ret;
63}
64
Marcelo Tosatti51c19b42012-11-27 23:28:57 -020065
Andy Lutomirski76480a62015-12-10 19:20:22 -080066#else
67
68notrace static long vdso_fallback_gettime(long clock, struct timespec *ts)
69{
70 long ret;
71
72 asm(
73 "mov %%ebx, %%edx \n"
74 "mov %2, %%ebx \n"
75 "call __kernel_vsyscall \n"
76 "mov %%edx, %%ebx \n"
77 : "=a" (ret)
78 : "0" (__NR_clock_gettime), "g" (clock), "c" (ts)
79 : "memory", "edx");
80 return ret;
81}
82
83notrace static long vdso_fallback_gtod(struct timeval *tv, struct timezone *tz)
84{
85 long ret;
86
87 asm(
88 "mov %%ebx, %%edx \n"
89 "mov %2, %%ebx \n"
90 "call __kernel_vsyscall \n"
91 "mov %%edx, %%ebx \n"
92 : "=a" (ret)
93 : "0" (__NR_gettimeofday), "g" (tv), "c" (tz)
94 : "memory", "edx");
95 return ret;
96}
97
98#endif
99
100#ifdef CONFIG_PARAVIRT_CLOCK
Andy Lutomirskidac16fb2015-12-10 19:20:20 -0800101static notrace const struct pvclock_vsyscall_time_info *get_pvti0(void)
Marcelo Tosatti51c19b42012-11-27 23:28:57 -0200102{
Andy Lutomirskidac16fb2015-12-10 19:20:20 -0800103 return (const struct pvclock_vsyscall_time_info *)&pvclock_page;
Marcelo Tosatti51c19b42012-11-27 23:28:57 -0200104}
105
106static notrace cycle_t vread_pvclock(int *mode)
107{
Andy Lutomirskidac16fb2015-12-10 19:20:20 -0800108 const struct pvclock_vcpu_time_info *pvti = &get_pvti0()->pvti;
Marcelo Tosatti51c19b42012-11-27 23:28:57 -0200109 cycle_t ret;
Andy Lutomirski6b078f52015-12-10 19:20:19 -0800110 u64 tsc, pvti_tsc;
111 u64 last, delta, pvti_system_time;
112 u32 version, pvti_tsc_to_system_mul, pvti_tsc_shift;
Marcelo Tosatti51c19b42012-11-27 23:28:57 -0200113
114 /*
Andy Lutomirski6b078f52015-12-10 19:20:19 -0800115 * Note: The kernel and hypervisor must guarantee that cpu ID
116 * number maps 1:1 to per-CPU pvclock time info.
Paolo Bonzini73459e22015-04-23 13:20:18 +0200117 *
Andy Lutomirski6b078f52015-12-10 19:20:19 -0800118 * Because the hypervisor is entirely unaware of guest userspace
119 * preemption, it cannot guarantee that per-CPU pvclock time
120 * info is updated if the underlying CPU changes or that that
121 * version is increased whenever underlying CPU changes.
122 *
123 * On KVM, we are guaranteed that pvti updates for any vCPU are
124 * atomic as seen by *all* vCPUs. This is an even stronger
125 * guarantee than we get with a normal seqlock.
126 *
127 * On Xen, we don't appear to have that guarantee, but Xen still
128 * supplies a valid seqlock using the version field.
Andy Lutomirski78fd8c72016-01-04 15:14:28 -0800129 *
Andy Lutomirski6b078f52015-12-10 19:20:19 -0800130 * We only do pvclock vdso timing at all if
131 * PVCLOCK_TSC_STABLE_BIT is set, and we interpret that bit to
132 * mean that all vCPUs have matching pvti and that the TSC is
133 * synced, so we can just look at vCPU 0's pvti.
Marcelo Tosatti51c19b42012-11-27 23:28:57 -0200134 */
Marcelo Tosatti51c19b42012-11-27 23:28:57 -0200135
Andy Lutomirski6b078f52015-12-10 19:20:19 -0800136 do {
137 version = pvti->version;
138
Andy Lutomirski76480a62015-12-10 19:20:22 -0800139 smp_rmb();
Andy Lutomirski6b078f52015-12-10 19:20:19 -0800140
Andy Lutomirski78fd8c72016-01-04 15:14:28 -0800141 if (unlikely(!(pvti->flags & PVCLOCK_TSC_STABLE_BIT))) {
142 *mode = VCLOCK_NONE;
143 return 0;
144 }
145
Andy Lutomirski76480a62015-12-10 19:20:22 -0800146 tsc = rdtsc_ordered();
Andy Lutomirski6b078f52015-12-10 19:20:19 -0800147 pvti_tsc_to_system_mul = pvti->tsc_to_system_mul;
148 pvti_tsc_shift = pvti->tsc_shift;
149 pvti_system_time = pvti->system_time;
150 pvti_tsc = pvti->tsc_timestamp;
151
152 /* Make sure that the version double-check is last. */
153 smp_rmb();
154 } while (unlikely((version & 1) || version != pvti->version));
155
156 delta = tsc - pvti_tsc;
157 ret = pvti_system_time +
158 pvclock_scale_delta(delta, pvti_tsc_to_system_mul,
159 pvti_tsc_shift);
Marcelo Tosatti51c19b42012-11-27 23:28:57 -0200160
Andy Lutomirski76480a62015-12-10 19:20:22 -0800161 /* refer to vread_tsc() comment for rationale */
Stefani Seibold7c031562014-03-17 23:22:10 +0100162 last = gtod->cycle_last;
Marcelo Tosatti51c19b42012-11-27 23:28:57 -0200163
164 if (likely(ret >= last))
165 return ret;
166
167 return last;
168}
169#endif
170
Stefani Seibold411f7902014-03-17 23:22:03 +0100171notrace static cycle_t vread_tsc(void)
Andi Kleen2aae9502007-07-21 17:10:01 +0200172{
Andy Lutomirski03b97302015-06-25 18:44:08 +0200173 cycle_t ret = (cycle_t)rdtsc_ordered();
174 u64 last = gtod->cycle_last;
Stefani Seibold411f7902014-03-17 23:22:03 +0100175
176 if (likely(ret >= last))
177 return ret;
178
179 /*
180 * GCC likes to generate cmov here, but this branch is extremely
Adam Buchbinder6a6256f2016-02-23 15:34:30 -0800181 * predictable (it's just a function of time and the likely is
Stefani Seibold411f7902014-03-17 23:22:03 +0100182 * very likely) and there's a data dependence, so force GCC
183 * to generate a branch instead. I don't barrier() because
184 * we don't actually need a barrier, and if this function
185 * ever gets inlined it will generate worse code.
186 */
187 asm volatile ("");
188 return last;
Andi Kleen2aae9502007-07-21 17:10:01 +0200189}
190
Marcelo Tosatti51c19b42012-11-27 23:28:57 -0200191notrace static inline u64 vgetsns(int *mode)
Andi Kleen2aae9502007-07-21 17:10:01 +0200192{
Stefani Seibold7a59ed42014-03-17 23:22:09 +0100193 u64 v;
Andy Lutomirski98d0ac32011-07-14 06:47:22 -0400194 cycles_t cycles;
Stefani Seibold7c031562014-03-17 23:22:10 +0100195
196 if (gtod->vclock_mode == VCLOCK_TSC)
Andy Lutomirski98d0ac32011-07-14 06:47:22 -0400197 cycles = vread_tsc();
Stefani Seibold7a59ed42014-03-17 23:22:09 +0100198#ifdef CONFIG_HPET_TIMER
Stefani Seibold7c031562014-03-17 23:22:10 +0100199 else if (gtod->vclock_mode == VCLOCK_HPET)
Andy Lutomirski98d0ac32011-07-14 06:47:22 -0400200 cycles = vread_hpet();
Stefani Seibold7a59ed42014-03-17 23:22:09 +0100201#endif
Marcelo Tosatti51c19b42012-11-27 23:28:57 -0200202#ifdef CONFIG_PARAVIRT_CLOCK
Stefani Seibold7c031562014-03-17 23:22:10 +0100203 else if (gtod->vclock_mode == VCLOCK_PVCLOCK)
Marcelo Tosatti51c19b42012-11-27 23:28:57 -0200204 cycles = vread_pvclock(mode);
205#endif
John Stultza939e812012-03-01 22:11:09 -0800206 else
207 return 0;
Stefani Seibold7c031562014-03-17 23:22:10 +0100208 v = (cycles - gtod->cycle_last) & gtod->mask;
209 return v * gtod->mult;
Andi Kleen2aae9502007-07-21 17:10:01 +0200210}
211
Andy Lutomirski5f293472012-03-22 21:15:52 -0700212/* Code size doesn't matter (vdso is 4k anyway) and this is faster. */
213notrace static int __always_inline do_realtime(struct timespec *ts)
Andi Kleen2aae9502007-07-21 17:10:01 +0200214{
John Stultz650ea022012-09-04 16:14:46 -0400215 unsigned long seq;
216 u64 ns;
John Stultza939e812012-03-01 22:11:09 -0800217 int mode;
218
Andi Kleen2aae9502007-07-21 17:10:01 +0200219 do {
Stefani Seibold7c031562014-03-17 23:22:10 +0100220 seq = gtod_read_begin(gtod);
221 mode = gtod->vclock_mode;
Andi Kleen2aae9502007-07-21 17:10:01 +0200222 ts->tv_sec = gtod->wall_time_sec;
John Stultz650ea022012-09-04 16:14:46 -0400223 ns = gtod->wall_time_snsec;
Marcelo Tosatti51c19b42012-11-27 23:28:57 -0200224 ns += vgetsns(&mode);
Stefani Seibold7c031562014-03-17 23:22:10 +0100225 ns >>= gtod->shift;
226 } while (unlikely(gtod_read_retry(gtod, seq)));
John Stultza939e812012-03-01 22:11:09 -0800227
Stefani Seibold7c031562014-03-17 23:22:10 +0100228 ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
229 ts->tv_nsec = ns;
230
John Stultza939e812012-03-01 22:11:09 -0800231 return mode;
Andi Kleen2aae9502007-07-21 17:10:01 +0200232}
233
Stefani Seibold7a59ed42014-03-17 23:22:09 +0100234notrace static int __always_inline do_monotonic(struct timespec *ts)
Andi Kleen2aae9502007-07-21 17:10:01 +0200235{
John Stultz650ea022012-09-04 16:14:46 -0400236 unsigned long seq;
237 u64 ns;
John Stultza939e812012-03-01 22:11:09 -0800238 int mode;
239
Andi Kleen2aae9502007-07-21 17:10:01 +0200240 do {
Stefani Seibold7c031562014-03-17 23:22:10 +0100241 seq = gtod_read_begin(gtod);
242 mode = gtod->vclock_mode;
Andy Lutomirski91ec87d2012-03-22 21:15:51 -0700243 ts->tv_sec = gtod->monotonic_time_sec;
John Stultz650ea022012-09-04 16:14:46 -0400244 ns = gtod->monotonic_time_snsec;
Marcelo Tosatti51c19b42012-11-27 23:28:57 -0200245 ns += vgetsns(&mode);
Stefani Seibold7c031562014-03-17 23:22:10 +0100246 ns >>= gtod->shift;
247 } while (unlikely(gtod_read_retry(gtod, seq)));
248
249 ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
250 ts->tv_nsec = ns;
Andy Lutomirski0f51f282011-05-23 09:31:27 -0400251
John Stultza939e812012-03-01 22:11:09 -0800252 return mode;
Andi Kleen2aae9502007-07-21 17:10:01 +0200253}
254
Stefani Seiboldce39c642014-03-17 23:22:04 +0100255notrace static void do_realtime_coarse(struct timespec *ts)
john stultzda15cfd2009-08-19 19:13:34 -0700256{
257 unsigned long seq;
258 do {
Stefani Seibold7c031562014-03-17 23:22:10 +0100259 seq = gtod_read_begin(gtod);
260 ts->tv_sec = gtod->wall_time_coarse_sec;
261 ts->tv_nsec = gtod->wall_time_coarse_nsec;
262 } while (unlikely(gtod_read_retry(gtod, seq)));
john stultzda15cfd2009-08-19 19:13:34 -0700263}
264
Stefani Seiboldce39c642014-03-17 23:22:04 +0100265notrace static void do_monotonic_coarse(struct timespec *ts)
john stultzda15cfd2009-08-19 19:13:34 -0700266{
Andy Lutomirski91ec87d2012-03-22 21:15:51 -0700267 unsigned long seq;
john stultzda15cfd2009-08-19 19:13:34 -0700268 do {
Stefani Seibold7c031562014-03-17 23:22:10 +0100269 seq = gtod_read_begin(gtod);
270 ts->tv_sec = gtod->monotonic_time_coarse_sec;
271 ts->tv_nsec = gtod->monotonic_time_coarse_nsec;
272 } while (unlikely(gtod_read_retry(gtod, seq)));
john stultzda15cfd2009-08-19 19:13:34 -0700273}
274
Steven Rostedt23adec52008-05-12 21:20:41 +0200275notrace int __vdso_clock_gettime(clockid_t clock, struct timespec *ts)
Andi Kleen2aae9502007-07-21 17:10:01 +0200276{
Andy Lutomirski0d7b8542011-06-05 13:50:20 -0400277 switch (clock) {
278 case CLOCK_REALTIME:
Stefani Seiboldce39c642014-03-17 23:22:04 +0100279 if (do_realtime(ts) == VCLOCK_NONE)
280 goto fallback;
Andy Lutomirski0d7b8542011-06-05 13:50:20 -0400281 break;
282 case CLOCK_MONOTONIC:
Stefani Seiboldce39c642014-03-17 23:22:04 +0100283 if (do_monotonic(ts) == VCLOCK_NONE)
284 goto fallback;
Andy Lutomirski0d7b8542011-06-05 13:50:20 -0400285 break;
286 case CLOCK_REALTIME_COARSE:
Stefani Seiboldce39c642014-03-17 23:22:04 +0100287 do_realtime_coarse(ts);
288 break;
Andy Lutomirski0d7b8542011-06-05 13:50:20 -0400289 case CLOCK_MONOTONIC_COARSE:
Stefani Seiboldce39c642014-03-17 23:22:04 +0100290 do_monotonic_coarse(ts);
291 break;
292 default:
293 goto fallback;
Andy Lutomirski0d7b8542011-06-05 13:50:20 -0400294 }
295
John Stultza939e812012-03-01 22:11:09 -0800296 return 0;
Stefani Seiboldce39c642014-03-17 23:22:04 +0100297fallback:
298 return vdso_fallback_gettime(clock, ts);
Andi Kleen2aae9502007-07-21 17:10:01 +0200299}
300int clock_gettime(clockid_t, struct timespec *)
301 __attribute__((weak, alias("__vdso_clock_gettime")));
302
Steven Rostedt23adec52008-05-12 21:20:41 +0200303notrace int __vdso_gettimeofday(struct timeval *tv, struct timezone *tz)
Andi Kleen2aae9502007-07-21 17:10:01 +0200304{
John Stultza939e812012-03-01 22:11:09 -0800305 if (likely(tv != NULL)) {
Stefani Seibold0df1ea22014-03-17 23:22:06 +0100306 if (unlikely(do_realtime((struct timespec *)tv) == VCLOCK_NONE))
307 return vdso_fallback_gtod(tv, tz);
John Stultza939e812012-03-01 22:11:09 -0800308 tv->tv_usec /= 1000;
Andi Kleen2aae9502007-07-21 17:10:01 +0200309 }
John Stultza939e812012-03-01 22:11:09 -0800310 if (unlikely(tz != NULL)) {
Stefani Seibold7c031562014-03-17 23:22:10 +0100311 tz->tz_minuteswest = gtod->tz_minuteswest;
312 tz->tz_dsttime = gtod->tz_dsttime;
John Stultza939e812012-03-01 22:11:09 -0800313 }
314
John Stultza939e812012-03-01 22:11:09 -0800315 return 0;
Andi Kleen2aae9502007-07-21 17:10:01 +0200316}
317int gettimeofday(struct timeval *, struct timezone *)
318 __attribute__((weak, alias("__vdso_gettimeofday")));
Andy Lutomirskif144a6b2011-05-23 09:31:30 -0400319
Andy Lutomirski0d7b8542011-06-05 13:50:20 -0400320/*
321 * This will break when the xtime seconds get inaccurate, but that is
322 * unlikely
323 */
Andy Lutomirskif144a6b2011-05-23 09:31:30 -0400324notrace time_t __vdso_time(time_t *t)
325{
Stefani Seibold7a59ed42014-03-17 23:22:09 +0100326 /* This is atomic on x86 so we don't need any locks. */
Stefani Seiboldaf8c93d2014-03-17 23:22:05 +0100327 time_t result = ACCESS_ONCE(gtod->wall_time_sec);
Andy Lutomirskif144a6b2011-05-23 09:31:30 -0400328
329 if (t)
330 *t = result;
331 return result;
332}
333int time(time_t *t)
334 __attribute__((weak, alias("__vdso_time")));