blob: 0577825cf89bad5ecdac456fa13d12f4b8e2fa0e [file] [log] [blame]
Thomas Gleixner250c2272007-10-11 11:17:24 +02001/*
Dave Jones835c34a2007-10-12 21:10:53 -04002 * check TSC synchronization.
Thomas Gleixner250c2272007-10-11 11:17:24 +02003 *
4 * Copyright (C) 2006, Red Hat, Inc., Ingo Molnar
5 *
6 * We check whether all boot CPUs have their TSC's synchronized,
7 * print a warning if not and turn off the TSC clock-source.
8 *
9 * The warp-check is point-to-point between two CPUs, the CPU
10 * initiating the bootup is the 'source CPU', the freshly booting
11 * CPU is the 'target CPU'.
12 *
13 * Only two CPUs may participate - they can enter in any order.
14 * ( The serial nature of the boot logic and the CPU hotplug lock
15 * protects against more than 2 CPUs entering this code. )
16 */
17#include <linux/spinlock.h>
18#include <linux/kernel.h>
19#include <linux/init.h>
20#include <linux/smp.h>
21#include <linux/nmi.h>
22#include <asm/tsc.h>
23
24/*
25 * Entry/exit counters that make sure that both CPUs
26 * run the measurement code at once:
27 */
28static __cpuinitdata atomic_t start_count;
29static __cpuinitdata atomic_t stop_count;
30
31/*
32 * We use a raw spinlock in this exceptional case, because
33 * we want to have the fastest, inlined, non-debug version
34 * of a critical section, to be able to prove TSC time-warps:
35 */
36static __cpuinitdata raw_spinlock_t sync_lock = __RAW_SPIN_LOCK_UNLOCKED;
37static __cpuinitdata cycles_t last_tsc;
38static __cpuinitdata cycles_t max_warp;
39static __cpuinitdata int nr_warps;
40
41/*
42 * TSC-warp measurement loop running on both CPUs:
43 */
44static __cpuinit void check_tsc_warp(void)
45{
46 cycles_t start, now, prev, end;
47 int i;
48
Andi Kleen6d63de82008-01-30 13:32:39 +010049 start = get_cycles();
Thomas Gleixner250c2272007-10-11 11:17:24 +020050 /*
51 * The measurement runs for 20 msecs:
52 */
53 end = start + tsc_khz * 20ULL;
54 now = start;
55
56 for (i = 0; ; i++) {
57 /*
58 * We take the global lock, measure TSC, save the
59 * previous TSC that was measured (possibly on
60 * another CPU) and update the previous TSC timestamp.
61 */
62 __raw_spin_lock(&sync_lock);
63 prev = last_tsc;
Andi Kleen6d63de82008-01-30 13:32:39 +010064 now = get_cycles();
Thomas Gleixner250c2272007-10-11 11:17:24 +020065 last_tsc = now;
66 __raw_spin_unlock(&sync_lock);
67
68 /*
69 * Be nice every now and then (and also check whether
Ingo Molnardf435102008-01-30 13:33:23 +010070 * measurement is done [we also insert a 10 million
Thomas Gleixner250c2272007-10-11 11:17:24 +020071 * loops safety exit, so we dont lock up in case the
72 * TSC readout is totally broken]):
73 */
74 if (unlikely(!(i & 7))) {
Ingo Molnardf435102008-01-30 13:33:23 +010075 if (now > end || i > 10000000)
Thomas Gleixner250c2272007-10-11 11:17:24 +020076 break;
77 cpu_relax();
78 touch_nmi_watchdog();
79 }
80 /*
81 * Outside the critical section we can now see whether
82 * we saw a time-warp of the TSC going backwards:
83 */
84 if (unlikely(prev > now)) {
85 __raw_spin_lock(&sync_lock);
86 max_warp = max(max_warp, prev - now);
87 nr_warps++;
88 __raw_spin_unlock(&sync_lock);
89 }
Ingo Molnarad8ca492008-01-30 13:33:24 +010090 }
91 if (!(now-start)) {
92 printk("Warning: zero tsc calibration delta: %Ld [max: %Ld]\n",
93 now-start, end-start);
94 WARN_ON(1);
Thomas Gleixner250c2272007-10-11 11:17:24 +020095 }
96}
97
98/*
99 * Source CPU calls into this - it waits for the freshly booted
100 * target CPU to arrive and then starts the measurement:
101 */
102void __cpuinit check_tsc_sync_source(int cpu)
103{
104 int cpus = 2;
105
106 /*
107 * No need to check if we already know that the TSC is not
108 * synchronized:
109 */
110 if (unsynchronized_tsc())
111 return;
112
113 printk(KERN_INFO "checking TSC synchronization [CPU#%d -> CPU#%d]:",
114 smp_processor_id(), cpu);
115
116 /*
117 * Reset it - in case this is a second bootup:
118 */
119 atomic_set(&stop_count, 0);
120
121 /*
122 * Wait for the target to arrive:
123 */
124 while (atomic_read(&start_count) != cpus-1)
125 cpu_relax();
126 /*
127 * Trigger the target to continue into the measurement too:
128 */
129 atomic_inc(&start_count);
130
131 check_tsc_warp();
132
133 while (atomic_read(&stop_count) != cpus-1)
134 cpu_relax();
135
Thomas Gleixner250c2272007-10-11 11:17:24 +0200136 if (nr_warps) {
137 printk("\n");
138 printk(KERN_WARNING "Measured %Ld cycles TSC warp between CPUs,"
139 " turning off TSC clock.\n", max_warp);
140 mark_tsc_unstable("check_tsc_sync_source failed");
Thomas Gleixner250c2272007-10-11 11:17:24 +0200141 } else {
142 printk(" passed.\n");
143 }
144
145 /*
Mike Galbraith4c6b8b42008-01-30 13:30:04 +0100146 * Reset it - just in case we boot another CPU later:
147 */
148 atomic_set(&start_count, 0);
149 nr_warps = 0;
150 max_warp = 0;
151 last_tsc = 0;
152
153 /*
Thomas Gleixner250c2272007-10-11 11:17:24 +0200154 * Let the target continue with the bootup:
155 */
156 atomic_inc(&stop_count);
157}
158
159/*
160 * Freshly booted CPUs call into this:
161 */
162void __cpuinit check_tsc_sync_target(void)
163{
164 int cpus = 2;
165
166 if (unsynchronized_tsc())
167 return;
168
169 /*
170 * Register this CPU's participation and wait for the
171 * source CPU to start the measurement:
172 */
173 atomic_inc(&start_count);
174 while (atomic_read(&start_count) != cpus)
175 cpu_relax();
176
177 check_tsc_warp();
178
179 /*
180 * Ok, we are done:
181 */
182 atomic_inc(&stop_count);
183
184 /*
185 * Wait for the source CPU to print stuff:
186 */
187 while (atomic_read(&stop_count) != cpus)
188 cpu_relax();
189}
190#undef NR_LOOPS
191