xref: /illumos-gate/usr/src/uts/i86pc/os/timestamp.c (revision 575694f6)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  *
26  * Copyright 2012 Nexenta Systems, Inc. All rights reserved.
27  * Copyright (c) 2014, 2016 by Delphix. All rights reserved.
28  * Copyright 2020 Joyent, Inc.
29  */
30 
31 #include <sys/types.h>
32 #include <sys/param.h>
33 #include <sys/systm.h>
34 #include <sys/disp.h>
35 #include <sys/var.h>
36 #include <sys/cmn_err.h>
37 #include <sys/debug.h>
38 #include <sys/x86_archext.h>
39 #include <sys/archsystm.h>
40 #include <sys/cpuvar.h>
41 #include <sys/psm_defs.h>
42 #include <sys/clock.h>
43 #include <sys/atomic.h>
44 #include <sys/lockstat.h>
45 #include <sys/smp_impldefs.h>
46 #include <sys/dtrace.h>
47 #include <sys/time.h>
48 #include <sys/panic.h>
49 #include <sys/cpu.h>
50 #include <sys/sdt.h>
51 #include <sys/comm_page.h>
52 #include <sys/bootconf.h>
53 #include <sys/kobj.h>
54 #include <sys/kobj_lex.h>
55 #include <sys/tsc.h>
56 #include <sys/prom_debug.h>
57 #include <util/qsort.h>
58 
59 /*
60  * Using the Pentium's TSC register for gethrtime()
61  * ------------------------------------------------
62  *
63  * The Pentium family, like many chip architectures, has a high-resolution
64  * timestamp counter ("TSC") which increments once per CPU cycle.  The contents
65  * of the timestamp counter are read with the RDTSC instruction.
66  *
67  * As with its UltraSPARC equivalent (the %tick register), TSC's cycle count
68  * must be translated into nanoseconds in order to implement gethrtime().
69  * We avoid inducing floating point operations in this conversion by
70  * implementing the same nsec_scale algorithm as that found in the sun4u
71  * platform code.  The sun4u NATIVE_TIME_TO_NSEC_SCALE block comment contains
72  * a detailed description of the algorithm; the comment is not reproduced
73  * here.  This implementation differs only in its value for NSEC_SHIFT:
74  * we implement an NSEC_SHIFT of 5 (instead of sun4u's 4) to allow for
75  * 60 MHz Pentiums.
76  *
77  * While TSC and %tick are both cycle counting registers, TSC's functionality
78  * falls short in several critical ways:
79  *
80  *  (a)	TSCs on different CPUs are not guaranteed to be in sync.  While in
81  *	practice they often _are_ in sync, this isn't guaranteed by the
82  *	architecture.
83  *
84  *  (b)	The TSC cannot be reliably set to an arbitrary value.  The architecture
85  *	only supports writing the low 32-bits of TSC, making it impractical
86  *	to rewrite.
87  *
88  *  (c)	The architecture doesn't have the capacity to interrupt based on
89  *	arbitrary values of TSC; there is no TICK_CMPR equivalent.
90  *
91  * Together, (a) and (b) imply that software must track the skew between
92  * TSCs and account for it (it is assumed that while there may exist skew,
93  * there does not exist drift).  To determine the skew between CPUs, we
94  * have newly onlined CPUs call tsc_sync_slave(), while the CPU performing
95  * the online operation calls tsc_sync_master().
96  *
97  * In the absence of time-of-day clock adjustments, gethrtime() must stay in
98  * sync with gettimeofday().  This is problematic; given (c), the software
99  * cannot drive its time-of-day source from TSC, and yet they must somehow be
100  * kept in sync.  We implement this by having a routine, tsc_tick(), which
101  * is called once per second from the interrupt which drives time-of-day.
102  *
103  * Note that the hrtime base for gethrtime, tsc_hrtime_base, is modified
104  * atomically with nsec_scale under CLOCK_LOCK.  This assures that time
105  * monotonically increases.
106  */
107 
108 #define	NSEC_SHIFT 5
109 
110 static uint_t nsec_unscale;
111 
112 /*
113  * These two variables used to be grouped together inside of a structure that
114  * lived on a single cache line. A regression (bug ID 4623398) caused the
115  * compiler to emit code that "optimized" away the while-loops below. The
116  * result was that no synchronization between the onlining and onlined CPUs
117  * took place.
118  */
119 static volatile int tsc_ready;
120 static volatile int tsc_sync_go;
121 
122 /*
123  * Used as indices into the tsc_sync_snaps[] array.
124  */
125 #define	TSC_MASTER		0
126 #define	TSC_SLAVE		1
127 
128 /*
129  * Used in the tsc_master_sync()/tsc_slave_sync() rendezvous.
130  */
131 #define	TSC_SYNC_STOP		1
132 #define	TSC_SYNC_GO		2
133 #define	TSC_SYNC_DONE		3
134 #define	SYNC_ITERATIONS		10
135 
136 #define	TSC_CONVERT_AND_ADD(tsc, hrt, scale) {		\
137 	unsigned int *_l = (unsigned int *)&(tsc);	\
138 	(hrt) += mul32(_l[1], scale) << NSEC_SHIFT;	\
139 	(hrt) += mul32(_l[0], scale) >> (32 - NSEC_SHIFT); \
140 }
141 
142 #define	TSC_CONVERT(tsc, hrt, scale) {			\
143 	unsigned int *_l = (unsigned int *)&(tsc);	\
144 	(hrt) = mul32(_l[1], scale) << NSEC_SHIFT;	\
145 	(hrt) += mul32(_l[0], scale) >> (32 - NSEC_SHIFT); \
146 }
147 
148 int tsc_master_slave_sync_needed = 1;
149 
150 typedef struct tsc_sync {
151 	volatile hrtime_t master_tsc, slave_tsc;
152 } tsc_sync_t;
153 static tsc_sync_t *tscp;
154 
155 static hrtime_t	tsc_last_jumped = 0;
156 static int	tsc_jumped = 0;
157 static uint32_t	tsc_wayback = 0;
158 /*
159  * The cap of 1 second was chosen since it is the frequency at which the
160  * tsc_tick() function runs which means that when gethrtime() is called it
161  * should never be more than 1 second since tsc_last was updated.
162  */
163 static hrtime_t tsc_resume_cap_ns = NANOSEC;	 /* 1s */
164 
165 static hrtime_t	shadow_tsc_hrtime_base;
166 static hrtime_t	shadow_tsc_last;
167 static uint_t	shadow_nsec_scale;
168 static uint32_t	shadow_hres_lock;
169 int get_tsc_ready();
170 
171 /*
172  * Allow an operator specify an explicit TSC calibration source
173  * via /etc/system e.g. `set tsc_calibration="pit"`
174  */
175 char *tsc_calibration;
176 
177 /*
178  * The source that was used to calibrate the TSC. This is currently just
179  * for diagnostic purposes.
180  */
181 static tsc_calibrate_t *tsc_calibration_source;
182 
183 /* The TSC frequency after calibration */
184 static uint64_t tsc_freq;
185 
186 static inline hrtime_t
tsc_protect(hrtime_t a)187 tsc_protect(hrtime_t a)
188 {
189 	if (a > tsc_resume_cap) {
190 		atomic_inc_32(&tsc_wayback);
191 		DTRACE_PROBE3(tsc__wayback, htrime_t, a, hrtime_t, tsc_last,
192 		    uint32_t, tsc_wayback);
193 		return (tsc_resume_cap);
194 	}
195 	return (a);
196 }
197 
198 hrtime_t
tsc_gethrtime(void)199 tsc_gethrtime(void)
200 {
201 	uint32_t old_hres_lock;
202 	hrtime_t tsc, hrt;
203 
204 	do {
205 		old_hres_lock = hres_lock;
206 
207 		if ((tsc = tsc_read()) >= tsc_last) {
208 			/*
209 			 * It would seem to be obvious that this is true
210 			 * (that is, the past is less than the present),
211 			 * but it isn't true in the presence of suspend/resume
212 			 * cycles.  If we manage to call gethrtime()
213 			 * after a resume, but before the first call to
214 			 * tsc_tick(), we will see the jump.  In this case,
215 			 * we will simply use the value in TSC as the delta.
216 			 */
217 			tsc -= tsc_last;
218 		} else if (tsc >= tsc_last - 2*tsc_max_delta) {
219 			/*
220 			 * There is a chance that tsc_tick() has just run on
221 			 * another CPU, and we have drifted just enough so that
222 			 * we appear behind tsc_last.  In this case, force the
223 			 * delta to be zero.
224 			 */
225 			tsc = 0;
226 		} else {
227 			/*
228 			 * If we reach this else clause we assume that we have
229 			 * gone through a suspend/resume cycle and use the
230 			 * current tsc value as the delta.
231 			 *
232 			 * In rare cases we can reach this else clause due to
233 			 * a lack of monotonicity in the TSC value.  In such
234 			 * cases using the current TSC value as the delta would
235 			 * cause us to return a value ~2x of what it should
236 			 * be.  To protect against these cases we cap the
237 			 * suspend/resume delta at tsc_resume_cap.
238 			 */
239 			tsc = tsc_protect(tsc);
240 		}
241 
242 		hrt = tsc_hrtime_base;
243 
244 		TSC_CONVERT_AND_ADD(tsc, hrt, nsec_scale);
245 	} while ((old_hres_lock & ~1) != hres_lock);
246 
247 	return (hrt);
248 }
249 
250 hrtime_t
tsc_gethrtime_delta(void)251 tsc_gethrtime_delta(void)
252 {
253 	uint32_t old_hres_lock;
254 	hrtime_t tsc, hrt;
255 	ulong_t flags;
256 
257 	do {
258 		old_hres_lock = hres_lock;
259 
260 		/*
261 		 * We need to disable interrupts here to assure that we
262 		 * don't migrate between the call to tsc_read() and
263 		 * adding the CPU's TSC tick delta. Note that disabling
264 		 * and reenabling preemption is forbidden here because
265 		 * we may be in the middle of a fast trap. In the amd64
266 		 * kernel we cannot tolerate preemption during a fast
267 		 * trap. See _update_sregs().
268 		 */
269 
270 		flags = clear_int_flag();
271 		tsc = tsc_read() + tsc_sync_tick_delta[CPU->cpu_id];
272 		restore_int_flag(flags);
273 
274 		/* See comments in tsc_gethrtime() above */
275 
276 		if (tsc >= tsc_last) {
277 			tsc -= tsc_last;
278 		} else if (tsc >= tsc_last - 2 * tsc_max_delta) {
279 			tsc = 0;
280 		} else {
281 			tsc = tsc_protect(tsc);
282 		}
283 
284 		hrt = tsc_hrtime_base;
285 
286 		TSC_CONVERT_AND_ADD(tsc, hrt, nsec_scale);
287 	} while ((old_hres_lock & ~1) != hres_lock);
288 
289 	return (hrt);
290 }
291 
292 hrtime_t
tsc_gethrtime_tick_delta(void)293 tsc_gethrtime_tick_delta(void)
294 {
295 	hrtime_t hrt;
296 	ulong_t flags;
297 
298 	flags = clear_int_flag();
299 	hrt = tsc_sync_tick_delta[CPU->cpu_id];
300 	restore_int_flag(flags);
301 
302 	return (hrt);
303 }
304 
305 /* Calculate the hrtime while exposing the parameters of that calculation. */
306 hrtime_t
tsc_gethrtime_params(uint64_t * tscp,uint32_t * scalep,uint8_t * shiftp)307 tsc_gethrtime_params(uint64_t *tscp, uint32_t *scalep, uint8_t *shiftp)
308 {
309 	uint32_t old_hres_lock, scale;
310 	hrtime_t tsc, last, base;
311 
312 	do {
313 		old_hres_lock = hres_lock;
314 
315 		if (gethrtimef == tsc_gethrtime_delta) {
316 			ulong_t flags;
317 
318 			flags = clear_int_flag();
319 			tsc = tsc_read() + tsc_sync_tick_delta[CPU->cpu_id];
320 			restore_int_flag(flags);
321 		} else {
322 			tsc = tsc_read();
323 		}
324 
325 		last = tsc_last;
326 		base = tsc_hrtime_base;
327 		scale = nsec_scale;
328 
329 	} while ((old_hres_lock & ~1) != hres_lock);
330 
331 	/* See comments in tsc_gethrtime() above */
332 	if (tsc >= last) {
333 		tsc -= last;
334 	} else if (tsc >= last - 2 * tsc_max_delta) {
335 		tsc = 0;
336 	} else {
337 		tsc = tsc_protect(tsc);
338 	}
339 
340 	TSC_CONVERT_AND_ADD(tsc, base, nsec_scale);
341 
342 	if (tscp != NULL) {
343 		/*
344 		 * Do not simply communicate the delta applied to the hrtime
345 		 * base, but rather the effective TSC measurement.
346 		 */
347 		*tscp = tsc + last;
348 	}
349 	if (scalep != NULL) {
350 		*scalep = scale;
351 	}
352 	if (shiftp != NULL) {
353 		*shiftp = NSEC_SHIFT;
354 	}
355 
356 	return (base);
357 }
358 
359 /*
360  * This is similar to tsc_gethrtime_delta, but it cannot actually spin on
361  * hres_lock.  As a result, it caches all of the variables it needs; if the
362  * variables don't change, it's done.
363  */
364 hrtime_t
dtrace_gethrtime(void)365 dtrace_gethrtime(void)
366 {
367 	uint32_t old_hres_lock;
368 	hrtime_t tsc, hrt;
369 	ulong_t flags;
370 
371 	do {
372 		old_hres_lock = hres_lock;
373 
374 		/*
375 		 * Interrupts are disabled to ensure that the thread isn't
376 		 * migrated between the tsc_read() and adding the CPU's
377 		 * TSC tick delta.
378 		 */
379 		flags = clear_int_flag();
380 
381 		tsc = tsc_read();
382 
383 		if (gethrtimef == tsc_gethrtime_delta)
384 			tsc += tsc_sync_tick_delta[CPU->cpu_id];
385 
386 		restore_int_flag(flags);
387 
388 		/*
389 		 * See the comments in tsc_gethrtime(), above.
390 		 */
391 		if (tsc >= tsc_last)
392 			tsc -= tsc_last;
393 		else if (tsc >= tsc_last - 2*tsc_max_delta)
394 			tsc = 0;
395 		else
396 			tsc = tsc_protect(tsc);
397 
398 		hrt = tsc_hrtime_base;
399 
400 		TSC_CONVERT_AND_ADD(tsc, hrt, nsec_scale);
401 
402 		if ((old_hres_lock & ~1) == hres_lock)
403 			break;
404 
405 		/*
406 		 * If we're here, the clock lock is locked -- or it has been
407 		 * unlocked and locked since we looked.  This may be due to
408 		 * tsc_tick() running on another CPU -- or it may be because
409 		 * some code path has ended up in dtrace_probe() with
410 		 * CLOCK_LOCK held.  We'll try to determine that we're in
411 		 * the former case by taking another lap if the lock has
412 		 * changed since when we first looked at it.
413 		 */
414 		if (old_hres_lock != hres_lock)
415 			continue;
416 
417 		/*
418 		 * So the lock was and is locked.  We'll use the old data
419 		 * instead.
420 		 */
421 		old_hres_lock = shadow_hres_lock;
422 
423 		/*
424 		 * Again, disable interrupts to ensure that the thread
425 		 * isn't migrated between the tsc_read() and adding
426 		 * the CPU's TSC tick delta.
427 		 */
428 		flags = clear_int_flag();
429 
430 		tsc = tsc_read();
431 
432 		if (gethrtimef == tsc_gethrtime_delta)
433 			tsc += tsc_sync_tick_delta[CPU->cpu_id];
434 
435 		restore_int_flag(flags);
436 
437 		/*
438 		 * See the comments in tsc_gethrtime(), above.
439 		 */
440 		if (tsc >= shadow_tsc_last)
441 			tsc -= shadow_tsc_last;
442 		else if (tsc >= shadow_tsc_last - 2 * tsc_max_delta)
443 			tsc = 0;
444 		else
445 			tsc = tsc_protect(tsc);
446 
447 		hrt = shadow_tsc_hrtime_base;
448 
449 		TSC_CONVERT_AND_ADD(tsc, hrt, shadow_nsec_scale);
450 	} while ((old_hres_lock & ~1) != shadow_hres_lock);
451 
452 	return (hrt);
453 }
454 
455 hrtime_t
tsc_gethrtimeunscaled(void)456 tsc_gethrtimeunscaled(void)
457 {
458 	uint32_t old_hres_lock;
459 	hrtime_t tsc;
460 
461 	do {
462 		old_hres_lock = hres_lock;
463 
464 		/* See tsc_tick(). */
465 		tsc = tsc_read() + tsc_last_jumped;
466 	} while ((old_hres_lock & ~1) != hres_lock);
467 
468 	return (tsc);
469 }
470 
471 /*
472  * Convert a nanosecond based timestamp to tsc
473  */
474 uint64_t
tsc_unscalehrtime(hrtime_t nsec)475 tsc_unscalehrtime(hrtime_t nsec)
476 {
477 	hrtime_t tsc;
478 
479 	if (tsc_gethrtime_enable) {
480 		TSC_CONVERT(nsec, tsc, nsec_unscale);
481 		return (tsc);
482 	}
483 	return ((uint64_t)nsec);
484 }
485 
486 /* Convert a tsc timestamp to nanoseconds */
487 void
tsc_scalehrtime(hrtime_t * tsc)488 tsc_scalehrtime(hrtime_t *tsc)
489 {
490 	hrtime_t hrt;
491 	hrtime_t mytsc;
492 
493 	if (tsc == NULL)
494 		return;
495 	mytsc = *tsc;
496 
497 	TSC_CONVERT(mytsc, hrt, nsec_scale);
498 	*tsc  = hrt;
499 }
500 
501 hrtime_t
tsc_gethrtimeunscaled_delta(void)502 tsc_gethrtimeunscaled_delta(void)
503 {
504 	hrtime_t hrt;
505 	ulong_t flags;
506 
507 	/*
508 	 * Similarly to tsc_gethrtime_delta, we need to disable preemption
509 	 * to prevent migration between the call to tsc_gethrtimeunscaled
510 	 * and adding the CPU's hrtime delta. Note that disabling and
511 	 * reenabling preemption is forbidden here because we may be in the
512 	 * middle of a fast trap. In the amd64 kernel we cannot tolerate
513 	 * preemption during a fast trap. See _update_sregs().
514 	 */
515 
516 	flags = clear_int_flag();
517 	hrt = tsc_gethrtimeunscaled() + tsc_sync_tick_delta[CPU->cpu_id];
518 	restore_int_flag(flags);
519 
520 	return (hrt);
521 }
522 
523 /*
524  * TSC Sync Master
525  *
526  * Typically called on the boot CPU, this attempts to quantify TSC skew between
527  * different CPUs.  If an appreciable difference is found, gethrtimef will be
528  * changed to point to tsc_gethrtime_delta().
529  *
530  * Calculating skews is precise only when the master and slave TSCs are read
531  * simultaneously; however, there is no algorithm that can read both CPUs in
532  * perfect simultaneity.  The proposed algorithm is an approximate method based
533  * on the behaviour of cache management.  The slave CPU continuously polls the
534  * TSC while reading a global variable updated by the master CPU.  The latest
535  * TSC reading is saved when the master's update (forced via mfence) reaches
536  * visibility on the slave.  The master will also take a TSC reading
537  * immediately following the mfence.
538  *
539  * While the delay between cache line invalidation on the slave and mfence
540  * completion on the master is not repeatable, the error is heuristically
541  * assumed to be 1/4th of the write time recorded by the master.  Multiple
542  * samples are taken to control for the variance caused by external factors
543  * such as bus contention.  Each sample set is independent per-CPU to control
544  * for differing memory latency on NUMA systems.
545  *
546  * TSC sync is disabled in the context of virtualization because the CPUs
547  * assigned to the guest are virtual CPUs which means the real CPUs on which
548  * guest runs keep changing during life time of guest OS. So we would end up
549  * calculating TSC skews for a set of CPUs during boot whereas the guest
550  * might migrate to a different set of physical CPUs at a later point of
551  * time.
552  */
553 void
tsc_sync_master(processorid_t slave)554 tsc_sync_master(processorid_t slave)
555 {
556 	ulong_t flags, source, min_write_time = ~0UL;
557 	hrtime_t write_time, mtsc_after, last_delta = 0;
558 	tsc_sync_t *tsc = tscp;
559 	int cnt;
560 	int hwtype;
561 
562 	hwtype = get_hwenv();
563 	if (!tsc_master_slave_sync_needed || (hwtype & HW_VIRTUAL) != 0)
564 		return;
565 
566 	flags = clear_int_flag();
567 	source = CPU->cpu_id;
568 
569 	for (cnt = 0; cnt < SYNC_ITERATIONS; cnt++) {
570 		while (tsc_sync_go != TSC_SYNC_GO)
571 			SMT_PAUSE();
572 
573 		tsc->master_tsc = tsc_read();
574 		membar_enter();
575 		mtsc_after = tsc_read();
576 		while (tsc_sync_go != TSC_SYNC_DONE)
577 			SMT_PAUSE();
578 		write_time =  mtsc_after - tsc->master_tsc;
579 		if (write_time <= min_write_time) {
580 			hrtime_t tdelta;
581 
582 			tdelta = tsc->slave_tsc - mtsc_after;
583 			if (tdelta < 0)
584 				tdelta = -tdelta;
585 			/*
586 			 * If the margin exists, subtract 1/4th of the measured
587 			 * write time from the master's TSC value.  This is an
588 			 * estimate of how late the mfence completion came
589 			 * after the slave noticed the cache line change.
590 			 */
591 			if (tdelta > (write_time/4)) {
592 				tdelta = tsc->slave_tsc -
593 				    (mtsc_after - (write_time/4));
594 			} else {
595 				tdelta = tsc->slave_tsc - mtsc_after;
596 			}
597 			last_delta = tsc_sync_tick_delta[source] - tdelta;
598 			tsc_sync_tick_delta[slave] = last_delta;
599 			min_write_time = write_time;
600 		}
601 
602 		tsc->master_tsc = tsc->slave_tsc = write_time = 0;
603 		membar_enter();
604 		tsc_sync_go = TSC_SYNC_STOP;
605 	}
606 
607 	/*
608 	 * Only enable the delta variants of the TSC functions if the measured
609 	 * skew is greater than the fastest write time.
610 	 */
611 	last_delta = (last_delta < 0) ? -last_delta : last_delta;
612 	if (last_delta > min_write_time) {
613 		gethrtimef = tsc_gethrtime_delta;
614 		gethrtimeunscaledf = tsc_gethrtimeunscaled_delta;
615 		tsc_ncpu = NCPU;
616 	}
617 	restore_int_flag(flags);
618 }
619 
620 /*
621  * TSC Sync Slave
622  *
623  * Called by a CPU which has just been onlined.  It is expected that the CPU
624  * performing the online operation will call tsc_sync_master().
625  *
626  * Like tsc_sync_master, this logic is skipped on virtualized platforms.
627  */
628 void
tsc_sync_slave(void)629 tsc_sync_slave(void)
630 {
631 	ulong_t flags;
632 	hrtime_t s1;
633 	tsc_sync_t *tsc = tscp;
634 	int cnt;
635 	int hwtype;
636 
637 	hwtype = get_hwenv();
638 	if (!tsc_master_slave_sync_needed || (hwtype & HW_VIRTUAL) != 0)
639 		return;
640 
641 	flags = clear_int_flag();
642 
643 	for (cnt = 0; cnt < SYNC_ITERATIONS; cnt++) {
644 		/* Re-fill the cache line */
645 		s1 = tsc->master_tsc;
646 		membar_enter();
647 		tsc_sync_go = TSC_SYNC_GO;
648 		do {
649 			/*
650 			 * Do not put an SMT_PAUSE here.  If the master and
651 			 * slave are the same hyper-threaded CPU, we want the
652 			 * master to yield as quickly as possible to the slave.
653 			 */
654 			s1 = tsc_read();
655 		} while (tsc->master_tsc == 0);
656 		tsc->slave_tsc = s1;
657 		membar_enter();
658 		tsc_sync_go = TSC_SYNC_DONE;
659 
660 		while (tsc_sync_go != TSC_SYNC_STOP)
661 			SMT_PAUSE();
662 	}
663 
664 	restore_int_flag(flags);
665 }
666 
667 /*
668  * Called once per second on a CPU from the cyclic subsystem's
669  * CY_HIGH_LEVEL interrupt.  (No longer just cpu0-only)
670  */
671 void
tsc_tick(void)672 tsc_tick(void)
673 {
674 	hrtime_t now, delta;
675 	ushort_t spl;
676 
677 	/*
678 	 * Before we set the new variables, we set the shadow values.  This
679 	 * allows for lock free operation in dtrace_gethrtime().
680 	 */
681 	lock_set_spl((lock_t *)&shadow_hres_lock + HRES_LOCK_OFFSET,
682 	    ipltospl(CBE_HIGH_PIL), &spl);
683 
684 	shadow_tsc_hrtime_base = tsc_hrtime_base;
685 	shadow_tsc_last = tsc_last;
686 	shadow_nsec_scale = nsec_scale;
687 
688 	shadow_hres_lock++;
689 	splx(spl);
690 
691 	CLOCK_LOCK(&spl);
692 
693 	now = tsc_read();
694 
695 	if (gethrtimef == tsc_gethrtime_delta)
696 		now += tsc_sync_tick_delta[CPU->cpu_id];
697 
698 	if (now < tsc_last) {
699 		/*
700 		 * The TSC has just jumped into the past.  We assume that
701 		 * this is due to a suspend/resume cycle, and we're going
702 		 * to use the _current_ value of TSC as the delta.  This
703 		 * will keep tsc_hrtime_base correct.  We're also going to
704 		 * assume that rate of tsc does not change after a suspend
705 		 * resume (i.e nsec_scale remains the same).
706 		 */
707 		delta = now;
708 		delta = tsc_protect(delta);
709 		tsc_last_jumped += tsc_last;
710 		tsc_jumped = 1;
711 	} else {
712 		/*
713 		 * Determine the number of TSC ticks since the last clock
714 		 * tick, and add that to the hrtime base.
715 		 */
716 		delta = now - tsc_last;
717 	}
718 
719 	TSC_CONVERT_AND_ADD(delta, tsc_hrtime_base, nsec_scale);
720 	tsc_last = now;
721 
722 	CLOCK_UNLOCK(spl);
723 }
724 
725 void
tsc_hrtimeinit(uint64_t cpu_freq_hz)726 tsc_hrtimeinit(uint64_t cpu_freq_hz)
727 {
728 	extern int gethrtime_hires;
729 	longlong_t tsc;
730 	ulong_t flags;
731 
732 	/*
733 	 * cpu_freq_hz is the measured cpu frequency in hertz
734 	 */
735 
736 	/*
737 	 * We can't accommodate CPUs slower than 31.25 MHz.
738 	 */
739 	ASSERT(cpu_freq_hz > NANOSEC / (1 << NSEC_SHIFT));
740 	nsec_scale =
741 	    (uint_t)(((uint64_t)NANOSEC << (32 - NSEC_SHIFT)) / cpu_freq_hz);
742 	nsec_unscale =
743 	    (uint_t)(((uint64_t)cpu_freq_hz << (32 - NSEC_SHIFT)) / NANOSEC);
744 
745 	flags = clear_int_flag();
746 	tsc = tsc_read();
747 	(void) tsc_gethrtime();
748 	tsc_max_delta = tsc_read() - tsc;
749 	restore_int_flag(flags);
750 	gethrtimef = tsc_gethrtime;
751 	gethrtimeunscaledf = tsc_gethrtimeunscaled;
752 	scalehrtimef = tsc_scalehrtime;
753 	unscalehrtimef = tsc_unscalehrtime;
754 	hrtime_tick = tsc_tick;
755 	gethrtime_hires = 1;
756 	/*
757 	 * Being part of the comm page, tsc_ncpu communicates the published
758 	 * length of the tsc_sync_tick_delta array.  This is kept zeroed to
759 	 * ignore the absent delta data while the TSCs are synced.
760 	 */
761 	tsc_ncpu = 0;
762 	/*
763 	 * Allocate memory for the structure used in the tsc sync logic.
764 	 * This structure should be aligned on a multiple of cache line size.
765 	 */
766 	tscp = kmem_zalloc(PAGESIZE, KM_SLEEP);
767 
768 	/*
769 	 * Convert the TSC resume cap ns value into its unscaled TSC value.
770 	 * See tsc_gethrtime().
771 	 */
772 	if (tsc_resume_cap == 0)
773 		TSC_CONVERT(tsc_resume_cap_ns, tsc_resume_cap, nsec_unscale);
774 }
775 
776 int
get_tsc_ready()777 get_tsc_ready()
778 {
779 	return (tsc_ready);
780 }
781 
782 /*
783  * Adjust all the deltas by adding the passed value to the array and activate
784  * the "delta" versions of the gethrtime functions.  It is possible that the
785  * adjustment could be negative.  Such may occur if the SunOS instance was
786  * moved by a virtual manager to a machine with a higher value of TSC.
787  */
788 void
tsc_adjust_delta(hrtime_t tdelta)789 tsc_adjust_delta(hrtime_t tdelta)
790 {
791 	int		i;
792 
793 	for (i = 0; i < NCPU; i++) {
794 		tsc_sync_tick_delta[i] += tdelta;
795 	}
796 
797 	gethrtimef = tsc_gethrtime_delta;
798 	gethrtimeunscaledf = tsc_gethrtimeunscaled_delta;
799 	tsc_ncpu = NCPU;
800 }
801 
802 /*
803  * Functions to manage TSC and high-res time on suspend and resume.
804  */
805 
806 /* tod_ops from "uts/i86pc/io/todpc_subr.c" */
807 extern tod_ops_t *tod_ops;
808 
809 static uint64_t tsc_saved_tsc = 0; /* 1 in 2^64 chance this'll screw up! */
810 static timestruc_t tsc_saved_ts;
811 static int	tsc_needs_resume = 0;	/* We only want to do this once. */
812 int		tsc_delta_onsuspend = 0;
813 int		tsc_adjust_seconds = 1;
814 int		tsc_suspend_count = 0;
815 int		tsc_resume_in_cyclic = 0;
816 
817 /*
818  * Take snapshots of the current time and do any other pre-suspend work.
819  */
820 void
tsc_suspend(void)821 tsc_suspend(void)
822 {
823 	/*
824 	 * We need to collect the time at which we suspended here so we know
825 	 * now much should be added during the resume.  This is called by each
826 	 * CPU, so reentry must be properly handled.
827 	 */
828 	if (tsc_gethrtime_enable) {
829 		/*
830 		 * Perform the tsc_read after acquiring the lock to make it as
831 		 * accurate as possible in the face of contention.
832 		 */
833 		mutex_enter(&tod_lock);
834 		tsc_saved_tsc = tsc_read();
835 		tsc_saved_ts = TODOP_GET(tod_ops);
836 		mutex_exit(&tod_lock);
837 		/* We only want to do this once. */
838 		if (tsc_needs_resume == 0) {
839 			if (tsc_delta_onsuspend) {
840 				tsc_adjust_delta(tsc_saved_tsc);
841 			} else {
842 				tsc_adjust_delta(nsec_scale);
843 			}
844 			tsc_suspend_count++;
845 		}
846 	}
847 
848 	invalidate_cache();
849 	tsc_needs_resume = 1;
850 }
851 
852 /*
853  * Restore all timestamp state based on the snapshots taken at suspend time.
854  */
855 void
tsc_resume(void)856 tsc_resume(void)
857 {
858 	/*
859 	 * We only need to (and want to) do this once.  So let the first
860 	 * caller handle this (we are locked by the cpu lock), as it
861 	 * is preferential that we get the earliest sync.
862 	 */
863 	if (tsc_needs_resume) {
864 		/*
865 		 * If using the TSC, adjust the delta based on how long
866 		 * we were sleeping (or away).  We also adjust for
867 		 * migration and a grown TSC.
868 		 */
869 		if (tsc_saved_tsc != 0) {
870 			timestruc_t	ts;
871 			hrtime_t	now, sleep_tsc = 0;
872 			int		sleep_sec;
873 			extern void	tsc_tick(void);
874 			extern uint64_t cpu_freq_hz;
875 
876 			/* tsc_read() MUST be before TODOP_GET() */
877 			mutex_enter(&tod_lock);
878 			now = tsc_read();
879 			ts = TODOP_GET(tod_ops);
880 			mutex_exit(&tod_lock);
881 
882 			/* Compute seconds of sleep time */
883 			sleep_sec = ts.tv_sec - tsc_saved_ts.tv_sec;
884 
885 			/*
886 			 * If the saved sec is less that or equal to
887 			 * the current ts, then there is likely a
888 			 * problem with the clock.  Assume at least
889 			 * one second has passed, so that time goes forward.
890 			 */
891 			if (sleep_sec <= 0) {
892 				sleep_sec = 1;
893 			}
894 
895 			/* How many TSC's should have occured while sleeping */
896 			if (tsc_adjust_seconds)
897 				sleep_tsc = sleep_sec * cpu_freq_hz;
898 
899 			/*
900 			 * We also want to subtract from the "sleep_tsc"
901 			 * the current value of tsc_read(), so that our
902 			 * adjustment accounts for the amount of time we
903 			 * have been resumed _or_ an adjustment based on
904 			 * the fact that we didn't actually power off the
905 			 * CPU (migration is another issue, but _should_
906 			 * also comply with this calculation).  If the CPU
907 			 * never powered off, then:
908 			 *    'now == sleep_tsc + saved_tsc'
909 			 * and the delta will effectively be "0".
910 			 */
911 			sleep_tsc -= now;
912 			if (tsc_delta_onsuspend) {
913 				tsc_adjust_delta(sleep_tsc);
914 			} else {
915 				tsc_adjust_delta(tsc_saved_tsc + sleep_tsc);
916 			}
917 			tsc_saved_tsc = 0;
918 
919 			tsc_tick();
920 		}
921 		tsc_needs_resume = 0;
922 	}
923 
924 }
925 
926 static int
tsc_calibrate_cmp(const void * a,const void * b)927 tsc_calibrate_cmp(const void *a, const void *b)
928 {
929 	const tsc_calibrate_t * const *a1 = a;
930 	const tsc_calibrate_t * const *b1 = b;
931 	const tsc_calibrate_t *l = *a1;
932 	const tsc_calibrate_t *r = *b1;
933 
934 	/* Sort from highest preference to lowest preference */
935 	if (l->tscc_preference > r->tscc_preference)
936 		return (-1);
937 	if (l->tscc_preference < r->tscc_preference)
938 		return (1);
939 
940 	/* For equal preference sources, sort alphabetically */
941 	int c = strcmp(l->tscc_source, r->tscc_source);
942 
943 	if (c < 0)
944 		return (-1);
945 	if (c > 0)
946 		return (1);
947 	return (0);
948 }
949 
950 SET_DECLARE(tsc_calibration_set, tsc_calibrate_t);
951 
952 static tsc_calibrate_t *
tsc_calibrate_get_force(const char * source)953 tsc_calibrate_get_force(const char *source)
954 {
955 	tsc_calibrate_t **tsccpp;
956 
957 	VERIFY3P(source, !=, NULL);
958 
959 	SET_FOREACH(tsccpp, tsc_calibration_set) {
960 		tsc_calibrate_t *tsccp = *tsccpp;
961 
962 		if (strcasecmp(source, tsccp->tscc_source) == 0)
963 			return (tsccp);
964 	}
965 
966 	/*
967 	 * If an operator explicitly gave a TSC value and we didn't find it,
968 	 * we should let them know.
969 	 */
970 	cmn_err(CE_NOTE,
971 	    "Explicit TSC calibration source '%s' not found; using default",
972 	    source);
973 
974 	return (NULL);
975 }
976 
977 /*
978  * As described in tscc_pit.c, as an intertim measure as we transition to
979  * alternate calibration sources besides the PIT, we still want to gather
980  * what the values would have been had we used the PIT. Therefore, if we're
981  * using a source other than the PIT, we explicitly run the PIT calibration
982  * which will store the TSC frequency as measured by the PIT for the
983  * benefit of the APIC code (as well as any potential diagnostics).
984  */
985 static void
tsc_pit_also(void)986 tsc_pit_also(void)
987 {
988 	tsc_calibrate_t *pit = tsc_calibrate_get_force("PIT");
989 	uint64_t dummy;
990 
991 	/* We should always have the PIT as a possible calibration source */
992 	VERIFY3P(pit, !=, NULL);
993 
994 	/* If we used the PIT to calibrate, we don't need to run again */
995 	if (tsc_calibration_source == pit)
996 		return;
997 
998 	/*
999 	 * Since we're not using the PIT as the actual TSC calibration source,
1000 	 * we don't care about the results or saving the result -- tscc_pit.c
1001 	 * saves the frequency in a global for the benefit of the APIC code.
1002 	 */
1003 	(void) pit->tscc_calibrate(&dummy);
1004 }
1005 
1006 uint64_t
tsc_calibrate(void)1007 tsc_calibrate(void)
1008 {
1009 	tsc_calibrate_t **tsccpp, *force;
1010 	size_t tsc_set_size;
1011 	int tsc_name_len;
1012 
1013 	/*
1014 	 * Every x86 system since the Pentium has TSC support. Since we
1015 	 * only support 64-bit x86 systems, there should always be a TSC
1016 	 * present, and something's horribly wrong if it's missing.
1017 	 */
1018 	if (!is_x86_feature(x86_featureset, X86FSET_TSC))
1019 		panic("System does not have TSC support");
1020 
1021 	/*
1022 	 * If we already successfully calibrated the TSC, no need to do
1023 	 * it again.
1024 	 */
1025 	if (tsc_freq > 0)
1026 		return (tsc_freq);
1027 
1028 	PRM_POINT("Calibrating the TSC...");
1029 
1030 	/*
1031 	 * Allow an operator to explicitly specify a calibration source via
1032 	 * `set tsc_calibration=foo` in the bootloader or
1033 	 * `set tsc_calibration="foo"` in /etc/system (preferring a bootloader
1034 	 * supplied value over /etc/system).
1035 	 *
1036 	 * If no source is given, or the specified source is not found, we
1037 	 * fallback to trying all of the known sources in order by preference
1038 	 * (high preference value to low preference value) until one succeeds.
1039 	 */
1040 	tsc_name_len = BOP_GETPROPLEN(bootops, "tsc_calibration");
1041 	if (tsc_name_len > 0) {
1042 		/* Overwrite any /etc/system supplied value */
1043 		if (tsc_calibration != NULL) {
1044 			size_t len = strlen(tsc_calibration) + 1;
1045 
1046 			kobj_free_string(tsc_calibration, len);
1047 		}
1048 
1049 		tsc_calibration = kmem_zalloc(tsc_name_len + 1, KM_SLEEP);
1050 		BOP_GETPROP(bootops, "tsc_calibration", tsc_calibration);
1051 	}
1052 
1053 	if (tsc_calibration != NULL &&
1054 	    (force = tsc_calibrate_get_force(tsc_calibration)) != NULL) {
1055 		if (tsc_name_len > 0) {
1056 			PRM_POINT("Forcing bootloader specified TSC calibration"
1057 			    " source");
1058 		} else {
1059 			PRM_POINT("Forcing /etc/system specified TSC "
1060 			    "calibration source");
1061 		}
1062 		PRM_DEBUGS(force->tscc_source);
1063 
1064 		if (!force->tscc_calibrate(&tsc_freq))
1065 			panic("Failed to calibrate the TSC");
1066 
1067 		tsc_calibration_source = force;
1068 
1069 		/*
1070 		 * We've saved the tsc_calibration_t that matched the value
1071 		 * of tsc_calibration at this point, so we can release the
1072 		 * memory for the value now.
1073 		 */
1074 		if (tsc_name_len > 0) {
1075 			kmem_free(tsc_calibration, tsc_name_len + 1);
1076 		} else if (tsc_calibration != NULL) {
1077 			size_t len = strlen(tsc_calibration) + 1;
1078 
1079 			kobj_free_string(tsc_calibration, len);
1080 		}
1081 		tsc_calibration = NULL;
1082 
1083 		tsc_pit_also();
1084 		return (tsc_freq);
1085 	}
1086 
1087 	/*
1088 	 * While we could sort the set contents in place, we'll make a copy
1089 	 * of the set and avoid modifying the original set.
1090 	 */
1091 	tsc_set_size = SET_COUNT(tsc_calibration_set) *
1092 	    sizeof (tsc_calibrate_t **);
1093 	tsccpp = kmem_zalloc(tsc_set_size, KM_SLEEP);
1094 	bcopy(SET_BEGIN(tsc_calibration_set), tsccpp, tsc_set_size);
1095 
1096 	/*
1097 	 * Sort by preference, highest to lowest
1098 	 */
1099 	qsort(tsccpp, SET_COUNT(tsc_calibration_set),
1100 	    sizeof (tsc_calibrate_t **), tsc_calibrate_cmp);
1101 
1102 	for (uint_t i = 0; i < SET_COUNT(tsc_calibration_set); i++) {
1103 		PRM_DEBUGS(tsccpp[i]->tscc_source);
1104 		if (tsccpp[i]->tscc_calibrate(&tsc_freq)) {
1105 			VERIFY3U(tsc_freq, >, 0);
1106 
1107 			cmn_err(CE_CONT,
1108 			    "?TSC calibrated using %s; freq is %lu MHz\n",
1109 			    tsccpp[i]->tscc_source, tsc_freq / 1000000);
1110 
1111 			/*
1112 			 * Note that tsccpp is just a (sorted) array of
1113 			 * pointers to the tsc_calibration_t's (from the
1114 			 * linker set). The actual tsc_calibration_t's aren't
1115 			 * kmem_alloc()ed (being part of the linker set), so
1116 			 * it's safe to keep a pointer to the one that was
1117 			 * used for calibration (intended for diagnostic
1118 			 * purposes).
1119 			 */
1120 			tsc_calibration_source = tsccpp[i];
1121 
1122 			kmem_free(tsccpp, tsc_set_size);
1123 			tsc_pit_also();
1124 			return (tsc_freq);
1125 		}
1126 	}
1127 
1128 	/*
1129 	 * In case it's useful, we don't free tsccpp -- we're about to panic
1130 	 * anyway.
1131 	 */
1132 	panic("Failed to calibrate TSC");
1133 }
1134 
1135 uint64_t
tsc_get_freq(void)1136 tsc_get_freq(void)
1137 {
1138 	VERIFY(tsc_freq > 0);
1139 	return (tsc_freq);
1140 }
1141