xref: /illumos-gate/usr/src/uts/i86pc/os/timestamp.c (revision b9bfdccda8a7c490cc133ece2ab92d914e10b8d7)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 #include <sys/types.h>
28 #include <sys/param.h>
29 #include <sys/systm.h>
30 #include <sys/disp.h>
31 #include <sys/var.h>
32 #include <sys/cmn_err.h>
33 #include <sys/debug.h>
34 #include <sys/x86_archext.h>
35 #include <sys/archsystm.h>
36 #include <sys/cpuvar.h>
37 #include <sys/psm_defs.h>
38 #include <sys/clock.h>
39 #include <sys/atomic.h>
40 #include <sys/lockstat.h>
41 #include <sys/smp_impldefs.h>
42 #include <sys/dtrace.h>
43 #include <sys/time.h>
44 #include <sys/panic.h>
45 #include <sys/cpu.h>
46 
47 /*
48  * Using the Pentium's TSC register for gethrtime()
49  * ------------------------------------------------
50  *
51  * The Pentium family, like many chip architectures, has a high-resolution
52  * timestamp counter ("TSC") which increments once per CPU cycle.  The contents
53  * of the timestamp counter are read with the RDTSC instruction.
54  *
55  * As with its UltraSPARC equivalent (the %tick register), TSC's cycle count
56  * must be translated into nanoseconds in order to implement gethrtime().
57  * We avoid inducing floating point operations in this conversion by
58  * implementing the same nsec_scale algorithm as that found in the sun4u
59  * platform code.  The sun4u NATIVE_TIME_TO_NSEC_SCALE block comment contains
60  * a detailed description of the algorithm; the comment is not reproduced
61  * here.  This implementation differs only in its value for NSEC_SHIFT:
62  * we implement an NSEC_SHIFT of 5 (instead of sun4u's 4) to allow for
63  * 60 MHz Pentiums.
64  *
65  * While TSC and %tick are both cycle counting registers, TSC's functionality
66  * falls short in several critical ways:
67  *
68  *  (a)	TSCs on different CPUs are not guaranteed to be in sync.  While in
69  *	practice they often _are_ in sync, this isn't guaranteed by the
70  *	architecture.
71  *
72  *  (b)	The TSC cannot be reliably set to an arbitrary value.  The architecture
73  *	only supports writing the low 32-bits of TSC, making it impractical
74  *	to rewrite.
75  *
76  *  (c)	The architecture doesn't have the capacity to interrupt based on
77  *	arbitrary values of TSC; there is no TICK_CMPR equivalent.
78  *
79  * Together, (a) and (b) imply that software must track the skew between
80  * TSCs and account for it (it is assumed that while there may exist skew,
81  * there does not exist drift).  To determine the skew between CPUs, we
82  * have newly onlined CPUs call tsc_sync_slave(), while the CPU performing
83  * the online operation calls tsc_sync_master().
84  *
85  * In the absence of time-of-day clock adjustments, gethrtime() must stay in
86  * sync with gettimeofday().  This is problematic; given (c), the software
87  * cannot drive its time-of-day source from TSC, and yet they must somehow be
88  * kept in sync.  We implement this by having a routine, tsc_tick(), which
89  * is called once per second from the interrupt which drives time-of-day.
90  *
91  * Note that the hrtime base for gethrtime, tsc_hrtime_base, is modified
92  * atomically with nsec_scale under CLOCK_LOCK.  This assures that time
93  * monotonically increases.
94  */
95 
96 #define	NSEC_SHIFT 5
97 
98 static uint_t nsec_scale;
99 
100 /*
101  * These two variables used to be grouped together inside of a structure that
102  * lived on a single cache line. A regression (bug ID 4623398) caused the
103  * compiler to emit code that "optimized" away the while-loops below. The
104  * result was that no synchronization between the onlining and onlined CPUs
105  * took place.
106  */
107 static volatile int tsc_ready;
108 static volatile int tsc_sync_go;
109 
110 /*
111  * Used as indices into the tsc_sync_snaps[] array.
112  */
113 #define	TSC_MASTER		0
114 #define	TSC_SLAVE		1
115 
116 /*
117  * Used in the tsc_master_sync()/tsc_slave_sync() rendezvous.
118  */
119 #define	TSC_SYNC_STOP		1
120 #define	TSC_SYNC_GO		2
121 #define	TSC_SYNC_DONE		3
122 #define	SYNC_ITERATIONS		10
123 
124 #define	TSC_CONVERT_AND_ADD(tsc, hrt, scale) {	 	\
125 	unsigned int *_l = (unsigned int *)&(tsc); 	\
126 	(hrt) += mul32(_l[1], scale) << NSEC_SHIFT; 	\
127 	(hrt) += mul32(_l[0], scale) >> (32 - NSEC_SHIFT); \
128 }
129 
130 #define	TSC_CONVERT(tsc, hrt, scale) { 			\
131 	unsigned int *_l = (unsigned int *)&(tsc); 	\
132 	(hrt) = mul32(_l[1], scale) << NSEC_SHIFT; 	\
133 	(hrt) += mul32(_l[0], scale) >> (32 - NSEC_SHIFT); \
134 }
135 
136 int tsc_master_slave_sync_needed = 1;
137 
138 static int	tsc_max_delta;
139 static hrtime_t tsc_sync_tick_delta[NCPU];
140 typedef struct tsc_sync {
141 	volatile hrtime_t master_tsc, slave_tsc;
142 } tsc_sync_t;
143 static tsc_sync_t *tscp;
144 static hrtime_t largest_tsc_delta = 0;
145 static ulong_t shortest_write_time = ~0UL;
146 
147 static hrtime_t	tsc_last = 0;
148 static hrtime_t	tsc_last_jumped = 0;
149 static hrtime_t	tsc_hrtime_base = 0;
150 static int	tsc_jumped = 0;
151 
152 static hrtime_t	shadow_tsc_hrtime_base;
153 static hrtime_t	shadow_tsc_last;
154 static uint_t	shadow_nsec_scale;
155 static uint32_t	shadow_hres_lock;
156 int get_tsc_ready();
157 
158 hrtime_t
159 tsc_gethrtime(void)
160 {
161 	uint32_t old_hres_lock;
162 	hrtime_t tsc, hrt;
163 
164 	do {
165 		old_hres_lock = hres_lock;
166 
167 		if ((tsc = tsc_read()) >= tsc_last) {
168 			/*
169 			 * It would seem to be obvious that this is true
170 			 * (that is, the past is less than the present),
171 			 * but it isn't true in the presence of suspend/resume
172 			 * cycles.  If we manage to call gethrtime()
173 			 * after a resume, but before the first call to
174 			 * tsc_tick(), we will see the jump.  In this case,
175 			 * we will simply use the value in TSC as the delta.
176 			 */
177 			tsc -= tsc_last;
178 		} else if (tsc >= tsc_last - 2*tsc_max_delta) {
179 			/*
180 			 * There is a chance that tsc_tick() has just run on
181 			 * another CPU, and we have drifted just enough so that
182 			 * we appear behind tsc_last.  In this case, force the
183 			 * delta to be zero.
184 			 */
185 			tsc = 0;
186 		}
187 
188 		hrt = tsc_hrtime_base;
189 
190 		TSC_CONVERT_AND_ADD(tsc, hrt, nsec_scale);
191 	} while ((old_hres_lock & ~1) != hres_lock);
192 
193 	return (hrt);
194 }
195 
196 hrtime_t
197 tsc_gethrtime_delta(void)
198 {
199 	uint32_t old_hres_lock;
200 	hrtime_t tsc, hrt;
201 	ulong_t flags;
202 
203 	do {
204 		old_hres_lock = hres_lock;
205 
206 		/*
207 		 * We need to disable interrupts here to assure that we
208 		 * don't migrate between the call to tsc_read() and
209 		 * adding the CPU's TSC tick delta. Note that disabling
210 		 * and reenabling preemption is forbidden here because
211 		 * we may be in the middle of a fast trap. In the amd64
212 		 * kernel we cannot tolerate preemption during a fast
213 		 * trap. See _update_sregs().
214 		 */
215 
216 		flags = clear_int_flag();
217 		tsc = tsc_read() + tsc_sync_tick_delta[CPU->cpu_id];
218 		restore_int_flag(flags);
219 
220 		/* See comments in tsc_gethrtime() above */
221 
222 		if (tsc >= tsc_last) {
223 			tsc -= tsc_last;
224 		} else if (tsc >= tsc_last - 2 * tsc_max_delta) {
225 			tsc = 0;
226 		}
227 
228 		hrt = tsc_hrtime_base;
229 
230 		TSC_CONVERT_AND_ADD(tsc, hrt, nsec_scale);
231 	} while ((old_hres_lock & ~1) != hres_lock);
232 
233 	return (hrt);
234 }
235 
236 /*
237  * This is similar to the above, but it cannot actually spin on hres_lock.
238  * As a result, it caches all of the variables it needs; if the variables
239  * don't change, it's done.
240  */
241 hrtime_t
242 dtrace_gethrtime(void)
243 {
244 	uint32_t old_hres_lock;
245 	hrtime_t tsc, hrt;
246 	ulong_t flags;
247 
248 	do {
249 		old_hres_lock = hres_lock;
250 
251 		/*
252 		 * Interrupts are disabled to ensure that the thread isn't
253 		 * migrated between the tsc_read() and adding the CPU's
254 		 * TSC tick delta.
255 		 */
256 		flags = clear_int_flag();
257 
258 		tsc = tsc_read();
259 
260 		if (gethrtimef == tsc_gethrtime_delta)
261 			tsc += tsc_sync_tick_delta[CPU->cpu_id];
262 
263 		restore_int_flag(flags);
264 
265 		/*
266 		 * See the comments in tsc_gethrtime(), above.
267 		 */
268 		if (tsc >= tsc_last)
269 			tsc -= tsc_last;
270 		else if (tsc >= tsc_last - 2*tsc_max_delta)
271 			tsc = 0;
272 
273 		hrt = tsc_hrtime_base;
274 
275 		TSC_CONVERT_AND_ADD(tsc, hrt, nsec_scale);
276 
277 		if ((old_hres_lock & ~1) == hres_lock)
278 			break;
279 
280 		/*
281 		 * If we're here, the clock lock is locked -- or it has been
282 		 * unlocked and locked since we looked.  This may be due to
283 		 * tsc_tick() running on another CPU -- or it may be because
284 		 * some code path has ended up in dtrace_probe() with
285 		 * CLOCK_LOCK held.  We'll try to determine that we're in
286 		 * the former case by taking another lap if the lock has
287 		 * changed since when we first looked at it.
288 		 */
289 		if (old_hres_lock != hres_lock)
290 			continue;
291 
292 		/*
293 		 * So the lock was and is locked.  We'll use the old data
294 		 * instead.
295 		 */
296 		old_hres_lock = shadow_hres_lock;
297 
298 		/*
299 		 * Again, disable interrupts to ensure that the thread
300 		 * isn't migrated between the tsc_read() and adding
301 		 * the CPU's TSC tick delta.
302 		 */
303 		flags = clear_int_flag();
304 
305 		tsc = tsc_read();
306 
307 		if (gethrtimef == tsc_gethrtime_delta)
308 			tsc += tsc_sync_tick_delta[CPU->cpu_id];
309 
310 		restore_int_flag(flags);
311 
312 		/*
313 		 * See the comments in tsc_gethrtime(), above.
314 		 */
315 		if (tsc >= shadow_tsc_last)
316 			tsc -= shadow_tsc_last;
317 		else if (tsc >= shadow_tsc_last - 2 * tsc_max_delta)
318 			tsc = 0;
319 
320 		hrt = shadow_tsc_hrtime_base;
321 
322 		TSC_CONVERT_AND_ADD(tsc, hrt, shadow_nsec_scale);
323 	} while ((old_hres_lock & ~1) != shadow_hres_lock);
324 
325 	return (hrt);
326 }
327 
328 hrtime_t
329 tsc_gethrtimeunscaled(void)
330 {
331 	uint32_t old_hres_lock;
332 	hrtime_t tsc;
333 
334 	do {
335 		old_hres_lock = hres_lock;
336 
337 		/* See tsc_tick(). */
338 		tsc = tsc_read() + tsc_last_jumped;
339 	} while ((old_hres_lock & ~1) != hres_lock);
340 
341 	return (tsc);
342 }
343 
344 
345 /* Convert a tsc timestamp to nanoseconds */
346 void
347 tsc_scalehrtime(hrtime_t *tsc)
348 {
349 	hrtime_t hrt;
350 	hrtime_t mytsc;
351 
352 	if (tsc == NULL)
353 		return;
354 	mytsc = *tsc;
355 
356 	TSC_CONVERT(mytsc, hrt, nsec_scale);
357 	*tsc  = hrt;
358 }
359 
360 hrtime_t
361 tsc_gethrtimeunscaled_delta(void)
362 {
363 	hrtime_t hrt;
364 	ulong_t flags;
365 
366 	/*
367 	 * Similarly to tsc_gethrtime_delta, we need to disable preemption
368 	 * to prevent migration between the call to tsc_gethrtimeunscaled
369 	 * and adding the CPU's hrtime delta. Note that disabling and
370 	 * reenabling preemption is forbidden here because we may be in the
371 	 * middle of a fast trap. In the amd64 kernel we cannot tolerate
372 	 * preemption during a fast trap. See _update_sregs().
373 	 */
374 
375 	flags = clear_int_flag();
376 	hrt = tsc_gethrtimeunscaled() + tsc_sync_tick_delta[CPU->cpu_id];
377 	restore_int_flag(flags);
378 
379 	return (hrt);
380 }
381 
382 /*
383  * Called by the master in the TSC sync operation (usually the boot CPU).
384  * If the slave is discovered to have a skew, gethrtimef will be changed to
385  * point to tsc_gethrtime_delta(). Calculating skews is precise only when
386  * the master and slave TSCs are read simultaneously; however, there is no
387  * algorithm that can read both CPUs in perfect simultaneity. The proposed
388  * algorithm is an approximate method based on the behaviour of cache
389  * management. The slave CPU continuously reads TSC and then reads a global
390  * variable which the master CPU updates. The moment the master's update reaches
391  * the slave's visibility (being forced by an mfence operation) we use the TSC
392  * reading taken on the slave. A corresponding TSC read will be taken on the
393  * master as soon as possible after finishing the mfence operation. But the
394  * delay between causing the slave to notice the invalid cache line and the
395  * competion of mfence is not repeatable. This error is heuristically assumed
396  * to be 1/4th of the total write time as being measured by the two TSC reads
397  * on the master sandwiching the mfence. Furthermore, due to the nature of
398  * bus arbitration, contention on memory bus, etc., the time taken for the write
399  * to reflect globally can vary a lot. So instead of taking a single reading,
400  * a set of readings are taken and the one with least write time is chosen
401  * to calculate the final skew.
402  *
403  * TSC sync is disabled in the context of virtualization because the CPUs
404  * assigned to the guest are virtual CPUs which means the real CPUs on which
405  * guest runs keep changing during life time of guest OS. So we would end up
406  * calculating TSC skews for a set of CPUs during boot whereas the guest
407  * might migrate to a different set of physical CPUs at a later point of
408  * time.
409  */
410 void
411 tsc_sync_master(processorid_t slave)
412 {
413 	ulong_t flags, source, min_write_time = ~0UL;
414 	hrtime_t write_time, x, mtsc_after, tdelta;
415 	tsc_sync_t *tsc = tscp;
416 	int cnt;
417 	int hwtype;
418 
419 	hwtype = get_hwenv();
420 	if (!tsc_master_slave_sync_needed || hwtype == HW_XEN_HVM ||
421 	    hwtype == HW_VMWARE)
422 		return;
423 
424 	flags = clear_int_flag();
425 	source = CPU->cpu_id;
426 
427 	for (cnt = 0; cnt < SYNC_ITERATIONS; cnt++) {
428 		while (tsc_sync_go != TSC_SYNC_GO)
429 			SMT_PAUSE();
430 
431 		tsc->master_tsc = tsc_read();
432 		membar_enter();
433 		mtsc_after = tsc_read();
434 		while (tsc_sync_go != TSC_SYNC_DONE)
435 			SMT_PAUSE();
436 		write_time =  mtsc_after - tsc->master_tsc;
437 		if (write_time <= min_write_time) {
438 			min_write_time = write_time;
439 			/*
440 			 * Apply heuristic adjustment only if the calculated
441 			 * delta is > 1/4th of the write time.
442 			 */
443 			x = tsc->slave_tsc - mtsc_after;
444 			if (x < 0)
445 				x = -x;
446 			if (x > (min_write_time/4))
447 				/*
448 				 * Subtract 1/4th of the measured write time
449 				 * from the master's TSC value, as an estimate
450 				 * of how late the mfence completion came
451 				 * after the slave noticed the cache line
452 				 * change.
453 				 */
454 				tdelta = tsc->slave_tsc -
455 				    (mtsc_after - (min_write_time/4));
456 			else
457 				tdelta = tsc->slave_tsc - mtsc_after;
458 			tsc_sync_tick_delta[slave] =
459 			    tsc_sync_tick_delta[source] - tdelta;
460 		}
461 
462 		tsc->master_tsc = tsc->slave_tsc = write_time = 0;
463 		membar_enter();
464 		tsc_sync_go = TSC_SYNC_STOP;
465 	}
466 	if (tdelta < 0)
467 		tdelta = -tdelta;
468 	if (tdelta > largest_tsc_delta)
469 		largest_tsc_delta = tdelta;
470 	if (min_write_time < shortest_write_time)
471 		shortest_write_time = min_write_time;
472 	/*
473 	 * Enable delta variants of tsc functions if the largest of all chosen
474 	 * deltas is > smallest of the write time.
475 	 */
476 	if (largest_tsc_delta > shortest_write_time) {
477 		gethrtimef = tsc_gethrtime_delta;
478 		gethrtimeunscaledf = tsc_gethrtimeunscaled_delta;
479 	}
480 	restore_int_flag(flags);
481 }
482 
483 /*
484  * Called by a CPU which has just been onlined.  It is expected that the CPU
485  * performing the online operation will call tsc_sync_master().
486  *
487  * TSC sync is disabled in the context of virtualization. See comments
488  * above tsc_sync_master.
489  */
490 void
491 tsc_sync_slave(void)
492 {
493 	ulong_t flags;
494 	hrtime_t s1;
495 	tsc_sync_t *tsc = tscp;
496 	int cnt;
497 	int hwtype;
498 
499 	hwtype = get_hwenv();
500 	if (!tsc_master_slave_sync_needed || hwtype == HW_XEN_HVM ||
501 	    hwtype == HW_VMWARE)
502 		return;
503 
504 	flags = clear_int_flag();
505 
506 	for (cnt = 0; cnt < SYNC_ITERATIONS; cnt++) {
507 		/* Re-fill the cache line */
508 		s1 = tsc->master_tsc;
509 		membar_enter();
510 		tsc_sync_go = TSC_SYNC_GO;
511 		do {
512 			/*
513 			 * Do not put an SMT_PAUSE here. For instance,
514 			 * if the master and slave are really the same
515 			 * hyper-threaded CPU, then you want the master
516 			 * to yield to the slave as quickly as possible here,
517 			 * but not the other way.
518 			 */
519 			s1 = tsc_read();
520 		} while (tsc->master_tsc == 0);
521 		tsc->slave_tsc = s1;
522 		membar_enter();
523 		tsc_sync_go = TSC_SYNC_DONE;
524 
525 		while (tsc_sync_go != TSC_SYNC_STOP)
526 			SMT_PAUSE();
527 	}
528 
529 	restore_int_flag(flags);
530 }
531 
532 /*
533  * Called once per second on a CPU from the cyclic subsystem's
534  * CY_HIGH_LEVEL interrupt.  (No longer just cpu0-only)
535  */
536 void
537 tsc_tick(void)
538 {
539 	hrtime_t now, delta;
540 	ushort_t spl;
541 
542 	/*
543 	 * Before we set the new variables, we set the shadow values.  This
544 	 * allows for lock free operation in dtrace_gethrtime().
545 	 */
546 	lock_set_spl((lock_t *)&shadow_hres_lock + HRES_LOCK_OFFSET,
547 	    ipltospl(CBE_HIGH_PIL), &spl);
548 
549 	shadow_tsc_hrtime_base = tsc_hrtime_base;
550 	shadow_tsc_last = tsc_last;
551 	shadow_nsec_scale = nsec_scale;
552 
553 	shadow_hres_lock++;
554 	splx(spl);
555 
556 	CLOCK_LOCK(&spl);
557 
558 	now = tsc_read();
559 
560 	if (gethrtimef == tsc_gethrtime_delta)
561 		now += tsc_sync_tick_delta[CPU->cpu_id];
562 
563 	if (now < tsc_last) {
564 		/*
565 		 * The TSC has just jumped into the past.  We assume that
566 		 * this is due to a suspend/resume cycle, and we're going
567 		 * to use the _current_ value of TSC as the delta.  This
568 		 * will keep tsc_hrtime_base correct.  We're also going to
569 		 * assume that rate of tsc does not change after a suspend
570 		 * resume (i.e nsec_scale remains the same).
571 		 */
572 		delta = now;
573 		tsc_last_jumped += tsc_last;
574 		tsc_jumped = 1;
575 	} else {
576 		/*
577 		 * Determine the number of TSC ticks since the last clock
578 		 * tick, and add that to the hrtime base.
579 		 */
580 		delta = now - tsc_last;
581 	}
582 
583 	TSC_CONVERT_AND_ADD(delta, tsc_hrtime_base, nsec_scale);
584 	tsc_last = now;
585 
586 	CLOCK_UNLOCK(spl);
587 }
588 
589 void
590 tsc_hrtimeinit(uint64_t cpu_freq_hz)
591 {
592 	extern int gethrtime_hires;
593 	longlong_t tsc;
594 	ulong_t flags;
595 
596 	/*
597 	 * cpu_freq_hz is the measured cpu frequency in hertz
598 	 */
599 
600 	/*
601 	 * We can't accommodate CPUs slower than 31.25 MHz.
602 	 */
603 	ASSERT(cpu_freq_hz > NANOSEC / (1 << NSEC_SHIFT));
604 	nsec_scale =
605 	    (uint_t)(((uint64_t)NANOSEC << (32 - NSEC_SHIFT)) / cpu_freq_hz);
606 
607 	flags = clear_int_flag();
608 	tsc = tsc_read();
609 	(void) tsc_gethrtime();
610 	tsc_max_delta = tsc_read() - tsc;
611 	restore_int_flag(flags);
612 	gethrtimef = tsc_gethrtime;
613 	gethrtimeunscaledf = tsc_gethrtimeunscaled;
614 	scalehrtimef = tsc_scalehrtime;
615 	hrtime_tick = tsc_tick;
616 	gethrtime_hires = 1;
617 	/*
618 	 * Allocate memory for the structure used in the tsc sync logic.
619 	 * This structure should be aligned on a multiple of cache line size.
620 	 */
621 	tscp = kmem_zalloc(PAGESIZE, KM_SLEEP);
622 }
623 
624 int
625 get_tsc_ready()
626 {
627 	return (tsc_ready);
628 }
629 
630 /*
631  * Adjust all the deltas by adding the passed value to the array.
632  * Then use the "delt" versions of the the gethrtime functions.
633  * Note that 'tdelta' _could_ be a negative number, which should
634  * reduce the values in the array (used, for example, if the Solaris
635  * instance was moved by a virtual manager to a machine with a higher
636  * value of tsc).
637  */
638 void
639 tsc_adjust_delta(hrtime_t tdelta)
640 {
641 	int		i;
642 
643 	for (i = 0; i < NCPU; i++) {
644 		tsc_sync_tick_delta[i] += tdelta;
645 	}
646 
647 	gethrtimef = tsc_gethrtime_delta;
648 	gethrtimeunscaledf = tsc_gethrtimeunscaled_delta;
649 }
650 
651 /*
652  * Functions to manage TSC and high-res time on suspend and resume.
653  */
654 
655 /*
656  * declarations needed for time adjustment
657  */
658 extern void	rtcsync(void);
659 extern tod_ops_t *tod_ops;
660 /* There must be a better way than exposing nsec_scale! */
661 extern uint_t	nsec_scale;
662 static uint64_t tsc_saved_tsc = 0; /* 1 in 2^64 chance this'll screw up! */
663 static timestruc_t tsc_saved_ts;
664 static int	tsc_needs_resume = 0;	/* We only want to do this once. */
665 int		tsc_delta_onsuspend = 0;
666 int		tsc_adjust_seconds = 1;
667 int		tsc_suspend_count = 0;
668 int		tsc_resume_in_cyclic = 0;
669 
670 /*
671  * Let timestamp.c know that we are suspending.  It needs to take
672  * snapshots of the current time, and do any pre-suspend work.
673  */
674 void
675 tsc_suspend(void)
676 {
677 /*
678  * What we need to do here, is to get the time we suspended, so that we
679  * know how much we should add to the resume.
680  * This routine is called by each CPU, so we need to handle reentry.
681  */
682 	if (tsc_gethrtime_enable) {
683 		/*
684 		 * We put the tsc_read() inside the lock as it
685 		 * as no locking constraints, and it puts the
686 		 * aquired value closer to the time stamp (in
687 		 * case we delay getting the lock).
688 		 */
689 		mutex_enter(&tod_lock);
690 		tsc_saved_tsc = tsc_read();
691 		tsc_saved_ts = TODOP_GET(tod_ops);
692 		mutex_exit(&tod_lock);
693 		/* We only want to do this once. */
694 		if (tsc_needs_resume == 0) {
695 			if (tsc_delta_onsuspend) {
696 				tsc_adjust_delta(tsc_saved_tsc);
697 			} else {
698 				tsc_adjust_delta(nsec_scale);
699 			}
700 			tsc_suspend_count++;
701 		}
702 	}
703 
704 	invalidate_cache();
705 	tsc_needs_resume = 1;
706 }
707 
708 /*
709  * Restore all timestamp state based on the snapshots taken at
710  * suspend time.
711  */
712 void
713 tsc_resume(void)
714 {
715 	/*
716 	 * We only need to (and want to) do this once.  So let the first
717 	 * caller handle this (we are locked by the cpu lock), as it
718 	 * is preferential that we get the earliest sync.
719 	 */
720 	if (tsc_needs_resume) {
721 		/*
722 		 * If using the TSC, adjust the delta based on how long
723 		 * we were sleeping (or away).  We also adjust for
724 		 * migration and a grown TSC.
725 		 */
726 		if (tsc_saved_tsc != 0) {
727 			timestruc_t	ts;
728 			hrtime_t	now, sleep_tsc = 0;
729 			int		sleep_sec;
730 			extern void	tsc_tick(void);
731 			extern uint64_t cpu_freq_hz;
732 
733 			/* tsc_read() MUST be before TODOP_GET() */
734 			mutex_enter(&tod_lock);
735 			now = tsc_read();
736 			ts = TODOP_GET(tod_ops);
737 			mutex_exit(&tod_lock);
738 
739 			/* Compute seconds of sleep time */
740 			sleep_sec = ts.tv_sec - tsc_saved_ts.tv_sec;
741 
742 			/*
743 			 * If the saved sec is less that or equal to
744 			 * the current ts, then there is likely a
745 			 * problem with the clock.  Assume at least
746 			 * one second has passed, so that time goes forward.
747 			 */
748 			if (sleep_sec <= 0) {
749 				sleep_sec = 1;
750 			}
751 
752 			/* How many TSC's should have occured while sleeping */
753 			if (tsc_adjust_seconds)
754 				sleep_tsc = sleep_sec * cpu_freq_hz;
755 
756 			/*
757 			 * We also want to subtract from the "sleep_tsc"
758 			 * the current value of tsc_read(), so that our
759 			 * adjustment accounts for the amount of time we
760 			 * have been resumed _or_ an adjustment based on
761 			 * the fact that we didn't actually power off the
762 			 * CPU (migration is another issue, but _should_
763 			 * also comply with this calculation).  If the CPU
764 			 * never powered off, then:
765 			 *    'now == sleep_tsc + saved_tsc'
766 			 * and the delta will effectively be "0".
767 			 */
768 			sleep_tsc -= now;
769 			if (tsc_delta_onsuspend) {
770 				tsc_adjust_delta(sleep_tsc);
771 			} else {
772 				tsc_adjust_delta(tsc_saved_tsc + sleep_tsc);
773 			}
774 			tsc_saved_tsc = 0;
775 
776 			tsc_tick();
777 		}
778 		tsc_needs_resume = 0;
779 	}
780 
781 }
782