1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22/*
23 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
24 * Use is subject to license terms.
25 *
26 * Copyright 2012 Nexenta Systems, Inc. All rights reserved.
27 * Copyright (c) 2014, 2016 by Delphix. All rights reserved.
28 * Copyright 2018 Joyent, Inc.
29 */
30
31#include <sys/types.h>
32#include <sys/param.h>
33#include <sys/systm.h>
34#include <sys/disp.h>
35#include <sys/var.h>
36#include <sys/cmn_err.h>
37#include <sys/debug.h>
38#include <sys/x86_archext.h>
39#include <sys/archsystm.h>
40#include <sys/cpuvar.h>
41#include <sys/psm_defs.h>
42#include <sys/clock.h>
43#include <sys/atomic.h>
44#include <sys/lockstat.h>
45#include <sys/smp_impldefs.h>
46#include <sys/dtrace.h>
47#include <sys/time.h>
48#include <sys/panic.h>
49#include <sys/cpu.h>
50#include <sys/sdt.h>
51#include <sys/comm_page.h>
52
53/*
54 * Using the Pentium's TSC register for gethrtime()
55 * ------------------------------------------------
56 *
57 * The Pentium family, like many chip architectures, has a high-resolution
58 * timestamp counter ("TSC") which increments once per CPU cycle.  The contents
59 * of the timestamp counter are read with the RDTSC instruction.
60 *
61 * As with its UltraSPARC equivalent (the %tick register), TSC's cycle count
62 * must be translated into nanoseconds in order to implement gethrtime().
63 * We avoid inducing floating point operations in this conversion by
64 * implementing the same nsec_scale algorithm as that found in the sun4u
65 * platform code.  The sun4u NATIVE_TIME_TO_NSEC_SCALE block comment contains
66 * a detailed description of the algorithm; the comment is not reproduced
67 * here.  This implementation differs only in its value for NSEC_SHIFT:
68 * we implement an NSEC_SHIFT of 5 (instead of sun4u's 4) to allow for
69 * 60 MHz Pentiums.
70 *
71 * While TSC and %tick are both cycle counting registers, TSC's functionality
72 * falls short in several critical ways:
73 *
74 *  (a)	TSCs on different CPUs are not guaranteed to be in sync.  While in
75 *	practice they often _are_ in sync, this isn't guaranteed by the
76 *	architecture.
77 *
78 *  (b)	The TSC cannot be reliably set to an arbitrary value.  The architecture
79 *	only supports writing the low 32-bits of TSC, making it impractical
80 *	to rewrite.
81 *
82 *  (c)	The architecture doesn't have the capacity to interrupt based on
83 *	arbitrary values of TSC; there is no TICK_CMPR equivalent.
84 *
85 * Together, (a) and (b) imply that software must track the skew between
86 * TSCs and account for it (it is assumed that while there may exist skew,
87 * there does not exist drift).  To determine the skew between CPUs, we
88 * have newly onlined CPUs call tsc_sync_slave(), while the CPU performing
89 * the online operation calls tsc_sync_master().
90 *
91 * In the absence of time-of-day clock adjustments, gethrtime() must stay in
92 * sync with gettimeofday().  This is problematic; given (c), the software
93 * cannot drive its time-of-day source from TSC, and yet they must somehow be
94 * kept in sync.  We implement this by having a routine, tsc_tick(), which
95 * is called once per second from the interrupt which drives time-of-day.
96 *
97 * Note that the hrtime base for gethrtime, tsc_hrtime_base, is modified
98 * atomically with nsec_scale under CLOCK_LOCK.  This assures that time
99 * monotonically increases.
100 */
101
102#define	NSEC_SHIFT 5
103
104static uint_t nsec_unscale;
105
106/*
107 * These two variables used to be grouped together inside of a structure that
108 * lived on a single cache line. A regression (bug ID 4623398) caused the
109 * compiler to emit code that "optimized" away the while-loops below. The
110 * result was that no synchronization between the onlining and onlined CPUs
111 * took place.
112 */
113static volatile int tsc_ready;
114static volatile int tsc_sync_go;
115
116/*
117 * Used as indices into the tsc_sync_snaps[] array.
118 */
119#define	TSC_MASTER		0
120#define	TSC_SLAVE		1
121
122/*
123 * Used in the tsc_master_sync()/tsc_slave_sync() rendezvous.
124 */
125#define	TSC_SYNC_STOP		1
126#define	TSC_SYNC_GO		2
127#define	TSC_SYNC_DONE		3
128#define	SYNC_ITERATIONS		10
129
130#define	TSC_CONVERT_AND_ADD(tsc, hrt, scale) {	 	\
131	unsigned int *_l = (unsigned int *)&(tsc); 	\
132	(hrt) += mul32(_l[1], scale) << NSEC_SHIFT; 	\
133	(hrt) += mul32(_l[0], scale) >> (32 - NSEC_SHIFT); \
134}
135
136#define	TSC_CONVERT(tsc, hrt, scale) { 			\
137	unsigned int *_l = (unsigned int *)&(tsc); 	\
138	(hrt) = mul32(_l[1], scale) << NSEC_SHIFT; 	\
139	(hrt) += mul32(_l[0], scale) >> (32 - NSEC_SHIFT); \
140}
141
142int tsc_master_slave_sync_needed = 1;
143
144typedef struct tsc_sync {
145	volatile hrtime_t master_tsc, slave_tsc;
146} tsc_sync_t;
147static tsc_sync_t *tscp;
148
149static hrtime_t	tsc_last_jumped = 0;
150static int	tsc_jumped = 0;
151static uint32_t	tsc_wayback = 0;
152/*
153 * The cap of 1 second was chosen since it is the frequency at which the
154 * tsc_tick() function runs which means that when gethrtime() is called it
155 * should never be more than 1 second since tsc_last was updated.
156 */
157static hrtime_t tsc_resume_cap_ns = NANOSEC;	 /* 1s */
158
159static hrtime_t	shadow_tsc_hrtime_base;
160static hrtime_t	shadow_tsc_last;
161static uint_t	shadow_nsec_scale;
162static uint32_t	shadow_hres_lock;
163int get_tsc_ready();
164
165static inline
166hrtime_t tsc_protect(hrtime_t a) {
167	if (a > tsc_resume_cap) {
168		atomic_inc_32(&tsc_wayback);
169		DTRACE_PROBE3(tsc__wayback, htrime_t, a, hrtime_t, tsc_last,
170		    uint32_t, tsc_wayback);
171		return (tsc_resume_cap);
172	}
173	return (a);
174}
175
176hrtime_t
177tsc_gethrtime(void)
178{
179	uint32_t old_hres_lock;
180	hrtime_t tsc, hrt;
181
182	do {
183		old_hres_lock = hres_lock;
184
185		if ((tsc = tsc_read()) >= tsc_last) {
186			/*
187			 * It would seem to be obvious that this is true
188			 * (that is, the past is less than the present),
189			 * but it isn't true in the presence of suspend/resume
190			 * cycles.  If we manage to call gethrtime()
191			 * after a resume, but before the first call to
192			 * tsc_tick(), we will see the jump.  In this case,
193			 * we will simply use the value in TSC as the delta.
194			 */
195			tsc -= tsc_last;
196		} else if (tsc >= tsc_last - 2*tsc_max_delta) {
197			/*
198			 * There is a chance that tsc_tick() has just run on
199			 * another CPU, and we have drifted just enough so that
200			 * we appear behind tsc_last.  In this case, force the
201			 * delta to be zero.
202			 */
203			tsc = 0;
204		} else {
205			/*
206			 * If we reach this else clause we assume that we have
207			 * gone through a suspend/resume cycle and use the
208			 * current tsc value as the delta.
209			 *
210			 * In rare cases we can reach this else clause due to
211			 * a lack of monotonicity in the TSC value.  In such
212			 * cases using the current TSC value as the delta would
213			 * cause us to return a value ~2x of what it should
214			 * be.  To protect against these cases we cap the
215			 * suspend/resume delta at tsc_resume_cap.
216			 */
217			tsc = tsc_protect(tsc);
218		}
219
220		hrt = tsc_hrtime_base;
221
222		TSC_CONVERT_AND_ADD(tsc, hrt, nsec_scale);
223	} while ((old_hres_lock & ~1) != hres_lock);
224
225	return (hrt);
226}
227
228hrtime_t
229tsc_gethrtime_delta(void)
230{
231	uint32_t old_hres_lock;
232	hrtime_t tsc, hrt;
233	ulong_t flags;
234
235	do {
236		old_hres_lock = hres_lock;
237
238		/*
239		 * We need to disable interrupts here to assure that we
240		 * don't migrate between the call to tsc_read() and
241		 * adding the CPU's TSC tick delta. Note that disabling
242		 * and reenabling preemption is forbidden here because
243		 * we may be in the middle of a fast trap. In the amd64
244		 * kernel we cannot tolerate preemption during a fast
245		 * trap. See _update_sregs().
246		 */
247
248		flags = clear_int_flag();
249		tsc = tsc_read() + tsc_sync_tick_delta[CPU->cpu_id];
250		restore_int_flag(flags);
251
252		/* See comments in tsc_gethrtime() above */
253
254		if (tsc >= tsc_last) {
255			tsc -= tsc_last;
256		} else if (tsc >= tsc_last - 2 * tsc_max_delta) {
257			tsc = 0;
258		} else {
259			tsc = tsc_protect(tsc);
260		}
261
262		hrt = tsc_hrtime_base;
263
264		TSC_CONVERT_AND_ADD(tsc, hrt, nsec_scale);
265	} while ((old_hres_lock & ~1) != hres_lock);
266
267	return (hrt);
268}
269
270hrtime_t
271tsc_gethrtime_tick_delta(void)
272{
273	hrtime_t hrt;
274	ulong_t flags;
275
276	flags = clear_int_flag();
277	hrt = tsc_sync_tick_delta[CPU->cpu_id];
278	restore_int_flag(flags);
279
280	return (hrt);
281}
282
283/* Calculate the hrtime while exposing the parameters of that calculation. */
284hrtime_t
285tsc_gethrtime_params(uint64_t *tscp, uint32_t *scalep, uint8_t *shiftp)
286{
287	uint32_t old_hres_lock, scale;
288	hrtime_t tsc, last, base;
289
290	do {
291		old_hres_lock = hres_lock;
292
293		if (gethrtimef == tsc_gethrtime_delta) {
294			ulong_t flags;
295
296			flags = clear_int_flag();
297			tsc = tsc_read() + tsc_sync_tick_delta[CPU->cpu_id];
298			restore_int_flag(flags);
299		} else {
300			tsc = tsc_read();
301		}
302
303		last = tsc_last;
304		base = tsc_hrtime_base;
305		scale = nsec_scale;
306
307	} while ((old_hres_lock & ~1) != hres_lock);
308
309	/* See comments in tsc_gethrtime() above */
310	if (tsc >= last) {
311		tsc -= last;
312	} else if (tsc >= last - 2 * tsc_max_delta) {
313		tsc = 0;
314	} else {
315		tsc = tsc_protect(tsc);
316	}
317
318	TSC_CONVERT_AND_ADD(tsc, base, nsec_scale);
319
320	if (tscp != NULL) {
321		/*
322		 * Do not simply communicate the delta applied to the hrtime
323		 * base, but rather the effective TSC measurement.
324		 */
325		*tscp = tsc + last;
326	}
327	if (scalep != NULL) {
328		*scalep = scale;
329	}
330	if (shiftp != NULL) {
331		*shiftp = NSEC_SHIFT;
332	}
333
334	return (base);
335}
336
337/*
338 * This is similar to tsc_gethrtime_delta, but it cannot actually spin on
339 * hres_lock.  As a result, it caches all of the variables it needs; if the
340 * variables don't change, it's done.
341 */
342hrtime_t
343dtrace_gethrtime(void)
344{
345	uint32_t old_hres_lock;
346	hrtime_t tsc, hrt;
347	ulong_t flags;
348
349	do {
350		old_hres_lock = hres_lock;
351
352		/*
353		 * Interrupts are disabled to ensure that the thread isn't
354		 * migrated between the tsc_read() and adding the CPU's
355		 * TSC tick delta.
356		 */
357		flags = clear_int_flag();
358
359		tsc = tsc_read();
360
361		if (gethrtimef == tsc_gethrtime_delta)
362			tsc += tsc_sync_tick_delta[CPU->cpu_id];
363
364		restore_int_flag(flags);
365
366		/*
367		 * See the comments in tsc_gethrtime(), above.
368		 */
369		if (tsc >= tsc_last)
370			tsc -= tsc_last;
371		else if (tsc >= tsc_last - 2*tsc_max_delta)
372			tsc = 0;
373		else
374			tsc = tsc_protect(tsc);
375
376		hrt = tsc_hrtime_base;
377
378		TSC_CONVERT_AND_ADD(tsc, hrt, nsec_scale);
379
380		if ((old_hres_lock & ~1) == hres_lock)
381			break;
382
383		/*
384		 * If we're here, the clock lock is locked -- or it has been
385		 * unlocked and locked since we looked.  This may be due to
386		 * tsc_tick() running on another CPU -- or it may be because
387		 * some code path has ended up in dtrace_probe() with
388		 * CLOCK_LOCK held.  We'll try to determine that we're in
389		 * the former case by taking another lap if the lock has
390		 * changed since when we first looked at it.
391		 */
392		if (old_hres_lock != hres_lock)
393			continue;
394
395		/*
396		 * So the lock was and is locked.  We'll use the old data
397		 * instead.
398		 */
399		old_hres_lock = shadow_hres_lock;
400
401		/*
402		 * Again, disable interrupts to ensure that the thread
403		 * isn't migrated between the tsc_read() and adding
404		 * the CPU's TSC tick delta.
405		 */
406		flags = clear_int_flag();
407
408		tsc = tsc_read();
409
410		if (gethrtimef == tsc_gethrtime_delta)
411			tsc += tsc_sync_tick_delta[CPU->cpu_id];
412
413		restore_int_flag(flags);
414
415		/*
416		 * See the comments in tsc_gethrtime(), above.
417		 */
418		if (tsc >= shadow_tsc_last)
419			tsc -= shadow_tsc_last;
420		else if (tsc >= shadow_tsc_last - 2 * tsc_max_delta)
421			tsc = 0;
422		else
423			tsc = tsc_protect(tsc);
424
425		hrt = shadow_tsc_hrtime_base;
426
427		TSC_CONVERT_AND_ADD(tsc, hrt, shadow_nsec_scale);
428	} while ((old_hres_lock & ~1) != shadow_hres_lock);
429
430	return (hrt);
431}
432
433hrtime_t
434tsc_gethrtimeunscaled(void)
435{
436	uint32_t old_hres_lock;
437	hrtime_t tsc;
438
439	do {
440		old_hres_lock = hres_lock;
441
442		/* See tsc_tick(). */
443		tsc = tsc_read() + tsc_last_jumped;
444	} while ((old_hres_lock & ~1) != hres_lock);
445
446	return (tsc);
447}
448
449/*
450 * Convert a nanosecond based timestamp to tsc
451 */
452uint64_t
453tsc_unscalehrtime(hrtime_t nsec)
454{
455	hrtime_t tsc;
456
457	if (tsc_gethrtime_enable) {
458		TSC_CONVERT(nsec, tsc, nsec_unscale);
459		return (tsc);
460	}
461	return ((uint64_t)nsec);
462}
463
464/* Convert a tsc timestamp to nanoseconds */
465void
466tsc_scalehrtime(hrtime_t *tsc)
467{
468	hrtime_t hrt;
469	hrtime_t mytsc;
470
471	if (tsc == NULL)
472		return;
473	mytsc = *tsc;
474
475	TSC_CONVERT(mytsc, hrt, nsec_scale);
476	*tsc  = hrt;
477}
478
479hrtime_t
480tsc_gethrtimeunscaled_delta(void)
481{
482	hrtime_t hrt;
483	ulong_t flags;
484
485	/*
486	 * Similarly to tsc_gethrtime_delta, we need to disable preemption
487	 * to prevent migration between the call to tsc_gethrtimeunscaled
488	 * and adding the CPU's hrtime delta. Note that disabling and
489	 * reenabling preemption is forbidden here because we may be in the
490	 * middle of a fast trap. In the amd64 kernel we cannot tolerate
491	 * preemption during a fast trap. See _update_sregs().
492	 */
493
494	flags = clear_int_flag();
495	hrt = tsc_gethrtimeunscaled() + tsc_sync_tick_delta[CPU->cpu_id];
496	restore_int_flag(flags);
497
498	return (hrt);
499}
500
501/*
502 * TSC Sync Master
503 *
504 * Typically called on the boot CPU, this attempts to quantify TSC skew between
505 * different CPUs.  If an appreciable difference is found, gethrtimef will be
506 * changed to point to tsc_gethrtime_delta().
507 *
508 * Calculating skews is precise only when the master and slave TSCs are read
509 * simultaneously; however, there is no algorithm that can read both CPUs in
510 * perfect simultaneity.  The proposed algorithm is an approximate method based
511 * on the behaviour of cache management.  The slave CPU continuously polls the
512 * TSC while reading a global variable updated by the master CPU.  The latest
513 * TSC reading is saved when the master's update (forced via mfence) reaches
514 * visibility on the slave.  The master will also take a TSC reading
515 * immediately following the mfence.
516 *
517 * While the delay between cache line invalidation on the slave and mfence
518 * completion on the master is not repeatable, the error is heuristically
519 * assumed to be 1/4th of the write time recorded by the master.  Multiple
520 * samples are taken to control for the variance caused by external factors
521 * such as bus contention.  Each sample set is independent per-CPU to control
522 * for differing memory latency on NUMA systems.
523 *
524 * TSC sync is disabled in the context of virtualization because the CPUs
525 * assigned to the guest are virtual CPUs which means the real CPUs on which
526 * guest runs keep changing during life time of guest OS. So we would end up
527 * calculating TSC skews for a set of CPUs during boot whereas the guest
528 * might migrate to a different set of physical CPUs at a later point of
529 * time.
530 */
531void
532tsc_sync_master(processorid_t slave)
533{
534	ulong_t flags, source, min_write_time = ~0UL;
535	hrtime_t write_time, mtsc_after, last_delta = 0;
536	tsc_sync_t *tsc = tscp;
537	int cnt;
538	int hwtype;
539
540	hwtype = get_hwenv();
541	if (!tsc_master_slave_sync_needed || (hwtype & HW_VIRTUAL) != 0)
542		return;
543
544	flags = clear_int_flag();
545	source = CPU->cpu_id;
546
547	for (cnt = 0; cnt < SYNC_ITERATIONS; cnt++) {
548		while (tsc_sync_go != TSC_SYNC_GO)
549			SMT_PAUSE();
550
551		tsc->master_tsc = tsc_read();
552		membar_enter();
553		mtsc_after = tsc_read();
554		while (tsc_sync_go != TSC_SYNC_DONE)
555			SMT_PAUSE();
556		write_time =  mtsc_after - tsc->master_tsc;
557		if (write_time <= min_write_time) {
558			hrtime_t tdelta;
559
560			tdelta = tsc->slave_tsc - mtsc_after;
561			if (tdelta < 0)
562				tdelta = -tdelta;
563			/*
564			 * If the margin exists, subtract 1/4th of the measured
565			 * write time from the master's TSC value.  This is an
566			 * estimate of how late the mfence completion came
567			 * after the slave noticed the cache line change.
568			 */
569			if (tdelta > (write_time/4)) {
570				tdelta = tsc->slave_tsc -
571				    (mtsc_after - (write_time/4));
572			} else {
573				tdelta = tsc->slave_tsc - mtsc_after;
574			}
575			last_delta = tsc_sync_tick_delta[source] - tdelta;
576			tsc_sync_tick_delta[slave] = last_delta;
577			min_write_time = write_time;
578		}
579
580		tsc->master_tsc = tsc->slave_tsc = write_time = 0;
581		membar_enter();
582		tsc_sync_go = TSC_SYNC_STOP;
583	}
584
585	/*
586	 * Only enable the delta variants of the TSC functions if the measured
587	 * skew is greater than the fastest write time.
588	 */
589	last_delta = (last_delta < 0) ? -last_delta : last_delta;
590	if (last_delta > min_write_time) {
591		gethrtimef = tsc_gethrtime_delta;
592		gethrtimeunscaledf = tsc_gethrtimeunscaled_delta;
593		tsc_ncpu = NCPU;
594	}
595	restore_int_flag(flags);
596}
597
598/*
599 * TSC Sync Slave
600 *
601 * Called by a CPU which has just been onlined.  It is expected that the CPU
602 * performing the online operation will call tsc_sync_master().
603 *
604 * Like tsc_sync_master, this logic is skipped on virtualized platforms.
605 */
606void
607tsc_sync_slave(void)
608{
609	ulong_t flags;
610	hrtime_t s1;
611	tsc_sync_t *tsc = tscp;
612	int cnt;
613	int hwtype;
614
615	hwtype = get_hwenv();
616	if (!tsc_master_slave_sync_needed || (hwtype & HW_VIRTUAL) != 0)
617		return;
618
619	flags = clear_int_flag();
620
621	for (cnt = 0; cnt < SYNC_ITERATIONS; cnt++) {
622		/* Re-fill the cache line */
623		s1 = tsc->master_tsc;
624		membar_enter();
625		tsc_sync_go = TSC_SYNC_GO;
626		do {
627			/*
628			 * Do not put an SMT_PAUSE here.  If the master and
629			 * slave are the same hyper-threaded CPU, we want the
630			 * master to yield as quickly as possible to the slave.
631			 */
632			s1 = tsc_read();
633		} while (tsc->master_tsc == 0);
634		tsc->slave_tsc = s1;
635		membar_enter();
636		tsc_sync_go = TSC_SYNC_DONE;
637
638		while (tsc_sync_go != TSC_SYNC_STOP)
639			SMT_PAUSE();
640	}
641
642	restore_int_flag(flags);
643}
644
645/*
646 * Called once per second on a CPU from the cyclic subsystem's
647 * CY_HIGH_LEVEL interrupt.  (No longer just cpu0-only)
648 */
649void
650tsc_tick(void)
651{
652	hrtime_t now, delta;
653	ushort_t spl;
654
655	/*
656	 * Before we set the new variables, we set the shadow values.  This
657	 * allows for lock free operation in dtrace_gethrtime().
658	 */
659	lock_set_spl((lock_t *)&shadow_hres_lock + HRES_LOCK_OFFSET,
660	    ipltospl(CBE_HIGH_PIL), &spl);
661
662	shadow_tsc_hrtime_base = tsc_hrtime_base;
663	shadow_tsc_last = tsc_last;
664	shadow_nsec_scale = nsec_scale;
665
666	shadow_hres_lock++;
667	splx(spl);
668
669	CLOCK_LOCK(&spl);
670
671	now = tsc_read();
672
673	if (gethrtimef == tsc_gethrtime_delta)
674		now += tsc_sync_tick_delta[CPU->cpu_id];
675
676	if (now < tsc_last) {
677		/*
678		 * The TSC has just jumped into the past.  We assume that
679		 * this is due to a suspend/resume cycle, and we're going
680		 * to use the _current_ value of TSC as the delta.  This
681		 * will keep tsc_hrtime_base correct.  We're also going to
682		 * assume that rate of tsc does not change after a suspend
683		 * resume (i.e nsec_scale remains the same).
684		 */
685		delta = now;
686		delta = tsc_protect(delta);
687		tsc_last_jumped += tsc_last;
688		tsc_jumped = 1;
689	} else {
690		/*
691		 * Determine the number of TSC ticks since the last clock
692		 * tick, and add that to the hrtime base.
693		 */
694		delta = now - tsc_last;
695	}
696
697	TSC_CONVERT_AND_ADD(delta, tsc_hrtime_base, nsec_scale);
698	tsc_last = now;
699
700	CLOCK_UNLOCK(spl);
701}
702
703void
704tsc_hrtimeinit(uint64_t cpu_freq_hz)
705{
706	extern int gethrtime_hires;
707	longlong_t tsc;
708	ulong_t flags;
709
710	/*
711	 * cpu_freq_hz is the measured cpu frequency in hertz
712	 */
713
714	/*
715	 * We can't accommodate CPUs slower than 31.25 MHz.
716	 */
717	ASSERT(cpu_freq_hz > NANOSEC / (1 << NSEC_SHIFT));
718	nsec_scale =
719	    (uint_t)(((uint64_t)NANOSEC << (32 - NSEC_SHIFT)) / cpu_freq_hz);
720	nsec_unscale =
721	    (uint_t)(((uint64_t)cpu_freq_hz << (32 - NSEC_SHIFT)) / NANOSEC);
722
723	flags = clear_int_flag();
724	tsc = tsc_read();
725	(void) tsc_gethrtime();
726	tsc_max_delta = tsc_read() - tsc;
727	restore_int_flag(flags);
728	gethrtimef = tsc_gethrtime;
729	gethrtimeunscaledf = tsc_gethrtimeunscaled;
730	scalehrtimef = tsc_scalehrtime;
731	unscalehrtimef = tsc_unscalehrtime;
732	hrtime_tick = tsc_tick;
733	gethrtime_hires = 1;
734	/*
735	 * Being part of the comm page, tsc_ncpu communicates the published
736	 * length of the tsc_sync_tick_delta array.  This is kept zeroed to
737	 * ignore the absent delta data while the TSCs are synced.
738	 */
739	tsc_ncpu = 0;
740	/*
741	 * Allocate memory for the structure used in the tsc sync logic.
742	 * This structure should be aligned on a multiple of cache line size.
743	 */
744	tscp = kmem_zalloc(PAGESIZE, KM_SLEEP);
745
746	/*
747	 * Convert the TSC resume cap ns value into its unscaled TSC value.
748	 * See tsc_gethrtime().
749	 */
750	if (tsc_resume_cap == 0)
751		TSC_CONVERT(tsc_resume_cap_ns, tsc_resume_cap, nsec_unscale);
752}
753
754int
755get_tsc_ready()
756{
757	return (tsc_ready);
758}
759
760/*
761 * Adjust all the deltas by adding the passed value to the array and activate
762 * the "delta" versions of the gethrtime functions.  It is possible that the
763 * adjustment could be negative.  Such may occur if the SunOS instance was
764 * moved by a virtual manager to a machine with a higher value of TSC.
765 */
766void
767tsc_adjust_delta(hrtime_t tdelta)
768{
769	int		i;
770
771	for (i = 0; i < NCPU; i++) {
772		tsc_sync_tick_delta[i] += tdelta;
773	}
774
775	gethrtimef = tsc_gethrtime_delta;
776	gethrtimeunscaledf = tsc_gethrtimeunscaled_delta;
777	tsc_ncpu = NCPU;
778}
779
780/*
781 * Functions to manage TSC and high-res time on suspend and resume.
782 */
783
784/* tod_ops from "uts/i86pc/io/todpc_subr.c" */
785extern tod_ops_t *tod_ops;
786
787static uint64_t tsc_saved_tsc = 0; /* 1 in 2^64 chance this'll screw up! */
788static timestruc_t tsc_saved_ts;
789static int	tsc_needs_resume = 0;	/* We only want to do this once. */
790int		tsc_delta_onsuspend = 0;
791int		tsc_adjust_seconds = 1;
792int		tsc_suspend_count = 0;
793int		tsc_resume_in_cyclic = 0;
794
795/*
796 * Take snapshots of the current time and do any other pre-suspend work.
797 */
798void
799tsc_suspend(void)
800{
801	/*
802	 * We need to collect the time at which we suspended here so we know
803	 * now much should be added during the resume.  This is called by each
804	 * CPU, so reentry must be properly handled.
805	 */
806	if (tsc_gethrtime_enable) {
807		/*
808		 * Perform the tsc_read after acquiring the lock to make it as
809		 * accurate as possible in the face of contention.
810		 */
811		mutex_enter(&tod_lock);
812		tsc_saved_tsc = tsc_read();
813		tsc_saved_ts = TODOP_GET(tod_ops);
814		mutex_exit(&tod_lock);
815		/* We only want to do this once. */
816		if (tsc_needs_resume == 0) {
817			if (tsc_delta_onsuspend) {
818				tsc_adjust_delta(tsc_saved_tsc);
819			} else {
820				tsc_adjust_delta(nsec_scale);
821			}
822			tsc_suspend_count++;
823		}
824	}
825
826	invalidate_cache();
827	tsc_needs_resume = 1;
828}
829
830/*
831 * Restore all timestamp state based on the snapshots taken at suspend time.
832 */
833void
834tsc_resume(void)
835{
836	/*
837	 * We only need to (and want to) do this once.  So let the first
838	 * caller handle this (we are locked by the cpu lock), as it
839	 * is preferential that we get the earliest sync.
840	 */
841	if (tsc_needs_resume) {
842		/*
843		 * If using the TSC, adjust the delta based on how long
844		 * we were sleeping (or away).  We also adjust for
845		 * migration and a grown TSC.
846		 */
847		if (tsc_saved_tsc != 0) {
848			timestruc_t	ts;
849			hrtime_t	now, sleep_tsc = 0;
850			int		sleep_sec;
851			extern void	tsc_tick(void);
852			extern uint64_t cpu_freq_hz;
853
854			/* tsc_read() MUST be before TODOP_GET() */
855			mutex_enter(&tod_lock);
856			now = tsc_read();
857			ts = TODOP_GET(tod_ops);
858			mutex_exit(&tod_lock);
859
860			/* Compute seconds of sleep time */
861			sleep_sec = ts.tv_sec - tsc_saved_ts.tv_sec;
862
863			/*
864			 * If the saved sec is less that or equal to
865			 * the current ts, then there is likely a
866			 * problem with the clock.  Assume at least
867			 * one second has passed, so that time goes forward.
868			 */
869			if (sleep_sec <= 0) {
870				sleep_sec = 1;
871			}
872
873			/* How many TSC's should have occured while sleeping */
874			if (tsc_adjust_seconds)
875				sleep_tsc = sleep_sec * cpu_freq_hz;
876
877			/*
878			 * We also want to subtract from the "sleep_tsc"
879			 * the current value of tsc_read(), so that our
880			 * adjustment accounts for the amount of time we
881			 * have been resumed _or_ an adjustment based on
882			 * the fact that we didn't actually power off the
883			 * CPU (migration is another issue, but _should_
884			 * also comply with this calculation).  If the CPU
885			 * never powered off, then:
886			 *    'now == sleep_tsc + saved_tsc'
887			 * and the delta will effectively be "0".
888			 */
889			sleep_tsc -= now;
890			if (tsc_delta_onsuspend) {
891				tsc_adjust_delta(sleep_tsc);
892			} else {
893				tsc_adjust_delta(tsc_saved_tsc + sleep_tsc);
894			}
895			tsc_saved_tsc = 0;
896
897			tsc_tick();
898		}
899		tsc_needs_resume = 0;
900	}
901
902}
903