12428aad8SPatrick Mooney /*
22428aad8SPatrick Mooney  * This file and its contents are supplied under the terms of the
32428aad8SPatrick Mooney  * Common Development and Distribution License ("CDDL"), version 1.0.
42428aad8SPatrick Mooney  * You may only use this file in accordance with the terms of version
52428aad8SPatrick Mooney  * 1.0 of the CDDL.
62428aad8SPatrick Mooney  *
72428aad8SPatrick Mooney  * A full copy of the text of the CDDL should have accompanied this
82428aad8SPatrick Mooney  * source.  A copy of the CDDL is also available via the Internet at
92428aad8SPatrick Mooney  * http://www.illumos.org/license/CDDL.
102428aad8SPatrick Mooney  */
112428aad8SPatrick Mooney 
122428aad8SPatrick Mooney /*
13*e121b61fSPatrick Mooney  * Copyright 2019 Joyent, Inc.
142428aad8SPatrick Mooney  */
152428aad8SPatrick Mooney 
162428aad8SPatrick Mooney #include <sys/comm_page.h>
172428aad8SPatrick Mooney #include <sys/tsc.h>
182428aad8SPatrick Mooney 
192428aad8SPatrick Mooney 
202428aad8SPatrick Mooney /*
212428aad8SPatrick Mooney  * Interrogate if querying the clock via the comm page is possible.
222428aad8SPatrick Mooney  */
232428aad8SPatrick Mooney int
__cp_can_gettime(comm_page_t * cp)242428aad8SPatrick Mooney __cp_can_gettime(comm_page_t *cp)
252428aad8SPatrick Mooney {
262428aad8SPatrick Mooney 	switch (cp->cp_tsc_type) {
272428aad8SPatrick Mooney 	case TSC_TSCP:
282428aad8SPatrick Mooney 	case TSC_RDTSC_LFENCE:
292428aad8SPatrick Mooney 	case TSC_RDTSC_CPUID:
302428aad8SPatrick Mooney 		return (1);
312428aad8SPatrick Mooney 	default:
322428aad8SPatrick Mooney 		break;
332428aad8SPatrick Mooney 	}
342428aad8SPatrick Mooney 	return (0);
352428aad8SPatrick Mooney }
362428aad8SPatrick Mooney 
372428aad8SPatrick Mooney #ifdef __amd64
382428aad8SPatrick Mooney 
392428aad8SPatrick Mooney /*
402428aad8SPatrick Mooney  * The functions used for calculating time (both monotonic and wall-clock) are
412428aad8SPatrick Mooney  * implemented in assembly on amd64.  This is primarily for stack conservation.
422428aad8SPatrick Mooney  */
432428aad8SPatrick Mooney 
442428aad8SPatrick Mooney #else /* i386 below */
452428aad8SPatrick Mooney 
462428aad8SPatrick Mooney /*
472428aad8SPatrick Mooney  * ASM-defined functions.
482428aad8SPatrick Mooney  */
492428aad8SPatrick Mooney extern hrtime_t __cp_tsc_read(comm_page_t *);
50*e121b61fSPatrick Mooney extern hrtime_t __cp_gethrtime_fasttrap();
512428aad8SPatrick Mooney 
522428aad8SPatrick Mooney /*
532428aad8SPatrick Mooney  * These are cloned from TSC and time related code in the kernel.  The should
542428aad8SPatrick Mooney  * be kept in sync in the case that the source values are changed.
552428aad8SPatrick Mooney  */
562428aad8SPatrick Mooney #define	NSEC_SHIFT	5
572428aad8SPatrick Mooney #define	ADJ_SHIFT	4
582428aad8SPatrick Mooney #define	NANOSEC		1000000000LL
592428aad8SPatrick Mooney 
602428aad8SPatrick Mooney #define	TSC_CONVERT_AND_ADD(tsc, hrt, scale) do {		\
61*e121b61fSPatrick Mooney 	uint32_t *_l = (uint32_t *)&(tsc);			\
622428aad8SPatrick Mooney 	uint64_t sc = (uint32_t)(scale);			\
632428aad8SPatrick Mooney 	(hrt) += (uint64_t)(_l[1] * sc) << NSEC_SHIFT;		\
642428aad8SPatrick Mooney 	(hrt) += (uint64_t)(_l[0] * sc) >> (32 - NSEC_SHIFT);	\
652428aad8SPatrick Mooney } while (0)
662428aad8SPatrick Mooney 
672428aad8SPatrick Mooney /*
682428aad8SPatrick Mooney  * Userspace version of tsc_gethrtime.
692428aad8SPatrick Mooney  * See: uts/i86pc/os/timestamp.c
702428aad8SPatrick Mooney  */
712428aad8SPatrick Mooney hrtime_t
__cp_gethrtime(comm_page_t * cp)722428aad8SPatrick Mooney __cp_gethrtime(comm_page_t *cp)
732428aad8SPatrick Mooney {
742428aad8SPatrick Mooney 	uint32_t old_hres_lock;
752428aad8SPatrick Mooney 	hrtime_t tsc, hrt, tsc_last;
762428aad8SPatrick Mooney 
772428aad8SPatrick Mooney 	/*
782428aad8SPatrick Mooney 	 * Several precautions must be taken when collecting the data necessary
792428aad8SPatrick Mooney 	 * to perform an accurate gethrtime calculation.
802428aad8SPatrick Mooney 	 *
812428aad8SPatrick Mooney 	 * While much of the TSC state stored in the comm page is unchanging
822428aad8SPatrick Mooney 	 * after boot, portions of it are periodically updated during OS ticks.
832428aad8SPatrick Mooney 	 * Changes to hres_lock during the course of the copy indicates a
842428aad8SPatrick Mooney 	 * potentially inconsistent snapshot, necessitating a loop.
852428aad8SPatrick Mooney 	 *
862428aad8SPatrick Mooney 	 * Even more complicated is the handling for TSCs which require sync
872428aad8SPatrick Mooney 	 * offsets between different CPUs.  Since userspace lacks the luxury of
882428aad8SPatrick Mooney 	 * disabling interrupts, a validation loop checking for CPU migrations
892428aad8SPatrick Mooney 	 * is used.  Pathological scheduling could, in theory, "outwit"
902428aad8SPatrick Mooney 	 * this check.  Such a possibility is considered an acceptable risk.
912428aad8SPatrick Mooney 	 *
922428aad8SPatrick Mooney 	 */
932428aad8SPatrick Mooney 	do {
942428aad8SPatrick Mooney 		old_hres_lock = cp->cp_hres_lock;
952428aad8SPatrick Mooney 		tsc_last = cp->cp_tsc_last;
962428aad8SPatrick Mooney 		hrt = cp->cp_tsc_hrtime_base;
972428aad8SPatrick Mooney 		tsc = __cp_tsc_read(cp);
98*e121b61fSPatrick Mooney 
99*e121b61fSPatrick Mooney 		/*
100*e121b61fSPatrick Mooney 		 * A TSC reading of 0 indicates the special case of an error
101*e121b61fSPatrick Mooney 		 * bail-out.  Rely on the fasttrap to supply an hrtime value.
102*e121b61fSPatrick Mooney 		 */
103*e121b61fSPatrick Mooney 		if (tsc == 0) {
104*e121b61fSPatrick Mooney 			return (__cp_gethrtime_fasttrap());
105*e121b61fSPatrick Mooney 		}
1062428aad8SPatrick Mooney 	} while ((old_hres_lock & ~1) != cp->cp_hres_lock);
1072428aad8SPatrick Mooney 
1082428aad8SPatrick Mooney 	if (tsc >= tsc_last) {
1092428aad8SPatrick Mooney 		tsc -= tsc_last;
1102428aad8SPatrick Mooney 	} else if (tsc >= tsc_last - (2 * cp->cp_tsc_max_delta)) {
1112428aad8SPatrick Mooney 		tsc = 0;
1122428aad8SPatrick Mooney 	} else if (tsc > cp->cp_tsc_resume_cap) {
1132428aad8SPatrick Mooney 		tsc = cp->cp_tsc_resume_cap;
1142428aad8SPatrick Mooney 	}
1152428aad8SPatrick Mooney 	TSC_CONVERT_AND_ADD(tsc, hrt, cp->cp_nsec_scale);
1162428aad8SPatrick Mooney 
1172428aad8SPatrick Mooney 	return (hrt);
1182428aad8SPatrick Mooney }
1192428aad8SPatrick Mooney 
1202428aad8SPatrick Mooney /*
1212428aad8SPatrick Mooney  * Userspace version of pc_gethrestime.
1222428aad8SPatrick Mooney  * See: uts/i86pc/os/machdep.c
1232428aad8SPatrick Mooney  */
1242428aad8SPatrick Mooney int
__cp_clock_gettime_realtime(comm_page_t * cp,timespec_t * tsp)1252428aad8SPatrick Mooney __cp_clock_gettime_realtime(comm_page_t *cp, timespec_t *tsp)
1262428aad8SPatrick Mooney {
1272428aad8SPatrick Mooney 	int lock_prev, nslt;
1282428aad8SPatrick Mooney 	timespec_t now;
1292428aad8SPatrick Mooney 	int64_t hres_adj;
1302428aad8SPatrick Mooney 
1312428aad8SPatrick Mooney loop:
1322428aad8SPatrick Mooney 	lock_prev = cp->cp_hres_lock;
1332428aad8SPatrick Mooney 	now.tv_sec = cp->cp_hrestime[0];
1342428aad8SPatrick Mooney 	now.tv_nsec = cp->cp_hrestime[1];
1352428aad8SPatrick Mooney 	nslt = (int)(__cp_gethrtime(cp) - cp->cp_hres_last_tick);
1362428aad8SPatrick Mooney 	hres_adj = cp->cp_hrestime_adj;
1372428aad8SPatrick Mooney 	if (nslt < 0) {
1382428aad8SPatrick Mooney 		/*
1392428aad8SPatrick Mooney 		 * Tick came between sampling hrtime and hres_last_tick;
1402428aad8SPatrick Mooney 		 */
1412428aad8SPatrick Mooney 		goto loop;
1422428aad8SPatrick Mooney 	}
1432428aad8SPatrick Mooney 	now.tv_nsec += nslt;
1442428aad8SPatrick Mooney 
1452428aad8SPatrick Mooney 	/*
1462428aad8SPatrick Mooney 	 * Apply hres_adj skew, if needed.
1472428aad8SPatrick Mooney 	 */
1482428aad8SPatrick Mooney 	if (hres_adj > 0) {
1492428aad8SPatrick Mooney 		nslt = (nslt >> ADJ_SHIFT);
1502428aad8SPatrick Mooney 		if (nslt > hres_adj)
1512428aad8SPatrick Mooney 			nslt = (int)hres_adj;
1522428aad8SPatrick Mooney 		now.tv_nsec += nslt;
1532428aad8SPatrick Mooney 	} else if (hres_adj < 0) {
1542428aad8SPatrick Mooney 		nslt = -(nslt >> ADJ_SHIFT);
1552428aad8SPatrick Mooney 		if (nslt < hres_adj)
1562428aad8SPatrick Mooney 			nslt = (int)hres_adj;
1572428aad8SPatrick Mooney 		now.tv_nsec += nslt;
1582428aad8SPatrick Mooney 	}
1592428aad8SPatrick Mooney 
1602428aad8SPatrick Mooney 	/*
1612428aad8SPatrick Mooney 	 * Rope in tv_nsec from any excessive adjustments.
1622428aad8SPatrick Mooney 	 */
1632428aad8SPatrick Mooney 	while ((unsigned long)now.tv_nsec >= NANOSEC) {
1642428aad8SPatrick Mooney 		now.tv_nsec -= NANOSEC;
1652428aad8SPatrick Mooney 		now.tv_sec++;
1662428aad8SPatrick Mooney 	}
1672428aad8SPatrick Mooney 
1682428aad8SPatrick Mooney 	if ((cp->cp_hres_lock & ~1) != lock_prev)
1692428aad8SPatrick Mooney 		goto loop;
1702428aad8SPatrick Mooney 
1712428aad8SPatrick Mooney 	*tsp = now;
1722428aad8SPatrick Mooney 	return (0);
1732428aad8SPatrick Mooney }
1742428aad8SPatrick Mooney 
1752428aad8SPatrick Mooney /*
1762428aad8SPatrick Mooney  * The __cp_clock_gettime_monotonic function expects that hrt2ts be present
1772428aad8SPatrick Mooney  * when the code is finally linked.
1782428aad8SPatrick Mooney  * (The amd64 version has no such requirement.)
1792428aad8SPatrick Mooney  */
1802428aad8SPatrick Mooney extern void hrt2ts(hrtime_t, timespec_t *);
1812428aad8SPatrick Mooney 
1822428aad8SPatrick Mooney int
__cp_clock_gettime_monotonic(comm_page_t * cp,timespec_t * tsp)1832428aad8SPatrick Mooney __cp_clock_gettime_monotonic(comm_page_t *cp, timespec_t *tsp)
1842428aad8SPatrick Mooney {
1852428aad8SPatrick Mooney 	hrtime_t hrt;
1862428aad8SPatrick Mooney 
1872428aad8SPatrick Mooney 	hrt = __cp_gethrtime(cp);
1882428aad8SPatrick Mooney 	hrt2ts(hrt, tsp);
1892428aad8SPatrick Mooney 	return (0);
1902428aad8SPatrick Mooney }
1912428aad8SPatrick Mooney 
1922428aad8SPatrick Mooney #endif /* __amd64 */
193