12428aadPatrick Mooney/*
22428aadPatrick Mooney * This file and its contents are supplied under the terms of the
32428aadPatrick Mooney * Common Development and Distribution License ("CDDL"), version 1.0.
42428aadPatrick Mooney * You may only use this file in accordance with the terms of version
52428aadPatrick Mooney * 1.0 of the CDDL.
62428aadPatrick Mooney *
72428aadPatrick Mooney * A full copy of the text of the CDDL should have accompanied this
82428aadPatrick Mooney * source.  A copy of the CDDL is also available via the Internet at
92428aadPatrick Mooney * http://www.illumos.org/license/CDDL.
102428aadPatrick Mooney */
112428aadPatrick Mooney
122428aadPatrick Mooney/*
13e121b61Patrick Mooney * Copyright 2019 Joyent, Inc.
142428aadPatrick Mooney */
152428aadPatrick Mooney
162428aadPatrick Mooney#include <sys/comm_page.h>
172428aadPatrick Mooney#include <sys/tsc.h>
182428aadPatrick Mooney
192428aadPatrick Mooney
202428aadPatrick Mooney/*
212428aadPatrick Mooney * Interrogate if querying the clock via the comm page is possible.
222428aadPatrick Mooney */
232428aadPatrick Mooneyint
242428aadPatrick Mooney__cp_can_gettime(comm_page_t *cp)
252428aadPatrick Mooney{
262428aadPatrick Mooney	switch (cp->cp_tsc_type) {
272428aadPatrick Mooney	case TSC_TSCP:
282428aadPatrick Mooney	case TSC_RDTSC_MFENCE:
292428aadPatrick Mooney	case TSC_RDTSC_LFENCE:
302428aadPatrick Mooney	case TSC_RDTSC_CPUID:
312428aadPatrick Mooney		return (1);
322428aadPatrick Mooney	default:
332428aadPatrick Mooney		break;
342428aadPatrick Mooney	}
352428aadPatrick Mooney	return (0);
362428aadPatrick Mooney}
372428aadPatrick Mooney
382428aadPatrick Mooney#ifdef __amd64
392428aadPatrick Mooney
402428aadPatrick Mooney/*
412428aadPatrick Mooney * The functions used for calculating time (both monotonic and wall-clock) are
422428aadPatrick Mooney * implemented in assembly on amd64.  This is primarily for stack conservation.
432428aadPatrick Mooney */
442428aadPatrick Mooney
452428aadPatrick Mooney#else /* i386 below */
462428aadPatrick Mooney
472428aadPatrick Mooney/*
482428aadPatrick Mooney * ASM-defined functions.
492428aadPatrick Mooney */
502428aadPatrick Mooneyextern hrtime_t __cp_tsc_read(comm_page_t *);
51e121b61Patrick Mooneyextern hrtime_t __cp_gethrtime_fasttrap();
522428aadPatrick Mooney
532428aadPatrick Mooney/*
542428aadPatrick Mooney * These are cloned from TSC and time related code in the kernel.  The should
552428aadPatrick Mooney * be kept in sync in the case that the source values are changed.
562428aadPatrick Mooney */
572428aadPatrick Mooney#define	NSEC_SHIFT	5
582428aadPatrick Mooney#define	ADJ_SHIFT	4
592428aadPatrick Mooney#define	NANOSEC		1000000000LL
602428aadPatrick Mooney
612428aadPatrick Mooney#define	TSC_CONVERT_AND_ADD(tsc, hrt, scale) do {		\
62e121b61Patrick Mooney	uint32_t *_l = (uint32_t *)&(tsc);			\
632428aadPatrick Mooney	uint64_t sc = (uint32_t)(scale);			\
642428aadPatrick Mooney	(hrt) += (uint64_t)(_l[1] * sc) << NSEC_SHIFT;		\
652428aadPatrick Mooney	(hrt) += (uint64_t)(_l[0] * sc) >> (32 - NSEC_SHIFT);	\
662428aadPatrick Mooney} while (0)
672428aadPatrick Mooney
682428aadPatrick Mooney/*
692428aadPatrick Mooney * Userspace version of tsc_gethrtime.
702428aadPatrick Mooney * See: uts/i86pc/os/timestamp.c
712428aadPatrick Mooney */
722428aadPatrick Mooneyhrtime_t
732428aadPatrick Mooney__cp_gethrtime(comm_page_t *cp)
742428aadPatrick Mooney{
752428aadPatrick Mooney	uint32_t old_hres_lock;
762428aadPatrick Mooney	hrtime_t tsc, hrt, tsc_last;
772428aadPatrick Mooney
782428aadPatrick Mooney	/*
792428aadPatrick Mooney	 * Several precautions must be taken when collecting the data necessary
802428aadPatrick Mooney	 * to perform an accurate gethrtime calculation.
812428aadPatrick Mooney	 *
822428aadPatrick Mooney	 * While much of the TSC state stored in the comm page is unchanging
832428aadPatrick Mooney	 * after boot, portions of it are periodically updated during OS ticks.
842428aadPatrick Mooney	 * Changes to hres_lock during the course of the copy indicates a
852428aadPatrick Mooney	 * potentially inconsistent snapshot, necessitating a loop.
862428aadPatrick Mooney	 *
872428aadPatrick Mooney	 * Even more complicated is the handling for TSCs which require sync
882428aadPatrick Mooney	 * offsets between different CPUs.  Since userspace lacks the luxury of
892428aadPatrick Mooney	 * disabling interrupts, a validation loop checking for CPU migrations
902428aadPatrick Mooney	 * is used.  Pathological scheduling could, in theory, "outwit"
912428aadPatrick Mooney	 * this check.  Such a possibility is considered an acceptable risk.
922428aadPatrick Mooney	 *
932428aadPatrick Mooney	 */
942428aadPatrick Mooney	do {
952428aadPatrick Mooney		old_hres_lock = cp->cp_hres_lock;
962428aadPatrick Mooney		tsc_last = cp->cp_tsc_last;
972428aadPatrick Mooney		hrt = cp->cp_tsc_hrtime_base;
982428aadPatrick Mooney		tsc = __cp_tsc_read(cp);
99e121b61Patrick Mooney
100e121b61Patrick Mooney		/*
101e121b61Patrick Mooney		 * A TSC reading of 0 indicates the special case of an error
102e121b61Patrick Mooney		 * bail-out.  Rely on the fasttrap to supply an hrtime value.
103e121b61Patrick Mooney		 */
104e121b61Patrick Mooney		if (tsc == 0) {
105e121b61Patrick Mooney			return (__cp_gethrtime_fasttrap());
106e121b61Patrick Mooney		}
1072428aadPatrick Mooney	} while ((old_hres_lock & ~1) != cp->cp_hres_lock);
1082428aadPatrick Mooney
1092428aadPatrick Mooney	if (tsc >= tsc_last) {
1102428aadPatrick Mooney		tsc -= tsc_last;
1112428aadPatrick Mooney	} else if (tsc >= tsc_last - (2 * cp->cp_tsc_max_delta)) {
1122428aadPatrick Mooney		tsc = 0;
1132428aadPatrick Mooney	} else if (tsc > cp->cp_tsc_resume_cap) {
1142428aadPatrick Mooney		tsc = cp->cp_tsc_resume_cap;
1152428aadPatrick Mooney	}
1162428aadPatrick Mooney	TSC_CONVERT_AND_ADD(tsc, hrt, cp->cp_nsec_scale);
1172428aadPatrick Mooney
1182428aadPatrick Mooney	return (hrt);
1192428aadPatrick Mooney}
1202428aadPatrick Mooney
1212428aadPatrick Mooney/*
1222428aadPatrick Mooney * Userspace version of pc_gethrestime.
1232428aadPatrick Mooney * See: uts/i86pc/os/machdep.c
1242428aadPatrick Mooney */
1252428aadPatrick Mooneyint
1262428aadPatrick Mooney__cp_clock_gettime_realtime(comm_page_t *cp, timespec_t *tsp)
1272428aadPatrick Mooney{
1282428aadPatrick Mooney	int lock_prev, nslt;
1292428aadPatrick Mooney	timespec_t now;
1302428aadPatrick Mooney	int64_t hres_adj;
1312428aadPatrick Mooney
1322428aadPatrick Mooneyloop:
1332428aadPatrick Mooney	lock_prev = cp->cp_hres_lock;
1342428aadPatrick Mooney	now.tv_sec = cp->cp_hrestime[0];
1352428aadPatrick Mooney	now.tv_nsec = cp->cp_hrestime[1];
1362428aadPatrick Mooney	nslt = (int)(__cp_gethrtime(cp) - cp->cp_hres_last_tick);
1372428aadPatrick Mooney	hres_adj = cp->cp_hrestime_adj;
1382428aadPatrick Mooney	if (nslt < 0) {
1392428aadPatrick Mooney		/*
1402428aadPatrick Mooney		 * Tick came between sampling hrtime and hres_last_tick;
1412428aadPatrick Mooney		 */
1422428aadPatrick Mooney		goto loop;
1432428aadPatrick Mooney	}
1442428aadPatrick Mooney	now.tv_nsec += nslt;
1452428aadPatrick Mooney
1462428aadPatrick Mooney	/*
1472428aadPatrick Mooney	 * Apply hres_adj skew, if needed.
1482428aadPatrick Mooney	 */
1492428aadPatrick Mooney	if (hres_adj > 0) {
1502428aadPatrick Mooney		nslt = (nslt >> ADJ_SHIFT);
1512428aadPatrick Mooney		if (nslt > hres_adj)
1522428aadPatrick Mooney			nslt = (int)hres_adj;
1532428aadPatrick Mooney		now.tv_nsec += nslt;
1542428aadPatrick Mooney	} else if (hres_adj < 0) {
1552428aadPatrick Mooney		nslt = -(nslt >> ADJ_SHIFT);
1562428aadPatrick Mooney		if (nslt < hres_adj)
1572428aadPatrick Mooney			nslt = (int)hres_adj;
1582428aadPatrick Mooney		now.tv_nsec += nslt;
1592428aadPatrick Mooney	}
1602428aadPatrick Mooney
1612428aadPatrick Mooney	/*
1622428aadPatrick Mooney	 * Rope in tv_nsec from any excessive adjustments.
1632428aadPatrick Mooney	 */
1642428aadPatrick Mooney	while ((unsigned long)now.tv_nsec >= NANOSEC) {
1652428aadPatrick Mooney		now.tv_nsec -= NANOSEC;
1662428aadPatrick Mooney		now.tv_sec++;
1672428aadPatrick Mooney	}
1682428aadPatrick Mooney
1692428aadPatrick Mooney	if ((cp->cp_hres_lock & ~1) != lock_prev)
1702428aadPatrick Mooney		goto loop;
1712428aadPatrick Mooney
1722428aadPatrick Mooney	*tsp = now;
1732428aadPatrick Mooney	return (0);
1742428aadPatrick Mooney}
1752428aadPatrick Mooney
1762428aadPatrick Mooney/*
1772428aadPatrick Mooney * The __cp_clock_gettime_monotonic function expects that hrt2ts be present
1782428aadPatrick Mooney * when the code is finally linked.
1792428aadPatrick Mooney * (The amd64 version has no such requirement.)
1802428aadPatrick Mooney */
1812428aadPatrick Mooneyextern void hrt2ts(hrtime_t, timespec_t *);
1822428aadPatrick Mooney
1832428aadPatrick Mooneyint
1842428aadPatrick Mooney__cp_clock_gettime_monotonic(comm_page_t *cp, timespec_t *tsp)
1852428aadPatrick Mooney{
1862428aadPatrick Mooney	hrtime_t hrt;
1872428aadPatrick Mooney
1882428aadPatrick Mooney	hrt = __cp_gethrtime(cp);
1892428aadPatrick Mooney	hrt2ts(hrt, tsp);
1902428aadPatrick Mooney	return (0);
1912428aadPatrick Mooney}
1922428aadPatrick Mooney
1932428aadPatrick Mooney#endif /* __amd64 */
194