12428aadPatrick Mooney/*
22428aadPatrick Mooney * This file and its contents are supplied under the terms of the
32428aadPatrick Mooney * Common Development and Distribution License ("CDDL"), version 1.0.
42428aadPatrick Mooney * You may only use this file in accordance with the terms of version
52428aadPatrick Mooney * 1.0 of the CDDL.
62428aadPatrick Mooney *
72428aadPatrick Mooney * A full copy of the text of the CDDL should have accompanied this
82428aadPatrick Mooney * source.  A copy of the CDDL is also available via the Internet at
92428aadPatrick Mooney * http://www.illumos.org/license/CDDL.
102428aadPatrick Mooney */
112428aadPatrick Mooney
122428aadPatrick Mooney/*
132428aadPatrick Mooney * Copyright 2016 Joyent, Inc.
142428aadPatrick Mooney */
152428aadPatrick Mooney
162428aadPatrick Mooney#include <sys/comm_page.h>
172428aadPatrick Mooney#include <sys/tsc.h>
182428aadPatrick Mooney
192428aadPatrick Mooney
202428aadPatrick Mooney/*
212428aadPatrick Mooney * Interrogate if querying the clock via the comm page is possible.
222428aadPatrick Mooney */
232428aadPatrick Mooneyint
242428aadPatrick Mooney__cp_can_gettime(comm_page_t *cp)
252428aadPatrick Mooney{
262428aadPatrick Mooney	switch (cp->cp_tsc_type) {
272428aadPatrick Mooney	case TSC_TSCP:
282428aadPatrick Mooney	case TSC_RDTSC_MFENCE:
292428aadPatrick Mooney	case TSC_RDTSC_LFENCE:
302428aadPatrick Mooney	case TSC_RDTSC_CPUID:
312428aadPatrick Mooney		return (1);
322428aadPatrick Mooney	default:
332428aadPatrick Mooney		break;
342428aadPatrick Mooney	}
352428aadPatrick Mooney	return (0);
362428aadPatrick Mooney}
372428aadPatrick Mooney
382428aadPatrick Mooney#ifdef __amd64
392428aadPatrick Mooney
402428aadPatrick Mooney/*
412428aadPatrick Mooney * The functions used for calculating time (both monotonic and wall-clock) are
422428aadPatrick Mooney * implemented in assembly on amd64.  This is primarily for stack conservation.
432428aadPatrick Mooney */
442428aadPatrick Mooney
452428aadPatrick Mooney#else /* i386 below */
462428aadPatrick Mooney
472428aadPatrick Mooney/*
482428aadPatrick Mooney * ASM-defined functions.
492428aadPatrick Mooney */
502428aadPatrick Mooneyextern hrtime_t __cp_tsc_read(comm_page_t *);
512428aadPatrick Mooney
522428aadPatrick Mooney/*
532428aadPatrick Mooney * These are cloned from TSC and time related code in the kernel.  The should
542428aadPatrick Mooney * be kept in sync in the case that the source values are changed.
552428aadPatrick Mooney */
562428aadPatrick Mooney#define	NSEC_SHIFT	5
572428aadPatrick Mooney#define	ADJ_SHIFT	4
582428aadPatrick Mooney#define	NANOSEC		1000000000LL
592428aadPatrick Mooney
602428aadPatrick Mooney#define	TSC_CONVERT_AND_ADD(tsc, hrt, scale) do {		\
612428aadPatrick Mooney	uint32_t *_l = (uint32_t *)&(tsc); 			\
622428aadPatrick Mooney	uint64_t sc = (uint32_t)(scale);			\
632428aadPatrick Mooney	(hrt) += (uint64_t)(_l[1] * sc) << NSEC_SHIFT;		\
642428aadPatrick Mooney	(hrt) += (uint64_t)(_l[0] * sc) >> (32 - NSEC_SHIFT);	\
652428aadPatrick Mooney} while (0)
662428aadPatrick Mooney
672428aadPatrick Mooney/*
682428aadPatrick Mooney * Userspace version of tsc_gethrtime.
692428aadPatrick Mooney * See: uts/i86pc/os/timestamp.c
702428aadPatrick Mooney */
712428aadPatrick Mooneyhrtime_t
722428aadPatrick Mooney__cp_gethrtime(comm_page_t *cp)
732428aadPatrick Mooney{
742428aadPatrick Mooney	uint32_t old_hres_lock;
752428aadPatrick Mooney	hrtime_t tsc, hrt, tsc_last;
762428aadPatrick Mooney
772428aadPatrick Mooney	/*
782428aadPatrick Mooney	 * Several precautions must be taken when collecting the data necessary
792428aadPatrick Mooney	 * to perform an accurate gethrtime calculation.
802428aadPatrick Mooney	 *
812428aadPatrick Mooney	 * While much of the TSC state stored in the comm page is unchanging
822428aadPatrick Mooney	 * after boot, portions of it are periodically updated during OS ticks.
832428aadPatrick Mooney	 * Changes to hres_lock during the course of the copy indicates a
842428aadPatrick Mooney	 * potentially inconsistent snapshot, necessitating a loop.
852428aadPatrick Mooney	 *
862428aadPatrick Mooney	 * Even more complicated is the handling for TSCs which require sync
872428aadPatrick Mooney	 * offsets between different CPUs.  Since userspace lacks the luxury of
882428aadPatrick Mooney	 * disabling interrupts, a validation loop checking for CPU migrations
892428aadPatrick Mooney	 * is used.  Pathological scheduling could, in theory, "outwit"
902428aadPatrick Mooney	 * this check.  Such a possibility is considered an acceptable risk.
912428aadPatrick Mooney	 *
922428aadPatrick Mooney	 */
932428aadPatrick Mooney	do {
942428aadPatrick Mooney		old_hres_lock = cp->cp_hres_lock;
952428aadPatrick Mooney		tsc_last = cp->cp_tsc_last;
962428aadPatrick Mooney		hrt = cp->cp_tsc_hrtime_base;
972428aadPatrick Mooney		tsc = __cp_tsc_read(cp);
982428aadPatrick Mooney	} while ((old_hres_lock & ~1) != cp->cp_hres_lock);
992428aadPatrick Mooney
1002428aadPatrick Mooney	if (tsc >= tsc_last) {
1012428aadPatrick Mooney		tsc -= tsc_last;
1022428aadPatrick Mooney	} else if (tsc >= tsc_last - (2 * cp->cp_tsc_max_delta)) {
1032428aadPatrick Mooney		tsc = 0;
1042428aadPatrick Mooney	} else if (tsc > cp->cp_tsc_resume_cap) {
1052428aadPatrick Mooney		tsc = cp->cp_tsc_resume_cap;
1062428aadPatrick Mooney	}
1072428aadPatrick Mooney	TSC_CONVERT_AND_ADD(tsc, hrt, cp->cp_nsec_scale);
1082428aadPatrick Mooney
1092428aadPatrick Mooney	return (hrt);
1102428aadPatrick Mooney}
1112428aadPatrick Mooney
1122428aadPatrick Mooney/*
1132428aadPatrick Mooney * Userspace version of pc_gethrestime.
1142428aadPatrick Mooney * See: uts/i86pc/os/machdep.c
1152428aadPatrick Mooney */
1162428aadPatrick Mooneyint
1172428aadPatrick Mooney__cp_clock_gettime_realtime(comm_page_t *cp, timespec_t *tsp)
1182428aadPatrick Mooney{
1192428aadPatrick Mooney	int lock_prev, nslt;
1202428aadPatrick Mooney	timespec_t now;
1212428aadPatrick Mooney	int64_t hres_adj;
1222428aadPatrick Mooney
1232428aadPatrick Mooneyloop:
1242428aadPatrick Mooney	lock_prev = cp->cp_hres_lock;
1252428aadPatrick Mooney	now.tv_sec = cp->cp_hrestime[0];
1262428aadPatrick Mooney	now.tv_nsec = cp->cp_hrestime[1];
1272428aadPatrick Mooney	nslt = (int)(__cp_gethrtime(cp) - cp->cp_hres_last_tick);
1282428aadPatrick Mooney	hres_adj = cp->cp_hrestime_adj;
1292428aadPatrick Mooney	if (nslt < 0) {
1302428aadPatrick Mooney		/*
1312428aadPatrick Mooney		 * Tick came between sampling hrtime and hres_last_tick;
1322428aadPatrick Mooney		 */
1332428aadPatrick Mooney		goto loop;
1342428aadPatrick Mooney	}
1352428aadPatrick Mooney	now.tv_nsec += nslt;
1362428aadPatrick Mooney
1372428aadPatrick Mooney	/*
1382428aadPatrick Mooney	 * Apply hres_adj skew, if needed.
1392428aadPatrick Mooney	 */
1402428aadPatrick Mooney	if (hres_adj > 0) {
1412428aadPatrick Mooney		nslt = (nslt >> ADJ_SHIFT);
1422428aadPatrick Mooney		if (nslt > hres_adj)
1432428aadPatrick Mooney			nslt = (int)hres_adj;
1442428aadPatrick Mooney		now.tv_nsec += nslt;
1452428aadPatrick Mooney	} else if (hres_adj < 0) {
1462428aadPatrick Mooney		nslt = -(nslt >> ADJ_SHIFT);
1472428aadPatrick Mooney		if (nslt < hres_adj)
1482428aadPatrick Mooney			nslt = (int)hres_adj;
1492428aadPatrick Mooney		now.tv_nsec += nslt;
1502428aadPatrick Mooney	}
1512428aadPatrick Mooney
1522428aadPatrick Mooney	/*
1532428aadPatrick Mooney	 * Rope in tv_nsec from any excessive adjustments.
1542428aadPatrick Mooney	 */
1552428aadPatrick Mooney	while ((unsigned long)now.tv_nsec >= NANOSEC) {
1562428aadPatrick Mooney		now.tv_nsec -= NANOSEC;
1572428aadPatrick Mooney		now.tv_sec++;
1582428aadPatrick Mooney	}
1592428aadPatrick Mooney
1602428aadPatrick Mooney	if ((cp->cp_hres_lock & ~1) != lock_prev)
1612428aadPatrick Mooney		goto loop;
1622428aadPatrick Mooney
1632428aadPatrick Mooney	*tsp = now;
1642428aadPatrick Mooney	return (0);
1652428aadPatrick Mooney}
1662428aadPatrick Mooney
1672428aadPatrick Mooney/*
1682428aadPatrick Mooney * The __cp_clock_gettime_monotonic function expects that hrt2ts be present
1692428aadPatrick Mooney * when the code is finally linked.
1702428aadPatrick Mooney * (The amd64 version has no such requirement.)
1712428aadPatrick Mooney */
1722428aadPatrick Mooneyextern void hrt2ts(hrtime_t, timespec_t *);
1732428aadPatrick Mooney
1742428aadPatrick Mooneyint
1752428aadPatrick Mooney__cp_clock_gettime_monotonic(comm_page_t *cp, timespec_t *tsp)
1762428aadPatrick Mooney{
1772428aadPatrick Mooney	hrtime_t hrt;
1782428aadPatrick Mooney
1792428aadPatrick Mooney	hrt = __cp_gethrtime(cp);
1802428aadPatrick Mooney	hrt2ts(hrt, tsp);
1812428aadPatrick Mooney	return (0);
1822428aadPatrick Mooney}
1832428aadPatrick Mooney
1842428aadPatrick Mooney#endif /* __amd64 */
185