1bf21cd93STycho Nightingale /*
2bf21cd93STycho Nightingale  * Copyright (c) 2004 Poul-Henning Kamp
3bf21cd93STycho Nightingale  * All rights reserved.
4bf21cd93STycho Nightingale  *
5bf21cd93STycho Nightingale  * Redistribution and use in source and binary forms, with or without
6bf21cd93STycho Nightingale  * modification, are permitted provided that the following conditions
7bf21cd93STycho Nightingale  * are met:
8bf21cd93STycho Nightingale  * 1. Redistributions of source code must retain the above copyright
9bf21cd93STycho Nightingale  *    notice, this list of conditions and the following disclaimer.
10bf21cd93STycho Nightingale  * 2. Redistributions in binary form must reproduce the above copyright
11bf21cd93STycho Nightingale  *    notice, this list of conditions and the following disclaimer in the
12bf21cd93STycho Nightingale  *    documentation and/or other materials provided with the distribution.
13bf21cd93STycho Nightingale  *
14bf21cd93STycho Nightingale  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15bf21cd93STycho Nightingale  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16bf21cd93STycho Nightingale  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17bf21cd93STycho Nightingale  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18bf21cd93STycho Nightingale  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19bf21cd93STycho Nightingale  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20bf21cd93STycho Nightingale  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21bf21cd93STycho Nightingale  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22bf21cd93STycho Nightingale  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23bf21cd93STycho Nightingale  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24bf21cd93STycho Nightingale  * SUCH DAMAGE.
25bf21cd93STycho Nightingale  *
26bf21cd93STycho Nightingale  * $FreeBSD: head/sys/kern/subr_unit.c 255057 2013-08-30 07:37:45Z kib $
27bf21cd93STycho Nightingale  */
28bf21cd93STycho Nightingale /*
29bf21cd93STycho Nightingale  * This file and its contents are supplied under the terms of the
30bf21cd93STycho Nightingale  * Common Development and Distribution License ("CDDL"), version 1.0.
31bf21cd93STycho Nightingale  * You may only use this file in accordance with the terms of version
32bf21cd93STycho Nightingale  * 1.0 of the CDDL.
33bf21cd93STycho Nightingale  *
34bf21cd93STycho Nightingale  * A full copy of the text of the CDDL should have accompanied this
35bf21cd93STycho Nightingale  * source.  A copy of the CDDL is also available via the Internet at
36bf21cd93STycho Nightingale  * http://www.illumos.org/license/CDDL.
37bf21cd93STycho Nightingale  *
38bf21cd93STycho Nightingale  * Copyright 2014 Pluribus Networks Inc.
394c87aefeSPatrick Mooney  * Copyright 2019 Joyent, Inc.
40ee8ae3faSPatrick Mooney  * Copyright 2020 Oxide Computer Company
41bf21cd93STycho Nightingale  */
42bf21cd93STycho Nightingale 
43bf21cd93STycho Nightingale #include <sys/types.h>
44bf21cd93STycho Nightingale #include <sys/archsystm.h>
45bf21cd93STycho Nightingale #include <sys/cpuset.h>
46bf21cd93STycho Nightingale #include <sys/fp.h>
47*8130f8e1SPatrick Mooney #include <sys/kmem.h>
48bf21cd93STycho Nightingale #include <sys/queue.h>
49bf21cd93STycho Nightingale #include <sys/spl.h>
50bf21cd93STycho Nightingale #include <sys/systm.h>
514c87aefeSPatrick Mooney #include <sys/ddidmareq.h>
524c87aefeSPatrick Mooney #include <sys/id_space.h>
534c87aefeSPatrick Mooney #include <sys/psm_defs.h>
544c87aefeSPatrick Mooney #include <sys/smp_impldefs.h>
554c87aefeSPatrick Mooney #include <sys/modhash.h>
564c87aefeSPatrick Mooney #include <sys/hma.h>
57bf21cd93STycho Nightingale 
58eb9a1df2SHans Rosenfeld #include <sys/x86_archext.h>
59eb9a1df2SHans Rosenfeld 
60bf21cd93STycho Nightingale #include <machine/cpufunc.h>
61bf21cd93STycho Nightingale #include <machine/md_var.h>
62bf21cd93STycho Nightingale #include <machine/specialreg.h>
63bf21cd93STycho Nightingale #include <machine/vmm.h>
64cf409e3fSDan Cross #include <machine/vmparam.h>
65bf21cd93STycho Nightingale #include <sys/vmm_impl.h>
664c87aefeSPatrick Mooney #include <sys/kernel.h>
67bf21cd93STycho Nightingale 
68bf21cd93STycho Nightingale #include <vm/as.h>
69bf21cd93STycho Nightingale #include <vm/seg_kmem.h>
70bf21cd93STycho Nightingale 
714c87aefeSPatrick Mooney SET_DECLARE(sysinit_set, struct sysinit);
724c87aefeSPatrick Mooney 
734c87aefeSPatrick Mooney void
sysinit(void)744c87aefeSPatrick Mooney sysinit(void)
754c87aefeSPatrick Mooney {
764c87aefeSPatrick Mooney 	struct sysinit **si;
774c87aefeSPatrick Mooney 
784c87aefeSPatrick Mooney 	SET_FOREACH(si, sysinit_set)
794c87aefeSPatrick Mooney 		(*si)->func((*si)->data);
804c87aefeSPatrick Mooney }
814c87aefeSPatrick Mooney 
822699b94cSPatrick Mooney uint8_t const bin2bcd_data[] = {
834c87aefeSPatrick Mooney 	0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09,
844c87aefeSPatrick Mooney 	0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19,
854c87aefeSPatrick Mooney 	0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29,
864c87aefeSPatrick Mooney 	0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39,
874c87aefeSPatrick Mooney 	0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49,
884c87aefeSPatrick Mooney 	0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59,
894c87aefeSPatrick Mooney 	0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69,
904c87aefeSPatrick Mooney 	0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79,
914c87aefeSPatrick Mooney 	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89,
924c87aefeSPatrick Mooney 	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99
934c87aefeSPatrick Mooney };
944c87aefeSPatrick Mooney 
95eb9a1df2SHans Rosenfeld void
invalidate_cache_all(void)960153d828SPatrick Mooney invalidate_cache_all(void)
97eb9a1df2SHans Rosenfeld {
98eb9a1df2SHans Rosenfeld 	cpuset_t cpuset;
99eb9a1df2SHans Rosenfeld 
100eb9a1df2SHans Rosenfeld 	kpreempt_disable();
101eb9a1df2SHans Rosenfeld 	cpuset_all_but(&cpuset, CPU->cpu_id);
102eb9a1df2SHans Rosenfeld 	xc_call((xc_arg_t)NULL, (xc_arg_t)NULL, (xc_arg_t)NULL,
103eb9a1df2SHans Rosenfeld 	    CPUSET2BV(cpuset), (xc_func_t)invalidate_cache);
104eb9a1df2SHans Rosenfeld 	invalidate_cache();
105eb9a1df2SHans Rosenfeld 	kpreempt_enable();
106eb9a1df2SHans Rosenfeld }
107eb9a1df2SHans Rosenfeld 
108bf21cd93STycho Nightingale vm_paddr_t
vtophys(void * va)1090153d828SPatrick Mooney vtophys(void *va)
110bf21cd93STycho Nightingale {
111bf21cd93STycho Nightingale 	pfn_t	pfn;
112bf21cd93STycho Nightingale 
1134c87aefeSPatrick Mooney 	/*
1144c87aefeSPatrick Mooney 	 * Since hat_getpfnum() may block on an htable mutex, this is not at
1154c87aefeSPatrick Mooney 	 * all safe to run from a critical_enter/kpreempt_disable context.
1164c87aefeSPatrick Mooney 	 * The FreeBSD analog does not have the same locking constraints, so
1174c87aefeSPatrick Mooney 	 * close attention must be paid wherever this is called.
1184c87aefeSPatrick Mooney 	 */
1194c87aefeSPatrick Mooney 	ASSERT(curthread->t_preempt == 0);
1204c87aefeSPatrick Mooney 
121bf21cd93STycho Nightingale 	pfn = hat_getpfnum(kas.a_hat, (caddr_t)va);
122bf21cd93STycho Nightingale 	ASSERT(pfn != PFN_INVALID);
123bf21cd93STycho Nightingale 	return (pfn << PAGE_SHIFT) | ((uintptr_t)va & PAGE_MASK);
124bf21cd93STycho Nightingale }
125bf21cd93STycho Nightingale 
126bf21cd93STycho Nightingale int
cpusetobj_ffs(const cpuset_t * set)127bf21cd93STycho Nightingale cpusetobj_ffs(const cpuset_t *set)
128bf21cd93STycho Nightingale {
1294c87aefeSPatrick Mooney 	uint_t large, small;
130bf21cd93STycho Nightingale 
1314c87aefeSPatrick Mooney 	/*
1324c87aefeSPatrick Mooney 	 * Rather than reaching into the cpuset_t ourselves, leave that task to
1334c87aefeSPatrick Mooney 	 * cpuset_bounds().  The simplicity is worth the extra wasted work to
1344c87aefeSPatrick Mooney 	 * find the upper bound.
1354c87aefeSPatrick Mooney 	 */
1364c87aefeSPatrick Mooney 	cpuset_bounds(set, &small, &large);
137bf21cd93STycho Nightingale 
1384c87aefeSPatrick Mooney 	if (small == CPUSET_NOTINSET) {
1394c87aefeSPatrick Mooney 		/* The FreeBSD version returns 0 if it find nothing */
1404c87aefeSPatrick Mooney 		return (0);
1414c87aefeSPatrick Mooney 	}
142bf21cd93STycho Nightingale 
1434c87aefeSPatrick Mooney 	ASSERT3U(small, <=, INT_MAX);
144bf21cd93STycho Nightingale 
1454c87aefeSPatrick Mooney 	/* Least significant bit index starts at 1 for valid results */
1464c87aefeSPatrick Mooney 	return (small + 1);
147bf21cd93STycho Nightingale }
148bf21cd93STycho Nightingale 
149*8130f8e1SPatrick Mooney struct vmm_ptp_item {
150*8130f8e1SPatrick Mooney 	void *vpi_vaddr;
151bf21cd93STycho Nightingale };
152*8130f8e1SPatrick Mooney static kmutex_t vmm_ptp_lock;
1534c87aefeSPatrick Mooney 
154*8130f8e1SPatrick Mooney static mod_hash_t *vmm_ptp_hash;
155*8130f8e1SPatrick Mooney uint_t vmm_ptp_hash_nchains = 16381;
156*8130f8e1SPatrick Mooney uint_t vmm_ptp_hash_size = PAGESIZE;
1574c87aefeSPatrick Mooney 
1584c87aefeSPatrick Mooney static void
vmm_ptp_hash_valdtor(mod_hash_val_t val)159*8130f8e1SPatrick Mooney vmm_ptp_hash_valdtor(mod_hash_val_t val)
1604c87aefeSPatrick Mooney {
161*8130f8e1SPatrick Mooney 	struct vmm_ptp_item *i = (struct vmm_ptp_item *)val;
1624c87aefeSPatrick Mooney 
163*8130f8e1SPatrick Mooney 	kmem_free(i->vpi_vaddr, PAGE_SIZE);
164*8130f8e1SPatrick Mooney 	kmem_free(i, sizeof (*i));
1654c87aefeSPatrick Mooney }
1664c87aefeSPatrick Mooney 
1674c87aefeSPatrick Mooney static void
vmm_ptp_init(void)168*8130f8e1SPatrick Mooney vmm_ptp_init(void)
1694c87aefeSPatrick Mooney {
170*8130f8e1SPatrick Mooney 	vmm_ptp_hash = mod_hash_create_ptrhash("vmm_ptp_hash",
171*8130f8e1SPatrick Mooney 	    vmm_ptp_hash_nchains, vmm_ptp_hash_valdtor, vmm_ptp_hash_size);
1724c87aefeSPatrick Mooney 
173*8130f8e1SPatrick Mooney 	VERIFY(vmm_ptp_hash != NULL);
1744c87aefeSPatrick Mooney }
1754c87aefeSPatrick Mooney 
1764c87aefeSPatrick Mooney static uint_t
vmm_ptp_check(mod_hash_key_t key,mod_hash_val_t * val,void * unused)177*8130f8e1SPatrick Mooney vmm_ptp_check(mod_hash_key_t key, mod_hash_val_t *val, void *unused)
1784c87aefeSPatrick Mooney {
179*8130f8e1SPatrick Mooney 	struct vmm_ptp_item *i = (struct vmm_ptp_item *)val;
1804c87aefeSPatrick Mooney 
181*8130f8e1SPatrick Mooney 	cmn_err(CE_PANIC, "!vmm_ptp_check: hash not empty: %p", i->vpi_vaddr);
1824c87aefeSPatrick Mooney 
1834c87aefeSPatrick Mooney 	return (MH_WALK_TERMINATE);
1844c87aefeSPatrick Mooney }
1854c87aefeSPatrick Mooney 
1864c87aefeSPatrick Mooney static void
vmm_ptp_cleanup(void)187*8130f8e1SPatrick Mooney vmm_ptp_cleanup(void)
1884c87aefeSPatrick Mooney {
189*8130f8e1SPatrick Mooney 	mod_hash_walk(vmm_ptp_hash, vmm_ptp_check, NULL);
190*8130f8e1SPatrick Mooney 	mod_hash_destroy_ptrhash(vmm_ptp_hash);
1914c87aefeSPatrick Mooney }
192bf21cd93STycho Nightingale 
193*8130f8e1SPatrick Mooney /*
194*8130f8e1SPatrick Mooney  * The logic in VT-d uses both kernel-virtual and direct-mapped addresses when
195*8130f8e1SPatrick Mooney  * freeing PTP pages.  Until the consuming code is improved to better track the
196*8130f8e1SPatrick Mooney  * pages it allocates, we keep the kernel-virtual addresses to those pages in a
197*8130f8e1SPatrick Mooney  * hash table for when they are freed.
198*8130f8e1SPatrick Mooney  */
199bf21cd93STycho Nightingale void *
vmm_ptp_alloc(void)200*8130f8e1SPatrick Mooney vmm_ptp_alloc(void)
201bf21cd93STycho Nightingale {
202*8130f8e1SPatrick Mooney 	void *p;
203*8130f8e1SPatrick Mooney 	struct vmm_ptp_item *i;
204bf21cd93STycho Nightingale 
205*8130f8e1SPatrick Mooney 	p = kmem_zalloc(PAGE_SIZE, KM_SLEEP);
206*8130f8e1SPatrick Mooney 	i = kmem_alloc(sizeof (struct vmm_ptp_item), KM_SLEEP);
207*8130f8e1SPatrick Mooney 	i->vpi_vaddr = p;
2084c87aefeSPatrick Mooney 
209*8130f8e1SPatrick Mooney 	mutex_enter(&vmm_ptp_lock);
210*8130f8e1SPatrick Mooney 	VERIFY(mod_hash_insert(vmm_ptp_hash,
2114c87aefeSPatrick Mooney 	    (mod_hash_key_t)PHYS_TO_DMAP(vtophys(p)), (mod_hash_val_t)i) == 0);
212*8130f8e1SPatrick Mooney 	mutex_exit(&vmm_ptp_lock);
213bf21cd93STycho Nightingale 
214bf21cd93STycho Nightingale 	return (p);
215bf21cd93STycho Nightingale }
216bf21cd93STycho Nightingale 
217bf21cd93STycho Nightingale void
vmm_ptp_free(void * addr)218*8130f8e1SPatrick Mooney vmm_ptp_free(void *addr)
219bf21cd93STycho Nightingale {
220*8130f8e1SPatrick Mooney 	mutex_enter(&vmm_ptp_lock);
221*8130f8e1SPatrick Mooney 	VERIFY(mod_hash_destroy(vmm_ptp_hash,
2224c87aefeSPatrick Mooney 	    (mod_hash_key_t)PHYS_TO_DMAP(vtophys(addr))) == 0);
223*8130f8e1SPatrick Mooney 	mutex_exit(&vmm_ptp_lock);
2244c87aefeSPatrick Mooney }
2254c87aefeSPatrick Mooney 
226*8130f8e1SPatrick Mooney /* Reach into i86pc/os/ddi_impl.c for these */
2274c87aefeSPatrick Mooney extern void *contig_alloc(size_t, ddi_dma_attr_t *, uintptr_t, int);
2284c87aefeSPatrick Mooney extern void contig_free(void *, size_t);
229bf21cd93STycho Nightingale 
2304c87aefeSPatrick Mooney void *
vmm_contig_alloc(size_t size)231*8130f8e1SPatrick Mooney vmm_contig_alloc(size_t size)
2324c87aefeSPatrick Mooney {
2334c87aefeSPatrick Mooney 	ddi_dma_attr_t attr = {
2344c87aefeSPatrick Mooney 		/* Using fastboot_dma_attr as a guide... */
235*8130f8e1SPatrick Mooney 		.dma_attr_version	= DMA_ATTR_V0,
236*8130f8e1SPatrick Mooney 		.dma_attr_addr_lo	= 0,
237*8130f8e1SPatrick Mooney 		.dma_attr_addr_hi	= ~0UL,
238*8130f8e1SPatrick Mooney 		.dma_attr_count_max	= 0x00000000FFFFFFFFULL,
239*8130f8e1SPatrick Mooney 		.dma_attr_align		= PAGE_SIZE,
240*8130f8e1SPatrick Mooney 		.dma_attr_burstsizes	= 1,
241*8130f8e1SPatrick Mooney 		.dma_attr_minxfer	= 1,
242*8130f8e1SPatrick Mooney 		.dma_attr_maxxfer	= 0x00000000FFFFFFFFULL,
243*8130f8e1SPatrick Mooney 		.dma_attr_seg		= 0x00000000FFFFFFFFULL, /* any */
244*8130f8e1SPatrick Mooney 		.dma_attr_sgllen	= 1,
245*8130f8e1SPatrick Mooney 		.dma_attr_granular	= PAGE_SIZE,
246*8130f8e1SPatrick Mooney 		.dma_attr_flags		= 0,
2474c87aefeSPatrick Mooney 	};
248*8130f8e1SPatrick Mooney 	void *res;
2494c87aefeSPatrick Mooney 
250*8130f8e1SPatrick Mooney 	res = contig_alloc(size, &attr, PAGE_SIZE, 1);
251*8130f8e1SPatrick Mooney 	if (res != NULL) {
252*8130f8e1SPatrick Mooney 		bzero(res, size);
2534c87aefeSPatrick Mooney 	}
254*8130f8e1SPatrick Mooney 
255*8130f8e1SPatrick Mooney 	return (res);
2564c87aefeSPatrick Mooney }
2574c87aefeSPatrick Mooney 
2584c87aefeSPatrick Mooney void
vmm_contig_free(void * addr,size_t size)259*8130f8e1SPatrick Mooney vmm_contig_free(void *addr, size_t size)
2604c87aefeSPatrick Mooney {
2614c87aefeSPatrick Mooney 	contig_free(addr, size);
262bf21cd93STycho Nightingale }
263bf21cd93STycho Nightingale 
264bf21cd93STycho Nightingale void
critical_enter(void)265bf21cd93STycho Nightingale critical_enter(void)
266bf21cd93STycho Nightingale {
267bf21cd93STycho Nightingale 	kpreempt_disable();
268bf21cd93STycho Nightingale }
269bf21cd93STycho Nightingale 
270bf21cd93STycho Nightingale void
critical_exit(void)271bf21cd93STycho Nightingale critical_exit(void)
272bf21cd93STycho Nightingale {
273bf21cd93STycho Nightingale 	kpreempt_enable();
274bf21cd93STycho Nightingale }
275bf21cd93STycho Nightingale 
276bf21cd93STycho Nightingale 
277bf21cd93STycho Nightingale static void
vmm_glue_callout_handler(void * arg)278bf21cd93STycho Nightingale vmm_glue_callout_handler(void *arg)
279bf21cd93STycho Nightingale {
280bf21cd93STycho Nightingale 	struct callout *c = arg;
281bf21cd93STycho Nightingale 
282ee8ae3faSPatrick Mooney 	if (callout_active(c)) {
283ee8ae3faSPatrick Mooney 		/*
284ee8ae3faSPatrick Mooney 		 * Record the handler fire time so that callout_pending() is
285ee8ae3faSPatrick Mooney 		 * able to detect if the callout becomes rescheduled during the
286ee8ae3faSPatrick Mooney 		 * course of the handler.
287ee8ae3faSPatrick Mooney 		 */
288ee8ae3faSPatrick Mooney 		c->c_fired = gethrtime();
289bf21cd93STycho Nightingale 		(c->c_func)(c->c_arg);
290bf21cd93STycho Nightingale 	}
291bf21cd93STycho Nightingale }
292bf21cd93STycho Nightingale 
293bf21cd93STycho Nightingale void
vmm_glue_callout_init(struct callout * c,int mpsafe)294bf21cd93STycho Nightingale vmm_glue_callout_init(struct callout *c, int mpsafe)
295bf21cd93STycho Nightingale {
296bf21cd93STycho Nightingale 	cyc_handler_t	hdlr;
297bf21cd93STycho Nightingale 	cyc_time_t	when;
298bf21cd93STycho Nightingale 
299bf21cd93STycho Nightingale 	hdlr.cyh_level = CY_LOW_LEVEL;
300bf21cd93STycho Nightingale 	hdlr.cyh_func = vmm_glue_callout_handler;
301bf21cd93STycho Nightingale 	hdlr.cyh_arg = c;
302bf21cd93STycho Nightingale 	when.cyt_when = CY_INFINITY;
303bf21cd93STycho Nightingale 	when.cyt_interval = CY_INFINITY;
304ee8ae3faSPatrick Mooney 	bzero(c, sizeof (*c));
305bf21cd93STycho Nightingale 
306bf21cd93STycho Nightingale 	mutex_enter(&cpu_lock);
3074c87aefeSPatrick Mooney 	c->c_cyc_id = cyclic_add(&hdlr, &when);
308bf21cd93STycho Nightingale 	mutex_exit(&cpu_lock);
309bf21cd93STycho Nightingale }
310bf21cd93STycho Nightingale 
3115103e761SPatrick Mooney void
callout_reset_hrtime(struct callout * c,hrtime_t target,void (* func)(void *),void * arg,int flags)3125103e761SPatrick Mooney callout_reset_hrtime(struct callout *c, hrtime_t target, void (*func)(void *),
3135103e761SPatrick Mooney     void *arg, int flags)
314bf21cd93STycho Nightingale {
315bf21cd93STycho Nightingale 	ASSERT(c->c_cyc_id != CYCLIC_NONE);
316bf21cd93STycho Nightingale 
317ee8ae3faSPatrick Mooney 	if ((flags & C_ABSOLUTE) == 0) {
318ee8ae3faSPatrick Mooney 		target += gethrtime();
319ee8ae3faSPatrick Mooney 	}
320ee8ae3faSPatrick Mooney 
321bf21cd93STycho Nightingale 	c->c_func = func;
322bf21cd93STycho Nightingale 	c->c_arg = arg;
323ee8ae3faSPatrick Mooney 	c->c_target = target;
324e0994bd2SPatrick Mooney 	(void) cyclic_reprogram(c->c_cyc_id, target);
325bf21cd93STycho Nightingale }
326bf21cd93STycho Nightingale 
327e0994bd2SPatrick Mooney void
vmm_glue_callout_stop(struct callout * c)328bf21cd93STycho Nightingale vmm_glue_callout_stop(struct callout *c)
329bf21cd93STycho Nightingale {
330bf21cd93STycho Nightingale 	ASSERT(c->c_cyc_id != CYCLIC_NONE);
331ee8ae3faSPatrick Mooney 
332ee8ae3faSPatrick Mooney 	c->c_target = 0;
333e0994bd2SPatrick Mooney 	(void) cyclic_reprogram(c->c_cyc_id, CY_INFINITY);
334bf21cd93STycho Nightingale }
335bf21cd93STycho Nightingale 
336e0994bd2SPatrick Mooney void
vmm_glue_callout_drain(struct callout * c)337bf21cd93STycho Nightingale vmm_glue_callout_drain(struct callout *c)
338bf21cd93STycho Nightingale {
339bf21cd93STycho Nightingale 	ASSERT(c->c_cyc_id != CYCLIC_NONE);
340ee8ae3faSPatrick Mooney 
341ee8ae3faSPatrick Mooney 	c->c_target = 0;
342bf21cd93STycho Nightingale 	mutex_enter(&cpu_lock);
343bf21cd93STycho Nightingale 	cyclic_remove(c->c_cyc_id);
344bf21cd93STycho Nightingale 	c->c_cyc_id = CYCLIC_NONE;
345bf21cd93STycho Nightingale 	mutex_exit(&cpu_lock);
346bf21cd93STycho Nightingale }
347bf21cd93STycho Nightingale 
348bf21cd93STycho Nightingale void
vmm_glue_callout_localize(struct callout * c)3494c87aefeSPatrick Mooney vmm_glue_callout_localize(struct callout *c)
350bf21cd93STycho Nightingale {
3514c87aefeSPatrick Mooney 	mutex_enter(&cpu_lock);
3524c87aefeSPatrick Mooney 	cyclic_move_here(c->c_cyc_id);
3534c87aefeSPatrick Mooney 	mutex_exit(&cpu_lock);
354bf21cd93STycho Nightingale }
355bf21cd93STycho Nightingale 
3565103e761SPatrick Mooney /*
3575103e761SPatrick Mooney  * Given an interval (in ns) and a frequency (in hz), calculate the number of
3585103e761SPatrick Mooney  * "ticks" at that frequency which cover the interval.
3595103e761SPatrick Mooney  */
3605103e761SPatrick Mooney uint64_t
hrt_freq_count(hrtime_t interval,uint32_t freq)3615103e761SPatrick Mooney hrt_freq_count(hrtime_t interval, uint32_t freq)
3625103e761SPatrick Mooney {
3635103e761SPatrick Mooney 	ASSERT3S(interval, >=, 0);
3645103e761SPatrick Mooney 	const uint64_t sec = interval / NANOSEC;
3655103e761SPatrick Mooney 	const uint64_t nsec = interval % NANOSEC;
3665103e761SPatrick Mooney 
3675103e761SPatrick Mooney 	return ((sec * freq) + ((nsec * freq) / NANOSEC));
3685103e761SPatrick Mooney }
3695103e761SPatrick Mooney 
3705103e761SPatrick Mooney /*
3715103e761SPatrick Mooney  * Given a frequency (in hz) and number of "ticks", calculate the interval
3725103e761SPatrick Mooney  * (in ns) which would be covered by those ticks.
3735103e761SPatrick Mooney  */
3745103e761SPatrick Mooney hrtime_t
hrt_freq_interval(uint32_t freq,uint64_t count)3755103e761SPatrick Mooney hrt_freq_interval(uint32_t freq, uint64_t count)
3765103e761SPatrick Mooney {
3775103e761SPatrick Mooney 	const uint64_t sec = count / freq;
3785103e761SPatrick Mooney 	const uint64_t frac = count % freq;
3795103e761SPatrick Mooney 
3805103e761SPatrick Mooney 	return ((NANOSEC * sec) + ((frac * NANOSEC) / freq));
3815103e761SPatrick Mooney }
3825103e761SPatrick Mooney 
3835103e761SPatrick Mooney 
3842699b94cSPatrick Mooney uint_t	cpu_high;		/* Highest arg to CPUID */
3852699b94cSPatrick Mooney uint_t	cpu_exthigh;		/* Highest arg to extended CPUID */
3862699b94cSPatrick Mooney uint_t	cpu_id;			/* Stepping ID */
387bf21cd93STycho Nightingale char	cpu_vendor[20];		/* CPU Origin code */
388bf21cd93STycho Nightingale 
389bf21cd93STycho Nightingale static void
vmm_cpuid_init(void)390bf21cd93STycho Nightingale vmm_cpuid_init(void)
391bf21cd93STycho Nightingale {
3922699b94cSPatrick Mooney 	uint_t regs[4];
393bf21cd93STycho Nightingale 
394bf21cd93STycho Nightingale 	do_cpuid(0, regs);
395bf21cd93STycho Nightingale 	cpu_high = regs[0];
3962699b94cSPatrick Mooney 	((uint_t *)&cpu_vendor)[0] = regs[1];
3972699b94cSPatrick Mooney 	((uint_t *)&cpu_vendor)[1] = regs[3];
3982699b94cSPatrick Mooney 	((uint_t *)&cpu_vendor)[2] = regs[2];
399bf21cd93STycho Nightingale 	cpu_vendor[12] = '\0';
400bf21cd93STycho Nightingale 
401bf21cd93STycho Nightingale 	do_cpuid(1, regs);
402bf21cd93STycho Nightingale 	cpu_id = regs[0];
403bf21cd93STycho Nightingale 
404bf21cd93STycho Nightingale 	do_cpuid(0x80000000, regs);
405bf21cd93STycho Nightingale 	cpu_exthigh = regs[0];
406bf21cd93STycho Nightingale }
407bf21cd93STycho Nightingale 
4084c87aefeSPatrick Mooney void
vmm_sol_glue_init(void)4094c87aefeSPatrick Mooney vmm_sol_glue_init(void)
410bf21cd93STycho Nightingale {
411*8130f8e1SPatrick Mooney 	vmm_ptp_init();
4124c87aefeSPatrick Mooney 	vmm_cpuid_init();
413bf21cd93STycho Nightingale }
414bf21cd93STycho Nightingale 
415bf21cd93STycho Nightingale void
vmm_sol_glue_cleanup(void)4164c87aefeSPatrick Mooney vmm_sol_glue_cleanup(void)
417bf21cd93STycho Nightingale {
418*8130f8e1SPatrick Mooney 	vmm_ptp_cleanup();
419bf21cd93STycho Nightingale }
420bf21cd93STycho Nightingale 
4214c87aefeSPatrick Mooney 
4224c87aefeSPatrick Mooney /* From FreeBSD's sys/kern/subr_clock.c */
4234c87aefeSPatrick Mooney 
4244c87aefeSPatrick Mooney /*-
4254c87aefeSPatrick Mooney  * Copyright (c) 1988 University of Utah.
4264c87aefeSPatrick Mooney  * Copyright (c) 1982, 1990, 1993
4274c87aefeSPatrick Mooney  *	The Regents of the University of California.  All rights reserved.
4284c87aefeSPatrick Mooney  *
4294c87aefeSPatrick Mooney  * This code is derived from software contributed to Berkeley by
4304c87aefeSPatrick Mooney  * the Systems Programming Group of the University of Utah Computer
4314c87aefeSPatrick Mooney  * Science Department.
4324c87aefeSPatrick Mooney  *
4334c87aefeSPatrick Mooney  * Redistribution and use in source and binary forms, with or without
4344c87aefeSPatrick Mooney  * modification, are permitted provided that the following conditions
4354c87aefeSPatrick Mooney  * are met:
4364c87aefeSPatrick Mooney  * 1. Redistributions of source code must retain the above copyright
4374c87aefeSPatrick Mooney  *    notice, this list of conditions and the following disclaimer.
4384c87aefeSPatrick Mooney  * 2. Redistributions in binary form must reproduce the above copyright
4394c87aefeSPatrick Mooney  *    notice, this list of conditions and the following disclaimer in the
4404c87aefeSPatrick Mooney  *    documentation and/or other materials provided with the distribution.
4414c87aefeSPatrick Mooney  * 4. Neither the name of the University nor the names of its contributors
4424c87aefeSPatrick Mooney  *    may be used to endorse or promote products derived from this software
4434c87aefeSPatrick Mooney  *    without specific prior written permission.
4444c87aefeSPatrick Mooney  *
4454c87aefeSPatrick Mooney  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
4464c87aefeSPatrick Mooney  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
4474c87aefeSPatrick Mooney  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
4484c87aefeSPatrick Mooney  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
4494c87aefeSPatrick Mooney  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
4504c87aefeSPatrick Mooney  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
4514c87aefeSPatrick Mooney  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
4524c87aefeSPatrick Mooney  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
4534c87aefeSPatrick Mooney  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
4544c87aefeSPatrick Mooney  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
4554c87aefeSPatrick Mooney  * SUCH DAMAGE.
4564c87aefeSPatrick Mooney  *
4574c87aefeSPatrick Mooney  *	from: Utah $Hdr: clock.c 1.18 91/01/21$
4584c87aefeSPatrick Mooney  *	from: @(#)clock.c	8.2 (Berkeley) 1/12/94
4594c87aefeSPatrick Mooney  *	from: NetBSD: clock_subr.c,v 1.6 2001/07/07 17:04:02 thorpej Exp
4604c87aefeSPatrick Mooney  *	and
4614c87aefeSPatrick Mooney  *	from: src/sys/i386/isa/clock.c,v 1.176 2001/09/04
4624c87aefeSPatrick Mooney  */
4634c87aefeSPatrick Mooney 
4644c87aefeSPatrick Mooney #include <sys/clock.h>
4654c87aefeSPatrick Mooney 
4662699b94cSPatrick Mooney /*
4674c87aefeSPatrick Mooney  * Generic routines to convert between a POSIX date
4684c87aefeSPatrick Mooney  * (seconds since 1/1/1970) and yr/mo/day/hr/min/sec
4694c87aefeSPatrick Mooney  * Derived from NetBSD arch/hp300/hp300/clock.c
4704c87aefeSPatrick Mooney  */
4714c87aefeSPatrick Mooney 
4724c87aefeSPatrick Mooney #define	FEBRUARY	2
47384971882SPatrick Mooney #define	days_in_year(y)		(leapyear(y) ? 366 : 365)
4744c87aefeSPatrick Mooney #define	days_in_month(y, m) \
4754c87aefeSPatrick Mooney 	(month_days[(m) - 1] + (m == FEBRUARY ? leapyear(y) : 0))
4764c87aefeSPatrick Mooney /* Day of week. Days are counted from 1/1/1970, which was a Thursday */
4774c87aefeSPatrick Mooney #define	day_of_week(days)	(((days) + 4) % 7)
4784c87aefeSPatrick Mooney 
4794c87aefeSPatrick Mooney static const int month_days[12] = {
4804c87aefeSPatrick Mooney 	31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31
4814c87aefeSPatrick Mooney };
4824c87aefeSPatrick Mooney 
4834c87aefeSPatrick Mooney 
4844c87aefeSPatrick Mooney /*
4854c87aefeSPatrick Mooney  * This inline avoids some unnecessary modulo operations
4864c87aefeSPatrick Mooney  * as compared with the usual macro:
4874c87aefeSPatrick Mooney  *   ( ((year % 4) == 0 &&
4884c87aefeSPatrick Mooney  *      (year % 100) != 0) ||
4894c87aefeSPatrick Mooney  *     ((year % 400) == 0) )
4904c87aefeSPatrick Mooney  * It is otherwise equivalent.
4914c87aefeSPatrick Mooney  */
4924c87aefeSPatrick Mooney static int
leapyear(int year)4934c87aefeSPatrick Mooney leapyear(int year)
494bf21cd93STycho Nightingale {
4954c87aefeSPatrick Mooney 	int rv = 0;
4964c87aefeSPatrick Mooney 
4974c87aefeSPatrick Mooney 	if ((year & 3) == 0) {
4984c87aefeSPatrick Mooney 		rv = 1;
4994c87aefeSPatrick Mooney 		if ((year % 100) == 0) {
5004c87aefeSPatrick Mooney 			rv = 0;
5014c87aefeSPatrick Mooney 			if ((year % 400) == 0)
5024c87aefeSPatrick Mooney 				rv = 1;
5034c87aefeSPatrick Mooney 		}
504bf21cd93STycho Nightingale 	}
5054c87aefeSPatrick Mooney 	return (rv);
506bf21cd93STycho Nightingale }
507bf21cd93STycho Nightingale 
5084c87aefeSPatrick Mooney int
clock_ct_to_ts(struct clocktime * ct,struct timespec * ts)5094c87aefeSPatrick Mooney clock_ct_to_ts(struct clocktime *ct, struct timespec *ts)
510bf21cd93STycho Nightingale {
5114c87aefeSPatrick Mooney 	int i, year, days;
5124c87aefeSPatrick Mooney 
5134c87aefeSPatrick Mooney 	year = ct->year;
5144c87aefeSPatrick Mooney 
5154c87aefeSPatrick Mooney #ifdef __FreeBSD__
5164c87aefeSPatrick Mooney 	if (ct_debug) {
5174c87aefeSPatrick Mooney 		printf("ct_to_ts(");
5184c87aefeSPatrick Mooney 		print_ct(ct);
5194c87aefeSPatrick Mooney 		printf(")");
5204c87aefeSPatrick Mooney 	}
5214c87aefeSPatrick Mooney #endif
5224c87aefeSPatrick Mooney 
5234c87aefeSPatrick Mooney 	/* Sanity checks. */
5244c87aefeSPatrick Mooney 	if (ct->mon < 1 || ct->mon > 12 || ct->day < 1 ||
5254c87aefeSPatrick Mooney 	    ct->day > days_in_month(year, ct->mon) ||
5262699b94cSPatrick Mooney 	    ct->hour > 23 || ct->min > 59 || ct->sec > 59 ||
5272699b94cSPatrick Mooney 	    (sizeof (time_t) == 4 && year > 2037)) {	/* time_t overflow */
5284c87aefeSPatrick Mooney #ifdef __FreeBSD__
5294c87aefeSPatrick Mooney 		if (ct_debug)
5304c87aefeSPatrick Mooney 			printf(" = EINVAL\n");
5314c87aefeSPatrick Mooney #endif
5324c87aefeSPatrick Mooney 		return (EINVAL);
5334c87aefeSPatrick Mooney 	}
5344c87aefeSPatrick Mooney 
5354c87aefeSPatrick Mooney 	/*
5364c87aefeSPatrick Mooney 	 * Compute days since start of time
5374c87aefeSPatrick Mooney 	 * First from years, then from months.
5384c87aefeSPatrick Mooney 	 */
5394c87aefeSPatrick Mooney 	days = 0;
5404c87aefeSPatrick Mooney 	for (i = POSIX_BASE_YEAR; i < year; i++)
5414c87aefeSPatrick Mooney 		days += days_in_year(i);
5424c87aefeSPatrick Mooney 
5434c87aefeSPatrick Mooney 	/* Months */
5444c87aefeSPatrick Mooney 	for (i = 1; i < ct->mon; i++)
54584971882SPatrick Mooney 		days += days_in_month(year, i);
5464c87aefeSPatrick Mooney 	days += (ct->day - 1);
5474c87aefeSPatrick Mooney 
5484c87aefeSPatrick Mooney 	ts->tv_sec = (((time_t)days * 24 + ct->hour) * 60 + ct->min) * 60 +
5494c87aefeSPatrick Mooney 	    ct->sec;
5504c87aefeSPatrick Mooney 	ts->tv_nsec = ct->nsec;
5514c87aefeSPatrick Mooney 
5524c87aefeSPatrick Mooney #ifdef __FreeBSD__
5534c87aefeSPatrick Mooney 	if (ct_debug)
5544c87aefeSPatrick Mooney 		printf(" = %ld.%09ld\n", (long)ts->tv_sec, (long)ts->tv_nsec);
5554c87aefeSPatrick Mooney #endif
5564c87aefeSPatrick Mooney 	return (0);
557bf21cd93STycho Nightingale }
558bf21cd93STycho Nightingale 
559bf21cd93STycho Nightingale void
clock_ts_to_ct(struct timespec * ts,struct clocktime * ct)5604c87aefeSPatrick Mooney clock_ts_to_ct(struct timespec *ts, struct clocktime *ct)
5614c87aefeSPatrick Mooney {
5624c87aefeSPatrick Mooney 	int i, year, days;
5634c87aefeSPatrick Mooney 	time_t rsec;	/* remainder seconds */
5644c87aefeSPatrick Mooney 	time_t secs;
5654c87aefeSPatrick Mooney 
5664c87aefeSPatrick Mooney 	secs = ts->tv_sec;
5674c87aefeSPatrick Mooney 	days = secs / SECDAY;
5684c87aefeSPatrick Mooney 	rsec = secs % SECDAY;
5694c87aefeSPatrick Mooney 
5704c87aefeSPatrick Mooney 	ct->dow = day_of_week(days);
5714c87aefeSPatrick Mooney 
5724c87aefeSPatrick Mooney 	/* Subtract out whole years, counting them in i. */
5734c87aefeSPatrick Mooney 	for (year = POSIX_BASE_YEAR; days >= days_in_year(year); year++)
5744c87aefeSPatrick Mooney 		days -= days_in_year(year);
5754c87aefeSPatrick Mooney 	ct->year = year;
5764c87aefeSPatrick Mooney 
5774c87aefeSPatrick Mooney 	/* Subtract out whole months, counting them in i. */
5784c87aefeSPatrick Mooney 	for (i = 1; days >= days_in_month(year, i); i++)
5794c87aefeSPatrick Mooney 		days -= days_in_month(year, i);
5804c87aefeSPatrick Mooney 	ct->mon = i;
5814c87aefeSPatrick Mooney 
5824c87aefeSPatrick Mooney 	/* Days are what is left over (+1) from all that. */
5834c87aefeSPatrick Mooney 	ct->day = days + 1;
5844c87aefeSPatrick Mooney 
5854c87aefeSPatrick Mooney 	/* Hours, minutes, seconds are easy */
5864c87aefeSPatrick Mooney 	ct->hour = rsec / 3600;
5874c87aefeSPatrick Mooney 	rsec = rsec % 3600;
5884c87aefeSPatrick Mooney 	ct->min  = rsec / 60;
5894c87aefeSPatrick Mooney 	rsec = rsec % 60;
5904c87aefeSPatrick Mooney 	ct->sec  = rsec;
5914c87aefeSPatrick Mooney 	ct->nsec = ts->tv_nsec;
5924c87aefeSPatrick Mooney #ifdef __FreeBSD__
5934c87aefeSPatrick Mooney 	if (ct_debug) {
5944c87aefeSPatrick Mooney 		printf("ts_to_ct(%ld.%09ld) = ",
5954c87aefeSPatrick Mooney 		    (long)ts->tv_sec, (long)ts->tv_nsec);
5964c87aefeSPatrick Mooney 		print_ct(ct);
5974c87aefeSPatrick Mooney 		printf("\n");
5984c87aefeSPatrick Mooney 	}
5994c87aefeSPatrick Mooney #endif
600bf21cd93STycho Nightingale }
6019250eb13SPatrick Mooney 
6029250eb13SPatrick Mooney /* Equivalent to the FreeBSD rdtsc(), but with any necessary per-cpu offset */
6039250eb13SPatrick Mooney uint64_t
rdtsc_offset(void)6049250eb13SPatrick Mooney rdtsc_offset(void)
6059250eb13SPatrick Mooney {
6069250eb13SPatrick Mooney 	/*
6079250eb13SPatrick Mooney 	 * The timestamp logic will decide if a delta need be applied to the
6089250eb13SPatrick Mooney 	 * unscaled hrtime reading (effectively rdtsc), but we do require it be
6099250eb13SPatrick Mooney 	 * backed by the TSC itself.
6109250eb13SPatrick Mooney 	 */
6119250eb13SPatrick Mooney 	extern hrtime_t (*gethrtimeunscaledf)(void);
6129250eb13SPatrick Mooney 	extern hrtime_t tsc_gethrtimeunscaled(void);
6139250eb13SPatrick Mooney 	extern hrtime_t tsc_gethrtimeunscaled_delta(void);
6149250eb13SPatrick Mooney 
6159250eb13SPatrick Mooney 	ASSERT(*gethrtimeunscaledf == tsc_gethrtimeunscaled ||
6169250eb13SPatrick Mooney 	    *gethrtimeunscaledf == tsc_gethrtimeunscaled_delta);
6179250eb13SPatrick Mooney 	return ((uint64_t)gethrtimeunscaledf());
6189250eb13SPatrick Mooney }
619