1bf21cd93STycho Nightingale /* 2bf21cd93STycho Nightingale * Copyright (c) 2004 Poul-Henning Kamp 3bf21cd93STycho Nightingale * All rights reserved. 4bf21cd93STycho Nightingale * 5bf21cd93STycho Nightingale * Redistribution and use in source and binary forms, with or without 6bf21cd93STycho Nightingale * modification, are permitted provided that the following conditions 7bf21cd93STycho Nightingale * are met: 8bf21cd93STycho Nightingale * 1. Redistributions of source code must retain the above copyright 9bf21cd93STycho Nightingale * notice, this list of conditions and the following disclaimer. 10bf21cd93STycho Nightingale * 2. Redistributions in binary form must reproduce the above copyright 11bf21cd93STycho Nightingale * notice, this list of conditions and the following disclaimer in the 12bf21cd93STycho Nightingale * documentation and/or other materials provided with the distribution. 13bf21cd93STycho Nightingale * 14bf21cd93STycho Nightingale * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15bf21cd93STycho Nightingale * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16bf21cd93STycho Nightingale * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17bf21cd93STycho Nightingale * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18bf21cd93STycho Nightingale * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19bf21cd93STycho Nightingale * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20bf21cd93STycho Nightingale * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21bf21cd93STycho Nightingale * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22bf21cd93STycho Nightingale * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23bf21cd93STycho Nightingale * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24bf21cd93STycho Nightingale * SUCH DAMAGE. 25bf21cd93STycho Nightingale * 26bf21cd93STycho Nightingale * $FreeBSD: head/sys/kern/subr_unit.c 255057 2013-08-30 07:37:45Z kib $ 27bf21cd93STycho Nightingale */ 28bf21cd93STycho Nightingale /* 29bf21cd93STycho Nightingale * This file and its contents are supplied under the terms of the 30bf21cd93STycho Nightingale * Common Development and Distribution License ("CDDL"), version 1.0. 31bf21cd93STycho Nightingale * You may only use this file in accordance with the terms of version 32bf21cd93STycho Nightingale * 1.0 of the CDDL. 33bf21cd93STycho Nightingale * 34bf21cd93STycho Nightingale * A full copy of the text of the CDDL should have accompanied this 35bf21cd93STycho Nightingale * source. A copy of the CDDL is also available via the Internet at 36bf21cd93STycho Nightingale * http://www.illumos.org/license/CDDL. 37bf21cd93STycho Nightingale * 38bf21cd93STycho Nightingale * Copyright 2014 Pluribus Networks Inc. 394c87aefeSPatrick Mooney * Copyright 2019 Joyent, Inc. 40ee8ae3faSPatrick Mooney * Copyright 2020 Oxide Computer Company 41bf21cd93STycho Nightingale */ 42bf21cd93STycho Nightingale 43bf21cd93STycho Nightingale #include <sys/types.h> 44bf21cd93STycho Nightingale #include <sys/archsystm.h> 45bf21cd93STycho Nightingale #include <sys/cpuset.h> 46bf21cd93STycho Nightingale #include <sys/fp.h> 47*8130f8e1SPatrick Mooney #include <sys/kmem.h> 48bf21cd93STycho Nightingale #include <sys/queue.h> 49bf21cd93STycho Nightingale #include <sys/spl.h> 50bf21cd93STycho Nightingale #include <sys/systm.h> 514c87aefeSPatrick Mooney #include <sys/ddidmareq.h> 524c87aefeSPatrick Mooney #include <sys/id_space.h> 534c87aefeSPatrick Mooney #include <sys/psm_defs.h> 544c87aefeSPatrick Mooney #include <sys/smp_impldefs.h> 554c87aefeSPatrick Mooney #include <sys/modhash.h> 564c87aefeSPatrick Mooney #include <sys/hma.h> 57bf21cd93STycho Nightingale 58eb9a1df2SHans Rosenfeld #include <sys/x86_archext.h> 59eb9a1df2SHans Rosenfeld 60bf21cd93STycho Nightingale #include <machine/cpufunc.h> 61bf21cd93STycho Nightingale #include <machine/md_var.h> 62bf21cd93STycho Nightingale #include <machine/specialreg.h> 63bf21cd93STycho Nightingale #include <machine/vmm.h> 64cf409e3fSDan Cross #include <machine/vmparam.h> 65bf21cd93STycho Nightingale #include <sys/vmm_impl.h> 664c87aefeSPatrick Mooney #include <sys/kernel.h> 67bf21cd93STycho Nightingale 68bf21cd93STycho Nightingale #include <vm/as.h> 69bf21cd93STycho Nightingale #include <vm/seg_kmem.h> 70bf21cd93STycho Nightingale 714c87aefeSPatrick Mooney SET_DECLARE(sysinit_set, struct sysinit); 724c87aefeSPatrick Mooney 734c87aefeSPatrick Mooney void 744c87aefeSPatrick Mooney sysinit(void) 754c87aefeSPatrick Mooney { 764c87aefeSPatrick Mooney struct sysinit **si; 774c87aefeSPatrick Mooney 784c87aefeSPatrick Mooney SET_FOREACH(si, sysinit_set) 794c87aefeSPatrick Mooney (*si)->func((*si)->data); 804c87aefeSPatrick Mooney } 814c87aefeSPatrick Mooney 822699b94cSPatrick Mooney uint8_t const bin2bcd_data[] = { 834c87aefeSPatrick Mooney 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 844c87aefeSPatrick Mooney 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 854c87aefeSPatrick Mooney 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 864c87aefeSPatrick Mooney 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 874c87aefeSPatrick Mooney 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 884c87aefeSPatrick Mooney 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 894c87aefeSPatrick Mooney 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 904c87aefeSPatrick Mooney 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 914c87aefeSPatrick Mooney 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 924c87aefeSPatrick Mooney 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99 934c87aefeSPatrick Mooney }; 944c87aefeSPatrick Mooney 95eb9a1df2SHans Rosenfeld void 960153d828SPatrick Mooney invalidate_cache_all(void) 97eb9a1df2SHans Rosenfeld { 98eb9a1df2SHans Rosenfeld cpuset_t cpuset; 99eb9a1df2SHans Rosenfeld 100eb9a1df2SHans Rosenfeld kpreempt_disable(); 101eb9a1df2SHans Rosenfeld cpuset_all_but(&cpuset, CPU->cpu_id); 102eb9a1df2SHans Rosenfeld xc_call((xc_arg_t)NULL, (xc_arg_t)NULL, (xc_arg_t)NULL, 103eb9a1df2SHans Rosenfeld CPUSET2BV(cpuset), (xc_func_t)invalidate_cache); 104eb9a1df2SHans Rosenfeld invalidate_cache(); 105eb9a1df2SHans Rosenfeld kpreempt_enable(); 106eb9a1df2SHans Rosenfeld } 107eb9a1df2SHans Rosenfeld 108bf21cd93STycho Nightingale vm_paddr_t 1090153d828SPatrick Mooney vtophys(void *va) 110bf21cd93STycho Nightingale { 111bf21cd93STycho Nightingale pfn_t pfn; 112bf21cd93STycho Nightingale 1134c87aefeSPatrick Mooney /* 1144c87aefeSPatrick Mooney * Since hat_getpfnum() may block on an htable mutex, this is not at 1154c87aefeSPatrick Mooney * all safe to run from a critical_enter/kpreempt_disable context. 1164c87aefeSPatrick Mooney * The FreeBSD analog does not have the same locking constraints, so 1174c87aefeSPatrick Mooney * close attention must be paid wherever this is called. 1184c87aefeSPatrick Mooney */ 1194c87aefeSPatrick Mooney ASSERT(curthread->t_preempt == 0); 1204c87aefeSPatrick Mooney 121bf21cd93STycho Nightingale pfn = hat_getpfnum(kas.a_hat, (caddr_t)va); 122bf21cd93STycho Nightingale ASSERT(pfn != PFN_INVALID); 123bf21cd93STycho Nightingale return (pfn << PAGE_SHIFT) | ((uintptr_t)va & PAGE_MASK); 124bf21cd93STycho Nightingale } 125bf21cd93STycho Nightingale 126bf21cd93STycho Nightingale int 127bf21cd93STycho Nightingale cpusetobj_ffs(const cpuset_t *set) 128bf21cd93STycho Nightingale { 1294c87aefeSPatrick Mooney uint_t large, small; 130bf21cd93STycho Nightingale 1314c87aefeSPatrick Mooney /* 1324c87aefeSPatrick Mooney * Rather than reaching into the cpuset_t ourselves, leave that task to 1334c87aefeSPatrick Mooney * cpuset_bounds(). The simplicity is worth the extra wasted work to 1344c87aefeSPatrick Mooney * find the upper bound. 1354c87aefeSPatrick Mooney */ 1364c87aefeSPatrick Mooney cpuset_bounds(set, &small, &large); 137bf21cd93STycho Nightingale 1384c87aefeSPatrick Mooney if (small == CPUSET_NOTINSET) { 1394c87aefeSPatrick Mooney /* The FreeBSD version returns 0 if it find nothing */ 1404c87aefeSPatrick Mooney return (0); 1414c87aefeSPatrick Mooney } 142bf21cd93STycho Nightingale 1434c87aefeSPatrick Mooney ASSERT3U(small, <=, INT_MAX); 144bf21cd93STycho Nightingale 1454c87aefeSPatrick Mooney /* Least significant bit index starts at 1 for valid results */ 1464c87aefeSPatrick Mooney return (small + 1); 147bf21cd93STycho Nightingale } 148bf21cd93STycho Nightingale 149*8130f8e1SPatrick Mooney struct vmm_ptp_item { 150*8130f8e1SPatrick Mooney void *vpi_vaddr; 151bf21cd93STycho Nightingale }; 152*8130f8e1SPatrick Mooney static kmutex_t vmm_ptp_lock; 1534c87aefeSPatrick Mooney 154*8130f8e1SPatrick Mooney static mod_hash_t *vmm_ptp_hash; 155*8130f8e1SPatrick Mooney uint_t vmm_ptp_hash_nchains = 16381; 156*8130f8e1SPatrick Mooney uint_t vmm_ptp_hash_size = PAGESIZE; 1574c87aefeSPatrick Mooney 1584c87aefeSPatrick Mooney static void 159*8130f8e1SPatrick Mooney vmm_ptp_hash_valdtor(mod_hash_val_t val) 1604c87aefeSPatrick Mooney { 161*8130f8e1SPatrick Mooney struct vmm_ptp_item *i = (struct vmm_ptp_item *)val; 1624c87aefeSPatrick Mooney 163*8130f8e1SPatrick Mooney kmem_free(i->vpi_vaddr, PAGE_SIZE); 164*8130f8e1SPatrick Mooney kmem_free(i, sizeof (*i)); 1654c87aefeSPatrick Mooney } 1664c87aefeSPatrick Mooney 1674c87aefeSPatrick Mooney static void 168*8130f8e1SPatrick Mooney vmm_ptp_init(void) 1694c87aefeSPatrick Mooney { 170*8130f8e1SPatrick Mooney vmm_ptp_hash = mod_hash_create_ptrhash("vmm_ptp_hash", 171*8130f8e1SPatrick Mooney vmm_ptp_hash_nchains, vmm_ptp_hash_valdtor, vmm_ptp_hash_size); 1724c87aefeSPatrick Mooney 173*8130f8e1SPatrick Mooney VERIFY(vmm_ptp_hash != NULL); 1744c87aefeSPatrick Mooney } 1754c87aefeSPatrick Mooney 1764c87aefeSPatrick Mooney static uint_t 177*8130f8e1SPatrick Mooney vmm_ptp_check(mod_hash_key_t key, mod_hash_val_t *val, void *unused) 1784c87aefeSPatrick Mooney { 179*8130f8e1SPatrick Mooney struct vmm_ptp_item *i = (struct vmm_ptp_item *)val; 1804c87aefeSPatrick Mooney 181*8130f8e1SPatrick Mooney cmn_err(CE_PANIC, "!vmm_ptp_check: hash not empty: %p", i->vpi_vaddr); 1824c87aefeSPatrick Mooney 1834c87aefeSPatrick Mooney return (MH_WALK_TERMINATE); 1844c87aefeSPatrick Mooney } 1854c87aefeSPatrick Mooney 1864c87aefeSPatrick Mooney static void 187*8130f8e1SPatrick Mooney vmm_ptp_cleanup(void) 1884c87aefeSPatrick Mooney { 189*8130f8e1SPatrick Mooney mod_hash_walk(vmm_ptp_hash, vmm_ptp_check, NULL); 190*8130f8e1SPatrick Mooney mod_hash_destroy_ptrhash(vmm_ptp_hash); 1914c87aefeSPatrick Mooney } 192bf21cd93STycho Nightingale 193*8130f8e1SPatrick Mooney /* 194*8130f8e1SPatrick Mooney * The logic in VT-d uses both kernel-virtual and direct-mapped addresses when 195*8130f8e1SPatrick Mooney * freeing PTP pages. Until the consuming code is improved to better track the 196*8130f8e1SPatrick Mooney * pages it allocates, we keep the kernel-virtual addresses to those pages in a 197*8130f8e1SPatrick Mooney * hash table for when they are freed. 198*8130f8e1SPatrick Mooney */ 199bf21cd93STycho Nightingale void * 200*8130f8e1SPatrick Mooney vmm_ptp_alloc(void) 201bf21cd93STycho Nightingale { 202*8130f8e1SPatrick Mooney void *p; 203*8130f8e1SPatrick Mooney struct vmm_ptp_item *i; 204bf21cd93STycho Nightingale 205*8130f8e1SPatrick Mooney p = kmem_zalloc(PAGE_SIZE, KM_SLEEP); 206*8130f8e1SPatrick Mooney i = kmem_alloc(sizeof (struct vmm_ptp_item), KM_SLEEP); 207*8130f8e1SPatrick Mooney i->vpi_vaddr = p; 2084c87aefeSPatrick Mooney 209*8130f8e1SPatrick Mooney mutex_enter(&vmm_ptp_lock); 210*8130f8e1SPatrick Mooney VERIFY(mod_hash_insert(vmm_ptp_hash, 2114c87aefeSPatrick Mooney (mod_hash_key_t)PHYS_TO_DMAP(vtophys(p)), (mod_hash_val_t)i) == 0); 212*8130f8e1SPatrick Mooney mutex_exit(&vmm_ptp_lock); 213bf21cd93STycho Nightingale 214bf21cd93STycho Nightingale return (p); 215bf21cd93STycho Nightingale } 216bf21cd93STycho Nightingale 217bf21cd93STycho Nightingale void 218*8130f8e1SPatrick Mooney vmm_ptp_free(void *addr) 219bf21cd93STycho Nightingale { 220*8130f8e1SPatrick Mooney mutex_enter(&vmm_ptp_lock); 221*8130f8e1SPatrick Mooney VERIFY(mod_hash_destroy(vmm_ptp_hash, 2224c87aefeSPatrick Mooney (mod_hash_key_t)PHYS_TO_DMAP(vtophys(addr))) == 0); 223*8130f8e1SPatrick Mooney mutex_exit(&vmm_ptp_lock); 2244c87aefeSPatrick Mooney } 2254c87aefeSPatrick Mooney 226*8130f8e1SPatrick Mooney /* Reach into i86pc/os/ddi_impl.c for these */ 2274c87aefeSPatrick Mooney extern void *contig_alloc(size_t, ddi_dma_attr_t *, uintptr_t, int); 2284c87aefeSPatrick Mooney extern void contig_free(void *, size_t); 229bf21cd93STycho Nightingale 2304c87aefeSPatrick Mooney void * 231*8130f8e1SPatrick Mooney vmm_contig_alloc(size_t size) 2324c87aefeSPatrick Mooney { 2334c87aefeSPatrick Mooney ddi_dma_attr_t attr = { 2344c87aefeSPatrick Mooney /* Using fastboot_dma_attr as a guide... */ 235*8130f8e1SPatrick Mooney .dma_attr_version = DMA_ATTR_V0, 236*8130f8e1SPatrick Mooney .dma_attr_addr_lo = 0, 237*8130f8e1SPatrick Mooney .dma_attr_addr_hi = ~0UL, 238*8130f8e1SPatrick Mooney .dma_attr_count_max = 0x00000000FFFFFFFFULL, 239*8130f8e1SPatrick Mooney .dma_attr_align = PAGE_SIZE, 240*8130f8e1SPatrick Mooney .dma_attr_burstsizes = 1, 241*8130f8e1SPatrick Mooney .dma_attr_minxfer = 1, 242*8130f8e1SPatrick Mooney .dma_attr_maxxfer = 0x00000000FFFFFFFFULL, 243*8130f8e1SPatrick Mooney .dma_attr_seg = 0x00000000FFFFFFFFULL, /* any */ 244*8130f8e1SPatrick Mooney .dma_attr_sgllen = 1, 245*8130f8e1SPatrick Mooney .dma_attr_granular = PAGE_SIZE, 246*8130f8e1SPatrick Mooney .dma_attr_flags = 0, 2474c87aefeSPatrick Mooney }; 248*8130f8e1SPatrick Mooney void *res; 2494c87aefeSPatrick Mooney 250*8130f8e1SPatrick Mooney res = contig_alloc(size, &attr, PAGE_SIZE, 1); 251*8130f8e1SPatrick Mooney if (res != NULL) { 252*8130f8e1SPatrick Mooney bzero(res, size); 2534c87aefeSPatrick Mooney } 254*8130f8e1SPatrick Mooney 255*8130f8e1SPatrick Mooney return (res); 2564c87aefeSPatrick Mooney } 2574c87aefeSPatrick Mooney 2584c87aefeSPatrick Mooney void 259*8130f8e1SPatrick Mooney vmm_contig_free(void *addr, size_t size) 2604c87aefeSPatrick Mooney { 2614c87aefeSPatrick Mooney contig_free(addr, size); 262bf21cd93STycho Nightingale } 263bf21cd93STycho Nightingale 264bf21cd93STycho Nightingale void 265bf21cd93STycho Nightingale critical_enter(void) 266bf21cd93STycho Nightingale { 267bf21cd93STycho Nightingale kpreempt_disable(); 268bf21cd93STycho Nightingale } 269bf21cd93STycho Nightingale 270bf21cd93STycho Nightingale void 271bf21cd93STycho Nightingale critical_exit(void) 272bf21cd93STycho Nightingale { 273bf21cd93STycho Nightingale kpreempt_enable(); 274bf21cd93STycho Nightingale } 275bf21cd93STycho Nightingale 276bf21cd93STycho Nightingale 277bf21cd93STycho Nightingale static void 278bf21cd93STycho Nightingale vmm_glue_callout_handler(void *arg) 279bf21cd93STycho Nightingale { 280bf21cd93STycho Nightingale struct callout *c = arg; 281bf21cd93STycho Nightingale 282ee8ae3faSPatrick Mooney if (callout_active(c)) { 283ee8ae3faSPatrick Mooney /* 284ee8ae3faSPatrick Mooney * Record the handler fire time so that callout_pending() is 285ee8ae3faSPatrick Mooney * able to detect if the callout becomes rescheduled during the 286ee8ae3faSPatrick Mooney * course of the handler. 287ee8ae3faSPatrick Mooney */ 288ee8ae3faSPatrick Mooney c->c_fired = gethrtime(); 289bf21cd93STycho Nightingale (c->c_func)(c->c_arg); 290bf21cd93STycho Nightingale } 291bf21cd93STycho Nightingale } 292bf21cd93STycho Nightingale 293bf21cd93STycho Nightingale void 294bf21cd93STycho Nightingale vmm_glue_callout_init(struct callout *c, int mpsafe) 295bf21cd93STycho Nightingale { 296bf21cd93STycho Nightingale cyc_handler_t hdlr; 297bf21cd93STycho Nightingale cyc_time_t when; 298bf21cd93STycho Nightingale 299bf21cd93STycho Nightingale hdlr.cyh_level = CY_LOW_LEVEL; 300bf21cd93STycho Nightingale hdlr.cyh_func = vmm_glue_callout_handler; 301bf21cd93STycho Nightingale hdlr.cyh_arg = c; 302bf21cd93STycho Nightingale when.cyt_when = CY_INFINITY; 303bf21cd93STycho Nightingale when.cyt_interval = CY_INFINITY; 304ee8ae3faSPatrick Mooney bzero(c, sizeof (*c)); 305bf21cd93STycho Nightingale 306bf21cd93STycho Nightingale mutex_enter(&cpu_lock); 3074c87aefeSPatrick Mooney c->c_cyc_id = cyclic_add(&hdlr, &when); 308bf21cd93STycho Nightingale mutex_exit(&cpu_lock); 309bf21cd93STycho Nightingale } 310bf21cd93STycho Nightingale 3115103e761SPatrick Mooney void 3125103e761SPatrick Mooney callout_reset_hrtime(struct callout *c, hrtime_t target, void (*func)(void *), 3135103e761SPatrick Mooney void *arg, int flags) 314bf21cd93STycho Nightingale { 315bf21cd93STycho Nightingale ASSERT(c->c_cyc_id != CYCLIC_NONE); 316bf21cd93STycho Nightingale 317ee8ae3faSPatrick Mooney if ((flags & C_ABSOLUTE) == 0) { 318ee8ae3faSPatrick Mooney target += gethrtime(); 319ee8ae3faSPatrick Mooney } 320ee8ae3faSPatrick Mooney 321bf21cd93STycho Nightingale c->c_func = func; 322bf21cd93STycho Nightingale c->c_arg = arg; 323ee8ae3faSPatrick Mooney c->c_target = target; 324e0994bd2SPatrick Mooney (void) cyclic_reprogram(c->c_cyc_id, target); 325bf21cd93STycho Nightingale } 326bf21cd93STycho Nightingale 327e0994bd2SPatrick Mooney void 328bf21cd93STycho Nightingale vmm_glue_callout_stop(struct callout *c) 329bf21cd93STycho Nightingale { 330bf21cd93STycho Nightingale ASSERT(c->c_cyc_id != CYCLIC_NONE); 331ee8ae3faSPatrick Mooney 332ee8ae3faSPatrick Mooney c->c_target = 0; 333e0994bd2SPatrick Mooney (void) cyclic_reprogram(c->c_cyc_id, CY_INFINITY); 334bf21cd93STycho Nightingale } 335bf21cd93STycho Nightingale 336e0994bd2SPatrick Mooney void 337bf21cd93STycho Nightingale vmm_glue_callout_drain(struct callout *c) 338bf21cd93STycho Nightingale { 339bf21cd93STycho Nightingale ASSERT(c->c_cyc_id != CYCLIC_NONE); 340ee8ae3faSPatrick Mooney 341ee8ae3faSPatrick Mooney c->c_target = 0; 342bf21cd93STycho Nightingale mutex_enter(&cpu_lock); 343bf21cd93STycho Nightingale cyclic_remove(c->c_cyc_id); 344bf21cd93STycho Nightingale c->c_cyc_id = CYCLIC_NONE; 345bf21cd93STycho Nightingale mutex_exit(&cpu_lock); 346bf21cd93STycho Nightingale } 347bf21cd93STycho Nightingale 348bf21cd93STycho Nightingale void 3494c87aefeSPatrick Mooney vmm_glue_callout_localize(struct callout *c) 350bf21cd93STycho Nightingale { 3514c87aefeSPatrick Mooney mutex_enter(&cpu_lock); 3524c87aefeSPatrick Mooney cyclic_move_here(c->c_cyc_id); 3534c87aefeSPatrick Mooney mutex_exit(&cpu_lock); 354bf21cd93STycho Nightingale } 355bf21cd93STycho Nightingale 3565103e761SPatrick Mooney /* 3575103e761SPatrick Mooney * Given an interval (in ns) and a frequency (in hz), calculate the number of 3585103e761SPatrick Mooney * "ticks" at that frequency which cover the interval. 3595103e761SPatrick Mooney */ 3605103e761SPatrick Mooney uint64_t 3615103e761SPatrick Mooney hrt_freq_count(hrtime_t interval, uint32_t freq) 3625103e761SPatrick Mooney { 3635103e761SPatrick Mooney ASSERT3S(interval, >=, 0); 3645103e761SPatrick Mooney const uint64_t sec = interval / NANOSEC; 3655103e761SPatrick Mooney const uint64_t nsec = interval % NANOSEC; 3665103e761SPatrick Mooney 3675103e761SPatrick Mooney return ((sec * freq) + ((nsec * freq) / NANOSEC)); 3685103e761SPatrick Mooney } 3695103e761SPatrick Mooney 3705103e761SPatrick Mooney /* 3715103e761SPatrick Mooney * Given a frequency (in hz) and number of "ticks", calculate the interval 3725103e761SPatrick Mooney * (in ns) which would be covered by those ticks. 3735103e761SPatrick Mooney */ 3745103e761SPatrick Mooney hrtime_t 3755103e761SPatrick Mooney hrt_freq_interval(uint32_t freq, uint64_t count) 3765103e761SPatrick Mooney { 3775103e761SPatrick Mooney const uint64_t sec = count / freq; 3785103e761SPatrick Mooney const uint64_t frac = count % freq; 3795103e761SPatrick Mooney 3805103e761SPatrick Mooney return ((NANOSEC * sec) + ((frac * NANOSEC) / freq)); 3815103e761SPatrick Mooney } 3825103e761SPatrick Mooney 3835103e761SPatrick Mooney 3842699b94cSPatrick Mooney uint_t cpu_high; /* Highest arg to CPUID */ 3852699b94cSPatrick Mooney uint_t cpu_exthigh; /* Highest arg to extended CPUID */ 3862699b94cSPatrick Mooney uint_t cpu_id; /* Stepping ID */ 387bf21cd93STycho Nightingale char cpu_vendor[20]; /* CPU Origin code */ 388bf21cd93STycho Nightingale 389bf21cd93STycho Nightingale static void 390bf21cd93STycho Nightingale vmm_cpuid_init(void) 391bf21cd93STycho Nightingale { 3922699b94cSPatrick Mooney uint_t regs[4]; 393bf21cd93STycho Nightingale 394bf21cd93STycho Nightingale do_cpuid(0, regs); 395bf21cd93STycho Nightingale cpu_high = regs[0]; 3962699b94cSPatrick Mooney ((uint_t *)&cpu_vendor)[0] = regs[1]; 3972699b94cSPatrick Mooney ((uint_t *)&cpu_vendor)[1] = regs[3]; 3982699b94cSPatrick Mooney ((uint_t *)&cpu_vendor)[2] = regs[2]; 399bf21cd93STycho Nightingale cpu_vendor[12] = '\0'; 400bf21cd93STycho Nightingale 401bf21cd93STycho Nightingale do_cpuid(1, regs); 402bf21cd93STycho Nightingale cpu_id = regs[0]; 403bf21cd93STycho Nightingale 404bf21cd93STycho Nightingale do_cpuid(0x80000000, regs); 405bf21cd93STycho Nightingale cpu_exthigh = regs[0]; 406bf21cd93STycho Nightingale } 407bf21cd93STycho Nightingale 4084c87aefeSPatrick Mooney void 4094c87aefeSPatrick Mooney vmm_sol_glue_init(void) 410bf21cd93STycho Nightingale { 411*8130f8e1SPatrick Mooney vmm_ptp_init(); 4124c87aefeSPatrick Mooney vmm_cpuid_init(); 413bf21cd93STycho Nightingale } 414bf21cd93STycho Nightingale 415bf21cd93STycho Nightingale void 4164c87aefeSPatrick Mooney vmm_sol_glue_cleanup(void) 417bf21cd93STycho Nightingale { 418*8130f8e1SPatrick Mooney vmm_ptp_cleanup(); 419bf21cd93STycho Nightingale } 420bf21cd93STycho Nightingale 4214c87aefeSPatrick Mooney 4224c87aefeSPatrick Mooney /* From FreeBSD's sys/kern/subr_clock.c */ 4234c87aefeSPatrick Mooney 4244c87aefeSPatrick Mooney /*- 4254c87aefeSPatrick Mooney * Copyright (c) 1988 University of Utah. 4264c87aefeSPatrick Mooney * Copyright (c) 1982, 1990, 1993 4274c87aefeSPatrick Mooney * The Regents of the University of California. All rights reserved. 4284c87aefeSPatrick Mooney * 4294c87aefeSPatrick Mooney * This code is derived from software contributed to Berkeley by 4304c87aefeSPatrick Mooney * the Systems Programming Group of the University of Utah Computer 4314c87aefeSPatrick Mooney * Science Department. 4324c87aefeSPatrick Mooney * 4334c87aefeSPatrick Mooney * Redistribution and use in source and binary forms, with or without 4344c87aefeSPatrick Mooney * modification, are permitted provided that the following conditions 4354c87aefeSPatrick Mooney * are met: 4364c87aefeSPatrick Mooney * 1. Redistributions of source code must retain the above copyright 4374c87aefeSPatrick Mooney * notice, this list of conditions and the following disclaimer. 4384c87aefeSPatrick Mooney * 2. Redistributions in binary form must reproduce the above copyright 4394c87aefeSPatrick Mooney * notice, this list of conditions and the following disclaimer in the 4404c87aefeSPatrick Mooney * documentation and/or other materials provided with the distribution. 4414c87aefeSPatrick Mooney * 4. Neither the name of the University nor the names of its contributors 4424c87aefeSPatrick Mooney * may be used to endorse or promote products derived from this software 4434c87aefeSPatrick Mooney * without specific prior written permission. 4444c87aefeSPatrick Mooney * 4454c87aefeSPatrick Mooney * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 4464c87aefeSPatrick Mooney * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 4474c87aefeSPatrick Mooney * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 4484c87aefeSPatrick Mooney * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 4494c87aefeSPatrick Mooney * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 4504c87aefeSPatrick Mooney * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 4514c87aefeSPatrick Mooney * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 4524c87aefeSPatrick Mooney * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 4534c87aefeSPatrick Mooney * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 4544c87aefeSPatrick Mooney * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 4554c87aefeSPatrick Mooney * SUCH DAMAGE. 4564c87aefeSPatrick Mooney * 4574c87aefeSPatrick Mooney * from: Utah $Hdr: clock.c 1.18 91/01/21$ 4584c87aefeSPatrick Mooney * from: @(#)clock.c 8.2 (Berkeley) 1/12/94 4594c87aefeSPatrick Mooney * from: NetBSD: clock_subr.c,v 1.6 2001/07/07 17:04:02 thorpej Exp 4604c87aefeSPatrick Mooney * and 4614c87aefeSPatrick Mooney * from: src/sys/i386/isa/clock.c,v 1.176 2001/09/04 4624c87aefeSPatrick Mooney */ 4634c87aefeSPatrick Mooney 4644c87aefeSPatrick Mooney #include <sys/clock.h> 4654c87aefeSPatrick Mooney 4662699b94cSPatrick Mooney /* 4674c87aefeSPatrick Mooney * Generic routines to convert between a POSIX date 4684c87aefeSPatrick Mooney * (seconds since 1/1/1970) and yr/mo/day/hr/min/sec 4694c87aefeSPatrick Mooney * Derived from NetBSD arch/hp300/hp300/clock.c 4704c87aefeSPatrick Mooney */ 4714c87aefeSPatrick Mooney 4724c87aefeSPatrick Mooney #define FEBRUARY 2 47384971882SPatrick Mooney #define days_in_year(y) (leapyear(y) ? 366 : 365) 4744c87aefeSPatrick Mooney #define days_in_month(y, m) \ 4754c87aefeSPatrick Mooney (month_days[(m) - 1] + (m == FEBRUARY ? leapyear(y) : 0)) 4764c87aefeSPatrick Mooney /* Day of week. Days are counted from 1/1/1970, which was a Thursday */ 4774c87aefeSPatrick Mooney #define day_of_week(days) (((days) + 4) % 7) 4784c87aefeSPatrick Mooney 4794c87aefeSPatrick Mooney static const int month_days[12] = { 4804c87aefeSPatrick Mooney 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 4814c87aefeSPatrick Mooney }; 4824c87aefeSPatrick Mooney 4834c87aefeSPatrick Mooney 4844c87aefeSPatrick Mooney /* 4854c87aefeSPatrick Mooney * This inline avoids some unnecessary modulo operations 4864c87aefeSPatrick Mooney * as compared with the usual macro: 4874c87aefeSPatrick Mooney * ( ((year % 4) == 0 && 4884c87aefeSPatrick Mooney * (year % 100) != 0) || 4894c87aefeSPatrick Mooney * ((year % 400) == 0) ) 4904c87aefeSPatrick Mooney * It is otherwise equivalent. 4914c87aefeSPatrick Mooney */ 4924c87aefeSPatrick Mooney static int 4934c87aefeSPatrick Mooney leapyear(int year) 494bf21cd93STycho Nightingale { 4954c87aefeSPatrick Mooney int rv = 0; 4964c87aefeSPatrick Mooney 4974c87aefeSPatrick Mooney if ((year & 3) == 0) { 4984c87aefeSPatrick Mooney rv = 1; 4994c87aefeSPatrick Mooney if ((year % 100) == 0) { 5004c87aefeSPatrick Mooney rv = 0; 5014c87aefeSPatrick Mooney if ((year % 400) == 0) 5024c87aefeSPatrick Mooney rv = 1; 5034c87aefeSPatrick Mooney } 504bf21cd93STycho Nightingale } 5054c87aefeSPatrick Mooney return (rv); 506bf21cd93STycho Nightingale } 507bf21cd93STycho Nightingale 5084c87aefeSPatrick Mooney int 5094c87aefeSPatrick Mooney clock_ct_to_ts(struct clocktime *ct, struct timespec *ts) 510bf21cd93STycho Nightingale { 5114c87aefeSPatrick Mooney int i, year, days; 5124c87aefeSPatrick Mooney 5134c87aefeSPatrick Mooney year = ct->year; 5144c87aefeSPatrick Mooney 5154c87aefeSPatrick Mooney #ifdef __FreeBSD__ 5164c87aefeSPatrick Mooney if (ct_debug) { 5174c87aefeSPatrick Mooney printf("ct_to_ts("); 5184c87aefeSPatrick Mooney print_ct(ct); 5194c87aefeSPatrick Mooney printf(")"); 5204c87aefeSPatrick Mooney } 5214c87aefeSPatrick Mooney #endif 5224c87aefeSPatrick Mooney 5234c87aefeSPatrick Mooney /* Sanity checks. */ 5244c87aefeSPatrick Mooney if (ct->mon < 1 || ct->mon > 12 || ct->day < 1 || 5254c87aefeSPatrick Mooney ct->day > days_in_month(year, ct->mon) || 5262699b94cSPatrick Mooney ct->hour > 23 || ct->min > 59 || ct->sec > 59 || 5272699b94cSPatrick Mooney (sizeof (time_t) == 4 && year > 2037)) { /* time_t overflow */ 5284c87aefeSPatrick Mooney #ifdef __FreeBSD__ 5294c87aefeSPatrick Mooney if (ct_debug) 5304c87aefeSPatrick Mooney printf(" = EINVAL\n"); 5314c87aefeSPatrick Mooney #endif 5324c87aefeSPatrick Mooney return (EINVAL); 5334c87aefeSPatrick Mooney } 5344c87aefeSPatrick Mooney 5354c87aefeSPatrick Mooney /* 5364c87aefeSPatrick Mooney * Compute days since start of time 5374c87aefeSPatrick Mooney * First from years, then from months. 5384c87aefeSPatrick Mooney */ 5394c87aefeSPatrick Mooney days = 0; 5404c87aefeSPatrick Mooney for (i = POSIX_BASE_YEAR; i < year; i++) 5414c87aefeSPatrick Mooney days += days_in_year(i); 5424c87aefeSPatrick Mooney 5434c87aefeSPatrick Mooney /* Months */ 5444c87aefeSPatrick Mooney for (i = 1; i < ct->mon; i++) 54584971882SPatrick Mooney days += days_in_month(year, i); 5464c87aefeSPatrick Mooney days += (ct->day - 1); 5474c87aefeSPatrick Mooney 5484c87aefeSPatrick Mooney ts->tv_sec = (((time_t)days * 24 + ct->hour) * 60 + ct->min) * 60 + 5494c87aefeSPatrick Mooney ct->sec; 5504c87aefeSPatrick Mooney ts->tv_nsec = ct->nsec; 5514c87aefeSPatrick Mooney 5524c87aefeSPatrick Mooney #ifdef __FreeBSD__ 5534c87aefeSPatrick Mooney if (ct_debug) 5544c87aefeSPatrick Mooney printf(" = %ld.%09ld\n", (long)ts->tv_sec, (long)ts->tv_nsec); 5554c87aefeSPatrick Mooney #endif 5564c87aefeSPatrick Mooney return (0); 557bf21cd93STycho Nightingale } 558bf21cd93STycho Nightingale 559bf21cd93STycho Nightingale void 5604c87aefeSPatrick Mooney clock_ts_to_ct(struct timespec *ts, struct clocktime *ct) 5614c87aefeSPatrick Mooney { 5624c87aefeSPatrick Mooney int i, year, days; 5634c87aefeSPatrick Mooney time_t rsec; /* remainder seconds */ 5644c87aefeSPatrick Mooney time_t secs; 5654c87aefeSPatrick Mooney 5664c87aefeSPatrick Mooney secs = ts->tv_sec; 5674c87aefeSPatrick Mooney days = secs / SECDAY; 5684c87aefeSPatrick Mooney rsec = secs % SECDAY; 5694c87aefeSPatrick Mooney 5704c87aefeSPatrick Mooney ct->dow = day_of_week(days); 5714c87aefeSPatrick Mooney 5724c87aefeSPatrick Mooney /* Subtract out whole years, counting them in i. */ 5734c87aefeSPatrick Mooney for (year = POSIX_BASE_YEAR; days >= days_in_year(year); year++) 5744c87aefeSPatrick Mooney days -= days_in_year(year); 5754c87aefeSPatrick Mooney ct->year = year; 5764c87aefeSPatrick Mooney 5774c87aefeSPatrick Mooney /* Subtract out whole months, counting them in i. */ 5784c87aefeSPatrick Mooney for (i = 1; days >= days_in_month(year, i); i++) 5794c87aefeSPatrick Mooney days -= days_in_month(year, i); 5804c87aefeSPatrick Mooney ct->mon = i; 5814c87aefeSPatrick Mooney 5824c87aefeSPatrick Mooney /* Days are what is left over (+1) from all that. */ 5834c87aefeSPatrick Mooney ct->day = days + 1; 5844c87aefeSPatrick Mooney 5854c87aefeSPatrick Mooney /* Hours, minutes, seconds are easy */ 5864c87aefeSPatrick Mooney ct->hour = rsec / 3600; 5874c87aefeSPatrick Mooney rsec = rsec % 3600; 5884c87aefeSPatrick Mooney ct->min = rsec / 60; 5894c87aefeSPatrick Mooney rsec = rsec % 60; 5904c87aefeSPatrick Mooney ct->sec = rsec; 5914c87aefeSPatrick Mooney ct->nsec = ts->tv_nsec; 5924c87aefeSPatrick Mooney #ifdef __FreeBSD__ 5934c87aefeSPatrick Mooney if (ct_debug) { 5944c87aefeSPatrick Mooney printf("ts_to_ct(%ld.%09ld) = ", 5954c87aefeSPatrick Mooney (long)ts->tv_sec, (long)ts->tv_nsec); 5964c87aefeSPatrick Mooney print_ct(ct); 5974c87aefeSPatrick Mooney printf("\n"); 5984c87aefeSPatrick Mooney } 5994c87aefeSPatrick Mooney #endif 600bf21cd93STycho Nightingale } 6019250eb13SPatrick Mooney 6029250eb13SPatrick Mooney /* Equivalent to the FreeBSD rdtsc(), but with any necessary per-cpu offset */ 6039250eb13SPatrick Mooney uint64_t 6049250eb13SPatrick Mooney rdtsc_offset(void) 6059250eb13SPatrick Mooney { 6069250eb13SPatrick Mooney /* 6079250eb13SPatrick Mooney * The timestamp logic will decide if a delta need be applied to the 6089250eb13SPatrick Mooney * unscaled hrtime reading (effectively rdtsc), but we do require it be 6099250eb13SPatrick Mooney * backed by the TSC itself. 6109250eb13SPatrick Mooney */ 6119250eb13SPatrick Mooney extern hrtime_t (*gethrtimeunscaledf)(void); 6129250eb13SPatrick Mooney extern hrtime_t tsc_gethrtimeunscaled(void); 6139250eb13SPatrick Mooney extern hrtime_t tsc_gethrtimeunscaled_delta(void); 6149250eb13SPatrick Mooney 6159250eb13SPatrick Mooney ASSERT(*gethrtimeunscaledf == tsc_gethrtimeunscaled || 6169250eb13SPatrick Mooney *gethrtimeunscaledf == tsc_gethrtimeunscaled_delta); 6179250eb13SPatrick Mooney return ((uint64_t)gethrtimeunscaledf()); 6189250eb13SPatrick Mooney } 619