1bf21cd93STycho Nightingale /* 2bf21cd93STycho Nightingale * Copyright (c) 2004 Poul-Henning Kamp 3bf21cd93STycho Nightingale * All rights reserved. 4bf21cd93STycho Nightingale * 5bf21cd93STycho Nightingale * Redistribution and use in source and binary forms, with or without 6bf21cd93STycho Nightingale * modification, are permitted provided that the following conditions 7bf21cd93STycho Nightingale * are met: 8bf21cd93STycho Nightingale * 1. Redistributions of source code must retain the above copyright 9bf21cd93STycho Nightingale * notice, this list of conditions and the following disclaimer. 10bf21cd93STycho Nightingale * 2. Redistributions in binary form must reproduce the above copyright 11bf21cd93STycho Nightingale * notice, this list of conditions and the following disclaimer in the 12bf21cd93STycho Nightingale * documentation and/or other materials provided with the distribution. 13bf21cd93STycho Nightingale * 14bf21cd93STycho Nightingale * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15bf21cd93STycho Nightingale * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16bf21cd93STycho Nightingale * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17bf21cd93STycho Nightingale * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18bf21cd93STycho Nightingale * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19bf21cd93STycho Nightingale * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20bf21cd93STycho Nightingale * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21bf21cd93STycho Nightingale * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22bf21cd93STycho Nightingale * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23bf21cd93STycho Nightingale * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24bf21cd93STycho Nightingale * SUCH DAMAGE. 25bf21cd93STycho Nightingale * 26bf21cd93STycho Nightingale * $FreeBSD: head/sys/kern/subr_unit.c 255057 2013-08-30 07:37:45Z kib $ 27bf21cd93STycho Nightingale */ 28bf21cd93STycho Nightingale /* 29bf21cd93STycho Nightingale * This file and its contents are supplied under the terms of the 30bf21cd93STycho Nightingale * Common Development and Distribution License ("CDDL"), version 1.0. 31bf21cd93STycho Nightingale * You may only use this file in accordance with the terms of version 32bf21cd93STycho Nightingale * 1.0 of the CDDL. 33bf21cd93STycho Nightingale * 34bf21cd93STycho Nightingale * A full copy of the text of the CDDL should have accompanied this 35bf21cd93STycho Nightingale * source. A copy of the CDDL is also available via the Internet at 36bf21cd93STycho Nightingale * http://www.illumos.org/license/CDDL. 37bf21cd93STycho Nightingale * 38bf21cd93STycho Nightingale * Copyright 2014 Pluribus Networks Inc. 394c87aefeSPatrick Mooney * Copyright 2019 Joyent, Inc. 40ee8ae3faSPatrick Mooney * Copyright 2020 Oxide Computer Company 41bf21cd93STycho Nightingale */ 42bf21cd93STycho Nightingale 43bf21cd93STycho Nightingale #include <sys/types.h> 44bf21cd93STycho Nightingale #include <sys/archsystm.h> 45bf21cd93STycho Nightingale #include <sys/cpuset.h> 46bf21cd93STycho Nightingale #include <sys/fp.h> 47bf21cd93STycho Nightingale #include <sys/malloc.h> 48bf21cd93STycho Nightingale #include <sys/queue.h> 49bf21cd93STycho Nightingale #include <sys/spl.h> 50bf21cd93STycho Nightingale #include <sys/systm.h> 514c87aefeSPatrick Mooney #include <sys/ddidmareq.h> 524c87aefeSPatrick Mooney #include <sys/id_space.h> 534c87aefeSPatrick Mooney #include <sys/psm_defs.h> 544c87aefeSPatrick Mooney #include <sys/smp_impldefs.h> 554c87aefeSPatrick Mooney #include <sys/modhash.h> 564c87aefeSPatrick Mooney #include <sys/hma.h> 57bf21cd93STycho Nightingale 58eb9a1df2SHans Rosenfeld #include <sys/x86_archext.h> 59eb9a1df2SHans Rosenfeld 60bf21cd93STycho Nightingale #include <machine/cpufunc.h> 61bf21cd93STycho Nightingale #include <machine/md_var.h> 62bf21cd93STycho Nightingale #include <machine/specialreg.h> 63bf21cd93STycho Nightingale #include <machine/vmm.h> 64cf409e3fSDan Cross #include <machine/vmparam.h> 65bf21cd93STycho Nightingale #include <sys/vmm_impl.h> 664c87aefeSPatrick Mooney #include <sys/kernel.h> 67bf21cd93STycho Nightingale 68bf21cd93STycho Nightingale #include <vm/as.h> 69bf21cd93STycho Nightingale #include <vm/seg_kmem.h> 70bf21cd93STycho Nightingale 714c87aefeSPatrick Mooney SET_DECLARE(sysinit_set, struct sysinit); 724c87aefeSPatrick Mooney 734c87aefeSPatrick Mooney void 744c87aefeSPatrick Mooney sysinit(void) 754c87aefeSPatrick Mooney { 764c87aefeSPatrick Mooney struct sysinit **si; 774c87aefeSPatrick Mooney 784c87aefeSPatrick Mooney SET_FOREACH(si, sysinit_set) 794c87aefeSPatrick Mooney (*si)->func((*si)->data); 804c87aefeSPatrick Mooney } 814c87aefeSPatrick Mooney 822699b94cSPatrick Mooney uint8_t const bin2bcd_data[] = { 834c87aefeSPatrick Mooney 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 844c87aefeSPatrick Mooney 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 854c87aefeSPatrick Mooney 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 864c87aefeSPatrick Mooney 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 874c87aefeSPatrick Mooney 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 884c87aefeSPatrick Mooney 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 894c87aefeSPatrick Mooney 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 904c87aefeSPatrick Mooney 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 914c87aefeSPatrick Mooney 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 924c87aefeSPatrick Mooney 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99 934c87aefeSPatrick Mooney }; 944c87aefeSPatrick Mooney 95eb9a1df2SHans Rosenfeld void 960153d828SPatrick Mooney invalidate_cache_all(void) 97eb9a1df2SHans Rosenfeld { 98eb9a1df2SHans Rosenfeld cpuset_t cpuset; 99eb9a1df2SHans Rosenfeld 100eb9a1df2SHans Rosenfeld kpreempt_disable(); 101eb9a1df2SHans Rosenfeld cpuset_all_but(&cpuset, CPU->cpu_id); 102eb9a1df2SHans Rosenfeld xc_call((xc_arg_t)NULL, (xc_arg_t)NULL, (xc_arg_t)NULL, 103eb9a1df2SHans Rosenfeld CPUSET2BV(cpuset), (xc_func_t)invalidate_cache); 104eb9a1df2SHans Rosenfeld invalidate_cache(); 105eb9a1df2SHans Rosenfeld kpreempt_enable(); 106eb9a1df2SHans Rosenfeld } 107eb9a1df2SHans Rosenfeld 108bf21cd93STycho Nightingale vm_paddr_t 1090153d828SPatrick Mooney vtophys(void *va) 110bf21cd93STycho Nightingale { 111bf21cd93STycho Nightingale pfn_t pfn; 112bf21cd93STycho Nightingale 1134c87aefeSPatrick Mooney /* 1144c87aefeSPatrick Mooney * Since hat_getpfnum() may block on an htable mutex, this is not at 1154c87aefeSPatrick Mooney * all safe to run from a critical_enter/kpreempt_disable context. 1164c87aefeSPatrick Mooney * The FreeBSD analog does not have the same locking constraints, so 1174c87aefeSPatrick Mooney * close attention must be paid wherever this is called. 1184c87aefeSPatrick Mooney */ 1194c87aefeSPatrick Mooney ASSERT(curthread->t_preempt == 0); 1204c87aefeSPatrick Mooney 121bf21cd93STycho Nightingale pfn = hat_getpfnum(kas.a_hat, (caddr_t)va); 122bf21cd93STycho Nightingale ASSERT(pfn != PFN_INVALID); 123bf21cd93STycho Nightingale return (pfn << PAGE_SHIFT) | ((uintptr_t)va & PAGE_MASK); 124bf21cd93STycho Nightingale } 125bf21cd93STycho Nightingale 126bf21cd93STycho Nightingale int 127bf21cd93STycho Nightingale cpusetobj_ffs(const cpuset_t *set) 128bf21cd93STycho Nightingale { 1294c87aefeSPatrick Mooney uint_t large, small; 130bf21cd93STycho Nightingale 1314c87aefeSPatrick Mooney /* 1324c87aefeSPatrick Mooney * Rather than reaching into the cpuset_t ourselves, leave that task to 1334c87aefeSPatrick Mooney * cpuset_bounds(). The simplicity is worth the extra wasted work to 1344c87aefeSPatrick Mooney * find the upper bound. 1354c87aefeSPatrick Mooney */ 1364c87aefeSPatrick Mooney cpuset_bounds(set, &small, &large); 137bf21cd93STycho Nightingale 1384c87aefeSPatrick Mooney if (small == CPUSET_NOTINSET) { 1394c87aefeSPatrick Mooney /* The FreeBSD version returns 0 if it find nothing */ 1404c87aefeSPatrick Mooney return (0); 1414c87aefeSPatrick Mooney } 142bf21cd93STycho Nightingale 1434c87aefeSPatrick Mooney ASSERT3U(small, <=, INT_MAX); 144bf21cd93STycho Nightingale 1454c87aefeSPatrick Mooney /* Least significant bit index starts at 1 for valid results */ 1464c87aefeSPatrick Mooney return (small + 1); 147bf21cd93STycho Nightingale } 148bf21cd93STycho Nightingale 149bf21cd93STycho Nightingale struct kmem_item { 150bf21cd93STycho Nightingale void *addr; 151bf21cd93STycho Nightingale size_t size; 152bf21cd93STycho Nightingale }; 153bf21cd93STycho Nightingale static kmutex_t kmem_items_lock; 1544c87aefeSPatrick Mooney 1554c87aefeSPatrick Mooney static mod_hash_t *vmm_alloc_hash; 1564c87aefeSPatrick Mooney uint_t vmm_alloc_hash_nchains = 16381; 1574c87aefeSPatrick Mooney uint_t vmm_alloc_hash_size = PAGESIZE; 1584c87aefeSPatrick Mooney 1594c87aefeSPatrick Mooney static void 1604c87aefeSPatrick Mooney vmm_alloc_hash_valdtor(mod_hash_val_t val) 1614c87aefeSPatrick Mooney { 1624c87aefeSPatrick Mooney struct kmem_item *i = (struct kmem_item *)val; 1634c87aefeSPatrick Mooney 1644c87aefeSPatrick Mooney kmem_free(i->addr, i->size); 1654c87aefeSPatrick Mooney kmem_free(i, sizeof (struct kmem_item)); 1664c87aefeSPatrick Mooney } 1674c87aefeSPatrick Mooney 1684c87aefeSPatrick Mooney static void 1694c87aefeSPatrick Mooney vmm_alloc_init(void) 1704c87aefeSPatrick Mooney { 1714c87aefeSPatrick Mooney vmm_alloc_hash = mod_hash_create_ptrhash("vmm_alloc_hash", 1724c87aefeSPatrick Mooney vmm_alloc_hash_nchains, vmm_alloc_hash_valdtor, 1734c87aefeSPatrick Mooney vmm_alloc_hash_size); 1744c87aefeSPatrick Mooney 1754c87aefeSPatrick Mooney VERIFY(vmm_alloc_hash != NULL); 1764c87aefeSPatrick Mooney } 1774c87aefeSPatrick Mooney 1784c87aefeSPatrick Mooney static uint_t 1794c87aefeSPatrick Mooney vmm_alloc_check(mod_hash_key_t key, mod_hash_val_t *val, void *unused) 1804c87aefeSPatrick Mooney { 1814c87aefeSPatrick Mooney struct kmem_item *i = (struct kmem_item *)val; 1824c87aefeSPatrick Mooney 1839dc804b9SPatrick Mooney cmn_err(CE_PANIC, "!vmm_alloc_check: hash not empty: %p, %lu", i->addr, 1844c87aefeSPatrick Mooney i->size); 1854c87aefeSPatrick Mooney 1864c87aefeSPatrick Mooney return (MH_WALK_TERMINATE); 1874c87aefeSPatrick Mooney } 1884c87aefeSPatrick Mooney 1894c87aefeSPatrick Mooney static void 1904c87aefeSPatrick Mooney vmm_alloc_cleanup(void) 1914c87aefeSPatrick Mooney { 1924c87aefeSPatrick Mooney mod_hash_walk(vmm_alloc_hash, vmm_alloc_check, NULL); 1934c87aefeSPatrick Mooney mod_hash_destroy_ptrhash(vmm_alloc_hash); 1944c87aefeSPatrick Mooney } 195bf21cd93STycho Nightingale 196bf21cd93STycho Nightingale void * 197bf21cd93STycho Nightingale malloc(unsigned long size, struct malloc_type *mtp, int flags) 198bf21cd93STycho Nightingale { 199bf21cd93STycho Nightingale void *p; 200bf21cd93STycho Nightingale struct kmem_item *i; 201bf21cd93STycho Nightingale int kmem_flag = KM_SLEEP; 202bf21cd93STycho Nightingale 203bf21cd93STycho Nightingale if (flags & M_NOWAIT) 204bf21cd93STycho Nightingale kmem_flag = KM_NOSLEEP; 205bf21cd93STycho Nightingale 206bf21cd93STycho Nightingale if (flags & M_ZERO) { 2074c87aefeSPatrick Mooney p = kmem_zalloc(size, kmem_flag); 208bf21cd93STycho Nightingale } else { 2094c87aefeSPatrick Mooney p = kmem_alloc(size, kmem_flag); 2104c87aefeSPatrick Mooney } 2114c87aefeSPatrick Mooney 2124c87aefeSPatrick Mooney if (p == NULL) 2134c87aefeSPatrick Mooney return (NULL); 2144c87aefeSPatrick Mooney 2154c87aefeSPatrick Mooney i = kmem_zalloc(sizeof (struct kmem_item), kmem_flag); 2164c87aefeSPatrick Mooney 2174c87aefeSPatrick Mooney if (i == NULL) { 2184c87aefeSPatrick Mooney kmem_free(p, size); 2194c87aefeSPatrick Mooney return (NULL); 220bf21cd93STycho Nightingale } 221bf21cd93STycho Nightingale 222bf21cd93STycho Nightingale mutex_enter(&kmem_items_lock); 223bf21cd93STycho Nightingale i->addr = p; 224bf21cd93STycho Nightingale i->size = size; 225bf21cd93STycho Nightingale 2264c87aefeSPatrick Mooney VERIFY(mod_hash_insert(vmm_alloc_hash, 2274c87aefeSPatrick Mooney (mod_hash_key_t)PHYS_TO_DMAP(vtophys(p)), (mod_hash_val_t)i) == 0); 2284c87aefeSPatrick Mooney 229bf21cd93STycho Nightingale mutex_exit(&kmem_items_lock); 230bf21cd93STycho Nightingale 231bf21cd93STycho Nightingale return (p); 232bf21cd93STycho Nightingale } 233bf21cd93STycho Nightingale 234bf21cd93STycho Nightingale void 235bf21cd93STycho Nightingale free(void *addr, struct malloc_type *mtp) 236bf21cd93STycho Nightingale { 237bf21cd93STycho Nightingale mutex_enter(&kmem_items_lock); 2384c87aefeSPatrick Mooney VERIFY(mod_hash_destroy(vmm_alloc_hash, 2394c87aefeSPatrick Mooney (mod_hash_key_t)PHYS_TO_DMAP(vtophys(addr))) == 0); 240bf21cd93STycho Nightingale mutex_exit(&kmem_items_lock); 2414c87aefeSPatrick Mooney } 2424c87aefeSPatrick Mooney 2434c87aefeSPatrick Mooney extern void *contig_alloc(size_t, ddi_dma_attr_t *, uintptr_t, int); 2444c87aefeSPatrick Mooney extern void contig_free(void *, size_t); 245bf21cd93STycho Nightingale 2464c87aefeSPatrick Mooney void * 2474c87aefeSPatrick Mooney contigmalloc(unsigned long size, struct malloc_type *type, int flags, 2484c87aefeSPatrick Mooney vm_paddr_t low, vm_paddr_t high, unsigned long alignment, 2494c87aefeSPatrick Mooney vm_paddr_t boundary) 2504c87aefeSPatrick Mooney { 2514c87aefeSPatrick Mooney ddi_dma_attr_t attr = { 2524c87aefeSPatrick Mooney /* Using fastboot_dma_attr as a guide... */ 2534c87aefeSPatrick Mooney DMA_ATTR_V0, 2544c87aefeSPatrick Mooney low, /* dma_attr_addr_lo */ 2554c87aefeSPatrick Mooney high, /* dma_attr_addr_hi */ 2564c87aefeSPatrick Mooney 0x00000000FFFFFFFFULL, /* dma_attr_count_max */ 2574c87aefeSPatrick Mooney alignment, /* dma_attr_align */ 2584c87aefeSPatrick Mooney 1, /* dma_attr_burstsize */ 2594c87aefeSPatrick Mooney 1, /* dma_attr_minxfer */ 2604c87aefeSPatrick Mooney 0x00000000FFFFFFFFULL, /* dma_attr_maxxfer */ 2614c87aefeSPatrick Mooney 0x00000000FFFFFFFFULL, /* dma_attr_seg: any */ 2624c87aefeSPatrick Mooney 1, /* dma_attr_sgllen */ 2634c87aefeSPatrick Mooney alignment, /* dma_attr_granular */ 2644c87aefeSPatrick Mooney 0, /* dma_attr_flags */ 2654c87aefeSPatrick Mooney }; 2664c87aefeSPatrick Mooney int cansleep = (flags & M_WAITOK); 2674c87aefeSPatrick Mooney void *result; 2684c87aefeSPatrick Mooney 2694c87aefeSPatrick Mooney ASSERT(alignment == PAGESIZE); 2704c87aefeSPatrick Mooney 2714c87aefeSPatrick Mooney result = contig_alloc((size_t)size, &attr, alignment, cansleep); 2724c87aefeSPatrick Mooney 2734c87aefeSPatrick Mooney if (result != NULL && (flags & M_ZERO) != 0) { 2744c87aefeSPatrick Mooney bzero(result, size); 2754c87aefeSPatrick Mooney } 2764c87aefeSPatrick Mooney return (result); 2774c87aefeSPatrick Mooney } 2784c87aefeSPatrick Mooney 2794c87aefeSPatrick Mooney void 2804c87aefeSPatrick Mooney contigfree(void *addr, unsigned long size, struct malloc_type *type) 2814c87aefeSPatrick Mooney { 2824c87aefeSPatrick Mooney contig_free(addr, size); 283bf21cd93STycho Nightingale } 284bf21cd93STycho Nightingale 285bf21cd93STycho Nightingale void 286bf21cd93STycho Nightingale critical_enter(void) 287bf21cd93STycho Nightingale { 288bf21cd93STycho Nightingale kpreempt_disable(); 289bf21cd93STycho Nightingale } 290bf21cd93STycho Nightingale 291bf21cd93STycho Nightingale void 292bf21cd93STycho Nightingale critical_exit(void) 293bf21cd93STycho Nightingale { 294bf21cd93STycho Nightingale kpreempt_enable(); 295bf21cd93STycho Nightingale } 296bf21cd93STycho Nightingale 297bf21cd93STycho Nightingale 298bf21cd93STycho Nightingale static void 299bf21cd93STycho Nightingale vmm_glue_callout_handler(void *arg) 300bf21cd93STycho Nightingale { 301bf21cd93STycho Nightingale struct callout *c = arg; 302bf21cd93STycho Nightingale 303ee8ae3faSPatrick Mooney if (callout_active(c)) { 304ee8ae3faSPatrick Mooney /* 305ee8ae3faSPatrick Mooney * Record the handler fire time so that callout_pending() is 306ee8ae3faSPatrick Mooney * able to detect if the callout becomes rescheduled during the 307ee8ae3faSPatrick Mooney * course of the handler. 308ee8ae3faSPatrick Mooney */ 309ee8ae3faSPatrick Mooney c->c_fired = gethrtime(); 310bf21cd93STycho Nightingale (c->c_func)(c->c_arg); 311bf21cd93STycho Nightingale } 312bf21cd93STycho Nightingale } 313bf21cd93STycho Nightingale 314bf21cd93STycho Nightingale void 315bf21cd93STycho Nightingale vmm_glue_callout_init(struct callout *c, int mpsafe) 316bf21cd93STycho Nightingale { 317bf21cd93STycho Nightingale cyc_handler_t hdlr; 318bf21cd93STycho Nightingale cyc_time_t when; 319bf21cd93STycho Nightingale 320bf21cd93STycho Nightingale hdlr.cyh_level = CY_LOW_LEVEL; 321bf21cd93STycho Nightingale hdlr.cyh_func = vmm_glue_callout_handler; 322bf21cd93STycho Nightingale hdlr.cyh_arg = c; 323bf21cd93STycho Nightingale when.cyt_when = CY_INFINITY; 324bf21cd93STycho Nightingale when.cyt_interval = CY_INFINITY; 325ee8ae3faSPatrick Mooney bzero(c, sizeof (*c)); 326bf21cd93STycho Nightingale 327bf21cd93STycho Nightingale mutex_enter(&cpu_lock); 3284c87aefeSPatrick Mooney c->c_cyc_id = cyclic_add(&hdlr, &when); 329bf21cd93STycho Nightingale mutex_exit(&cpu_lock); 330bf21cd93STycho Nightingale } 331bf21cd93STycho Nightingale 3325103e761SPatrick Mooney void 3335103e761SPatrick Mooney callout_reset_hrtime(struct callout *c, hrtime_t target, void (*func)(void *), 3345103e761SPatrick Mooney void *arg, int flags) 335bf21cd93STycho Nightingale { 336bf21cd93STycho Nightingale ASSERT(c->c_cyc_id != CYCLIC_NONE); 337bf21cd93STycho Nightingale 338ee8ae3faSPatrick Mooney if ((flags & C_ABSOLUTE) == 0) { 339ee8ae3faSPatrick Mooney target += gethrtime(); 340ee8ae3faSPatrick Mooney } 341ee8ae3faSPatrick Mooney 342bf21cd93STycho Nightingale c->c_func = func; 343bf21cd93STycho Nightingale c->c_arg = arg; 344ee8ae3faSPatrick Mooney c->c_target = target; 345*e0994bd2SPatrick Mooney (void) cyclic_reprogram(c->c_cyc_id, target); 346bf21cd93STycho Nightingale } 347bf21cd93STycho Nightingale 348*e0994bd2SPatrick Mooney void 349bf21cd93STycho Nightingale vmm_glue_callout_stop(struct callout *c) 350bf21cd93STycho Nightingale { 351bf21cd93STycho Nightingale ASSERT(c->c_cyc_id != CYCLIC_NONE); 352ee8ae3faSPatrick Mooney 353ee8ae3faSPatrick Mooney c->c_target = 0; 354*e0994bd2SPatrick Mooney (void) cyclic_reprogram(c->c_cyc_id, CY_INFINITY); 355bf21cd93STycho Nightingale } 356bf21cd93STycho Nightingale 357*e0994bd2SPatrick Mooney void 358bf21cd93STycho Nightingale vmm_glue_callout_drain(struct callout *c) 359bf21cd93STycho Nightingale { 360bf21cd93STycho Nightingale ASSERT(c->c_cyc_id != CYCLIC_NONE); 361ee8ae3faSPatrick Mooney 362ee8ae3faSPatrick Mooney c->c_target = 0; 363bf21cd93STycho Nightingale mutex_enter(&cpu_lock); 364bf21cd93STycho Nightingale cyclic_remove(c->c_cyc_id); 365bf21cd93STycho Nightingale c->c_cyc_id = CYCLIC_NONE; 366bf21cd93STycho Nightingale mutex_exit(&cpu_lock); 367bf21cd93STycho Nightingale } 368bf21cd93STycho Nightingale 369bf21cd93STycho Nightingale void 3704c87aefeSPatrick Mooney vmm_glue_callout_localize(struct callout *c) 371bf21cd93STycho Nightingale { 3724c87aefeSPatrick Mooney mutex_enter(&cpu_lock); 3734c87aefeSPatrick Mooney cyclic_move_here(c->c_cyc_id); 3744c87aefeSPatrick Mooney mutex_exit(&cpu_lock); 375bf21cd93STycho Nightingale } 376bf21cd93STycho Nightingale 3775103e761SPatrick Mooney /* 3785103e761SPatrick Mooney * Given an interval (in ns) and a frequency (in hz), calculate the number of 3795103e761SPatrick Mooney * "ticks" at that frequency which cover the interval. 3805103e761SPatrick Mooney */ 3815103e761SPatrick Mooney uint64_t 3825103e761SPatrick Mooney hrt_freq_count(hrtime_t interval, uint32_t freq) 3835103e761SPatrick Mooney { 3845103e761SPatrick Mooney ASSERT3S(interval, >=, 0); 3855103e761SPatrick Mooney const uint64_t sec = interval / NANOSEC; 3865103e761SPatrick Mooney const uint64_t nsec = interval % NANOSEC; 3875103e761SPatrick Mooney 3885103e761SPatrick Mooney return ((sec * freq) + ((nsec * freq) / NANOSEC)); 3895103e761SPatrick Mooney } 3905103e761SPatrick Mooney 3915103e761SPatrick Mooney /* 3925103e761SPatrick Mooney * Given a frequency (in hz) and number of "ticks", calculate the interval 3935103e761SPatrick Mooney * (in ns) which would be covered by those ticks. 3945103e761SPatrick Mooney */ 3955103e761SPatrick Mooney hrtime_t 3965103e761SPatrick Mooney hrt_freq_interval(uint32_t freq, uint64_t count) 3975103e761SPatrick Mooney { 3985103e761SPatrick Mooney const uint64_t sec = count / freq; 3995103e761SPatrick Mooney const uint64_t frac = count % freq; 4005103e761SPatrick Mooney 4015103e761SPatrick Mooney return ((NANOSEC * sec) + ((frac * NANOSEC) / freq)); 4025103e761SPatrick Mooney } 4035103e761SPatrick Mooney 4045103e761SPatrick Mooney 4052699b94cSPatrick Mooney uint_t cpu_high; /* Highest arg to CPUID */ 4062699b94cSPatrick Mooney uint_t cpu_exthigh; /* Highest arg to extended CPUID */ 4072699b94cSPatrick Mooney uint_t cpu_id; /* Stepping ID */ 408bf21cd93STycho Nightingale char cpu_vendor[20]; /* CPU Origin code */ 409bf21cd93STycho Nightingale 410bf21cd93STycho Nightingale static void 411bf21cd93STycho Nightingale vmm_cpuid_init(void) 412bf21cd93STycho Nightingale { 4132699b94cSPatrick Mooney uint_t regs[4]; 414bf21cd93STycho Nightingale 415bf21cd93STycho Nightingale do_cpuid(0, regs); 416bf21cd93STycho Nightingale cpu_high = regs[0]; 4172699b94cSPatrick Mooney ((uint_t *)&cpu_vendor)[0] = regs[1]; 4182699b94cSPatrick Mooney ((uint_t *)&cpu_vendor)[1] = regs[3]; 4192699b94cSPatrick Mooney ((uint_t *)&cpu_vendor)[2] = regs[2]; 420bf21cd93STycho Nightingale cpu_vendor[12] = '\0'; 421bf21cd93STycho Nightingale 422bf21cd93STycho Nightingale do_cpuid(1, regs); 423bf21cd93STycho Nightingale cpu_id = regs[0]; 424bf21cd93STycho Nightingale 425bf21cd93STycho Nightingale do_cpuid(0x80000000, regs); 426bf21cd93STycho Nightingale cpu_exthigh = regs[0]; 427bf21cd93STycho Nightingale } 428bf21cd93STycho Nightingale 4294c87aefeSPatrick Mooney void 4304c87aefeSPatrick Mooney vmm_sol_glue_init(void) 431bf21cd93STycho Nightingale { 4324c87aefeSPatrick Mooney vmm_alloc_init(); 4334c87aefeSPatrick Mooney vmm_cpuid_init(); 434bf21cd93STycho Nightingale } 435bf21cd93STycho Nightingale 436bf21cd93STycho Nightingale void 4374c87aefeSPatrick Mooney vmm_sol_glue_cleanup(void) 438bf21cd93STycho Nightingale { 4394c87aefeSPatrick Mooney vmm_alloc_cleanup(); 440bf21cd93STycho Nightingale } 441bf21cd93STycho Nightingale 4424c87aefeSPatrick Mooney 4434c87aefeSPatrick Mooney /* From FreeBSD's sys/kern/subr_clock.c */ 4444c87aefeSPatrick Mooney 4454c87aefeSPatrick Mooney /*- 4464c87aefeSPatrick Mooney * Copyright (c) 1988 University of Utah. 4474c87aefeSPatrick Mooney * Copyright (c) 1982, 1990, 1993 4484c87aefeSPatrick Mooney * The Regents of the University of California. All rights reserved. 4494c87aefeSPatrick Mooney * 4504c87aefeSPatrick Mooney * This code is derived from software contributed to Berkeley by 4514c87aefeSPatrick Mooney * the Systems Programming Group of the University of Utah Computer 4524c87aefeSPatrick Mooney * Science Department. 4534c87aefeSPatrick Mooney * 4544c87aefeSPatrick Mooney * Redistribution and use in source and binary forms, with or without 4554c87aefeSPatrick Mooney * modification, are permitted provided that the following conditions 4564c87aefeSPatrick Mooney * are met: 4574c87aefeSPatrick Mooney * 1. Redistributions of source code must retain the above copyright 4584c87aefeSPatrick Mooney * notice, this list of conditions and the following disclaimer. 4594c87aefeSPatrick Mooney * 2. Redistributions in binary form must reproduce the above copyright 4604c87aefeSPatrick Mooney * notice, this list of conditions and the following disclaimer in the 4614c87aefeSPatrick Mooney * documentation and/or other materials provided with the distribution. 4624c87aefeSPatrick Mooney * 4. Neither the name of the University nor the names of its contributors 4634c87aefeSPatrick Mooney * may be used to endorse or promote products derived from this software 4644c87aefeSPatrick Mooney * without specific prior written permission. 4654c87aefeSPatrick Mooney * 4664c87aefeSPatrick Mooney * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 4674c87aefeSPatrick Mooney * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 4684c87aefeSPatrick Mooney * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 4694c87aefeSPatrick Mooney * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 4704c87aefeSPatrick Mooney * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 4714c87aefeSPatrick Mooney * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 4724c87aefeSPatrick Mooney * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 4734c87aefeSPatrick Mooney * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 4744c87aefeSPatrick Mooney * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 4754c87aefeSPatrick Mooney * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 4764c87aefeSPatrick Mooney * SUCH DAMAGE. 4774c87aefeSPatrick Mooney * 4784c87aefeSPatrick Mooney * from: Utah $Hdr: clock.c 1.18 91/01/21$ 4794c87aefeSPatrick Mooney * from: @(#)clock.c 8.2 (Berkeley) 1/12/94 4804c87aefeSPatrick Mooney * from: NetBSD: clock_subr.c,v 1.6 2001/07/07 17:04:02 thorpej Exp 4814c87aefeSPatrick Mooney * and 4824c87aefeSPatrick Mooney * from: src/sys/i386/isa/clock.c,v 1.176 2001/09/04 4834c87aefeSPatrick Mooney */ 4844c87aefeSPatrick Mooney 4854c87aefeSPatrick Mooney #include <sys/clock.h> 4864c87aefeSPatrick Mooney 4872699b94cSPatrick Mooney /* 4884c87aefeSPatrick Mooney * Generic routines to convert between a POSIX date 4894c87aefeSPatrick Mooney * (seconds since 1/1/1970) and yr/mo/day/hr/min/sec 4904c87aefeSPatrick Mooney * Derived from NetBSD arch/hp300/hp300/clock.c 4914c87aefeSPatrick Mooney */ 4924c87aefeSPatrick Mooney 4934c87aefeSPatrick Mooney #define FEBRUARY 2 49484971882SPatrick Mooney #define days_in_year(y) (leapyear(y) ? 366 : 365) 4954c87aefeSPatrick Mooney #define days_in_month(y, m) \ 4964c87aefeSPatrick Mooney (month_days[(m) - 1] + (m == FEBRUARY ? leapyear(y) : 0)) 4974c87aefeSPatrick Mooney /* Day of week. Days are counted from 1/1/1970, which was a Thursday */ 4984c87aefeSPatrick Mooney #define day_of_week(days) (((days) + 4) % 7) 4994c87aefeSPatrick Mooney 5004c87aefeSPatrick Mooney static const int month_days[12] = { 5014c87aefeSPatrick Mooney 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 5024c87aefeSPatrick Mooney }; 5034c87aefeSPatrick Mooney 5044c87aefeSPatrick Mooney 5054c87aefeSPatrick Mooney /* 5064c87aefeSPatrick Mooney * This inline avoids some unnecessary modulo operations 5074c87aefeSPatrick Mooney * as compared with the usual macro: 5084c87aefeSPatrick Mooney * ( ((year % 4) == 0 && 5094c87aefeSPatrick Mooney * (year % 100) != 0) || 5104c87aefeSPatrick Mooney * ((year % 400) == 0) ) 5114c87aefeSPatrick Mooney * It is otherwise equivalent. 5124c87aefeSPatrick Mooney */ 5134c87aefeSPatrick Mooney static int 5144c87aefeSPatrick Mooney leapyear(int year) 515bf21cd93STycho Nightingale { 5164c87aefeSPatrick Mooney int rv = 0; 5174c87aefeSPatrick Mooney 5184c87aefeSPatrick Mooney if ((year & 3) == 0) { 5194c87aefeSPatrick Mooney rv = 1; 5204c87aefeSPatrick Mooney if ((year % 100) == 0) { 5214c87aefeSPatrick Mooney rv = 0; 5224c87aefeSPatrick Mooney if ((year % 400) == 0) 5234c87aefeSPatrick Mooney rv = 1; 5244c87aefeSPatrick Mooney } 525bf21cd93STycho Nightingale } 5264c87aefeSPatrick Mooney return (rv); 527bf21cd93STycho Nightingale } 528bf21cd93STycho Nightingale 5294c87aefeSPatrick Mooney int 5304c87aefeSPatrick Mooney clock_ct_to_ts(struct clocktime *ct, struct timespec *ts) 531bf21cd93STycho Nightingale { 5324c87aefeSPatrick Mooney int i, year, days; 5334c87aefeSPatrick Mooney 5344c87aefeSPatrick Mooney year = ct->year; 5354c87aefeSPatrick Mooney 5364c87aefeSPatrick Mooney #ifdef __FreeBSD__ 5374c87aefeSPatrick Mooney if (ct_debug) { 5384c87aefeSPatrick Mooney printf("ct_to_ts("); 5394c87aefeSPatrick Mooney print_ct(ct); 5404c87aefeSPatrick Mooney printf(")"); 5414c87aefeSPatrick Mooney } 5424c87aefeSPatrick Mooney #endif 5434c87aefeSPatrick Mooney 5444c87aefeSPatrick Mooney /* Sanity checks. */ 5454c87aefeSPatrick Mooney if (ct->mon < 1 || ct->mon > 12 || ct->day < 1 || 5464c87aefeSPatrick Mooney ct->day > days_in_month(year, ct->mon) || 5472699b94cSPatrick Mooney ct->hour > 23 || ct->min > 59 || ct->sec > 59 || 5482699b94cSPatrick Mooney (sizeof (time_t) == 4 && year > 2037)) { /* time_t overflow */ 5494c87aefeSPatrick Mooney #ifdef __FreeBSD__ 5504c87aefeSPatrick Mooney if (ct_debug) 5514c87aefeSPatrick Mooney printf(" = EINVAL\n"); 5524c87aefeSPatrick Mooney #endif 5534c87aefeSPatrick Mooney return (EINVAL); 5544c87aefeSPatrick Mooney } 5554c87aefeSPatrick Mooney 5564c87aefeSPatrick Mooney /* 5574c87aefeSPatrick Mooney * Compute days since start of time 5584c87aefeSPatrick Mooney * First from years, then from months. 5594c87aefeSPatrick Mooney */ 5604c87aefeSPatrick Mooney days = 0; 5614c87aefeSPatrick Mooney for (i = POSIX_BASE_YEAR; i < year; i++) 5624c87aefeSPatrick Mooney days += days_in_year(i); 5634c87aefeSPatrick Mooney 5644c87aefeSPatrick Mooney /* Months */ 5654c87aefeSPatrick Mooney for (i = 1; i < ct->mon; i++) 56684971882SPatrick Mooney days += days_in_month(year, i); 5674c87aefeSPatrick Mooney days += (ct->day - 1); 5684c87aefeSPatrick Mooney 5694c87aefeSPatrick Mooney ts->tv_sec = (((time_t)days * 24 + ct->hour) * 60 + ct->min) * 60 + 5704c87aefeSPatrick Mooney ct->sec; 5714c87aefeSPatrick Mooney ts->tv_nsec = ct->nsec; 5724c87aefeSPatrick Mooney 5734c87aefeSPatrick Mooney #ifdef __FreeBSD__ 5744c87aefeSPatrick Mooney if (ct_debug) 5754c87aefeSPatrick Mooney printf(" = %ld.%09ld\n", (long)ts->tv_sec, (long)ts->tv_nsec); 5764c87aefeSPatrick Mooney #endif 5774c87aefeSPatrick Mooney return (0); 578bf21cd93STycho Nightingale } 579bf21cd93STycho Nightingale 580bf21cd93STycho Nightingale void 5814c87aefeSPatrick Mooney clock_ts_to_ct(struct timespec *ts, struct clocktime *ct) 5824c87aefeSPatrick Mooney { 5834c87aefeSPatrick Mooney int i, year, days; 5844c87aefeSPatrick Mooney time_t rsec; /* remainder seconds */ 5854c87aefeSPatrick Mooney time_t secs; 5864c87aefeSPatrick Mooney 5874c87aefeSPatrick Mooney secs = ts->tv_sec; 5884c87aefeSPatrick Mooney days = secs / SECDAY; 5894c87aefeSPatrick Mooney rsec = secs % SECDAY; 5904c87aefeSPatrick Mooney 5914c87aefeSPatrick Mooney ct->dow = day_of_week(days); 5924c87aefeSPatrick Mooney 5934c87aefeSPatrick Mooney /* Subtract out whole years, counting them in i. */ 5944c87aefeSPatrick Mooney for (year = POSIX_BASE_YEAR; days >= days_in_year(year); year++) 5954c87aefeSPatrick Mooney days -= days_in_year(year); 5964c87aefeSPatrick Mooney ct->year = year; 5974c87aefeSPatrick Mooney 5984c87aefeSPatrick Mooney /* Subtract out whole months, counting them in i. */ 5994c87aefeSPatrick Mooney for (i = 1; days >= days_in_month(year, i); i++) 6004c87aefeSPatrick Mooney days -= days_in_month(year, i); 6014c87aefeSPatrick Mooney ct->mon = i; 6024c87aefeSPatrick Mooney 6034c87aefeSPatrick Mooney /* Days are what is left over (+1) from all that. */ 6044c87aefeSPatrick Mooney ct->day = days + 1; 6054c87aefeSPatrick Mooney 6064c87aefeSPatrick Mooney /* Hours, minutes, seconds are easy */ 6074c87aefeSPatrick Mooney ct->hour = rsec / 3600; 6084c87aefeSPatrick Mooney rsec = rsec % 3600; 6094c87aefeSPatrick Mooney ct->min = rsec / 60; 6104c87aefeSPatrick Mooney rsec = rsec % 60; 6114c87aefeSPatrick Mooney ct->sec = rsec; 6124c87aefeSPatrick Mooney ct->nsec = ts->tv_nsec; 6134c87aefeSPatrick Mooney #ifdef __FreeBSD__ 6144c87aefeSPatrick Mooney if (ct_debug) { 6154c87aefeSPatrick Mooney printf("ts_to_ct(%ld.%09ld) = ", 6164c87aefeSPatrick Mooney (long)ts->tv_sec, (long)ts->tv_nsec); 6174c87aefeSPatrick Mooney print_ct(ct); 6184c87aefeSPatrick Mooney printf("\n"); 6194c87aefeSPatrick Mooney } 6204c87aefeSPatrick Mooney #endif 621bf21cd93STycho Nightingale } 6229250eb13SPatrick Mooney 6239250eb13SPatrick Mooney /* Equivalent to the FreeBSD rdtsc(), but with any necessary per-cpu offset */ 6249250eb13SPatrick Mooney uint64_t 6259250eb13SPatrick Mooney rdtsc_offset(void) 6269250eb13SPatrick Mooney { 6279250eb13SPatrick Mooney /* 6289250eb13SPatrick Mooney * The timestamp logic will decide if a delta need be applied to the 6299250eb13SPatrick Mooney * unscaled hrtime reading (effectively rdtsc), but we do require it be 6309250eb13SPatrick Mooney * backed by the TSC itself. 6319250eb13SPatrick Mooney */ 6329250eb13SPatrick Mooney extern hrtime_t (*gethrtimeunscaledf)(void); 6339250eb13SPatrick Mooney extern hrtime_t tsc_gethrtimeunscaled(void); 6349250eb13SPatrick Mooney extern hrtime_t tsc_gethrtimeunscaled_delta(void); 6359250eb13SPatrick Mooney 6369250eb13SPatrick Mooney ASSERT(*gethrtimeunscaledf == tsc_gethrtimeunscaled || 6379250eb13SPatrick Mooney *gethrtimeunscaledf == tsc_gethrtimeunscaled_delta); 6389250eb13SPatrick Mooney return ((uint64_t)gethrtimeunscaledf()); 6399250eb13SPatrick Mooney } 640