17c478bdstevel@tonic-gate/*
27c478bdstevel@tonic-gate * CDDL HEADER START
37c478bdstevel@tonic-gate *
47c478bdstevel@tonic-gate * The contents of this file are subject to the terms of the
5100b72fandrei * Common Development and Distribution License (the "License").
6100b72fandrei * You may not use this file except in compliance with the License.
77c478bdstevel@tonic-gate *
87c478bdstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bdstevel@tonic-gate * or http://www.opensolaris.org/os/licensing.
107c478bdstevel@tonic-gate * See the License for the specific language governing permissions
117c478bdstevel@tonic-gate * and limitations under the License.
127c478bdstevel@tonic-gate *
137c478bdstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each
147c478bdstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bdstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the
167c478bdstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying
177c478bdstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bdstevel@tonic-gate *
197c478bdstevel@tonic-gate * CDDL HEADER END
207c478bdstevel@tonic-gate */
21d669432Yuri Pankov
227c478bdstevel@tonic-gate/*
230db3240Stephen Hanson * Copyright (c) 1993, 2010, Oracle and/or its affiliates. All rights reserved.
249e3e4dfGarrett D'Amore * Copyright 2012 DEY Storage Systems, Inc.  All rights reserved.
25d669432Yuri Pankov * Copyright 2017 Nexenta Systems, Inc.
2674ecdb5John Levon * Copyright (c) 2018 Joyent, Inc.
27af868f4Matthew Ahrens * Copyright (c) 2015 by Delphix. All rights reserved.
28fdcca78Joshua M. Clulow * Copyright 2020 Oxide Computer Company
297c478bdstevel@tonic-gate */
30a311483Gerry Liu/*
31a311483Gerry Liu * Copyright (c) 2010, Intel Corporation.
32a311483Gerry Liu * All rights reserved.
33a311483Gerry Liu */
347c478bdstevel@tonic-gate
357c478bdstevel@tonic-gate#include <sys/types.h>
367c478bdstevel@tonic-gate#include <sys/t_lock.h>
377c478bdstevel@tonic-gate#include <sys/param.h>
387c478bdstevel@tonic-gate#include <sys/sysmacros.h>
397c478bdstevel@tonic-gate#include <sys/signal.h>
407c478bdstevel@tonic-gate#include <sys/systm.h>
417c478bdstevel@tonic-gate#include <sys/user.h>
427c478bdstevel@tonic-gate#include <sys/mman.h>
437c478bdstevel@tonic-gate#include <sys/vm.h>
447c478bdstevel@tonic-gate#include <sys/conf.h>
457c478bdstevel@tonic-gate#include <sys/avintr.h>
467c478bdstevel@tonic-gate#include <sys/autoconf.h>
477c478bdstevel@tonic-gate#include <sys/disp.h>
487c478bdstevel@tonic-gate#include <sys/class.h>
497c478bdstevel@tonic-gate#include <sys/bitmap.h>
507c478bdstevel@tonic-gate
517c478bdstevel@tonic-gate#include <sys/privregs.h>
527c478bdstevel@tonic-gate
537c478bdstevel@tonic-gate#include <sys/proc.h>
547c478bdstevel@tonic-gate#include <sys/buf.h>
557c478bdstevel@tonic-gate#include <sys/kmem.h>
56ae115bcmrj#include <sys/mem.h>
577c478bdstevel@tonic-gate#include <sys/kstat.h>
587c478bdstevel@tonic-gate
597c478bdstevel@tonic-gate#include <sys/reboot.h>
607c478bdstevel@tonic-gate
617c478bdstevel@tonic-gate#include <sys/cred.h>
627c478bdstevel@tonic-gate#include <sys/vnode.h>
637c478bdstevel@tonic-gate#include <sys/file.h>
647c478bdstevel@tonic-gate
657c478bdstevel@tonic-gate#include <sys/procfs.h>
667c478bdstevel@tonic-gate
677c478bdstevel@tonic-gate#include <sys/vfs.h>
687c478bdstevel@tonic-gate#include <sys/cmn_err.h>
697c478bdstevel@tonic-gate#include <sys/utsname.h>
707c478bdstevel@tonic-gate#include <sys/debug.h>
717c478bdstevel@tonic-gate#include <sys/kdi.h>
727c478bdstevel@tonic-gate
737c478bdstevel@tonic-gate#include <sys/dumphdr.h>
747c478bdstevel@tonic-gate#include <sys/bootconf.h>
75a311483Gerry Liu#include <sys/memlist_plat.h>
767c478bdstevel@tonic-gate#include <sys/varargs.h>
777c478bdstevel@tonic-gate#include <sys/promif.h>
78fdcca78Joshua M. Clulow#include <sys/prom_debug.h>
79843e198johnlev#include <sys/modctl.h>
807c478bdstevel@tonic-gate
817c478bdstevel@tonic-gate#include <sys/sunddi.h>
827c478bdstevel@tonic-gate#include <sys/sunndi.h>
837c478bdstevel@tonic-gate#include <sys/ndi_impldefs.h>
847c478bdstevel@tonic-gate#include <sys/ddidmareq.h>
857c478bdstevel@tonic-gate#include <sys/psw.h>
867c478bdstevel@tonic-gate#include <sys/regset.h>
877c478bdstevel@tonic-gate#include <sys/clock.h>
887c478bdstevel@tonic-gate#include <sys/pte.h>
897c478bdstevel@tonic-gate#include <sys/tss.h>
907c478bdstevel@tonic-gate#include <sys/stack.h>
917c478bdstevel@tonic-gate#include <sys/trap.h>
927c478bdstevel@tonic-gate#include <sys/fp.h>
931d03c31johnlev#include <vm/kboot_mmu.h>
947c478bdstevel@tonic-gate#include <vm/anon.h>
957c478bdstevel@tonic-gate#include <vm/as.h>
967c478bdstevel@tonic-gate#include <vm/page.h>
977c478bdstevel@tonic-gate#include <vm/seg.h>
987c478bdstevel@tonic-gate#include <vm/seg_dev.h>
997c478bdstevel@tonic-gate#include <vm/seg_kmem.h>
1007c478bdstevel@tonic-gate#include <vm/seg_kpm.h>
1017c478bdstevel@tonic-gate#include <vm/seg_map.h>
1027c478bdstevel@tonic-gate#include <vm/seg_vn.h>
1037c478bdstevel@tonic-gate#include <vm/seg_kp.h>
1047c478bdstevel@tonic-gate#include <sys/memnode.h>
1057c478bdstevel@tonic-gate#include <vm/vm_dep.h>
1067c478bdstevel@tonic-gate#include <sys/thread.h>
1077c478bdstevel@tonic-gate#include <sys/sysconf.h>
1087c478bdstevel@tonic-gate#include <sys/vm_machparam.h>
1097c478bdstevel@tonic-gate#include <sys/archsystm.h>
1107c478bdstevel@tonic-gate#include <sys/machsystm.h>
1117c478bdstevel@tonic-gate#include <vm/hat.h>
1127c478bdstevel@tonic-gate#include <vm/hat_i86.h>
1137c478bdstevel@tonic-gate#include <sys/pmem.h>
1147c478bdstevel@tonic-gate#include <sys/smp_impldefs.h>
1157c478bdstevel@tonic-gate#include <sys/x86_archext.h>
116a563a03bholler#include <sys/cpuvar.h>
1177c478bdstevel@tonic-gate#include <sys/segments.h>
1187c478bdstevel@tonic-gate#include <sys/clconf.h>
1197c478bdstevel@tonic-gate#include <sys/kobj.h>
1207c478bdstevel@tonic-gate#include <sys/kobj_lex.h>
1217c478bdstevel@tonic-gate#include <sys/cpc_impl.h>
1227aec1d6cindi#include <sys/cpu_module.h>
12384ab085mws#include <sys/smbios.h>
124ae115bcmrj#include <sys/debug_info.h>
1251d03c31johnlev#include <sys/bootinfo.h>
126a288e5aJoshua M. Clulow#include <sys/ddi_periodic.h>
1275679c89jv#include <sys/systeminfo.h>
1281939740Sherry Moore#include <sys/multiboot.h>
1290181461Keith M Wesolowski#include <sys/ramdisk.h>
130ae115bcmrj
131a311483Gerry Liu#ifdef	__xpv
1321d03c31johnlev
133843e198johnlev#include <sys/hypervisor.h>
134843e198johnlev#include <sys/xen_mmu.h>
135843e198johnlev#include <sys/evtchn_impl.h>
136843e198johnlev#include <sys/gnttab.h>
137843e198johnlev#include <sys/xpv_panic.h>
138843e198johnlev#include <xen/sys/xenbus_comms.h>
139843e198johnlev#include <xen/public/physdev.h>
1401d03c31johnlev
141843e198johnlevextern void xen_late_startup(void);
142ae115bcmrj
1431d03c31johnlevstruct xen_evt_data cpu0_evt_data;
1441d03c31johnlev
145a311483Gerry Liu#else	/* __xpv */
146a311483Gerry Liu#include <sys/memlist_impl.h>
147a311483Gerry Liu
148a311483Gerry Liuextern void mem_config_init(void);
1491d03c31johnlev#endif /* __xpv */
1507c478bdstevel@tonic-gate
1517c478bdstevel@tonic-gateextern void progressbar_init(void);
1529acbbeannextern void brand_init(void);
15306fb6a3dvextern void pcf_init(void);
1540e75152Eric Saxeextern void pg_init(void);
1556a59053Alex Wilsonextern void ssp_init(void);
1567c478bdstevel@tonic-gate
157d7d9365blakejextern int size_pse_array(pgcnt_t, int);
158d7d9365blakej
159ed5289fKen Erickson#if defined(_SOFT_HOSTID)
160ed5289fKen Erickson
161ed5289fKen Erickson#include <sys/rtc.h>
162ed5289fKen Erickson
163ed5289fKen Ericksonstatic int32_t set_soft_hostid(void);
164ed5289fKen Ericksonstatic char hostid_file[] = "/etc/hostid";
165ed5289fKen Erickson
166ed5289fKen Erickson#endif
167ed5289fKen Erickson
16894f1124Vikram Hegdevoid *gfx_devinfo_list;
169ed5289fKen Erickson
1703a634bfVikram Hegde#if defined(__amd64) && !defined(__xpv)
1713a634bfVikram Hegdeextern void immu_startup(void);
1723a634bfVikram Hegde#endif
1733a634bfVikram Hegde
1747c478bdstevel@tonic-gate/*
1757c478bdstevel@tonic-gate * XXX make declaration below "static" when drivers no longer use this
1767c478bdstevel@tonic-gate * interface.
1777c478bdstevel@tonic-gate */
1787c478bdstevel@tonic-gateextern caddr_t p0_va;	/* Virtual address for accessing physical page 0 */
1797c478bdstevel@tonic-gate
1807c478bdstevel@tonic-gate/*
1817c478bdstevel@tonic-gate * segkp
1827c478bdstevel@tonic-gate */
1837c478bdstevel@tonic-gateextern int segkp_fromheap;
1847c478bdstevel@tonic-gate
1857c478bdstevel@tonic-gatestatic void kvm_init(void);
1867c478bdstevel@tonic-gatestatic void startup_init(void);
1877c478bdstevel@tonic-gatestatic void startup_memlist(void);
188ae115bcmrjstatic void startup_kmem(void);
1897c478bdstevel@tonic-gatestatic void startup_modules(void);
1907c478bdstevel@tonic-gatestatic void startup_vm(void);
1917c478bdstevel@tonic-gatestatic void startup_end(void);
19235b1ab9josephbstatic void layout_kernel_va(void);
1937c478bdstevel@tonic-gate
1947c478bdstevel@tonic-gate/*
1957c478bdstevel@tonic-gate * Declare these as initialized data so we can patch them.
1967c478bdstevel@tonic-gate */
197f53ad21kchow#ifdef __i386
19835b1ab9josephb
199f53ad21kchow/*
200f53ad21kchow * Due to virtual address space limitations running in 32 bit mode, restrict
20135b1ab9josephb * the amount of physical memory configured to a max of PHYSMEM pages (16g).
202f53ad21kchow *
203f53ad21kchow * If the physical max memory size of 64g were allowed to be configured, the
204f53ad21kchow * size of user virtual address space will be less than 1g. A limited user
205f53ad21kchow * address space greatly reduces the range of applications that can run.
206f53ad21kchow *
20735b1ab9josephb * If more physical memory than PHYSMEM is required, users should preferably
20835b1ab9josephb * run in 64 bit mode which has far looser virtual address space limitations.
209f53ad21kchow *
210f53ad21kchow * If 64 bit mode is not available (as in IA32) and/or more physical memory
21135b1ab9josephb * than PHYSMEM is required in 32 bit mode, physmem can be set to the desired
212f53ad21kchow * value or to 0 (to configure all available memory) via eeprom(1M). kernelbase
213f53ad21kchow * should also be carefully tuned to balance out the need of the user
214f53ad21kchow * application while minimizing the risk of kernel heap exhaustion due to
215f53ad21kchow * kernelbase being set too high.
216f53ad21kchow */
21735b1ab9josephb#define	PHYSMEM	0x400000
218f53ad21kchow
21935b1ab9josephb#else /* __amd64 */
22035b1ab9josephb
22135b1ab9josephb/*
22235b1ab9josephb * For now we can handle memory with physical addresses up to about
22335b1ab9josephb * 64 Terabytes. This keeps the kernel above the VA hole, leaving roughly
22435b1ab9josephb * half the VA space for seg_kpm. When systems get bigger than 64TB this
22535b1ab9josephb * code will need revisiting. There is an implicit assumption that there
22635b1ab9josephb * are no *huge* holes in the physical address space too.
22735b1ab9josephb */
22835b1ab9josephb#define	TERABYTE		(1ul << 40)
22935b1ab9josephb#define	PHYSMEM_MAX64		mmu_btop(64 * TERABYTE)
23035b1ab9josephb#define	PHYSMEM			PHYSMEM_MAX64
23135b1ab9josephb#define	AMD64_VA_HOLE_END	0xFFFF800000000000ul
23235b1ab9josephb
23335b1ab9josephb#endif /* __amd64 */
23435b1ab9josephb
23535b1ab9josephbpgcnt_t physmem = PHYSMEM;
2367c478bdstevel@tonic-gatepgcnt_t obp_pages;	/* Memory used by PROM for its text and data */
2377c478bdstevel@tonic-gate
2387c478bdstevel@tonic-gatechar *kobj_file_buf;
2397c478bdstevel@tonic-gateint kobj_file_bufsize;	/* set in /etc/system */
2407c478bdstevel@tonic-gate
2417c478bdstevel@tonic-gate/* Global variables for MP support. Used in mp_startup */
2421939740Sherry Moorecaddr_t	rm_platter_va = 0;
2437c478bdstevel@tonic-gateuint32_t rm_platter_pa;
2447c478bdstevel@tonic-gate
24583f9b80kchowint	auto_lpg_disable = 1;
24683f9b80kchow
2477c478bdstevel@tonic-gate/*
2487c478bdstevel@tonic-gate * Some CPUs have holes in the middle of the 64-bit virtual address range.
2497c478bdstevel@tonic-gate */
2507c478bdstevel@tonic-gateuintptr_t hole_start, hole_end;
2517c478bdstevel@tonic-gate
2527c478bdstevel@tonic-gate/*
2537c478bdstevel@tonic-gate * kpm mapping window
2547c478bdstevel@tonic-gate */
2557c478bdstevel@tonic-gatecaddr_t kpm_vbase;
2567c478bdstevel@tonic-gatesize_t  kpm_size;
25735b1ab9josephbstatic int kpm_desired;
25835b1ab9josephb#ifdef __amd64
25935b1ab9josephbstatic uintptr_t segkpm_base = (uintptr_t)SEGKPM_BASE;
26035b1ab9josephb#endif
2617c478bdstevel@tonic-gate
2627c478bdstevel@tonic-gate/*
2637c478bdstevel@tonic-gate * Configuration parameters set at boot time.
2647c478bdstevel@tonic-gate */
2657c478bdstevel@tonic-gate
2667c478bdstevel@tonic-gatecaddr_t econtig;		/* end of first block of contiguous kernel */
2677c478bdstevel@tonic-gate
2687c478bdstevel@tonic-gatestruct bootops		*bootops = 0;	/* passed in from boot */
2697c478bdstevel@tonic-gatestruct bootops		**bootopsp;
2707c478bdstevel@tonic-gatestruct boot_syscalls	*sysp;		/* passed in from boot */
2717c478bdstevel@tonic-gate
2727c478bdstevel@tonic-gatechar bootblock_fstype[16];
2737c478bdstevel@tonic-gate
2747c478bdstevel@tonic-gatechar kern_bootargs[OBP_MAXPATHLEN];
2751939740Sherry Moorechar kern_bootfile[OBP_MAXPATHLEN];
2767c478bdstevel@tonic-gate
2777c478bdstevel@tonic-gate/*
278ad23a2djohansen * ZFS zio segment.  This allows us to exclude large portions of ZFS data that
279ad23a2djohansen * gets cached in kmem caches on the heap.  If this is set to zero, we allocate
280ad23a2djohansen * zio buffers from their own segment, otherwise they are allocated from the
281ad23a2djohansen * heap.  The optimization of allocating zio buffers from their own segment is
282ad23a2djohansen * only valid on 64-bit kernels.
283ad23a2djohansen */
284ad23a2djohansen#if defined(__amd64)
285ad23a2djohansenint segzio_fromheap = 0;
286ad23a2djohansen#else
287ad23a2djohansenint segzio_fromheap = 1;
288ad23a2djohansen#endif
289ad23a2djohansen
290ad23a2djohansen/*
2913ce2fcdRobert Mustacchi * Give folks an escape hatch for disabling SMAP via kmdb. Doesn't work
2923ce2fcdRobert Mustacchi * post-boot.
2933ce2fcdRobert Mustacchi */
2943ce2fcdRobert Mustacchiint disable_smap = 0;
2953ce2fcdRobert Mustacchi
2963ce2fcdRobert Mustacchi/*
2977c478bdstevel@tonic-gate * new memory fragmentations are possible in startup() due to BOP_ALLOCs. this
2987c478bdstevel@tonic-gate * depends on number of BOP_ALLOC calls made and requested size, memory size
2997c478bdstevel@tonic-gate * combination and whether boot.bin memory needs to be freed.
3007c478bdstevel@tonic-gate */
3017c478bdstevel@tonic-gate#define	POSS_NEW_FRAGMENTS	12
3027c478bdstevel@tonic-gate
3037c478bdstevel@tonic-gate/*
3047c478bdstevel@tonic-gate * VM data structures
3057c478bdstevel@tonic-gate */
3067c478bdstevel@tonic-gatelong page_hashsz;		/* Size of page hash table (power of two) */
307cb15d5dPeter Rivalunsigned int page_hashsz_shift;	/* log2(page_hashsz) */
3087c478bdstevel@tonic-gatestruct page *pp_base;		/* Base of initial system page struct array */
3097c478bdstevel@tonic-gatestruct page **page_hash;	/* Page hash table */
310d7d9365blakejpad_mutex_t *pse_mutex;		/* Locks protecting pp->p_selock */
311d7d9365blakejsize_t pse_table_size;		/* Number of mutexes in pse_mutex[] */
312d7d9365blakejint pse_shift;			/* log2(pse_table_size) */
3137c478bdstevel@tonic-gatestruct seg ktextseg;		/* Segment used for kernel executable image */
3147c478bdstevel@tonic-gatestruct seg kvalloc;		/* Segment used for "valloc" mapping */
3157c478bdstevel@tonic-gatestruct seg kpseg;		/* Segment used for pageable kernel virt mem */
3167c478bdstevel@tonic-gatestruct seg kmapseg;		/* Segment used for generic kernel mappings */
3177c478bdstevel@tonic-gatestruct seg kdebugseg;		/* Segment used for the kernel debugger */
3187c478bdstevel@tonic-gate
3197c478bdstevel@tonic-gatestruct seg *segkmap = &kmapseg;	/* Kernel generic mapping segment */
320ae115bcmrjstatic struct seg *segmap = &kmapseg;	/* easier to use name for in here */
321ae115bcmrj
3227c478bdstevel@tonic-gatestruct seg *segkp = &kpseg;	/* Pageable kernel virtual memory segment */
3237c478bdstevel@tonic-gate
3247c478bdstevel@tonic-gatestruct seg kvseg_core;		/* Segment used for the core heap */
3257c478bdstevel@tonic-gatestruct seg kpmseg;		/* Segment used for physical mapping */
3267c478bdstevel@tonic-gatestruct seg *segkpm = &kpmseg;	/* 64bit kernel physical mapping segment */
3277c478bdstevel@tonic-gate
3287c478bdstevel@tonic-gatecaddr_t segkp_base;		/* Base address of segkp */
329ad23a2djohansencaddr_t segzio_base;		/* Base address of segzio */
3307c478bdstevel@tonic-gatepgcnt_t segkpsize = btop(SEGKPDEFSIZE);	/* size of segkp segment in pages */
33104909c8John Levoncaddr_t segkvmm_base;
33204909c8John Levonpgcnt_t segkvmmsize;
33304909c8John Levonpgcnt_t segziosize;
3347c478bdstevel@tonic-gate
335ae115bcmrj/*
336a311483Gerry Liu * A static DR page_t VA map is reserved that can map the page structures
337a311483Gerry Liu * for a domain's entire RA space. The pages that back this space are
338a311483Gerry Liu * dynamically allocated and need not be physically contiguous.  The DR
339a311483Gerry Liu * map size is derived from KPM size.
340a311483Gerry Liu * This mechanism isn't used by x86 yet, so just stubs here.
341a311483Gerry Liu */
342a311483Gerry Liuint ppvm_enable = 0;		/* Static virtual map for page structs */
343a311483Gerry Liupage_t *ppvm_base = NULL;	/* Base of page struct map */
344a311483Gerry Liupgcnt_t ppvm_size = 0;		/* Size of page struct map */
345a311483Gerry Liu
346a311483Gerry Liu/*
347ae115bcmrj * VA range available to the debugger
348ae115bcmrj */
349ae115bcmrjconst caddr_t kdi_segdebugbase = (const caddr_t)SEGDEBUGBASE;
350ae115bcmrjconst size_t kdi_segdebugsize = SEGDEBUGSIZE;
351