1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22/*
23 * Copyright (c) 1993, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright 2012 DEY Storage Systems, Inc.  All rights reserved.
25 * Copyright 2017 Nexenta Systems, Inc.
26 * Copyright (c) 2018 Joyent, Inc.
27 * Copyright (c) 2015 by Delphix. All rights reserved.
28 * Copyright 2020 Oxide Computer Company
29 */
30/*
31 * Copyright (c) 2010, Intel Corporation.
32 * All rights reserved.
33 */
34
35#include <sys/types.h>
36#include <sys/t_lock.h>
37#include <sys/param.h>
38#include <sys/sysmacros.h>
39#include <sys/signal.h>
40#include <sys/systm.h>
41#include <sys/user.h>
42#include <sys/mman.h>
43#include <sys/vm.h>
44#include <sys/conf.h>
45#include <sys/avintr.h>
46#include <sys/autoconf.h>
47#include <sys/disp.h>
48#include <sys/class.h>
49#include <sys/bitmap.h>
50
51#include <sys/privregs.h>
52
53#include <sys/proc.h>
54#include <sys/buf.h>
55#include <sys/kmem.h>
56#include <sys/mem.h>
57#include <sys/kstat.h>
58
59#include <sys/reboot.h>
60
61#include <sys/cred.h>
62#include <sys/vnode.h>
63#include <sys/file.h>
64
65#include <sys/procfs.h>
66
67#include <sys/vfs.h>
68#include <sys/cmn_err.h>
69#include <sys/utsname.h>
70#include <sys/debug.h>
71#include <sys/kdi.h>
72
73#include <sys/dumphdr.h>
74#include <sys/bootconf.h>
75#include <sys/memlist_plat.h>
76#include <sys/varargs.h>
77#include <sys/promif.h>
78#include <sys/prom_debug.h>
79#include <sys/modctl.h>
80
81#include <sys/sunddi.h>
82#include <sys/sunndi.h>
83#include <sys/ndi_impldefs.h>
84#include <sys/ddidmareq.h>
85#include <sys/psw.h>
86#include <sys/regset.h>
87#include <sys/clock.h>
88#include <sys/pte.h>
89#include <sys/tss.h>
90#include <sys/stack.h>
91#include <sys/trap.h>
92#include <sys/fp.h>
93#include <vm/kboot_mmu.h>
94#include <vm/anon.h>
95#include <vm/as.h>
96#include <vm/page.h>
97#include <vm/seg.h>
98#include <vm/seg_dev.h>
99#include <vm/seg_kmem.h>
100#include <vm/seg_kpm.h>
101#include <vm/seg_map.h>
102#include <vm/seg_vn.h>
103#include <vm/seg_kp.h>
104#include <sys/memnode.h>
105#include <vm/vm_dep.h>
106#include <sys/thread.h>
107#include <sys/sysconf.h>
108#include <sys/vm_machparam.h>
109#include <sys/archsystm.h>
110#include <sys/machsystm.h>
111#include <vm/hat.h>
112#include <vm/hat_i86.h>
113#include <sys/pmem.h>
114#include <sys/smp_impldefs.h>
115#include <sys/x86_archext.h>
116#include <sys/cpuvar.h>
117#include <sys/segments.h>
118#include <sys/clconf.h>
119#include <sys/kobj.h>
120#include <sys/kobj_lex.h>
121#include <sys/cpc_impl.h>
122#include <sys/cpu_module.h>
123#include <sys/smbios.h>
124#include <sys/debug_info.h>
125#include <sys/bootinfo.h>
126#include <sys/ddi_periodic.h>
127#include <sys/systeminfo.h>
128#include <sys/multiboot.h>
129#include <sys/ramdisk.h>
130
131#ifdef	__xpv
132
133#include <sys/hypervisor.h>
134#include <sys/xen_mmu.h>
135#include <sys/evtchn_impl.h>
136#include <sys/gnttab.h>
137#include <sys/xpv_panic.h>
138#include <xen/sys/xenbus_comms.h>
139#include <xen/public/physdev.h>
140
141extern void xen_late_startup(void);
142
143struct xen_evt_data cpu0_evt_data;
144
145#else	/* __xpv */
146#include <sys/memlist_impl.h>
147
148extern void mem_config_init(void);
149#endif /* __xpv */
150
151extern void progressbar_init(void);
152extern void brand_init(void);
153extern void pcf_init(void);
154extern void pg_init(void);
155extern void ssp_init(void);
156
157extern int size_pse_array(pgcnt_t, int);
158
159#if defined(_SOFT_HOSTID)
160
161#include <sys/rtc.h>
162
163static int32_t set_soft_hostid(void);
164static char hostid_file[] = "/etc/hostid";
165
166#endif
167
168void *gfx_devinfo_list;
169
170#if defined(__amd64) && !defined(__xpv)
171extern void immu_startup(void);
172#endif
173
174/*
175 * XXX make declaration below "static" when drivers no longer use this
176 * interface.
177 */
178extern caddr_t p0_va;	/* Virtual address for accessing physical page 0 */
179
180/*
181 * segkp
182 */
183extern int segkp_fromheap;
184
185static void kvm_init(void);
186static void startup_init(void);
187static void startup_memlist(void);
188static void startup_kmem(void);
189static void startup_modules(void);
190static void startup_vm(void);
191static void startup_end(void);
192static void layout_kernel_va(void);
193
194/*
195 * Declare these as initialized data so we can patch them.
196 */
197#ifdef __i386
198
199/*
200 * Due to virtual address space limitations running in 32 bit mode, restrict
201 * the amount of physical memory configured to a max of PHYSMEM pages (16g).
202 *
203 * If the physical max memory size of 64g were allowed to be configured, the
204 * size of user virtual address space will be less than 1g. A limited user
205 * address space greatly reduces the range of applications that can run.
206 *
207 * If more physical memory than PHYSMEM is required, users should preferably
208 * run in 64 bit mode which has far looser virtual address space limitations.
209 *
210 * If 64 bit mode is not available (as in IA32) and/or more physical memory
211 * than PHYSMEM is required in 32 bit mode, physmem can be set to the desired
212 * value or to 0 (to configure all available memory) via eeprom(1M). kernelbase
213 * should also be carefully tuned to balance out the need of the user
214 * application while minimizing the risk of kernel heap exhaustion due to
215 * kernelbase being set too high.
216 */
217#define	PHYSMEM	0x400000
218
219#else /* __amd64 */
220
221/*
222 * For now we can handle memory with physical addresses up to about
223 * 64 Terabytes. This keeps the kernel above the VA hole, leaving roughly
224 * half the VA space for seg_kpm. When systems get bigger than 64TB this
225 * code will need revisiting. There is an implicit assumption that there
226 * are no *huge* holes in the physical address space too.
227 */
228#define	TERABYTE		(1ul << 40)
229#define	PHYSMEM_MAX64		mmu_btop(64 * TERABYTE)
230#define	PHYSMEM			PHYSMEM_MAX64
231#define	AMD64_VA_HOLE_END	0xFFFF800000000000ul
232
233#endif /* __amd64 */
234
235pgcnt_t physmem = PHYSMEM;
236pgcnt_t obp_pages;	/* Memory used by PROM for its text and data */
237
238char *kobj_file_buf;
239int kobj_file_bufsize;	/* set in /etc/system */
240
241/* Global variables for MP support. Used in mp_startup */
242caddr_t	rm_platter_va = 0;
243uint32_t rm_platter_pa;
244
245int	auto_lpg_disable = 1;
246
247/*
248 * Some CPUs have holes in the middle of the 64-bit virtual address range.
249 */
250uintptr_t hole_start, hole_end;
251
252/*
253 * kpm mapping window
254 */
255caddr_t kpm_vbase;
256size_t  kpm_size;
257static int kpm_desired;
258#ifdef __amd64
259static uintptr_t segkpm_base = (uintptr_t)SEGKPM_BASE;
260#endif
261
262/*
263 * Configuration parameters set at boot time.
264 */
265
266caddr_t econtig;		/* end of first block of contiguous kernel */
267
268struct bootops		*bootops = 0;	/* passed in from boot */
269struct bootops		**bootopsp;
270struct boot_syscalls	*sysp;		/* passed in from boot */
271
272char bootblock_fstype[16];
273
274char kern_bootargs[OBP_MAXPATHLEN];
275char kern_bootfile[OBP_MAXPATHLEN];
276
277/*
278 * ZFS zio segment.  This allows us to exclude large portions of ZFS data that
279 * gets cached in kmem caches on the heap.  If this is set to zero, we allocate
280 * zio buffers from their own segment, otherwise they are allocated from the
281 * heap.  The optimization of allocating zio buffers from their own segment is
282 * only valid on 64-bit kernels.
283 */
284#if defined(__amd64)
285int segzio_fromheap = 0;
286#else
287int segzio_fromheap = 1;
288#endif
289
290/*
291 * Give folks an escape hatch for disabling SMAP via kmdb. Doesn't work
292 * post-boot.
293 */
294int disable_smap = 0;
295
296/*
297 * new memory fragmentations are possible in startup() due to BOP_ALLOCs. this
298 * depends on number of BOP_ALLOC calls made and requested size, memory size
299 * combination and whether boot.bin memory needs to be freed.
300 */
301#define	POSS_NEW_FRAGMENTS	12
302
303/*
304 * VM data structures
305 */
306long page_hashsz;		/* Size of page hash table (power of two) */
307unsigned int page_hashsz_shift;	/* log2(page_hashsz) */
308struct page *pp_base;		/* Base of initial system page struct array */
309struct page **page_hash;	/* Page hash table */
310pad_mutex_t *pse_mutex;		/* Locks protecting pp->p_selock */
311size_t pse_table_size;		/* Number of mutexes in pse_mutex[] */
312int pse_shift;			/* log2(pse_table_size) */
313struct seg ktextseg;		/* Segment used for kernel executable image */
314struct seg kvalloc;		/* Segment used for "valloc" mapping */
315struct seg kpseg;		/* Segment used for pageable kernel virt mem */
316struct seg kmapseg;		/* Segment used for generic kernel mappings */
317struct seg kdebugseg;		/* Segment used for the kernel debugger */
318
319struct seg *segkmap = &kmapseg;	/* Kernel generic mapping segment */
320static struct seg *segmap = &kmapseg;	/* easier to use name for in here */
321
322struct seg *segkp = &kpseg;	/* Pageable kernel virtual memory segment */
323
324struct seg kvseg_core;		/* Segment used for the core heap */
325struct seg kpmseg;		/* Segment used for physical mapping */
326struct seg *segkpm = &kpmseg;	/* 64bit kernel physical mapping segment */
327
328caddr_t segkp_base;		/* Base address of segkp */
329caddr_t segzio_base;		/* Base address of segzio */
330pgcnt_t segkpsize = btop(SEGKPDEFSIZE);	/* size of segkp segment in pages */
331caddr_t segkvmm_base;
332pgcnt_t segkvmmsize;
333pgcnt_t segziosize;
334
335/*
336 * A static DR page_t VA map is reserved that can map the page structures
337 * for a domain's entire RA space. The pages that back this space are
338 * dynamically allocated and need not be physically contiguous.  The DR
339 * map size is derived from KPM size.
340 * This mechanism isn't used by x86 yet, so just stubs here.
341 */
342int ppvm_enable = 0;		/* Static virtual map for page structs */
343page_t *ppvm_base = NULL;	/* Base of page struct map */
344pgcnt_t ppvm_size = 0;		/* Size of page struct map */
345
346/*
347 * VA range available to the debugger
348 */
349const caddr_t kdi_segdebugbase = (const caddr_t)SEGDEBUGBASE;
350const size_t kdi_segdebugsize = SEGDEBUGSIZE;
351
352struct memseg *memseg_base;
353struct vnode unused_pages_vp;
354
355#define	FOURGB	0x100000000LL
356
357struct memlist *memlist;
358
359caddr_t s_text;		/* start of kernel text segment */
360caddr_t e_text;		/* end of kernel text segment */
361caddr_t s_data;		/* start of kernel data segment */
362caddr_t e_data;		/* end of kernel data segment */
363caddr_t modtext;	/* start of loadable module text reserved */
364caddr_t e_modtext;	/* end of loadable module text reserved */
365caddr_t moddata;	/* start of loadable module data reserved */
366caddr_t e_moddata;	/* end of loadable module data reserved */
367
368struct memlist *phys_install;	/* Total installed physical memory */
369struct memlist *phys_avail;	/* Total available physical memory */
370struct memlist *bios_rsvd;	/* Bios reserved memory */
371
372/*
373 * kphysm_init returns the number of pages that were processed
374 */
375static pgcnt_t kphysm_init(page_t *, pgcnt_t);
376
377#define	IO_PROP_SIZE	64	/* device property size */
378
379/*
380 * a couple useful roundup macros
381 */
382#define	ROUND_UP_PAGE(x)	\
383	((uintptr_t)P2ROUNDUP((uintptr_t)(x), (uintptr_t)MMU_PAGESIZE))
384#define	ROUND_UP_LPAGE(x)	\
385	((uintptr_t)P2ROUNDUP((uintptr_t)(x), mmu.level_size[1]))
386#define	ROUND_UP_4MEG(x)	\
387	((uintptr_t)P2ROUNDUP((uintptr_t)(x), (uintptr_t)FOUR_MEG))
388#define	ROUND_UP_TOPLEVEL(x)	\
389	((uintptr_t)P2ROUNDUP((uintptr_t)(x), mmu.level_size[mmu.max_level]))
390
391/*
392 *	32-bit Kernel's Virtual memory layout.
393 *		+-----------------------+
394 *		|			|
395 * 0xFFC00000  -|-----------------------|- ARGSBASE
396 *		|	debugger	|
397 * 0xFF800000  -|-----------------------|- SEGDEBUGBASE
398 *		|      Kernel Data	|
399 * 0xFEC00000  -|-----------------------|
400 *              |      Kernel Text	|
401 * 0xFE800000  -|-----------------------|- KERNEL_TEXT (0xFB400000 on Xen)
402 *		|---       GDT       ---|- GDT page (GDT_VA)
403 *		|---    debug info   ---|- debug info (DEBUG_INFO_VA)
404 *		|			|
405 *		|   page_t structures	|
406 *		|   memsegs, memlists,	|
407 *		|   page hash, etc.	|
408 * ---	       -|-----------------------|- ekernelheap, valloc_base (floating)
409 *		|			|  (segkp is just an arena in the heap)
410 *		|			|
411 *		|	kvseg		|
412 *		|			|
413 *		|			|
414 * ---         -|-----------------------|- kernelheap (floating)
415 *		|        Segkmap	|
416 * 0xC3002000  -|-----------------------|- segmap_start (floating)
417 *		|	Red Zone	|
418 * 0xC3000000  -|-----------------------|- kernelbase / userlimit (floating)
419 *		|			|			||
420 *		|     Shared objects	|			\/
421 *		|			|
422 *		:			:
423 *		|	user data	|
424 *		|-----------------------|
425 *		|	user text	|
426 * 0x08048000  -|-----------------------|
427 *		|	user stack	|
428 *		:			:
429 *		|	invalid		|
430 * 0x00000000	+-----------------------+
431 *
432 *
433 *		64-bit Kernel's Virtual memory layout. (assuming 64 bit app)
434 *			+-----------------------+
435 *			|			|
436 * 0xFFFFFFFF.FFC00000  |-----------------------|- ARGSBASE
437 *			|	debugger (?)	|
438 * 0xFFFFFFFF.FF800000  |-----------------------|- SEGDEBUGBASE
439 *			|      unused		|
440 *			+-----------------------+
441 *			|      Kernel Data	|
442 * 0xFFFFFFFF.FBC00000  |-----------------------|
443 *			|      Kernel Text	|
444 * 0xFFFFFFFF.FB800000  |-----------------------|- KERNEL_TEXT
445 *			|---    debug info   ---|- debug info (DEBUG_INFO_VA)
446 *			|---       GDT       ---|- GDT page (GDT_VA)
447 *			|---       IDT       ---|- IDT page (IDT_VA)
448 *			|---       LDT       ---|- LDT pages (LDT_VA)
449 *			|			|
450 *			|      Core heap	| (used for loadable modules)
451 * 0xFFFFFFFF.C0000000  |-----------------------|- core_base / ekernelheap
452 *			|	 Kernel		|
453 *			|	  heap		|
454 *			|			|
455 *			|			|
456 * 0xFFFFFXXX.XXX00000  |-----------------------|- kernelheap (floating)
457 *			|	 segmap		|
458 * 0xFFFFFXXX.XXX00000  |-----------------------|- segmap_start (floating)
459 *			|    device mappings	|
460 * 0xFFFFFXXX.XXX00000  |-----------------------|- toxic_addr (floating)
461 *			|	 segzio		|
462 * 0xFFFFFXXX.XXX00000  |-----------------------|- segzio_base (floating)
463 *			|        segkvmm	|
464 *			|			|
465 *			|			|
466 *			|			|
467 * 0xFFFFFXXX.XXX00000  |-----------------------|- segkvmm_base (floating)
468 *			|	 segkp		|
469 *			|-----------------------|- segkp_base (floating)
470 *			|   page_t structures	|  valloc_base + valloc_sz
471 *			|   memsegs, memlists,	|
472 *			|   page hash, etc.	|
473 * 0xFFFFFE00.00000000  |-----------------------|- valloc_base (lower if >256GB)
474 *			|	 segkpm		|
475 *			|			|
476 * 0xFFFFFD00.00000000  |-----------------------|- SEGKPM_BASE (lower if >256GB)
477 *			|	Red Zone	|
478 * 0xFFFFFC80.00000000  |-----------------------|- KERNELBASE (lower if >256GB)
479 * 0xFFFFFC7F.FFE00000  |-----------------------|- USERLIMIT (lower if >256GB)
480 *			|     User stack	|- User space memory
481 *			|			|
482 *			| shared objects, etc	|	(grows downwards)
483 *			:			:
484 *			|			|
485 * 0xFFFF8000.00000000  |-----------------------|
486 *			|			|
487 *			| VA Hole / unused	|
488 *			|			|
489 * 0x00008000.00000000  |-----------------------|
490 *			|			|
491 *			|			|
492 *			:			:
493 *			|	user heap	|	(grows upwards)
494 *			|			|
495 *			|	user data	|
496 *			|-----------------------|
497 *			|	user text	|
498 * 0x00000000.04000000  |-----------------------|
499 *			|	invalid		|
500 * 0x00000000.00000000	+-----------------------+
501 *
502 * A 32 bit app on the 64 bit kernel sees the same layout as on the 32 bit
503 * kernel, except that userlimit is raised to 0xfe000000
504 *
505 * Floating values:
506 *
507 * valloc_base: start of the kernel's memory management/tracking data
508 * structures.  This region contains page_t structures for
509 * physical memory, memsegs, memlists, and the page hash.
510 *
511 * core_base: start of the kernel's "core" heap area on 64-bit systems.
512 * This area is intended to be used for global data as well as for module
513 * text/data that does not fit into the nucleus pages.  The core heap is
514 * restricted to a 2GB range, allowing every address within it to be
515 * accessed using rip-relative addressing
516 *
517 * ekernelheap: end of kernelheap and start of segmap.
518 *
519 * kernelheap: start of kernel heap.  On 32-bit systems, this starts right
520 * above a red zone that separates the user's address space from the
521 * kernel's.  On 64-bit systems, it sits above segkp and segkpm.
522 *
523 * segmap_start: start of segmap. The length of segmap can be modified
524 * through eeprom. The default length is 16MB on 32-bit systems and 64MB
525 * on 64-bit systems.
526 *
527 * kernelbase: On a 32-bit kernel the default value of 0xd4000000 will be
528 * decreased by 2X the size required for page_t.  This allows the kernel
529 * heap to grow in size with physical memory.  With sizeof(page_t) == 80
530 * bytes, the following shows the values of kernelbase and kernel heap
531 * sizes for different memory configurations (assuming default segmap and
532 * segkp sizes).
533 *
534 *	mem	size for	kernelbase	kernel heap
535 *	size	page_t's			size
536 *	----	---------	----------	-----------
537 *	1gb	0x01400000	0xd1800000	684MB
538 *	2gb	0x02800000	0xcf000000	704MB
539 *	4gb	0x05000000	0xca000000	744MB
540 *	6gb	0x07800000	0xc5000000	784MB
541 *	8gb	0x0a000000	0xc0000000	824MB
542 *	16gb	0x14000000	0xac000000	984MB
543 *	32gb	0x28000000	0x84000000	1304MB
544 *	64gb	0x50000000	0x34000000	1944MB (*)
545 *
546 * kernelbase is less than the abi minimum of 0xc0000000 for memory
547 * configurations above 8gb.
548 *
549 * (*) support for memory configurations above 32gb will require manual tuning
550 * of kernelbase to balance out the need of user applications.
551 */
552
553/* real-time-clock initialization parameters */
554extern time_t process_rtc_config_file(void);
555
556uintptr_t	kernelbase;
557uintptr_t	postbootkernelbase;	/* not set till boot loader is gone */
558uintptr_t	eprom_kernelbase;
559size_t		segmapsize;
560uintptr_t	segmap_start;
561int		segmapfreelists;
562pgcnt_t		npages;
563pgcnt_t		orig_npages;
564size_t		core_size;		/* size of "core" heap */
565uintptr_t	core_base;		/* base address of "core" heap */
566
567/*
568 * List of bootstrap pages. We mark these as allocated in startup.
569 * release_bootstrap() will free them when we're completely done with
570 * the bootstrap.
571 */
572static page_t *bootpages;
573
574/*
575 * boot time pages that have a vnode from the ramdisk will keep that forever.
576 */
577static page_t *rd_pages;
578
579/*
580 * Lower 64K
581 */
582static page_t *lower_pages = NULL;
583static int lower_pages_count = 0;
584
585struct system_hardware system_hardware;
586
587/*
588 * Enable some debugging messages concerning memory usage...
589 */
590static void
591print_memlist(char *title, struct memlist *mp)
592{
593	prom_printf("MEMLIST: %s:\n", title);
594	while (mp != NULL)  {
595		prom_printf("\tAddress 0x%" PRIx64 ", size 0x%" PRIx64 "\n",
596		    mp->ml_address, mp->ml_size);
597		mp = mp->ml_next;
598	}
599}
600
601/*
602 * XX64 need a comment here.. are these just default values, surely
603 * we read the "cpuid" type information to figure this out.
604 */
605int	l2cache_sz = 0x80000;
606int	l2cache_linesz = 0x40;
607int	l2cache_assoc = 1;
608
609static size_t	textrepl_min_gb = 10;
610
611/*
612 * on 64 bit we use a predifined VA range for mapping devices in the kernel
613 * on 32 bit the mappings are intermixed in the heap, so we use a bit map
614 */
615#ifdef __amd64
616
617vmem_t		*device_arena;
618uintptr_t	toxic_addr = (uintptr_t)NULL;
619size_t		toxic_size = 1024 * 1024 * 1024; /* Sparc uses 1 gig too */
620
621#else	/* __i386 */
622
623ulong_t		*toxic_bit_map;	/* one bit for each 4k of VA in heap_arena */
624size_t		toxic_bit_map_len = 0;	/* in bits */
625
626#endif	/* __i386 */
627
628int prom_debug;
629
630/*
631 * This structure is used to keep track of the intial allocations
632 * done in startup_memlist(). The value of NUM_ALLOCATIONS needs to
633 * be >= the number of ADD_TO_ALLOCATIONS() executed in the code.
634 */
635#define	NUM_ALLOCATIONS 8
636int num_allocations = 0;
637struct {
638	void **al_ptr;
639	size_t al_size;
640} allocations[NUM_ALLOCATIONS];
641size_t valloc_sz = 0;
642uintptr_t valloc_base;
643
644#define	ADD_TO_ALLOCATIONS(ptr, size) {					\
645		size = ROUND_UP_PAGE(size);				\
646		if (num_allocations == NUM_ALLOCATIONS)			\
647			panic("too many ADD_TO_ALLOCATIONS()");		\
648		allocations[num_allocations].al_ptr = (void**)&ptr;	\
649		allocations[num_allocations].al_size = size;		\
650		valloc_sz += size;					\
651		++num_allocations;					\
652	}
653
654/*
655 * Allocate all the initial memory needed by the page allocator.
656 */
657static void
658perform_allocations(void)
659{
660	caddr_t mem;
661	int i;
662	int valloc_align;
663
664	PRM_DEBUG(valloc_base);
665	PRM_DEBUG(valloc_sz);
666	valloc_align = mmu.level_size[mmu.max_page_level > 0];
667	mem = BOP_ALLOC(bootops, (caddr_t)valloc_base, valloc_sz, valloc_align);
668	if (mem != (caddr_t)valloc_base)
669		panic("BOP_ALLOC() failed");
670	bzero(mem, valloc_sz);
671	for (i = 0; i < num_allocations; ++i) {
672		*allocations[i].al_ptr = (void *)mem;
673		mem += allocations[i].al_size;
674	}
675}
676
677/*
678 * Set up and enable SMAP now before we start other CPUs, but after the kernel's
679 * VM has been set up so we can use hot_patch_kernel_text().
680 *
681 * We can only patch 1, 2, or 4 bytes, but not three bytes. So instead, we
682 * replace the four byte word at the patch point. See uts/intel/ia32/ml/copy.s
683 * for more information on what's going on here.
684 */
685static void
686startup_smap(void)
687{
688	int i;
689	uint32_t inst;
690	uint8_t *instp;
691	char sym[128];
692	struct modctl *modp;
693
694	extern int _smap_enable_patch_count;
695	extern int _smap_disable_patch_count;
696
697	if (disable_smap != 0)
698		remove_x86_feature(x86_featureset, X86FSET_SMAP);
699
700	if (is_x86_feature(x86_featureset, X86FSET_SMAP) == B_FALSE)
701		return;
702
703	for (i = 0; i < _smap_enable_patch_count; i++) {
704		int sizep;
705
706		VERIFY3U(i, <, _smap_enable_patch_count);
707		VERIFY(snprintf(sym, sizeof (sym), "_smap_enable_patch_%d", i) <
708		    sizeof (sym));
709		instp = (uint8_t *)(void *)kobj_getelfsym(sym, NULL, &sizep);
710		VERIFY(instp != 0);
711		inst = (instp[3] << 24) | (SMAP_CLAC_INSTR & 0x00ffffff);
712		hot_patch_kernel_text((caddr_t)instp, inst, 4);
713	}
714
715	for (i = 0; i < _smap_disable_patch_count; i++) {
716		int sizep;
717
718		VERIFY(snprintf(sym, sizeof (sym), "_smap_disable_patch_%d",
719		    i) < sizeof (sym));
720		instp = (uint8_t *)(void *)kobj_getelfsym(sym, NULL, &sizep);
721		VERIFY(instp != 0);
722		inst = (instp[3] << 24) | (SMAP_STAC_INSTR & 0x00ffffff);
723		hot_patch_kernel_text((caddr_t)instp, inst, 4);
724	}
725
726	/*
727	 * Hotinline calls to smap_enable and smap_disable within
728	 * unix module. Hotinlines in other modules are done on
729	 * mod_load().
730	 */
731	modp = mod_hold_by_name("unix");
732	do_hotinlines(modp->mod_mp);
733	mod_release_mod(modp);
734
735	setcr4(getcr4() | CR4_SMAP);
736	smap_enable();
737}
738
739/*
740 * Our world looks like this at startup time.
741 *
742 * In a 32-bit OS, boot loads the kernel text at 0xfe800000 and kernel data
743 * at 0xfec00000.  On a 64-bit OS, kernel text and data are loaded at
744 * 0xffffffff.fe800000 and 0xffffffff.fec00000 respectively.  Those
745 * addresses are fixed in the binary at link time.
746 *
747 * On the text page:
748 * unix/genunix/krtld/module text loads.
749 *
750 * On the data page:
751 * unix/genunix/krtld/module data loads.
752 *
753 * Machine-dependent startup code
754 */
755void
756startup(void)
757{
758#if !defined(__xpv)
759	extern void startup_pci_bios(void);
760#endif
761	extern cpuset_t cpu_ready_set;
762
763	/*
764	 * Make sure that nobody tries to use sekpm until we have
765	 * initialized it properly.
766	 */
767#if defined(__amd64)
768	kpm_desired = 1;
769#endif
770	kpm_enable = 0;
771	CPUSET_ONLY(cpu_ready_set, 0);	/* cpu 0 is boot cpu */
772
773#if defined(__xpv)	/* XXPV fix me! */
774	{
775		extern int segvn_use_regions;
776		segvn_use_regions = 0;
777	}
778#endif
779	ssp_init();
780	progressbar_init();
781	startup_init();
782#if defined(__xpv)
783	startup_xen_version();
784#endif
785	startup_memlist();
786	startup_kmem();
787	startup_vm();
788#if !defined(__xpv)
789	/*
790	 * Note we need to do this even on fast reboot in order to access
791	 * the irq routing table (used for pci labels).
792	 */
793	startup_pci_bios();
794	startup_smap();
795#endif
796#if defined(__xpv)
797	startup_xen_mca();
798#endif
799	startup_modules();
800
801	startup_end();
802}
803
804static void
805startup_init()
806{
807	PRM_POINT("startup_init() starting...");
808
809	/*
810	 * Complete the extraction of cpuid data
811	 */
812	cpuid_pass2(CPU);
813
814	(void) check_boot_version(BOP_GETVERSION(bootops));
815
816	/*
817	 * Check for prom_debug in boot environment
818	 */
819	if (BOP_GETPROPLEN(bootops, "prom_debug") >= 0) {
820		++prom_debug;
821		PRM_POINT("prom_debug found in boot enviroment");
822	}
823
824	/*
825	 * Collect node, cpu and memory configuration information.
826	 */
827	get_system_configuration();
828
829	/*
830	 * Halt if this is an unsupported processor.
831	 */
832	if (x86_type == X86_TYPE_486 || x86_type == X86_TYPE_CYRIX_486) {
833		printf("\n486 processor (\"%s\") detected.\n",
834		    CPU->cpu_brandstr);
835		halt("This processor is not supported by this release "
836		    "of Solaris.");
837	}
838
839	PRM_POINT("startup_init() done");
840}
841
842/*
843 * Callback for copy_memlist_filter() to filter nucleus, kadb/kmdb, (ie.
844 * everything mapped above KERNEL_TEXT) pages from phys_avail. Note it
845 * also filters out physical page zero.  There is some reliance on the
846 * boot loader allocating only a few contiguous physical memory chunks.
847 */
848static void
849avail_filter(uint64_t *addr, uint64_t *size)
850{
851	uintptr_t va;
852	uintptr_t next_va;
853	pfn_t pfn;
854	uint64_t pfn_addr;
855	uint64_t pfn_eaddr;
856	uint_t prot;
857	size_t len;
858	uint_t change;
859
860	if (prom_debug)
861		prom_printf("\tFilter: in: a=%" PRIx64 ", s=%" PRIx64 "\n",
862		    *addr, *size);
863
864	/*
865	 * page zero is required for BIOS.. never make it available
866	 */
867	if (*addr == 0) {
868		*addr += MMU_PAGESIZE;
869		*size -= MMU_PAGESIZE;
870	}
871
872	/*
873	 * First we trim from the front of the range. Since kbm_probe()
874	 * walks ranges in virtual order, but addr/size are physical, we need
875	 * to the list until no changes are seen.  This deals with the case
876	 * where page "p" is mapped at v, page "p + PAGESIZE" is mapped at w
877	 * but w < v.
878	 */
879	do {
880		change = 0;
881		for (va = KERNEL_TEXT;
882		    *size > 0 && kbm_probe(&va, &len, &pfn, &prot) != 0;
883		    va = next_va) {
884
885			next_va = va + len;
886			pfn_addr = pfn_to_pa(pfn);
887			pfn_eaddr = pfn_addr + len;
888
889			if (pfn_addr <= *addr && pfn_eaddr > *addr) {
890				change = 1;
891				while (*size > 0 && len > 0) {
892					*addr += MMU_PAGESIZE;
893					*size -= MMU_PAGESIZE;
894					len -= MMU_PAGESIZE;
895				}
896			}
897		}
898		if (change && prom_debug)
899			prom_printf("\t\ttrim: a=%" PRIx64 ", s=%" PRIx64 "\n",
900			    *addr, *size);
901	} while (change);
902
903	/*
904	 * Trim pages from the end of the range.
905	 */
906	for (va = KERNEL_TEXT;
907	    *size > 0 && kbm_probe(&va, &len, &pfn, &prot) != 0;
908	    va = next_va) {
909
910		next_va = va + len;
911		pfn_addr = pfn_to_pa(pfn);
912
913		if (pfn_addr >= *addr && pfn_addr < *addr + *size)
914			*size = pfn_addr - *addr;
915	}
916
917	if (prom_debug)
918		prom_printf("\tFilter out: a=%" PRIx64 ", s=%" PRIx64 "\n",
919		    *addr, *size);
920}
921
922static void
923kpm_init()
924{
925	struct segkpm_crargs b;
926
927	/*
928	 * These variables were all designed for sfmmu in which segkpm is
929	 * mapped using a single pagesize - either 8KB or 4MB.  On x86, we
930	 * might use 2+ page sizes on a single machine, so none of these
931	 * variables have a single correct value.  They are set up as if we
932	 * always use a 4KB pagesize, which should do no harm.  In the long
933	 * run, we should get rid of KPM's assumption that only a single
934	 * pagesize is used.
935	 */
936	kpm_pgshft = MMU_PAGESHIFT;
937	kpm_pgsz =  MMU_PAGESIZE;
938	kpm_pgoff = MMU_PAGEOFFSET;
939	kpmp2pshft = 0;
940	kpmpnpgs = 1;
941	ASSERT(((uintptr_t)kpm_vbase & (kpm_pgsz - 1)) == 0);
942
943	PRM_POINT("about to create segkpm");
944	rw_enter(&kas.a_lock, RW_WRITER);
945
946	if (seg_attach(&kas, kpm_vbase, kpm_size, segkpm) < 0)
947		panic("cannot attach segkpm");
948
949	b.prot = PROT_READ | PROT_WRITE;
950	b.nvcolors = 1;
951
952	if (segkpm_create(segkpm, (caddr_t)&b) != 0)
953		panic("segkpm_create segkpm");
954
955	rw_exit(&kas.a_lock);
956
957	kpm_enable = 1;
958
959	/*
960	 * As the KPM was disabled while setting up the system, go back and fix
961	 * CPU zero's access to its user page table. This is a bit gross, but
962	 * we have a chicken and egg problem otherwise.
963	 */
964	ASSERT(CPU->cpu_hat_info->hci_user_l3ptes == NULL);
965	CPU->cpu_hat_info->hci_user_l3ptes =
966	    (x86pte_t *)hat_kpm_mapin_pfn(CPU->cpu_hat_info->hci_user_l3pfn);
967}
968
969/*
970 * The debug info page provides enough information to allow external
971 * inspectors (e.g. when running under a hypervisor) to bootstrap
972 * themselves into allowing full-blown kernel debugging.
973 */
974static void
975init_debug_info(void)
976{
977	caddr_t mem;
978	debug_info_t *di;
979
980#ifndef __lint
981	ASSERT(sizeof (debug_info_t) < MMU_PAGESIZE);
982#endif
983
984	mem = BOP_ALLOC(bootops, (caddr_t)DEBUG_INFO_VA, MMU_PAGESIZE,
985	    MMU_PAGESIZE);
986
987	if (mem != (caddr_t)DEBUG_INFO_VA)
988		panic("BOP_ALLOC() failed");
989	bzero(mem, MMU_PAGESIZE);
990
991	di = (debug_info_t *)mem;
992
993	di->di_magic = DEBUG_INFO_MAGIC;
994	di->di_version = DEBUG_INFO_VERSION;
995	di->di_modules = (uintptr_t)&modules;
996	di->di_s_text = (uintptr_t)s_text;
997	di->di_e_text = (uintptr_t)e_text;
998	di->di_s_data = (uintptr_t)s_data;
999	di->di_e_data = (uintptr_t)e_data;
1000	di->di_hat_htable_off = offsetof(hat_t, hat_htable);
1001	di->di_ht_pfn_off = offsetof(htable_t, ht_pfn);
1002}
1003
1004/*
1005 * Build the memlists and other kernel essential memory system data structures.
1006 * This is everything at valloc_base.
1007 */
1008static void
1009startup_memlist(void)
1010{
1011	size_t memlist_sz;
1012	size_t memseg_sz;
1013	size_t pagehash_sz;
1014	size_t pp_sz;
1015	uintptr_t va;
1016	size_t len;
1017	uint_t prot;
1018	pfn_t pfn;
1019	int memblocks;
1020	pfn_t rsvd_high_pfn;
1021	pgcnt_t rsvd_pgcnt;
1022	size_t rsvdmemlist_sz;
1023	int rsvdmemblocks;
1024	caddr_t pagecolor_mem;
1025	size_t pagecolor_memsz;
1026	caddr_t page_ctrs_mem;
1027	size_t page_ctrs_size;
1028	size_t pse_table_alloc_size;
1029	struct memlist *current;
1030	extern void startup_build_mem_nodes(struct memlist *);
1031
1032	/* XX64 fix these - they should be in include files */
1033	extern size_t page_coloring_init(uint_t, int, int);
1034	extern void page_coloring_setup(caddr_t);
1035
1036	PRM_POINT("startup_memlist() starting...");
1037
1038	/*
1039	 * Use leftover large page nucleus text/data space for loadable modules.
1040	 * Use at most MODTEXT/MODDATA.
1041	 */
1042	len = kbm_nucleus_size;
1043	ASSERT(len > MMU_PAGESIZE);
1044
1045	moddata = (caddr_t)ROUND_UP_PAGE(e_data);
1046	e_moddata = (caddr_t)P2ROUNDUP((uintptr_t)e_data, (uintptr_t)len);
1047	if (e_moddata - moddata > MODDATA)
1048		e_moddata = moddata + MODDATA;
1049
1050	modtext = (caddr_t)ROUND_UP_PAGE(e_text);
1051	e_modtext = (caddr_t)P2ROUNDUP((uintptr_t)e_text, (uintptr_t)len);
1052	if (e_modtext - modtext > MODTEXT)
1053		e_modtext = modtext + MODTEXT;
1054
1055	econtig = e_moddata;
1056
1057	PRM_DEBUG(modtext);
1058	PRM_DEBUG(e_modtext);
1059	PRM_DEBUG(moddata);
1060	PRM_DEBUG(e_moddata);
1061	PRM_DEBUG(econtig);
1062
1063	/*
1064	 * Examine the boot loader physical memory map to find out:
1065	 * - total memory in system - physinstalled
1066	 * - the max physical address - physmax
1067	 * - the number of discontiguous segments of memory.
1068	 */
1069	if (prom_debug)
1070		print_memlist("boot physinstalled",
1071		    bootops->boot_mem->physinstalled);
1072	installed_top_size_ex(bootops->boot_mem->physinstalled, &physmax,
1073	    &physinstalled, &memblocks);
1074	PRM_DEBUG(physmax);
1075	PRM_DEBUG(physinstalled);
1076	PRM_DEBUG(memblocks);
1077
1078	/*
1079	 * We no longer support any form of memory DR.
1080	 */
1081	plat_dr_physmax = 0;
1082
1083	/*
1084	 * Examine the bios reserved memory to find out:
1085	 * - the number of discontiguous segments of memory.
1086	 */
1087	if (prom_debug)
1088		print_memlist("boot reserved mem",
1089		    bootops->boot_mem->rsvdmem);
1090	installed_top_size_ex(bootops->boot_mem->rsvdmem, &rsvd_high_pfn,
1091	    &rsvd_pgcnt, &rsvdmemblocks);
1092	PRM_DEBUG(rsvd_high_pfn);
1093	PRM_DEBUG(rsvd_pgcnt);
1094	PRM_DEBUG(rsvdmemblocks);
1095
1096	/*
1097	 * Initialize hat's mmu parameters.
1098	 * Check for enforce-prot-exec in boot environment. It's used to
1099	 * enable/disable support for the page table entry NX bit.
1100	 * The default is to enforce PROT_EXEC on processors that support NX.
1101	 * Boot seems to round up the "len", but 8 seems to be big enough.
1102	 */
1103	mmu_init();
1104
1105#ifdef	__i386
1106	/*
1107	 * physmax is lowered if there is more memory than can be
1108	 * physically addressed in 32 bit (PAE/non-PAE) modes.
1109	 */
1110	if (mmu.pae_hat) {
1111		if (PFN_ABOVE64G(physmax)) {
1112			physinstalled -= (physmax - (PFN_64G - 1));
1113			physmax = PFN_64G - 1;
1114		}
1115	} else {
1116		if (PFN_ABOVE4G(physmax)) {
1117			physinstalled -= (physmax - (PFN_4G - 1));
1118			physmax = PFN_4G - 1;
1119		}
1120	}
1121#endif
1122
1123	startup_build_mem_nodes(bootops->boot_mem->physinstalled);
1124
1125	if (BOP_GETPROPLEN(bootops, "enforce-prot-exec") >= 0) {
1126		int len = BOP_GETPROPLEN(bootops, "enforce-prot-exec");
1127		char value[8];
1128
1129		if (len < 8)
1130			(void) BOP_GETPROP(bootops, "enforce-prot-exec", value);
1131		else
1132			(void) strcpy(value, "");
1133		if (strcmp(value, "off") == 0)
1134			mmu.pt_nx = 0;
1135	}
1136	PRM_DEBUG(mmu.pt_nx);
1137
1138	/*
1139	 * We will need page_t's for every page in the system, except for
1140	 * memory mapped at or above above the start of the kernel text segment.
1141	 *
1142	 * pages above e_modtext are attributed to kernel debugger (obp_pages)
1143	 */
1144	npages = physinstalled - 1; /* avail_filter() skips page 0, so "- 1" */
1145	obp_pages = 0;
1146	va = KERNEL_TEXT;
1147	while (kbm_probe(&va, &len, &pfn, &prot) != 0) {
1148		npages -= len >> MMU_PAGESHIFT;
1149		if (va >= (uintptr_t)e_moddata)
1150			obp_pages += len >> MMU_PAGESHIFT;
1151		va += len;
1152	}
1153	PRM_DEBUG(npages);
1154	PRM_DEBUG(obp_pages);
1155
1156	/*
1157	 * If physmem is patched to be non-zero, use it instead of the computed
1158	 * value unless it is larger than the actual amount of memory on hand.
1159	 */
1160	if (physmem == 0 || physmem > npages) {
1161		physmem = npages;
1162	} else if (physmem < npages) {
1163		orig_npages = npages;
1164		npages = physmem;
1165	}
1166	PRM_DEBUG(physmem);
1167
1168	/*
1169	 * We now compute the sizes of all the  initial allocations for
1170	 * structures the kernel needs in order do kmem_alloc(). These
1171	 * include:
1172	 *	memsegs
1173	 *	memlists
1174	 *	page hash table
1175	 *	page_t's
1176	 *	page coloring data structs
1177	 */
1178	memseg_sz = sizeof (struct memseg) * (memblocks + POSS_NEW_FRAGMENTS);
1179	ADD_TO_ALLOCATIONS(memseg_base, memseg_sz);
1180	PRM_DEBUG(memseg_sz);
1181
1182	/*
1183	 * Reserve space for memlists. There's no real good way to know exactly
1184	 * how much room we'll need, but this should be a good upper bound.
1185	 */
1186	memlist_sz = ROUND_UP_PAGE(2 * sizeof (struct memlist) *
1187	    (memblocks + POSS_NEW_FRAGMENTS));
1188	ADD_TO_ALLOCATIONS(memlist, memlist_sz);
1189	PRM_DEBUG(memlist_sz);
1190
1191	/*
1192	 * Reserve space for bios reserved memlists.
1193	 */
1194	rsvdmemlist_sz = ROUND_UP_PAGE(2 * sizeof (struct memlist) *
1195	    (rsvdmemblocks + POSS_NEW_FRAGMENTS));
1196	ADD_TO_ALLOCATIONS(bios_rsvd, rsvdmemlist_sz);
1197	PRM_DEBUG(rsvdmemlist_sz);
1198
1199	/* LINTED */
1200	ASSERT(P2SAMEHIGHBIT((1 << PP_SHIFT), sizeof (struct page)));
1201	/*
1202	 * The page structure hash table size is a power of 2
1203	 * such that the average hash chain length is PAGE_HASHAVELEN.
1204	 */
1205	page_hashsz = npages / PAGE_HASHAVELEN;
1206	page_hashsz_shift = highbit(page_hashsz);
1207	page_hashsz = 1 << page_hashsz_shift;
1208	pagehash_sz = sizeof (struct page *) * page_hashsz;
1209	ADD_TO_ALLOCATIONS(page_hash, pagehash_sz);
1210	PRM_DEBUG(pagehash_sz);
1211
1212	/*
1213	 * Set aside room for the page structures themselves.
1214	 */
1215	PRM_DEBUG(npages);
1216	pp_sz = sizeof (struct page) * npages;
1217	ADD_TO_ALLOCATIONS(pp_base, pp_sz);
1218	PRM_DEBUG(pp_sz);
1219
1220	/*
1221	 * determine l2 cache info and memory size for page coloring
1222	 */
1223	(void) getl2cacheinfo(CPU,
1224	    &l2cache_sz, &l2cache_linesz, &l2cache_assoc);
1225	pagecolor_memsz =
1226	    page_coloring_init(l2cache_sz, l2cache_linesz, l2cache_assoc);
1227	ADD_TO_ALLOCATIONS(pagecolor_mem, pagecolor_memsz);
1228	PRM_DEBUG(pagecolor_memsz);
1229
1230	page_ctrs_size = page_ctrs_sz();
1231	ADD_TO_ALLOCATIONS(page_ctrs_mem, page_ctrs_size);
1232	PRM_DEBUG(page_ctrs_size);
1233
1234	/*
1235	 * Allocate the array that protects pp->p_selock.
1236	 */
1237	pse_shift = size_pse_array(physmem, max_ncpus);
1238	pse_table_size = 1 << pse_shift;
1239	pse_table_alloc_size = pse_table_size * sizeof (pad_mutex_t);
1240	ADD_TO_ALLOCATIONS(pse_mutex, pse_table_alloc_size);
1241
1242	valloc_sz = ROUND_UP_LPAGE(valloc_sz);
1243	valloc_base = VALLOC_BASE;
1244
1245	/*
1246	 * The signicant memory-sized regions are roughly sized as follows in
1247	 * the default layout with max physmem:
1248	 *  segkpm: 1x physmem allocated (but 1Tb room, below VALLOC_BASE)
1249	 *  segzio: 1.5x physmem
1250	 *  segkvmm: 4x physmem
1251	 *  heap: whatever's left up to COREHEAP_BASE, at least 1.5x physmem
1252	 *
1253	 * The idea is that we leave enough room to avoid fragmentation issues,
1254	 * so we would like the VA arenas to have some extra.
1255	 *
1256	 * Ignoring the loose change of segkp, valloc, and such, this means that
1257	 * as COREHEAP_BASE-VALLOC_BASE=2Tb, we can accommodate a physmem up to
1258	 * about (2Tb / 7.0), rounded down to 256Gb in the check below.
1259	 *
1260	 * Note that KPM lives below VALLOC_BASE, but we want to include it in
1261	 * adjustments, hence the 8 below.
1262	 *
1263	 * Beyond 256Gb, we push segkpm_base (and hence kernelbase and
1264	 * _userlimit) down to accommodate the VA requirements above.
1265	 */
1266	if (physmax + 1 > mmu_btop(TERABYTE / 4)) {
1267		uint64_t physmem_bytes = mmu_ptob(physmax + 1);
1268		uint64_t adjustment = 8 * (physmem_bytes - (TERABYTE / 4));
1269
1270		PRM_DEBUG(adjustment);
1271
1272		/*
1273		 * segkpm_base is always aligned on a L3 PTE boundary.
1274		 */
1275		segkpm_base -= P2ROUNDUP(adjustment, KERNEL_REDZONE_SIZE);
1276
1277		/*
1278		 * But make sure we leave some space for user apps above hole.
1279		 */
1280		segkpm_base = MAX(segkpm_base, AMD64_VA_HOLE_END + TERABYTE);
1281
1282		ASSERT(segkpm_base <= SEGKPM_BASE);
1283
1284		valloc_base = segkpm_base + P2ROUNDUP(physmem_bytes, ONE_GIG);
1285		if (valloc_base < segkpm_base)
1286			panic("not enough kernel VA to support memory size");
1287	}
1288
1289	PRM_DEBUG(segkpm_base);
1290	PRM_DEBUG(valloc_base);
1291
1292	/*
1293	 * do all the initial allocations
1294	 */
1295	perform_allocations();
1296
1297	/*
1298	 * Build phys_install and phys_avail in kernel memspace.
1299	 * - phys_install should be all memory in the system.
1300	 * - phys_avail is phys_install minus any memory mapped before this
1301	 *    point above KERNEL_TEXT.
1302	 */
1303	current = phys_install = memlist;
1304	copy_memlist_filter(bootops->boot_mem->physinstalled, &current, NULL);
1305	if ((caddr_t)current > (caddr_t)memlist + memlist_sz)
1306		panic("physinstalled was too big!");
1307	if (prom_debug)
1308		print_memlist("phys_install", phys_install);
1309
1310	phys_avail = current;
1311	PRM_POINT("Building phys_avail:\n");
1312	copy_memlist_filter(bootops->boot_mem->physinstalled, &current,
1313	    avail_filter);
1314	if ((caddr_t)current > (caddr_t)memlist + memlist_sz)
1315		panic("physavail was too big!");
1316	if (prom_debug)
1317		print_memlist("phys_avail", phys_avail);
1318#ifndef	__xpv
1319	/*
1320	 * Free unused memlist items, which may be used by memory DR driver
1321	 * at runtime.
1322	 */
1323	if ((caddr_t)current < (caddr_t)memlist + memlist_sz) {
1324		memlist_free_block((caddr_t)current,
1325		    (caddr_t)memlist + memlist_sz - (caddr_t)current);
1326	}
1327#endif
1328
1329	/*
1330	 * Build bios reserved memspace
1331	 */
1332	current = bios_rsvd;
1333	copy_memlist_filter(bootops->boot_mem->rsvdmem, &current, NULL);
1334	if ((caddr_t)current > (caddr_t)bios_rsvd + rsvdmemlist_sz)
1335		panic("bios_rsvd was too big!");
1336	if (prom_debug)
1337		print_memlist("bios_rsvd", bios_rsvd);
1338#ifndef	__xpv
1339	/*
1340	 * Free unused memlist items, which may be used by memory DR driver
1341	 * at runtime.
1342	 */
1343	if ((caddr_t)current < (caddr_t)bios_rsvd + rsvdmemlist_sz) {
1344		memlist_free_block((caddr_t)current,
1345		    (caddr_t)bios_rsvd + rsvdmemlist_sz - (caddr_t)current);
1346	}
1347#endif
1348
1349	/*
1350	 * setup page coloring
1351	 */
1352	page_coloring_setup(pagecolor_mem);
1353	page_lock_init();	/* currently a no-op */
1354
1355	/*
1356	 * free page list counters
1357	 */
1358	(void) page_ctrs_alloc(page_ctrs_mem);
1359
1360	/*
1361	 * Size the pcf array based on the number of cpus in the box at
1362	 * boot time.
1363	 */
1364
1365	pcf_init();
1366
1367	/*
1368	 * Initialize the page structures from the memory lists.
1369	 */
1370	availrmem_initial = availrmem = freemem = 0;
1371	PRM_POINT("Calling kphysm_init()...");
1372	npages = kphysm_init(pp_base, npages);
1373	PRM_POINT("kphysm_init() done");
1374	PRM_DEBUG(npages);
1375
1376	init_debug_info();
1377
1378	/*
1379	 * Now that page_t's have been initialized, remove all the
1380	 * initial allocation pages from the kernel free page lists.
1381	 */
1382	boot_mapin((caddr_t)valloc_base, valloc_sz);
1383	boot_mapin((caddr_t)MISC_VA_BASE, MISC_VA_SIZE);
1384	PRM_POINT("startup_memlist() done");
1385
1386	PRM_DEBUG(valloc_sz);
1387
1388#if defined(__amd64)
1389	if ((availrmem >> (30 - MMU_PAGESHIFT)) >=
1390	    textrepl_min_gb && l2cache_sz <= 2 << 20) {
1391		extern size_t textrepl_size_thresh;
1392		textrepl_size_thresh = (16 << 20) - 1;
1393	}
1394#endif
1395}
1396
1397/*
1398 * Layout the kernel's part of address space and initialize kmem allocator.
1399 */
1400static void
1401startup_kmem(void)
1402{
1403	extern void page_set_colorequiv_arr(void);
1404#if !defined(__xpv)
1405	extern uint64_t kpti_kbase;
1406#endif
1407
1408	PRM_POINT("startup_kmem() starting...");
1409
1410#if defined(__amd64)
1411	if (eprom_kernelbase && eprom_kernelbase != KERNELBASE)
1412		cmn_err(CE_NOTE, "!kernelbase cannot be changed on 64-bit "
1413		    "systems.");
1414	kernelbase = segkpm_base - KERNEL_REDZONE_SIZE;
1415	core_base = (uintptr_t)COREHEAP_BASE;
1416	core_size = (size_t)MISC_VA_BASE - COREHEAP_BASE;
1417#else	/* __i386 */
1418	/*
1419	 * We configure kernelbase based on:
1420	 *
1421	 * 1. user specified kernelbase via eeprom command. Value cannot exceed
1422	 *    KERNELBASE_MAX. we large page align eprom_kernelbase
1423	 *
1424	 * 2. Default to KERNELBASE and adjust to 2X less the size for page_t.
1425	 *    On large memory systems we must lower kernelbase to allow
1426	 *    enough room for page_t's for all of memory.
1427	 *
1428	 * The value set here, might be changed a little later.
1429	 */
1430	if (eprom_kernelbase) {
1431		kernelbase = eprom_kernelbase & mmu.level_mask[1];
1432		if (kernelbase > KERNELBASE_MAX)
1433			kernelbase = KERNELBASE_MAX;
1434	} else {
1435		kernelbase = (uintptr_t)KERNELBASE;
1436		kernelbase -= ROUND_UP_4MEG(2 * valloc_sz);
1437	}
1438	ASSERT((kernelbase & mmu.level_offset[1]) == 0);
1439	core_base = valloc_base;
1440	core_size = 0;
1441#endif	/* __i386 */
1442
1443	PRM_DEBUG(core_base);
1444	PRM_DEBUG(core_size);
1445	PRM_DEBUG(kernelbase);
1446
1447#if defined(__i386)
1448	segkp_fromheap = 1;
1449#endif	/* __i386 */
1450
1451	ekernelheap = (char *)core_base;
1452	PRM_DEBUG(ekernelheap);
1453
1454	/*
1455	 * Now that we know the real value of kernelbase,
1456	 * update variables that were initialized with a value of
1457	 * KERNELBASE (in common/conf/param.c).
1458	 *
1459	 * XXX	The problem with this sort of hackery is that the
1460	 *	compiler just may feel like putting the const declarations
1461	 *	(in param.c) into the .text section.  Perhaps they should
1462	 *	just be declared as variables there?
1463	 */
1464
1465	*(uintptr_t *)&_kernelbase = kernelbase;
1466	*(uintptr_t *)&_userlimit = kernelbase;
1467#if defined(__amd64)
1468	*(uintptr_t *)&_userlimit -= KERNELBASE - USERLIMIT;
1469#if !defined(__xpv)
1470	kpti_kbase = kernelbase;
1471#endif
1472#else
1473	*(uintptr_t *)&_userlimit32 = _userlimit;
1474#endif
1475	PRM_DEBUG(_kernelbase);
1476	PRM_DEBUG(_userlimit);
1477	PRM_DEBUG(_userlimit32);
1478
1479	/* We have to re-do this now that we've modified _userlimit. */
1480	mmu_calc_user_slots();
1481
1482	layout_kernel_va();
1483
1484#if defined(__i386)
1485	/*
1486	 * If segmap is too large we can push the bottom of the kernel heap
1487	 * higher than the base.  Or worse, it could exceed the top of the
1488	 * VA space entirely, causing it to wrap around.
1489	 */
1490	if (kernelheap >= ekernelheap || (uintptr_t)kernelheap < kernelbase)
1491		panic("too little address space available for kernelheap,"
1492		    " use eeprom for lower kernelbase or smaller segmapsize");
1493#endif	/* __i386 */
1494
1495	/*
1496	 * Initialize the kernel heap. Note 3rd argument must be > 1st.
1497	 */
1498	kernelheap_init(kernelheap, ekernelheap,
1499	    kernelheap + MMU_PAGESIZE,
1500	    (void *)core_base, (void *)(core_base + core_size));
1501
1502#if defined(__xpv)
1503	/*
1504	 * Link pending events struct into cpu struct
1505	 */
1506	CPU->cpu_m.mcpu_evt_pend = &cpu0_evt_data;
1507#endif
1508	/*
1509	 * Initialize kernel memory allocator.
1510	 */
1511	kmem_init();
1512
1513	/*
1514	 * Factor in colorequiv to check additional 'equivalent' bins
1515	 */
1516	page_set_colorequiv_arr();
1517
1518	/*
1519	 * print this out early so that we know what's going on
1520	 */
1521	print_x86_featureset(x86_featureset);
1522
1523	/*
1524	 * Initialize bp_mapin().
1525	 */
1526	bp_init(MMU_PAGESIZE, HAT_STORECACHING_OK);
1527
1528	/*
1529	 * orig_npages is non-zero if physmem has been configured for less
1530	 * than the available memory.
1531	 */
1532	if (orig_npages) {
1533		cmn_err(CE_WARN, "!%slimiting physmem to 0x%lx of 0x%lx pages",
1534		    (npages == PHYSMEM ? "Due to virtual address space " : ""),
1535		    npages, orig_npages);
1536	}
1537#if defined(__i386)
1538	if (eprom_kernelbase && (eprom_kernelbase != kernelbase))
1539		cmn_err(CE_WARN, "kernelbase value, User specified 0x%lx, "
1540		    "System using 0x%lx",
1541		    (uintptr_t)eprom_kernelbase, (uintptr_t)kernelbase);
1542#endif
1543
1544#ifdef	KERNELBASE_ABI_MIN
1545	if (kernelbase < (uintptr_t)KERNELBASE_ABI_MIN) {
1546		cmn_err(CE_NOTE, "!kernelbase set to 0x%lx, system is not "
1547		    "i386 ABI compliant.", (uintptr_t)kernelbase);
1548	}
1549#endif
1550
1551#ifndef __xpv
1552	if (plat_dr_support_memory()) {
1553		mem_config_init();
1554	}
1555#else	/* __xpv */
1556	/*
1557	 * Some of the xen start information has to be relocated up
1558	 * into the kernel's permanent address space.
1559	 */
1560	PRM_POINT("calling xen_relocate_start_info()");
1561	xen_relocate_start_info();
1562	PRM_POINT("xen_relocate_start_info() done");
1563
1564	/*
1565	 * (Update the vcpu pointer in our cpu structure to point into
1566	 * the relocated shared info.)
1567	 */
1568	CPU->cpu_m.mcpu_vcpu_info =
1569	    &HYPERVISOR_shared_info->vcpu_info[CPU->cpu_id];
1570#endif	/* __xpv */
1571
1572	PRM_POINT("startup_kmem() done");
1573}
1574
1575#ifndef __xpv
1576/*
1577 * If we have detected that we are running in an HVM environment, we need
1578 * to prepend the PV driver directory to the module search path.
1579 */
1580#define	HVM_MOD_DIR "/platform/i86hvm/kernel"
1581static void
1582update_default_path()
1583{
1584	char *current, *newpath;
1585	int newlen;
1586
1587	/*
1588	 * We are about to resync with krtld.  krtld will reset its
1589	 * internal module search path iff Solaris has set default_path.
1590	 * We want to be sure we're prepending this new directory to the
1591	 * right search path.
1592	 */
1593	current = (default_path == NULL) ? kobj_module_path : default_path;
1594
1595	newlen = strlen(HVM_MOD_DIR) + strlen(current) + 2;
1596	newpath = kmem_alloc(newlen, KM_SLEEP);
1597	(void) strcpy(newpath, HVM_MOD_DIR);
1598	(void) strcat(newpath, " ");
1599	(void) strcat(newpath, current);
1600
1601	default_path = newpath;
1602}
1603#endif
1604
1605static void
1606startup_modules(void)
1607{
1608	int cnt;
1609	extern void prom_setup(void);
1610	int32_t v, h;
1611	char d[11];
1612	char *cp;
1613	cmi_hdl_t hdl;
1614
1615	PRM_POINT("startup_modules() starting...");
1616
1617#ifndef __xpv
1618	/*
1619	 * Initialize ten-micro second timer so that drivers will
1620	 * not get short changed in their init phase. This was
1621	 * not getting called until clkinit which, on fast cpu's
1622	 * caused the drv_usecwait to be way too short.
1623	 */
1624	microfind();
1625
1626	if ((get_hwenv() & HW_XEN_HVM) != 0)
1627		update_default_path();
1628#endif
1629
1630	/*
1631	 * Read the GMT lag from /etc/rtc_config.
1632	 */
1633	sgmtl(process_rtc_config_file());
1634
1635	/*
1636	 * Calculate default settings of system parameters based upon
1637	 * maxusers, yet allow to be overridden via the /etc/system file.
1638	 */
1639	param_calc(0);
1640
1641	mod_setup();
1642
1643	/*
1644	 * Initialize system parameters.
1645	 */
1646	param_init();
1647
1648	/*
1649	 * Initialize the default brands
1650	 */
1651	brand_init();
1652
1653	/*
1654	 * maxmem is the amount of physical memory we're playing with.
1655	 */
1656	maxmem = physmem;
1657
1658	/*
1659	 * Initialize segment management stuff.
1660	 */
1661	seg_init();
1662
1663	if (modload("fs", "specfs") == -1)
1664		halt("Can't load specfs");
1665
1666	if (modload("fs", "devfs") == -1)
1667		halt("Can't load devfs");
1668
1669	if (modload("fs", "dev") == -1)
1670		halt("Can't load dev");
1671
1672	if (modload("fs", "procfs") == -1)
1673		halt("Can't load procfs");
1674
1675	(void) modloadonly("sys", "lbl_edition");
1676
1677	dispinit();
1678
1679	/* Read cluster configuration data. */
1680	clconf_init();
1681
1682#if defined(__xpv)
1683	(void) ec_init();
1684	gnttab_init();
1685	(void) xs_early_init();
1686#endif /* __xpv */
1687
1688	/*
1689	 * Create a kernel device tree. First, create rootnex and
1690	 * then invoke bus specific code to probe devices.
1691	 */
1692	setup_ddi();
1693
1694#ifdef __xpv
1695	if (DOMAIN_IS_INITDOMAIN(xen_info))
1696#endif
1697	{
1698		id_t smid;
1699		smbios_system_t smsys;
1700		smbios_info_t sminfo;
1701		char *mfg;
1702		/*
1703		 * Load the System Management BIOS into the global ksmbios
1704		 * handle, if an SMBIOS is present on this system.
1705		 * Also set "si-hw-provider" property, if not already set.
1706		 */
1707		ksmbios = smbios_open(NULL, SMB_VERSION, ksmbios_flags, NULL);
1708		if (ksmbios != NULL &&
1709		    ((smid = smbios_info_system(ksmbios, &smsys)) != SMB_ERR) &&
1710		    (smbios_info_common(ksmbios, smid, &sminfo)) != SMB_ERR) {
1711			mfg = (char *)sminfo.smbi_manufacturer;
1712			if (BOP_GETPROPLEN(bootops, "si-hw-provider") < 0) {
1713				extern char hw_provider[];
1714				int i;
1715				for (i = 0; i < SYS_NMLN; i++) {
1716					if (isprint(mfg[i]))
1717						hw_provider[i] = mfg[i];
1718					else {
1719						hw_provider[i] = '\0';
1720						break;
1721					}
1722				}
1723				hw_provider[SYS_NMLN - 1] = '\0';
1724			}
1725		}
1726	}
1727
1728
1729	/*
1730	 * Originally clconf_init() apparently needed the hostid.  But
1731	 * this no longer appears to be true - it uses its own nodeid.
1732	 * By placing the hostid logic here, we are able to make use of
1733	 * the SMBIOS UUID.
1734	 */
1735	if ((h = set_soft_hostid()) == HW_INVALID_HOSTID) {
1736		cmn_err(CE_WARN, "Unable to set hostid");
1737	} else {
1738		for (v = h, cnt = 0; cnt < 10; cnt++) {
1739			d[cnt] = (char)(v % 10);
1740			v /= 10;
1741			if (v == 0)
1742				break;
1743		}
1744		for (cp = hw_serial; cnt >= 0; cnt--)
1745			*cp++ = d[cnt] + '0';
1746		*cp = 0;
1747	}
1748
1749	/*
1750	 * Set up the CPU module subsystem for the boot cpu in the native
1751	 * case, and all physical cpu resource in the xpv dom0 case.
1752	 * Modifies the device tree, so this must be done after
1753	 * setup_ddi().
1754	 */
1755#ifdef __xpv
1756	/*
1757	 * If paravirtualized and on dom0 then we initialize all physical
1758	 * cpu handles now;  if paravirtualized on a domU then do not
1759	 * initialize.
1760	 */
1761	if (DOMAIN_IS_INITDOMAIN(xen_info)) {
1762		xen_mc_lcpu_cookie_t cpi;
1763
1764		for (cpi = xen_physcpu_next(NULL); cpi != NULL;
1765		    cpi = xen_physcpu_next(cpi)) {
1766			if ((hdl = cmi_init(CMI_HDL_SOLARIS_xVM_MCA,
1767			    xen_physcpu_chipid(cpi), xen_physcpu_coreid(cpi),
1768			    xen_physcpu_strandid(cpi))) != NULL &&
1769			    is_x86_feature(x86_featureset, X86FSET_MCA))
1770				cmi_mca_init(hdl);
1771		}
1772	}
1773#else
1774	/*
1775	 * Initialize a handle for the boot cpu - others will initialize
1776	 * as they startup.
1777	 */
1778	if ((hdl = cmi_init(CMI_HDL_NATIVE, cmi_ntv_hwchipid(CPU),
1779	    cmi_ntv_hwcoreid(CPU), cmi_ntv_hwstrandid(CPU))) != NULL) {
1780		if (is_x86_feature(x86_featureset, X86FSET_MCA))
1781			cmi_mca_init(hdl);
1782		CPU->cpu_m.mcpu_cmi_hdl = hdl;
1783	}
1784#endif	/* __xpv */
1785
1786	/*
1787	 * Fake a prom tree such that /dev/openprom continues to work
1788	 */
1789	PRM_POINT("startup_modules: calling prom_setup...");
1790	prom_setup();
1791	PRM_POINT("startup_modules: done");
1792
1793	/*
1794	 * Load all platform specific modules
1795	 */
1796	PRM_POINT("startup_modules: calling psm_modload...");
1797	psm_modload();
1798
1799	PRM_POINT("startup_modules() done");
1800}
1801
1802/*
1803 * claim a "setaside" boot page for use in the kernel
1804 */
1805page_t *
1806boot_claim_page(pfn_t pfn)
1807{
1808	page_t *pp;
1809
1810	pp = page_numtopp_nolock(pfn);
1811	ASSERT(pp != NULL);
1812
1813	if (PP_ISBOOTPAGES(pp)) {
1814		if (pp->p_next != NULL)
1815			pp->p_next->p_prev = pp->p_prev;
1816		if (pp->p_prev == NULL)
1817			bootpages = pp->p_next;
1818		else
1819			pp->p_prev->p_next = pp->p_next;
1820	} else {
1821		/*
1822		 * htable_attach() expects a base pagesize page
1823		 */
1824		if (pp->p_szc != 0)
1825			page_boot_demote(pp);
1826		pp = page_numtopp(pfn, SE_EXCL);
1827	}
1828	return (pp);
1829}
1830
1831/*
1832 * Walk through the pagetables looking for pages mapped in by boot.  If the
1833 * setaside flag is set the pages are expected to be returned to the
1834 * kernel later in boot, so we add them to the bootpages list.
1835 */
1836static void
1837protect_boot_range(uintptr_t low, uintptr_t high, int setaside)
1838{
1839	uintptr_t va = low;
1840	size_t len;
1841	uint_t prot;
1842	pfn_t pfn;
1843	page_t *pp;
1844	pgcnt_t boot_protect_cnt = 0;
1845
1846	while (kbm_probe(&va, &len, &pfn, &prot) != 0 && va < high) {
1847		if (va + len >= high)
1848			panic("0x%lx byte mapping at 0x%p exceeds boot's "
1849			    "legal range.", len, (void *)va);
1850
1851		while (len > 0) {
1852			pp = page_numtopp_alloc(pfn);
1853			if (pp != NULL) {
1854				if (setaside == 0)
1855					panic("Unexpected mapping by boot.  "
1856					    "addr=%p pfn=%lx\n",
1857					    (void *)va, pfn);
1858
1859				pp->p_next = bootpages;
1860				pp->p_prev = NULL;
1861				PP_SETBOOTPAGES(pp);
1862				if (bootpages != NULL) {
1863					bootpages->p_prev = pp;
1864				}
1865				bootpages = pp;
1866				++boot_protect_cnt;
1867			}
1868
1869			++pfn;
1870			len -= MMU_PAGESIZE;
1871			va += MMU_PAGESIZE;
1872		}
1873	}
1874	PRM_DEBUG(boot_protect_cnt);
1875}
1876
1877/*
1878 * Establish the final size of the kernel's heap, size of segmap, segkp, etc.
1879 */
1880static void
1881layout_kernel_va(void)
1882{
1883	const size_t physmem_size = mmu_ptob(physmem);
1884	size_t size;
1885
1886	PRM_POINT("layout_kernel_va() starting...");
1887
1888	kpm_vbase = (caddr_t)segkpm_base;
1889	kpm_size = ROUND_UP_LPAGE(mmu_ptob(physmax + 1));
1890	if ((uintptr_t)kpm_vbase + kpm_size > (uintptr_t)valloc_base)
1891		panic("not enough room for kpm!");
1892	PRM_DEBUG(kpm_size);
1893	PRM_DEBUG(kpm_vbase);
1894
1895	segkp_base = (caddr_t)valloc_base + valloc_sz;
1896	if (!segkp_fromheap) {
1897		size = mmu_ptob(segkpsize);
1898
1899		/*
1900		 * determine size of segkp
1901		 */
1902		if (size < SEGKPMINSIZE || size > SEGKPMAXSIZE) {
1903			size = SEGKPDEFSIZE;
1904			cmn_err(CE_WARN, "!Illegal value for segkpsize. "
1905			    "segkpsize has been reset to %ld pages",
1906			    mmu_btop(size));
1907		}
1908		size = MIN(size, MAX(SEGKPMINSIZE, physmem_size));
1909
1910		segkpsize = mmu_btop(ROUND_UP_LPAGE(size));
1911	}
1912	PRM_DEBUG(segkp_base);
1913	PRM_DEBUG(segkpsize);
1914
1915	/*
1916	 * segkvmm: backing for vmm guest memory. Like segzio, we have a
1917	 * separate segment for two reasons: it makes it easy to skip our pages
1918	 * on kernel crash dumps, and it helps avoid fragmentation.  With this
1919	 * segment, we're expecting significantly-sized allocations only; we'll
1920	 * default to 4x the size of physmem.
1921	 */
1922	segkvmm_base = segkp_base + mmu_ptob(segkpsize);
1923	size = segkvmmsize != 0 ? mmu_ptob(segkvmmsize) : (physmem_size * 4);
1924
1925	size = MAX(size, SEGVMMMINSIZE);
1926	segkvmmsize = mmu_btop(ROUND_UP_LPAGE(size));
1927
1928	PRM_DEBUG(segkvmmsize);
1929	PRM_DEBUG(segkvmm_base);
1930
1931	/*
1932	 * segzio is used for ZFS cached data.  For segzio, we use 1.5x physmem.
1933	 */
1934	segzio_base = segkvmm_base + mmu_ptob(segkvmmsize);
1935	if (segzio_fromheap) {
1936		segziosize = 0;
1937	} else {
1938		size = (segziosize != 0) ? mmu_ptob(segziosize) :
1939		    (physmem_size * 3) / 2;
1940
1941		size = MAX(size, SEGZIOMINSIZE);
1942		segziosize = mmu_btop(ROUND_UP_LPAGE(size));
1943	}
1944	PRM_DEBUG(segziosize);
1945	PRM_DEBUG(segzio_base);
1946
1947	/*
1948	 * Put the range of VA for device mappings next, kmdb knows to not
1949	 * grep in this range of addresses.
1950	 */
1951	toxic_addr =
1952	    ROUND_UP_LPAGE((uintptr_t)segzio_base + mmu_ptob(segziosize));
1953	PRM_DEBUG(toxic_addr);
1954	segmap_start = ROUND_UP_LPAGE(toxic_addr + toxic_size);
1955
1956	/*
1957	 * Users can change segmapsize through eeprom. If the variable
1958	 * is tuned through eeprom, there is no upper bound on the
1959	 * size of segmap.
1960	 */
1961	segmapsize = MAX(ROUND_UP_LPAGE(segmapsize), SEGMAPDEFAULT);
1962
1963	PRM_DEBUG(segmap_start);
1964	PRM_DEBUG(segmapsize);
1965	kernelheap = (caddr_t)ROUND_UP_LPAGE(segmap_start + segmapsize);
1966	PRM_DEBUG(kernelheap);
1967	PRM_POINT("layout_kernel_va() done...");
1968}
1969
1970/*
1971 * Finish initializing the VM system, now that we are no longer
1972 * relying on the boot time memory allocators.
1973 */
1974static void
1975startup_vm(void)
1976{
1977	struct segmap_crargs a;
1978
1979	extern int use_brk_lpg, use_stk_lpg;
1980
1981	PRM_POINT("startup_vm() starting...");
1982
1983	/*
1984	 * Initialize the hat layer.
1985	 */
1986	hat_init();
1987
1988	/*
1989	 * Do final allocations of HAT data structures that need to
1990	 * be allocated before quiescing the boot loader.
1991	 */
1992	PRM_POINT("Calling hat_kern_alloc()...");
1993	hat_kern_alloc((caddr_t)segmap_start, segmapsize, ekernelheap);
1994	PRM_POINT("hat_kern_alloc() done");
1995
1996#ifndef __xpv
1997	/*
1998	 * Setup Page Attribute Table
1999	 */
2000	pat_sync();
2001#endif
2002
2003	/*
2004	 * The next two loops are done in distinct steps in order
2005	 * to be sure that any page that is doubly mapped (both above
2006	 * KERNEL_TEXT and below kernelbase) is dealt with correctly.
2007	 * Note this may never happen, but it might someday.
2008	 */
2009	bootpages = NULL;
2010	PRM_POINT("Protecting boot pages");
2011
2012	/*
2013	 * Protect any pages mapped above KERNEL_TEXT that somehow have
2014	 * page_t's. This can only happen if something weird allocated
2015	 * in this range (like kadb/kmdb).
2016	 */
2017	protect_boot_range(KERNEL_TEXT, (uintptr_t)-1, 0);
2018
2019	/*
2020	 * Before we can take over memory allocation/mapping from the boot
2021	 * loader we must remove from our free page lists any boot allocated
2022	 * pages that stay mapped until release_bootstrap().
2023	 */
2024	protect_boot_range(0, kernelbase, 1);
2025
2026
2027	/*
2028	 * Switch to running on regular HAT (not boot_mmu)
2029	 */
2030	PRM_POINT("Calling hat_kern_setup()...");
2031	hat_kern_setup();
2032
2033	/*
2034	 * It is no longer safe to call BOP_ALLOC(), so make sure we don't.
2035	 */
2036	bop_no_more_mem();
2037
2038	PRM_POINT("hat_kern_setup() done");
2039
2040	hat_cpu_online(CPU);
2041
2042	/*
2043	 * Initialize VM system
2044	 */
2045	PRM_POINT("Calling kvm_init()...");
2046	kvm_init();
2047	PRM_POINT("kvm_init() done");
2048
2049	/*
2050	 * Tell kmdb that the VM system is now working
2051	 */
2052	if (boothowto & RB_DEBUG)
2053		kdi_dvec_vmready();
2054
2055#if defined(__xpv)
2056	/*
2057	 * Populate the I/O pool on domain 0
2058	 */
2059	if (DOMAIN_IS_INITDOMAIN(xen_info)) {
2060		extern long populate_io_pool(void);
2061		long init_io_pool_cnt;
2062
2063		PRM_POINT("Populating reserve I/O page pool");
2064		init_io_pool_cnt = populate_io_pool();
2065		PRM_DEBUG(init_io_pool_cnt);
2066	}
2067#endif
2068	/*
2069	 * Mangle the brand string etc.
2070	 */
2071	cpuid_pass3(CPU);
2072
2073#if defined(__amd64)
2074
2075	/*
2076	 * Create the device arena for toxic (to dtrace/kmdb) mappings.
2077	 */
2078	device_arena = vmem_create("device", (void *)toxic_addr,
2079	    toxic_size, MMU_PAGESIZE, NULL, NULL, NULL, 0, VM_SLEEP);
2080
2081#else	/* __i386 */
2082
2083	/*
2084	 * allocate the bit map that tracks toxic pages
2085	 */
2086	toxic_bit_map_len = btop((ulong_t)(valloc_base - kernelbase));
2087	PRM_DEBUG(toxic_bit_map_len);
2088	toxic_bit_map =
2089	    kmem_zalloc(BT_SIZEOFMAP(toxic_bit_map_len), KM_NOSLEEP);
2090	ASSERT(toxic_bit_map != NULL);
2091	PRM_DEBUG(toxic_bit_map);
2092
2093#endif	/* __i386 */
2094
2095
2096	/*
2097	 * Now that we've got more VA, as well as the ability to allocate from
2098	 * it, tell the debugger.
2099	 */
2100	if (boothowto & RB_DEBUG)
2101		kdi_dvec_memavail();
2102
2103#if !defined(__xpv)
2104	/*
2105	 * Map page pfn=0 for drivers, such as kd, that need to pick up
2106	 * parameters left there by controllers/BIOS.
2107	 */
2108	PRM_POINT("setup up p0_va");
2109	p0_va = i86devmap(0, 1, PROT_READ);
2110	PRM_DEBUG(p0_va);
2111#endif
2112
2113	cmn_err(CE_CONT, "?mem = %luK (0x%lx)\n",
2114	    physinstalled << (MMU_PAGESHIFT - 10), ptob(physinstalled));
2115
2116	/*
2117	 * disable automatic large pages for small memory systems or
2118	 * when the disable flag is set.
2119	 *
2120	 * Do not yet consider page sizes larger than 2m/4m.
2121	 */
2122	if (!auto_lpg_disable && mmu.max_page_level > 0) {
2123		max_uheap_lpsize = LEVEL_SIZE(1);
2124		max_ustack_lpsize = LEVEL_SIZE(1);
2125		max_privmap_lpsize = LEVEL_SIZE(1);
2126		max_uidata_lpsize = LEVEL_SIZE(1);
2127		max_utext_lpsize = LEVEL_SIZE(1);
2128		max_shm_lpsize = LEVEL_SIZE(1);
2129	}
2130	if (physmem < privm_lpg_min_physmem || mmu.max_page_level == 0 ||
2131	    auto_lpg_disable) {
2132		use_brk_lpg = 0;
2133		use_stk_lpg = 0;
2134	}
2135	mcntl0_lpsize = LEVEL_SIZE(mmu.umax_page_level);
2136
2137	PRM_POINT("Calling hat_init_finish()...");
2138	hat_init_finish();
2139	PRM_POINT("hat_init_finish() done");
2140
2141	/*
2142	 * Initialize the segkp segment type.
2143	 */
2144	rw_enter(&kas.a_lock, RW_WRITER);
2145	PRM_POINT("Attaching segkp");
2146	if (segkp_fromheap) {
2147		segkp->s_as = &kas;
2148	} else if (seg_attach(&kas, (caddr_t)segkp_base, mmu_ptob(segkpsize),
2149	    segkp) < 0) {
2150		panic("startup: cannot attach segkp");
2151		/*NOTREACHED*/
2152	}
2153	PRM_POINT("Doing segkp_create()");
2154	if (segkp_create(segkp) != 0) {
2155		panic("startup: segkp_create failed");
2156		/*NOTREACHED*/
2157	}
2158	PRM_DEBUG(segkp);
2159	rw_exit(&kas.a_lock);
2160
2161	/*
2162	 * kpm segment
2163	 */
2164	segmap_kpm = 0;
2165	if (kpm_desired)
2166		kpm_init();
2167
2168	/*
2169	 * Now create segmap segment.
2170	 */
2171	rw_enter(&kas.a_lock, RW_WRITER);
2172	if (seg_attach(&kas, (caddr_t)segmap_start, segmapsize, segmap) < 0) {
2173		panic("cannot attach segmap");
2174		/*NOTREACHED*/
2175	}
2176	PRM_DEBUG(segmap);
2177
2178	a.prot = PROT_READ | PROT_WRITE;
2179	a.shmsize = 0;
2180	a.nfreelist = segmapfreelists;
2181
2182	if (segmap_create(segmap, (caddr_t)&a) != 0)
2183		panic("segmap_create segmap");
2184	rw_exit(&kas.a_lock);
2185
2186	setup_vaddr_for_ppcopy(CPU);
2187
2188	segdev_init();
2189#if defined(__xpv)
2190	if (DOMAIN_IS_INITDOMAIN(xen_info))
2191#endif
2192		pmem_init();
2193
2194	PRM_POINT("startup_vm() done");
2195}
2196
2197/*
2198 * Load a tod module for the non-standard tod part found on this system.
2199 */
2200static void
2201load_tod_module(char *todmod)
2202{
2203	if (modload("tod", todmod) == -1)
2204		halt("Can't load TOD module");
2205}
2206
2207static void
2208startup_end(void)
2209{
2210	int i;
2211	extern void setx86isalist(void);
2212	extern void cpu_event_init(void);
2213
2214	PRM_POINT("startup_end() starting...");
2215
2216	/*
2217	 * Perform tasks that get done after most of the VM
2218	 * initialization has been done but before the clock
2219	 * and other devices get started.
2220	 */
2221	kern_setup1();
2222
2223	/*
2224	 * Perform CPC initialization for this CPU.
2225	 */
2226	kcpc_hw_init(CPU);
2227
2228	/*
2229	 * Initialize cpu event framework.
2230	 */
2231	cpu_event_init();
2232
2233#if defined(OPTERON_WORKAROUND_6323525)
2234	if (opteron_workaround_6323525)
2235		patch_workaround_6323525();
2236#endif
2237	/*
2238	 * If needed, load TOD module now so that ddi_get_time(9F) etc. work
2239	 * (For now, "needed" is defined as set tod_module_name in /etc/system)
2240	 */
2241	if (tod_module_name != NULL) {
2242		PRM_POINT("load_tod_module()");
2243		load_tod_module(tod_module_name);
2244	}
2245
2246#if defined(__xpv)
2247	/*
2248	 * Forceload interposing TOD module for the hypervisor.
2249	 */
2250	PRM_POINT("load_tod_module()");
2251	load_tod_module("xpvtod");
2252#endif
2253
2254	/*
2255	 * Configure the system.
2256	 */
2257	PRM_POINT("Calling configure()...");
2258	configure();		/* set up devices */
2259	PRM_POINT("configure() done");
2260
2261	/*
2262	 * We can now setup for XSAVE because fpu_probe is done in configure().
2263	 */
2264	if (fp_save_mech == FP_XSAVE) {
2265		PRM_POINT("xsave_setup_msr()");
2266		xsave_setup_msr(CPU);
2267	}
2268
2269	/*
2270	 * Set the isa_list string to the defined instruction sets we
2271	 * support.
2272	 */
2273	setx86isalist();
2274	PRM_POINT("cpu_intr_alloc()");
2275	cpu_intr_alloc(CPU, NINTR_THREADS);
2276	PRM_POINT("psm_install()");
2277	psm_install();
2278
2279	/*
2280	 * We're done with bootops.  We don't unmap the bootstrap yet because
2281	 * we're still using bootsvcs.
2282	 */
2283	PRM_POINT("NULLing out bootops");
2284	*bootopsp = (struct bootops *)NULL;
2285	bootops = (struct bootops *)NULL;
2286
2287#if defined(__xpv)
2288	ec_init_debug_irq();
2289	xs_domu_init();
2290#endif
2291
2292#if !defined(__xpv)
2293	/*
2294	 * Intel IOMMU has been setup/initialized in ddi_impl.c
2295	 * Start it up now.
2296	 */
2297	immu_startup();
2298
2299	/*
2300	 * Now that we're no longer going to drop into real mode for a BIOS call
2301	 * via bootops, we can enable PCID (which requires CR0.PG).
2302	 */
2303	enable_pcid();
2304#endif
2305
2306	PRM_POINT("Enabling interrupts");
2307	(*picinitf)();
2308	sti();
2309#if defined(__xpv)
2310	ASSERT(CPU->cpu_m.mcpu_vcpu_info->evtchn_upcall_mask == 0);
2311	xen_late_startup();
2312#endif
2313
2314	(void) add_avsoftintr((void *)&softlevel1_hdl, 1, softlevel1,
2315	    "softlevel1", NULL, NULL); /* XXX to be moved later */
2316
2317	/*
2318	 * Register software interrupt handlers for ddi_periodic_add(9F).
2319	 * Software interrupts up to the level 10 are supported.
2320	 */
2321	for (i = DDI_IPL_1; i <= DDI_IPL_10; i++) {
2322		(void) add_avsoftintr((void *)&softlevel_hdl[i-1], i,
2323		    (avfunc)(uintptr_t)ddi_periodic_softintr, "ddi_periodic",
2324		    (caddr_t)(uintptr_t)i, NULL);
2325	}
2326
2327#if !defined(__xpv)
2328	if (modload("drv", "amd_iommu") < 0) {
2329		PRM_POINT("No AMD IOMMU present\n");
2330	} else if (ddi_hold_installed_driver(ddi_name_to_major(
2331	    "amd_iommu")) == NULL) {
2332		prom_printf("ERROR: failed to attach AMD IOMMU\n");
2333	}
2334#endif
2335	post_startup_cpu_fixups();
2336
2337	PRM_POINT("startup_end() done");
2338}
2339
2340/*
2341 * Don't remove the following 2 variables.  They are necessary
2342 * for reading the hostid from the legacy file (/kernel/misc/sysinit).
2343 */
2344char *_hs1107 = hw_serial;
2345ulong_t  _bdhs34;
2346
2347void
2348post_startup(void)
2349{
2350	extern void cpupm_init(cpu_t *);
2351	extern void cpu_event_init_cpu(cpu_t *);
2352
2353	/*
2354	 * Set the system wide, processor-specific flags to be passed
2355	 * to userland via the aux vector for performance hints and
2356	 * instruction set extensions.
2357	 */
2358	bind_hwcap();
2359
2360#ifdef __xpv
2361	if (DOMAIN_IS_INITDOMAIN(xen_info))
2362#endif
2363	{
2364#if defined(__xpv)
2365		xpv_panic_init();
2366#else
2367		/*
2368		 * Startup the memory scrubber.
2369		 * XXPV	This should be running somewhere ..
2370		 */
2371		if ((get_hwenv() & HW_VIRTUAL) == 0)
2372			memscrub_init();
2373#endif
2374	}
2375
2376	/*
2377	 * Complete CPU module initialization
2378	 */
2379	cmi_post_startup();
2380
2381	/*
2382	 * Perform forceloading tasks for /etc/system.
2383	 */
2384	(void) mod_sysctl(SYS_FORCELOAD, NULL);
2385
2386	/*
2387	 * ON4.0: Force /proc module in until clock interrupt handle fixed
2388	 * ON4.0: This must be fixed or restated in /etc/systems.
2389	 */
2390	(void) modload("fs", "procfs");
2391
2392	(void) i_ddi_attach_hw_nodes("pit_beep");
2393
2394#if defined(__i386)
2395	/*
2396	 * Check for required functional Floating Point hardware,
2397	 * unless FP hardware explicitly disabled.
2398	 */
2399	if (fpu_exists && (fpu_pentium_fdivbug || fp_kind == FP_NO))
2400		halt("No working FP hardware found");
2401#endif
2402
2403	maxmem = freemem;
2404
2405	cpu_event_init_cpu(CPU);
2406	cpupm_init(CPU);
2407	(void) mach_cpu_create_device_node(CPU, NULL);
2408
2409	pg_init();
2410}
2411
2412static int
2413pp_in_range(page_t *pp, uint64_t low_addr, uint64_t high_addr)
2414{
2415	return ((pp->p_pagenum >= btop(low_addr)) &&
2416	    (pp->p_pagenum < btopr(high_addr)));
2417}
2418
2419static int
2420pp_in_module(page_t *pp, const rd_existing_t *modranges)
2421{
2422	uint_t i;
2423
2424	for (i = 0; modranges[i].phys != 0; i++) {
2425		if (pp_in_range(pp, modranges[i].phys,
2426		    modranges[i].phys + modranges[i].size))
2427			return (1);
2428	}
2429
2430	return (0);
2431}
2432
2433void
2434release_bootstrap(void)
2435{
2436	int root_is_ramdisk;
2437	page_t *pp;
2438	extern void kobj_boot_unmountroot(void);
2439	extern dev_t rootdev;
2440	uint_t i;
2441	char propname[32];
2442	rd_existing_t *modranges;
2443#if !defined(__xpv)
2444	pfn_t	pfn;
2445#endif
2446
2447	/*
2448	 * Save the bootfs module ranges so that we can reserve them below
2449	 * for the real bootfs.
2450	 */
2451	modranges = kmem_alloc(sizeof (rd_existing_t) * MAX_BOOT_MODULES,
2452	    KM_SLEEP);
2453	for (i = 0; ; i++) {
2454		uint64_t start, size;
2455
2456		modranges[i].phys = 0;
2457
2458		(void) snprintf(propname, sizeof (propname),
2459		    "module-addr-%u", i);
2460		if (do_bsys_getproplen(NULL, propname) <= 0)
2461			break;
2462		(void) do_bsys_getprop(NULL, propname, &start);
2463
2464		(void) snprintf(propname, sizeof (propname),
2465		    "module-size-%u", i);
2466		if (do_bsys_getproplen(NULL, propname) <= 0)
2467			break;
2468		(void) do_bsys_getprop(NULL, propname, &size);
2469
2470		modranges[i].phys = start;
2471		modranges[i].size = size;
2472	}
2473
2474	/* unmount boot ramdisk and release kmem usage */
2475	kobj_boot_unmountroot();
2476
2477	/*
2478	 * We're finished using the boot loader so free its pages.
2479	 */
2480	PRM_POINT("Unmapping lower boot pages");
2481
2482	clear_boot_mappings(0, _userlimit);
2483
2484	postbootkernelbase = kernelbase;
2485
2486	/*
2487	 * If root isn't on ramdisk, destroy the hardcoded
2488	 * ramdisk node now and release the memory. Else,
2489	 * ramdisk memory is kept in rd_pages.
2490	 */
2491	root_is_ramdisk = (getmajor(rootdev) == ddi_name_to_major("ramdisk"));
2492	if (!root_is_ramdisk) {
2493		dev_info_t *dip = ddi_find_devinfo("ramdisk", -1, 0);
2494		ASSERT(dip && ddi_get_parent(dip) == ddi_root_node());
2495		ndi_rele_devi(dip);	/* held from ddi_find_devinfo */
2496		(void) ddi_remove_child(dip, 0);
2497	}
2498
2499	PRM_POINT("Releasing boot pages");
2500	while (bootpages) {
2501		extern uint64_t ramdisk_start, ramdisk_end;
2502		pp = bootpages;
2503		bootpages = pp->p_next;
2504
2505
2506		/* Keep pages for the lower 64K */
2507		if (pp_in_range(pp, 0, 0x40000)) {
2508			pp->p_next = lower_pages;
2509			lower_pages = pp;
2510			lower_pages_count++;
2511			continue;
2512		}
2513
2514		if (root_is_ramdisk && pp_in_range(pp, ramdisk_start,
2515		    ramdisk_end) || pp_in_module(pp, modranges)) {
2516			pp->p_next = rd_pages;
2517			rd_pages = pp;
2518			continue;
2519		}
2520		pp->p_next = (struct page *)0;
2521		pp->p_prev = (struct page *)0;
2522		PP_CLRBOOTPAGES(pp);
2523		page_free(pp, 1);
2524	}
2525	PRM_POINT("Boot pages released");
2526
2527	kmem_free(modranges, sizeof (rd_existing_t) * 99);
2528
2529#if !defined(__xpv)
2530/* XXPV -- note this following bunch of code needs to be revisited in Xen 3.0 */
2531	/*
2532	 * Find 1 page below 1 MB so that other processors can boot up or
2533	 * so that any processor can resume.
2534	 * Make sure it has a kernel VA as well as a 1:1 mapping.
2535	 * We should have just free'd one up.
2536	 */
2537
2538	/*
2539	 * 0x10 pages is 64K.  Leave the bottom 64K alone
2540	 * for BIOS.
2541	 */
2542	for (pfn = 0x10; pfn < btop(1*1024*1024); pfn++) {
2543		if (page_numtopp_alloc(pfn) == NULL)
2544			continue;
2545		rm_platter_va = i86devmap(pfn, 1,
2546		    PROT_READ | PROT_WRITE | PROT_EXEC);
2547		rm_platter_pa = ptob(pfn);
2548		break;
2549	}
2550	if (pfn == btop(1*1024*1024) && use_mp)
2551		panic("No page below 1M available for starting "
2552		    "other processors or for resuming from system-suspend");
2553#endif	/* !__xpv */
2554}
2555
2556/*
2557 * Initialize the platform-specific parts of a page_t.
2558 */
2559void
2560add_physmem_cb(page_t *pp, pfn_t pnum)
2561{
2562	pp->p_pagenum = pnum;
2563	pp->p_mapping = NULL;
2564	pp->p_embed = 0;
2565	pp->p_share = 0;
2566	pp->p_mlentry = 0;
2567}
2568
2569/*
2570 * kphysm_init() initializes physical memory.
2571 */
2572static pgcnt_t
2573kphysm_init(page_t *pp, pgcnt_t npages)
2574{
2575	struct memlist	*pmem;
2576	struct memseg	*cur_memseg;
2577	pfn_t		base_pfn;
2578	pfn_t		end_pfn;
2579	pgcnt_t		num;
2580	pgcnt_t		pages_done = 0;
2581	uint64_t	addr;
2582	uint64_t	size;
2583	extern pfn_t	ddiphysmin;
2584	extern int	mnode_xwa;
2585	int		ms = 0, me = 0;
2586
2587	ASSERT(page_hash != NULL && page_hashsz != 0);
2588
2589	cur_memseg = memseg_base;
2590	for (pmem = phys_avail; pmem && npages; pmem = pmem->ml_next) {
2591		/*
2592		 * In a 32 bit kernel can't use higher memory if we're
2593		 * not booting in PAE mode. This check takes care of that.
2594		 */
2595		addr = pmem->ml_address;
2596		size = pmem->ml_size;
2597		if (btop(addr) > physmax)
2598			continue;
2599
2600		/*
2601		 * align addr and size - they may not be at page boundaries
2602		 */
2603		if ((addr & MMU_PAGEOFFSET) != 0) {
2604			addr += MMU_PAGEOFFSET;
2605			addr &= ~(uint64_t)MMU_PAGEOFFSET;
2606			size -= addr - pmem->ml_address;
2607		}
2608
2609		/* only process pages below or equal to physmax */
2610		if ((btop(addr + size) - 1) > physmax)
2611			size = ptob(physmax - btop(addr) + 1);
2612
2613		num = btop(size);
2614		if (num == 0)
2615			continue;
2616
2617		if (num > npages)
2618			num = npages;
2619
2620		npages -= num;
2621		pages_done += num;
2622		base_pfn = btop(addr);
2623
2624		if (prom_debug)
2625			prom_printf("MEMSEG addr=0x%" PRIx64
2626			    " pgs=0x%lx pfn 0x%lx-0x%lx\n",
2627			    addr, num, base_pfn, base_pfn + num);
2628
2629		/*
2630		 * Ignore pages below ddiphysmin to simplify ddi memory
2631		 * allocation with non-zero addr_lo requests.
2632		 */
2633		if (base_pfn < ddiphysmin) {
2634			if (base_pfn + num <= ddiphysmin)
2635				continue;
2636			pp += (ddiphysmin - base_pfn);
2637			num -= (ddiphysmin - base_pfn);
2638			base_pfn = ddiphysmin;
2639		}
2640
2641		/*
2642		 * mnode_xwa is greater than 1 when large pages regions can
2643		 * cross memory node boundaries. To prevent the formation
2644		 * of these large pages, configure the memsegs based on the
2645		 * memory node ranges which had been made non-contiguous.
2646		 */
2647		end_pfn = base_pfn + num - 1;
2648		if (mnode_xwa > 1) {
2649			ms = PFN_2_MEM_NODE(base_pfn);
2650			me = PFN_2_MEM_NODE(end_pfn);
2651
2652			if (ms != me) {
2653				/*
2654				 * current range spans more than 1 memory node.
2655				 * Set num to only the pfn range in the start
2656				 * memory node.
2657				 */
2658				num = mem_node_config[ms].physmax - base_pfn
2659				    + 1;
2660				ASSERT(end_pfn > mem_node_config[ms].physmax);
2661			}
2662		}
2663
2664		for (;;) {
2665			/*
2666			 * Build the memsegs entry
2667			 */
2668			cur_memseg->pages = pp;
2669			cur_memseg->epages = pp + num;
2670			cur_memseg->pages_base = base_pfn;
2671			cur_memseg->pages_end = base_pfn + num;
2672
2673			/*
2674			 * Insert into memseg list in decreasing pfn range
2675			 * order. Low memory is typically more fragmented such
2676			 * that this ordering keeps the larger ranges at the
2677			 * front of the list for code that searches memseg.
2678			 * This ASSERTS that the memsegs coming in from boot
2679			 * are in increasing physical address order and not
2680			 * contiguous.
2681			 */
2682			if (memsegs != NULL) {
2683				ASSERT(cur_memseg->pages_base >=
2684				    memsegs->pages_end);
2685				cur_memseg->next = memsegs;
2686			}
2687			memsegs = cur_memseg;
2688
2689			/*
2690			 * add_physmem() initializes the PSM part of the page
2691			 * struct by calling the PSM back with add_physmem_cb().
2692			 * In addition it coalesces pages into larger pages as
2693			 * it initializes them.
2694			 */
2695			add_physmem(pp, num, base_pfn);
2696			cur_memseg++;
2697			availrmem_initial += num;
2698			availrmem += num;
2699
2700			pp += num;
2701			if (ms >= me)
2702				break;
2703
2704			/* process next memory node range */
2705			ms++;
2706			base_pfn = mem_node_config[ms].physbase;
2707
2708			if (mnode_xwa > 1) {
2709				num = MIN(mem_node_config[ms].physmax,
2710				    end_pfn) - base_pfn + 1;
2711			} else {
2712				num = mem_node_config[ms].physmax -
2713				    base_pfn + 1;
2714			}
2715		}
2716	}
2717
2718	PRM_DEBUG(availrmem_initial);
2719	PRM_DEBUG(availrmem);
2720	PRM_DEBUG(freemem);
2721	build_pfn_hash();
2722	return (pages_done);
2723}
2724
2725/*
2726 * Kernel VM initialization.
2727 */
2728static void
2729kvm_init(void)
2730{
2731	ASSERT((((uintptr_t)s_text) & MMU_PAGEOFFSET) == 0);
2732
2733	/*
2734	 * Put the kernel segments in kernel address space.
2735	 */
2736	rw_enter(&kas.a_lock, RW_WRITER);
2737	as_avlinit(&kas);
2738
2739	(void) seg_attach(&kas, s_text, e_moddata - s_text, &ktextseg);
2740	(void) segkmem_create(&ktextseg);
2741
2742	(void) seg_attach(&kas, (caddr_t)valloc_base, valloc_sz, &kvalloc);
2743	(void) segkmem_create(&kvalloc);
2744
2745	(void) seg_attach(&kas, kernelheap,
2746	    ekernelheap - kernelheap, &kvseg);
2747	(void) segkmem_create(&kvseg);
2748
2749	if (core_size > 0) {
2750		PRM_POINT("attaching kvseg_core");
2751		(void) seg_attach(&kas, (caddr_t)core_base, core_size,
2752		    &kvseg_core);
2753		(void) segkmem_create(&kvseg_core);
2754	}
2755
2756	PRM_POINT("attaching segkvmm");
2757	(void) seg_attach(&kas, segkvmm_base, mmu_ptob(segkvmmsize), &kvmmseg);
2758	(void) segkmem_create(&kvmmseg);
2759	segkmem_kvmm_init(segkvmm_base, mmu_ptob(segkvmmsize));
2760
2761	if (segziosize > 0) {
2762		PRM_POINT("attaching segzio");
2763		(void) seg_attach(&kas, segzio_base, mmu_ptob(segziosize),
2764		    &kzioseg);
2765		(void) segkmem_create(&kzioseg);
2766
2767		/* create zio area covering new segment */
2768		segkmem_zio_init(segzio_base, mmu_ptob(segziosize));
2769	}
2770
2771	(void) seg_attach(&kas, kdi_segdebugbase, kdi_segdebugsize, &kdebugseg);
2772	(void) segkmem_create(&kdebugseg);
2773
2774	rw_exit(&kas.a_lock);
2775
2776	/*
2777	 * Ensure that the red zone at kernelbase is never accessible.
2778	 */
2779	PRM_POINT("protecting redzone");
2780	(void) as_setprot(&kas, (caddr_t)kernelbase, KERNEL_REDZONE_SIZE, 0);
2781
2782	/*
2783	 * Make the text writable so that it can be hot patched by DTrace.
2784	 */
2785	(void) as_setprot(&kas, s_text, e_modtext - s_text,
2786	    PROT_READ | PROT_WRITE | PROT_EXEC);
2787
2788	/*
2789	 * Make data writable until end.
2790	 */
2791	(void) as_setprot(&kas, s_data, e_moddata - s_data,
2792	    PROT_READ | PROT_WRITE | PROT_EXEC);
2793}
2794
2795#ifndef __xpv
2796/*
2797 * Solaris adds an entry for Write Combining caching to the PAT
2798 */
2799static uint64_t pat_attr_reg = PAT_DEFAULT_ATTRIBUTE;
2800
2801void
2802pat_sync(void)
2803{
2804	ulong_t	cr0, cr0_orig, cr4;
2805
2806	if (!is_x86_feature(x86_featureset, X86FSET_PAT))
2807		return;
2808	cr0_orig = cr0 = getcr0();
2809	cr4 = getcr4();
2810
2811	/* disable caching and flush all caches and TLBs */
2812	cr0 |= CR0_CD;
2813	cr0 &= ~CR0_NW;
2814	setcr0(cr0);
2815	invalidate_cache();
2816	if (cr4 & CR4_PGE) {
2817		setcr4(cr4 & ~(ulong_t)CR4_PGE);
2818		setcr4(cr4);
2819	} else {
2820		reload_cr3();
2821	}
2822
2823	/* add our entry to the PAT */
2824	wrmsr(REG_PAT, pat_attr_reg);
2825
2826	/* flush TLBs and cache again, then reenable cr0 caching */
2827	if (cr4 & CR4_PGE) {
2828		setcr4(cr4 & ~(ulong_t)CR4_PGE);
2829		setcr4(cr4);
2830	} else {
2831		reload_cr3();
2832	}
2833	invalidate_cache();
2834	setcr0(cr0_orig);
2835}
2836
2837#endif /* !__xpv */
2838
2839#if defined(_SOFT_HOSTID)
2840/*
2841 * On platforms that do not have a hardware serial number, attempt
2842 * to set one based on the contents of /etc/hostid.  If this file does
2843 * not exist, assume that we are to generate a new hostid and set
2844 * it in the kernel, for subsequent saving by a userland process
2845 * once the system is up and the root filesystem is mounted r/w.
2846 *
2847 * In order to gracefully support upgrade on OpenSolaris, if
2848 * /etc/hostid does not exist, we will attempt to get a serial number
2849 * using the legacy method (/kernel/misc/sysinit).
2850 *
2851 * If that isn't present, we attempt to use an SMBIOS UUID, which is
2852 * a hardware serial number.  Note that we don't automatically trust
2853 * all SMBIOS UUIDs (some older platforms are defective and ship duplicate
2854 * UUIDs in violation of the standard), we check against a blacklist.
2855 *
2856 * In an attempt to make the hostid less prone to abuse
2857 * (for license circumvention, etc), we store it in /etc/hostid
2858 * in rot47 format.
2859 */
2860extern volatile unsigned long tenmicrodata;
2861static int atoi(char *);
2862
2863/*
2864 * Set this to non-zero in /etc/system if you think your SMBIOS returns a
2865 * UUID that is not unique. (Also report it so that the smbios_uuid_blacklist
2866 * array can be updated.)
2867 */
2868int smbios_broken_uuid = 0;
2869
2870/*
2871 * List of known bad UUIDs.  This is just the lower 32-bit values, since
2872 * that's what we use for the host id.  If your hostid falls here, you need
2873 * to contact your hardware OEM for a fix for your BIOS.
2874 */
2875static unsigned char
2876smbios_uuid_blacklist[][16] = {
2877
2878	{	/* Reported bad UUID (Google search) */
2879		0x00, 0x02, 0x00, 0x03, 0x00, 0x04, 0x00, 0x05,
2880		0x00, 0x06, 0x00, 0x07, 0x00, 0x08, 0x00, 0x09,
2881	},
2882	{	/* Known bad DELL UUID */
2883		0x4C, 0x4C, 0x45, 0x44, 0x00, 0x00, 0x20, 0x10,
2884		0x80, 0x20, 0x80, 0xC0, 0x4F, 0x20, 0x20, 0x20,
2885	},
2886	{	/* Uninitialized flash */
2887		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
2888		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
2889	},
2890	{	/* All zeros */
2891		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
2892		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
2893	},
2894};
2895
2896static int32_t
2897uuid_to_hostid(const uint8_t *uuid)
2898{
2899	/*
2900	 * Although the UUIDs are 128-bits, they may not distribute entropy
2901	 * evenly.  We would like to use SHA or MD5, but those are located
2902	 * in loadable modules and not available this early in boot.  As we
2903	 * don't need the values to be cryptographically strong, we just
2904	 * generate 32-bit vaue by xor'ing the various sequences together,
2905	 * which ensures that the entire UUID contributes to the hostid.
2906	 */
2907	uint32_t	id = 0;
2908
2909	/* first check against the blacklist */
2910	for (int i = 0; i < (sizeof (smbios_uuid_blacklist) / 16); i++) {
2911		if (bcmp(smbios_uuid_blacklist[0], uuid, 16) == 0) {
2912			cmn_err(CE_CONT, "?Broken SMBIOS UUID. "
2913			    "Contact BIOS manufacturer for repair.\n");
2914			return ((int32_t)HW_INVALID_HOSTID);
2915		}
2916	}
2917
2918	for (int i = 0; i < 16; i++)
2919		id ^= ((uuid[i]) << (8 * (i % sizeof (id))));
2920
2921	/* Make sure return value is positive */
2922	return (id & 0x7fffffff);
2923}
2924
2925static int32_t
2926set_soft_hostid(void)
2927{
2928	struct _buf *file;
2929	char tokbuf[MAXNAMELEN];
2930	token_t token;
2931	int done = 0;
2932	u_longlong_t tmp;
2933	int i;
2934	int32_t hostid = (int32_t)HW_INVALID_HOSTID;
2935	unsigned char *c;
2936	hrtime_t tsc;
2937	smbios_system_t smsys;
2938
2939	/*
2940	 * If /etc/hostid file not found, we'd like to get a pseudo
2941	 * random number to use at the hostid.  A nice way to do this
2942	 * is to read the real time clock.  To remain xen-compatible,
2943	 * we can't poke the real hardware, so we use tsc_read() to
2944	 * read the real time clock.  However, there is an ominous
2945	 * warning in tsc_read that says it can return zero, so we
2946	 * deal with that possibility by falling back to using the
2947	 * (hopefully random enough) value in tenmicrodata.
2948	 */
2949
2950	if ((file = kobj_open_file(hostid_file)) == (struct _buf *)-1) {
2951		/*
2952		 * hostid file not found - try to load sysinit module
2953		 * and see if it has a nonzero hostid value...use that
2954		 * instead of generating a new hostid here if so.
2955		 */
2956		if ((i = modload("misc", "sysinit")) != -1) {
2957			if (strlen(hw_serial) > 0)
2958				hostid = (int32_t)atoi(hw_serial);
2959			(void) modunload(i);
2960		}
2961
2962		/*
2963		 * We try to use the SMBIOS UUID. But not if it is blacklisted
2964		 * in /etc/system.
2965		 */
2966		if ((hostid == HW_INVALID_HOSTID) &&
2967		    (smbios_broken_uuid == 0) &&
2968		    (ksmbios != NULL) &&
2969		    (smbios_info_system(ksmbios, &smsys) != SMB_ERR) &&
2970		    (smsys.smbs_uuidlen >= 16)) {
2971			hostid = uuid_to_hostid(smsys.smbs_uuid);
2972		}
2973
2974		/*
2975		 * Generate a "random" hostid using the clock.  These
2976		 * hostids will change on each boot if the value is not
2977		 * saved to a persistent /etc/hostid file.
2978		 */
2979		if (hostid == HW_INVALID_HOSTID) {
2980			tsc = tsc_read();
2981			if (tsc == 0)	/* tsc_read can return zero sometimes */
2982				hostid = (int32_t)tenmicrodata & 0x0CFFFFF;
2983			else
2984				hostid = (int32_t)tsc & 0x0CFFFFF;
2985		}
2986	} else {
2987		/* hostid file found */
2988		while (!done) {
2989			token = kobj_lex(file, tokbuf, sizeof (tokbuf));
2990
2991			switch (token) {
2992			case POUND:
2993				/*
2994				 * skip comments
2995				 */
2996				kobj_find_eol(file);
2997				break;
2998			case STRING:
2999				/*
3000				 * un-rot47 - obviously this
3001				 * nonsense is ascii-specific
3002				 */
3003				for (c = (unsigned char *)tokbuf;
3004				    *c != '\0'; c++) {
3005					*c += 47;
3006					if (*c > '~')
3007						*c -= 94;
3008					else if (*c < '!')
3009						*c += 94;
3010				}
3011				/*
3012				 * now we should have a real number
3013				 */
3014
3015				if (kobj_getvalue(tokbuf, &tmp) != 0)
3016					kobj_file_err(CE_WARN, file,
3017					    "Bad value %s for hostid",
3018					    tokbuf);
3019				else
3020					hostid = (int32_t)tmp;
3021
3022				break;
3023			case EOF:
3024				done = 1;
3025				/* FALLTHROUGH */
3026			case NEWLINE:
3027				kobj_newline(file);
3028				break;
3029			default:
3030				break;
3031
3032			}
3033		}
3034		if (hostid == HW_INVALID_HOSTID) /* didn't find a hostid */
3035			kobj_file_err(CE_WARN, file,
3036			    "hostid missing or corrupt");
3037
3038		kobj_close_file(file);
3039	}
3040	/*
3041	 * hostid is now the value read from /etc/hostid, or the
3042	 * new hostid we generated in this routine or HW_INVALID_HOSTID if not
3043	 * set.
3044	 */
3045	return (hostid);
3046}
3047
3048static int
3049atoi(char *p)
3050{
3051	int i = 0;
3052
3053	while (*p != '\0')
3054		i = 10 * i + (*p++ - '0');
3055
3056	return (i);
3057}
3058
3059#endif /* _SOFT_HOSTID */
3060
3061void
3062get_system_configuration(void)
3063{
3064	char	prop[32];
3065	u_longlong_t nodes_ll, cpus_pernode_ll, lvalue;
3066
3067	if (BOP_GETPROPLEN(bootops, "nodes") > sizeof (prop) ||
3068	    BOP_GETPROP(bootops, "nodes", prop) < 0 ||
3069	    kobj_getvalue(prop, &nodes_ll) == -1 ||
3070	    nodes_ll > MAXNODES ||
3071	    BOP_GETPROPLEN(bootops, "cpus_pernode") > sizeof (prop) ||
3072	    BOP_GETPROP(bootops, "cpus_pernode", prop) < 0 ||
3073	    kobj_getvalue(prop, &cpus_pernode_ll) == -1) {
3074		system_hardware.hd_nodes = 1;
3075		system_hardware.hd_cpus_per_node = 0;
3076	} else {
3077		system_hardware.hd_nodes = (int)nodes_ll;
3078		system_hardware.hd_cpus_per_node = (int)cpus_pernode_ll;
3079	}
3080
3081	if (BOP_GETPROPLEN(bootops, "kernelbase") > sizeof (prop) ||
3082	    BOP_GETPROP(bootops, "kernelbase", prop) < 0 ||
3083	    kobj_getvalue(prop, &lvalue) == -1)
3084		eprom_kernelbase = 0;
3085	else
3086		eprom_kernelbase = (uintptr_t)lvalue;
3087
3088	if (BOP_GETPROPLEN(bootops, "segmapsize") > sizeof (prop) ||
3089	    BOP_GETPROP(bootops, "segmapsize", prop) < 0 ||
3090	    kobj_getvalue(prop, &lvalue) == -1)
3091		segmapsize = SEGMAPDEFAULT;
3092	else
3093		segmapsize = (uintptr_t)lvalue;
3094
3095	if (BOP_GETPROPLEN(bootops, "segmapfreelists") > sizeof (prop) ||
3096	    BOP_GETPROP(bootops, "segmapfreelists", prop) < 0 ||
3097	    kobj_getvalue(prop, &lvalue) == -1)
3098		segmapfreelists = 0;	/* use segmap driver default */
3099	else
3100		segmapfreelists = (int)lvalue;
3101
3102	/* physmem used to be here, but moved much earlier to fakebop.c */
3103}
3104
3105/*
3106 * Add to a memory list.
3107 * start = start of new memory segment
3108 * len = length of new memory segment in bytes
3109 * new = pointer to a new struct memlist
3110 * memlistp = memory list to which to add segment.
3111 */
3112void
3113memlist_add(
3114	uint64_t start,
3115	uint64_t len,
3116	struct memlist *new,
3117	struct memlist **memlistp)
3118{
3119	struct memlist *cur;
3120	uint64_t end = start + len;
3121
3122	new->ml_address = start;
3123	new->ml_size = len;
3124
3125	cur = *memlistp;
3126
3127	while (cur) {
3128		if (cur->ml_address >= end) {
3129			new->ml_next = cur;
3130			*memlistp = new;
3131			new->ml_prev = cur->ml_prev;
3132			cur->ml_prev = new;
3133			return;
3134		}
3135		ASSERT(cur->ml_address + cur->ml_size <= start);
3136		if (cur->ml_next == NULL) {
3137			cur->ml_next = new;
3138			new->ml_prev = cur;
3139			new->ml_next = NULL;
3140			return;
3141		}
3142		memlistp = &cur->ml_next;
3143		cur = cur->ml_next;
3144	}
3145}
3146
3147void
3148kobj_vmem_init(vmem_t **text_arena, vmem_t **data_arena)
3149{
3150	size_t tsize = e_modtext - modtext;
3151	size_t dsize = e_moddata - moddata;
3152
3153	*text_arena = vmem_create("module_text", tsize ? modtext : NULL, tsize,
3154	    1, segkmem_alloc, segkmem_free, heaptext_arena, 0, VM_SLEEP);
3155	*data_arena = vmem_create("module_data", dsize ? moddata : NULL, dsize,
3156	    1, segkmem_alloc, segkmem_free, heap32_arena, 0, VM_SLEEP);
3157}
3158
3159caddr_t
3160kobj_text_alloc(vmem_t *arena, size_t size)
3161{
3162	return (vmem_alloc(arena, size, VM_SLEEP | VM_BESTFIT));
3163}
3164
3165/*ARGSUSED*/
3166caddr_t
3167kobj_texthole_alloc(caddr_t addr, size_t size)
3168{
3169	panic("unexpected call to kobj_texthole_alloc()");
3170	/*NOTREACHED*/
3171	return (0);
3172}
3173
3174/*ARGSUSED*/
3175void
3176kobj_texthole_free(caddr_t addr, size_t size)
3177{
3178	panic("unexpected call to kobj_texthole_free()");
3179}
3180
3181/*
3182 * This is called just after configure() in startup().
3183 *
3184 * The ISALIST concept is a bit hopeless on Intel, because
3185 * there's no guarantee of an ever-more-capable processor
3186 * given that various parts of the instruction set may appear
3187 * and disappear between different implementations.
3188 *
3189 * While it would be possible to correct it and even enhance
3190 * it somewhat, the explicit hardware capability bitmask allows
3191 * more flexibility.
3192 *
3193 * So, we just leave this alone.
3194 */
3195void
3196setx86isalist(void)
3197{
3198	char *tp;
3199	size_t len;
3200	extern char *isa_list;
3201
3202#define	TBUFSIZE	1024
3203
3204	tp = kmem_alloc(TBUFSIZE, KM_SLEEP);
3205	*tp = '\0';
3206
3207#if defined(__amd64)
3208	(void) strcpy(tp, "amd64 ");
3209#endif
3210
3211	switch (x86_vendor) {
3212	case X86_VENDOR_Intel:
3213	case X86_VENDOR_AMD:
3214	case X86_VENDOR_TM:
3215		if (is_x86_feature(x86_featureset, X86FSET_CMOV)) {
3216			/*
3217			 * Pentium Pro or later
3218			 */
3219			(void) strcat(tp, "pentium_pro");
3220			(void) strcat(tp,
3221			    is_x86_feature(x86_featureset, X86FSET_MMX) ?
3222			    "+mmx pentium_pro " : " ");
3223		}
3224		/*FALLTHROUGH*/
3225	case X86_VENDOR_Cyrix:
3226		/*
3227		 * The Cyrix 6x86 does not have any Pentium features
3228		 * accessible while not at privilege level 0.
3229		 */
3230		if (is_x86_feature(x86_featureset, X86FSET_CPUID)) {
3231			(void) strcat(tp, "pentium");
3232			(void) strcat(tp,
3233			    is_x86_feature(x86_featureset, X86FSET_MMX) ?
3234			    "+mmx pentium " : " ");
3235		}
3236		break;
3237	default:
3238		break;
3239	}
3240	(void) strcat(tp, "i486 i386 i86");
3241	len = strlen(tp) + 1;   /* account for NULL at end of string */
3242	isa_list = strcpy(kmem_alloc(len, KM_SLEEP), tp);
3243	kmem_free(tp, TBUFSIZE);
3244
3245#undef TBUFSIZE
3246}
3247
3248
3249#ifdef __amd64
3250
3251void *
3252device_arena_alloc(size_t size, int vm_flag)
3253{
3254	return (vmem_alloc(device_arena, size, vm_flag));
3255}
3256
3257void
3258device_arena_free(void *vaddr, size_t size)
3259{
3260	vmem_free(device_arena, vaddr, size);
3261}
3262
3263#else /* __i386 */
3264
3265void *
3266device_arena_alloc(size_t size, int vm_flag)
3267{
3268	caddr_t	vaddr;
3269	uintptr_t v;
3270	size_t	start;
3271	size_t	end;
3272
3273	vaddr = vmem_alloc(heap_arena, size, vm_flag);
3274	if (vaddr == NULL)
3275		return (NULL);
3276
3277	v = (uintptr_t)vaddr;
3278	ASSERT(v >= kernelbase);
3279	ASSERT(v + size <= valloc_base);
3280
3281	start = btop(v - kernelbase);
3282	end = btop(v + size - 1 - kernelbase);
3283	ASSERT(start < toxic_bit_map_len);
3284	ASSERT(end < toxic_bit_map_len);
3285
3286	while (start <= end) {
3287		BT_ATOMIC_SET(toxic_bit_map, start);
3288		++start;
3289	}
3290	return (vaddr);
3291}
3292
3293void
3294device_arena_free(void *vaddr, size_t size)
3295{
3296	uintptr_t v = (uintptr_t)vaddr;
3297	size_t	start;
3298	size_t	end;
3299
3300	ASSERT(v >= kernelbase);
3301	ASSERT(v + size <= valloc_base);
3302
3303	start = btop(v - kernelbase);
3304	end = btop(v + size - 1 - kernelbase);
3305	ASSERT(start < toxic_bit_map_len);
3306	ASSERT(end < toxic_bit_map_len);
3307
3308	while (start <= end) {
3309		ASSERT(BT_TEST(toxic_bit_map, start) != 0);
3310		BT_ATOMIC_CLEAR(toxic_bit_map, start);
3311		++start;
3312	}
3313	vmem_free(heap_arena, vaddr, size);
3314}
3315
3316/*
3317 * returns 1st address in range that is in device arena, or NULL
3318 * if len is not NULL it returns the length of the toxic range
3319 */
3320void *
3321device_arena_contains(void *vaddr, size_t size, size_t *len)
3322{
3323	uintptr_t v = (uintptr_t)vaddr;
3324	uintptr_t eaddr = v + size;
3325	size_t start;
3326	size_t end;
3327
3328	/*
3329	 * if called very early by kmdb, just return NULL
3330	 */
3331	if (toxic_bit_map == NULL)
3332		return (NULL);
3333
3334	/*
3335	 * First check if we're completely outside the bitmap range.
3336	 */
3337	if (v >= valloc_base || eaddr < kernelbase)
3338		return (NULL);
3339
3340	/*
3341	 * Trim ends of search to look at only what the bitmap covers.
3342	 */
3343	if (v < kernelbase)
3344		v = kernelbase;
3345	start = btop(v - kernelbase);
3346	end = btop(eaddr - kernelbase);
3347	if (end >= toxic_bit_map_len)
3348		end = toxic_bit_map_len;
3349
3350	if (bt_range(toxic_bit_map, &start, &end, end) == 0)
3351		return (NULL);
3352
3353	v = kernelbase + ptob(start);
3354	if (len != NULL)
3355		*len = ptob(end - start);
3356	return ((void *)v);
3357}
3358
3359#endif	/* __i386 */
3360