xref: /illumos-gate/usr/src/uts/i86pc/os/startup.c (revision a5652762e5f7bf683d19f18542e5e39df63bad79)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #pragma ident	"%Z%%M%	%I%	%E% SMI"
27 
28 #include <sys/types.h>
29 #include <sys/t_lock.h>
30 #include <sys/param.h>
31 #include <sys/sysmacros.h>
32 #include <sys/signal.h>
33 #include <sys/systm.h>
34 #include <sys/user.h>
35 #include <sys/mman.h>
36 #include <sys/vm.h>
37 #include <sys/conf.h>
38 #include <sys/avintr.h>
39 #include <sys/autoconf.h>
40 #include <sys/disp.h>
41 #include <sys/class.h>
42 #include <sys/bitmap.h>
43 
44 #include <sys/privregs.h>
45 
46 #include <sys/proc.h>
47 #include <sys/buf.h>
48 #include <sys/kmem.h>
49 #include <sys/kstat.h>
50 
51 #include <sys/reboot.h>
52 
53 #include <sys/cred.h>
54 #include <sys/vnode.h>
55 #include <sys/file.h>
56 
57 #include <sys/procfs.h>
58 
59 #include <sys/vfs.h>
60 #include <sys/cmn_err.h>
61 #include <sys/utsname.h>
62 #include <sys/debug.h>
63 #include <sys/kdi.h>
64 
65 #include <sys/dumphdr.h>
66 #include <sys/bootconf.h>
67 #include <sys/varargs.h>
68 #include <sys/promif.h>
69 #include <sys/modctl.h>		/* for "procfs" hack */
70 
71 #include <sys/sunddi.h>
72 #include <sys/sunndi.h>
73 #include <sys/ndi_impldefs.h>
74 #include <sys/ddidmareq.h>
75 #include <sys/psw.h>
76 #include <sys/regset.h>
77 #include <sys/clock.h>
78 #include <sys/pte.h>
79 #include <sys/tss.h>
80 #include <sys/stack.h>
81 #include <sys/trap.h>
82 #include <sys/fp.h>
83 #include <vm/anon.h>
84 #include <vm/as.h>
85 #include <vm/page.h>
86 #include <vm/seg.h>
87 #include <vm/seg_dev.h>
88 #include <vm/seg_kmem.h>
89 #include <vm/seg_kpm.h>
90 #include <vm/seg_map.h>
91 #include <vm/seg_vn.h>
92 #include <vm/seg_kp.h>
93 #include <sys/memnode.h>
94 #include <vm/vm_dep.h>
95 #include <sys/thread.h>
96 #include <sys/sysconf.h>
97 #include <sys/vm_machparam.h>
98 #include <sys/archsystm.h>
99 #include <sys/machsystm.h>
100 #include <vm/hat.h>
101 #include <vm/hat_i86.h>
102 #include <sys/pmem.h>
103 #include <sys/smp_impldefs.h>
104 #include <sys/x86_archext.h>
105 #include <sys/segments.h>
106 #include <sys/clconf.h>
107 #include <sys/kobj.h>
108 #include <sys/kobj_lex.h>
109 #include <sys/cpc_impl.h>
110 #include <sys/chip.h>
111 #include <sys/x86_archext.h>
112 #include <sys/cpu_module.h>
113 #include <sys/smbios.h>
114 
115 extern void progressbar_init(void);
116 extern void progressbar_start(void);
117 
118 /*
119  * XXX make declaration below "static" when drivers no longer use this
120  * interface.
121  */
122 extern caddr_t p0_va;	/* Virtual address for accessing physical page 0 */
123 
124 /*
125  * segkp
126  */
127 extern int segkp_fromheap;
128 
129 static void kvm_init(void);
130 static void startup_init(void);
131 static void startup_memlist(void);
132 static void startup_modules(void);
133 static void startup_bop_gone(void);
134 static void startup_vm(void);
135 static void startup_end(void);
136 
137 /*
138  * Declare these as initialized data so we can patch them.
139  */
140 #ifdef __i386
141 /*
142  * Due to virtual address space limitations running in 32 bit mode, restrict
143  * the amount of physical memory configured to a max of PHYSMEM32 pages (16g).
144  *
145  * If the physical max memory size of 64g were allowed to be configured, the
146  * size of user virtual address space will be less than 1g. A limited user
147  * address space greatly reduces the range of applications that can run.
148  *
149  * If more physical memory than PHYSMEM32 is required, users should preferably
150  * run in 64 bit mode which has no virtual address space limitation issues.
151  *
152  * If 64 bit mode is not available (as in IA32) and/or more physical memory
153  * than PHYSMEM32 is required in 32 bit mode, physmem can be set to the desired
154  * value or to 0 (to configure all available memory) via eeprom(1M). kernelbase
155  * should also be carefully tuned to balance out the need of the user
156  * application while minimizing the risk of kernel heap exhaustion due to
157  * kernelbase being set too high.
158  */
159 #define	PHYSMEM32	0x400000
160 
161 pgcnt_t physmem = PHYSMEM32;
162 #else
163 pgcnt_t physmem = 0;	/* memory size in pages, patch if you want less */
164 #endif
165 pgcnt_t obp_pages;	/* Memory used by PROM for its text and data */
166 
167 char *kobj_file_buf;
168 int kobj_file_bufsize;	/* set in /etc/system */
169 
170 /* Global variables for MP support. Used in mp_startup */
171 caddr_t	rm_platter_va;
172 uint32_t rm_platter_pa;
173 
174 int	auto_lpg_disable = 1;
175 
176 /*
177  * Some CPUs have holes in the middle of the 64-bit virtual address range.
178  */
179 uintptr_t hole_start, hole_end;
180 
181 /*
182  * kpm mapping window
183  */
184 caddr_t kpm_vbase;
185 size_t  kpm_size;
186 static int kpm_desired = 0;		/* Do we want to try to use segkpm? */
187 
188 /*
189  * VA range that must be preserved for boot until we release all of its
190  * mappings.
191  */
192 #if defined(__amd64)
193 static void *kmem_setaside;
194 #endif
195 
196 /*
197  * Configuration parameters set at boot time.
198  */
199 
200 caddr_t econtig;		/* end of first block of contiguous kernel */
201 
202 struct bootops		*bootops = 0;	/* passed in from boot */
203 struct bootops		**bootopsp;
204 struct boot_syscalls	*sysp;		/* passed in from boot */
205 
206 char bootblock_fstype[16];
207 
208 char kern_bootargs[OBP_MAXPATHLEN];
209 
210 /*
211  * new memory fragmentations are possible in startup() due to BOP_ALLOCs. this
212  * depends on number of BOP_ALLOC calls made and requested size, memory size
213  * combination and whether boot.bin memory needs to be freed.
214  */
215 #define	POSS_NEW_FRAGMENTS	12
216 
217 /*
218  * VM data structures
219  */
220 long page_hashsz;		/* Size of page hash table (power of two) */
221 struct page *pp_base;		/* Base of initial system page struct array */
222 struct page **page_hash;	/* Page hash table */
223 struct seg ktextseg;		/* Segment used for kernel executable image */
224 struct seg kvalloc;		/* Segment used for "valloc" mapping */
225 struct seg kpseg;		/* Segment used for pageable kernel virt mem */
226 struct seg kmapseg;		/* Segment used for generic kernel mappings */
227 struct seg kdebugseg;		/* Segment used for the kernel debugger */
228 
229 struct seg *segkmap = &kmapseg;	/* Kernel generic mapping segment */
230 struct seg *segkp = &kpseg;	/* Pageable kernel virtual memory segment */
231 
232 #if defined(__amd64)
233 struct seg kvseg_core;		/* Segment used for the core heap */
234 struct seg kpmseg;		/* Segment used for physical mapping */
235 struct seg *segkpm = &kpmseg;	/* 64bit kernel physical mapping segment */
236 #else
237 struct seg *segkpm = NULL;	/* Unused on IA32 */
238 #endif
239 
240 caddr_t segkp_base;		/* Base address of segkp */
241 #if defined(__amd64)
242 pgcnt_t segkpsize = btop(SEGKPDEFSIZE);	/* size of segkp segment in pages */
243 #else
244 pgcnt_t segkpsize = 0;
245 #endif
246 
247 struct memseg *memseg_base;
248 struct vnode unused_pages_vp;
249 
250 #define	FOURGB	0x100000000LL
251 
252 struct memlist *memlist;
253 
254 caddr_t s_text;		/* start of kernel text segment */
255 caddr_t e_text;		/* end of kernel text segment */
256 caddr_t s_data;		/* start of kernel data segment */
257 caddr_t e_data;		/* end of kernel data segment */
258 caddr_t modtext;	/* start of loadable module text reserved */
259 caddr_t e_modtext;	/* end of loadable module text reserved */
260 caddr_t moddata;	/* start of loadable module data reserved */
261 caddr_t e_moddata;	/* end of loadable module data reserved */
262 
263 struct memlist *phys_install;	/* Total installed physical memory */
264 struct memlist *phys_avail;	/* Total available physical memory */
265 
266 static void memlist_add(uint64_t, uint64_t, struct memlist *,
267 	struct memlist **);
268 
269 /*
270  * kphysm_init returns the number of pages that were processed
271  */
272 static pgcnt_t kphysm_init(page_t *, struct memseg *, pgcnt_t, pgcnt_t);
273 
274 #define	IO_PROP_SIZE	64	/* device property size */
275 
276 /*
277  * a couple useful roundup macros
278  */
279 #define	ROUND_UP_PAGE(x)	\
280 	((uintptr_t)P2ROUNDUP((uintptr_t)(x), (uintptr_t)MMU_PAGESIZE))
281 #define	ROUND_UP_LPAGE(x)	\
282 	((uintptr_t)P2ROUNDUP((uintptr_t)(x), mmu.level_size[1]))
283 #define	ROUND_UP_4MEG(x)	\
284 	((uintptr_t)P2ROUNDUP((uintptr_t)(x), (uintptr_t)FOURMB_PAGESIZE))
285 #define	ROUND_UP_TOPLEVEL(x)	\
286 	((uintptr_t)P2ROUNDUP((uintptr_t)(x), mmu.level_size[mmu.max_level]))
287 
288 /*
289  *	32-bit Kernel's Virtual memory layout.
290  *		+-----------------------+
291  *		|	psm 1-1 map	|
292  *		|	exec args area	|
293  * 0xFFC00000  -|-----------------------|- ARGSBASE
294  *		|	debugger	|
295  * 0xFF800000  -|-----------------------|- SEGDEBUGBASE
296  *		|      Kernel Data	|
297  * 0xFEC00000  -|-----------------------|
298  *              |      Kernel Text	|
299  * 0xFE800000  -|-----------------------|- KERNEL_TEXT
300  * 		|     LUFS sinkhole	|
301  * 0xFE000000  -|-----------------------|- lufs_addr
302  * ---         -|-----------------------|- valloc_base + valloc_sz
303  * 		|   early pp structures	|
304  * 		|   memsegs, memlists, 	|
305  * 		|   page hash, etc.	|
306  * ---	       -|-----------------------|- valloc_base (floating)
307  * 		|     ptable_va    	|
308  * 0xFDFFE000  -|-----------------------|- ekernelheap, ptable_va
309  *		|			|  (segkp is an arena under the heap)
310  *		|			|
311  *		|	kvseg		|
312  *		|			|
313  *		|			|
314  * ---         -|-----------------------|- kernelheap (floating)
315  * 		|        Segkmap	|
316  * 0xC3002000  -|-----------------------|- segkmap_start (floating)
317  *		|	Red Zone	|
318  * 0xC3000000  -|-----------------------|- kernelbase / userlimit (floating)
319  *		|			|			||
320  *		|     Shared objects	|			\/
321  *		|			|
322  *		:			:
323  *		|	user data	|
324  *		|-----------------------|
325  *		|	user text	|
326  * 0x08048000  -|-----------------------|
327  *		|	user stack	|
328  *		:			:
329  *		|	invalid		|
330  * 0x00000000	+-----------------------+
331  *
332  *
333  *		64-bit Kernel's Virtual memory layout. (assuming 64 bit app)
334  *			+-----------------------+
335  *			|	psm 1-1 map	|
336  *			|	exec args area	|
337  * 0xFFFFFFFF.FFC00000  |-----------------------|- ARGSBASE
338  *			|	debugger (?)	|
339  * 0xFFFFFFFF.FF800000  |-----------------------|- SEGDEBUGBASE
340  *			|      unused    	|
341  *			+-----------------------+
342  *			|      Kernel Data	|
343  * 0xFFFFFFFF.FBC00000  |-----------------------|
344  *			|      Kernel Text	|
345  * 0xFFFFFFFF.FB800000  |-----------------------|- KERNEL_TEXT
346  * 			|     LUFS sinkhole	|
347  * 0xFFFFFFFF.FB000000 -|-----------------------|- lufs_addr
348  * ---                  |-----------------------|- valloc_base + valloc_sz
349  * 			|   early pp structures	|
350  * 			|   memsegs, memlists, 	|
351  * 			|   page hash, etc.	|
352  * ---                  |-----------------------|- valloc_base
353  * 			|     ptable_va    	|
354  * ---                  |-----------------------|- ptable_va
355  * 			|      Core heap	| (used for loadable modules)
356  * 0xFFFFFFFF.C0000000  |-----------------------|- core_base / ekernelheap
357  *			|	 Kernel		|
358  *			|	  heap		|
359  * 0xFFFFFXXX.XXX00000  |-----------------------|- kernelheap (floating)
360  *			|	 segkmap	|
361  * 0xFFFFFXXX.XXX00000  |-----------------------|- segkmap_start (floating)
362  *			|    device mappings	|
363  * 0xFFFFFXXX.XXX00000  |-----------------------|- toxic_addr (floating)
364  *			|	  segkp		|
365  * ---                  |-----------------------|- segkp_base
366  *			|	 segkpm		|
367  * 0xFFFFFE00.00000000  |-----------------------|
368  *			|	Red Zone	|
369  * 0xFFFFFD80.00000000  |-----------------------|- KERNELBASE
370  *			|     User stack	|- User space memory
371  * 			|			|
372  * 			| shared objects, etc	|	(grows downwards)
373  *			:			:
374  * 			|			|
375  * 0xFFFF8000.00000000  |-----------------------|
376  * 			|			|
377  * 			| VA Hole / unused	|
378  * 			|			|
379  * 0x00008000.00000000  |-----------------------|
380  *			|			|
381  *			|			|
382  *			:			:
383  *			|	user heap	|	(grows upwards)
384  *			|			|
385  *			|	user data	|
386  *			|-----------------------|
387  *			|	user text	|
388  * 0x00000000.04000000  |-----------------------|
389  *			|	invalid		|
390  * 0x00000000.00000000	+-----------------------+
391  *
392  * A 32 bit app on the 64 bit kernel sees the same layout as on the 32 bit
393  * kernel, except that userlimit is raised to 0xfe000000
394  *
395  * Floating values:
396  *
397  * valloc_base: start of the kernel's memory management/tracking data
398  * structures.  This region contains page_t structures for the lowest 4GB
399  * of physical memory, memsegs, memlists, and the page hash.
400  *
401  * core_base: start of the kernel's "core" heap area on 64-bit systems.
402  * This area is intended to be used for global data as well as for module
403  * text/data that does not fit into the nucleus pages.  The core heap is
404  * restricted to a 2GB range, allowing every address within it to be
405  * accessed using rip-relative addressing
406  *
407  * ekernelheap: end of kernelheap and start of segmap.
408  *
409  * kernelheap: start of kernel heap.  On 32-bit systems, this starts right
410  * above a red zone that separates the user's address space from the
411  * kernel's.  On 64-bit systems, it sits above segkp and segkpm.
412  *
413  * segkmap_start: start of segmap. The length of segmap can be modified
414  * by changing segmapsize in /etc/system (preferred) or eeprom (deprecated).
415  * The default length is 16MB on 32-bit systems and 64MB on 64-bit systems.
416  *
417  * kernelbase: On a 32-bit kernel the default value of 0xd4000000 will be
418  * decreased by 2X the size required for page_t.  This allows the kernel
419  * heap to grow in size with physical memory.  With sizeof(page_t) == 80
420  * bytes, the following shows the values of kernelbase and kernel heap
421  * sizes for different memory configurations (assuming default segmap and
422  * segkp sizes).
423  *
424  *	mem	size for	kernelbase	kernel heap
425  *	size	page_t's			size
426  *	----	---------	----------	-----------
427  *	1gb	0x01400000	0xd1800000	684MB
428  *	2gb	0x02800000	0xcf000000	704MB
429  *	4gb	0x05000000	0xca000000	744MB
430  *	6gb	0x07800000	0xc5000000	784MB
431  *	8gb	0x0a000000	0xc0000000	824MB
432  *	16gb	0x14000000	0xac000000	984MB
433  *	32gb	0x28000000	0x84000000	1304MB
434  *	64gb	0x50000000	0x34000000	1944MB (*)
435  *
436  * kernelbase is less than the abi minimum of 0xc0000000 for memory
437  * configurations above 8gb.
438  *
439  * (*) support for memory configurations above 32gb will require manual tuning
440  * of kernelbase to balance out the need of user applications.
441  */
442 
443 /* real-time-clock initialization parameters */
444 long gmt_lag;		/* offset in seconds of gmt to local time */
445 extern long process_rtc_config_file(void);
446 
447 char		*final_kernelheap;
448 char		*boot_kernelheap;
449 uintptr_t	kernelbase;
450 uintptr_t	eprom_kernelbase;
451 size_t		segmapsize;
452 static uintptr_t segmap_reserved;
453 uintptr_t	segkmap_start;
454 int		segmapfreelists;
455 pgcnt_t		boot_npages;
456 pgcnt_t		npages;
457 size_t		core_size;		/* size of "core" heap */
458 uintptr_t	core_base;		/* base address of "core" heap */
459 
460 /*
461  * List of bootstrap pages. We mark these as allocated in startup.
462  * release_bootstrap() will free them when we're completely done with
463  * the bootstrap.
464  */
465 static page_t *bootpages, *rd_pages;
466 
467 struct system_hardware system_hardware;
468 
469 /*
470  * Enable some debugging messages concerning memory usage...
471  *
472  * XX64 There should only be one print routine once memlist usage between
473  * vmx and the kernel is cleaned up and there is a single memlist structure
474  * shared between kernel and boot.
475  */
476 static void
477 print_boot_memlist(char *title, struct memlist *mp)
478 {
479 	prom_printf("MEMLIST: %s:\n", title);
480 	while (mp != NULL)  {
481 		prom_printf("\tAddress 0x%" PRIx64 ", size 0x%" PRIx64 "\n",
482 		    mp->address, mp->size);
483 		mp = mp->next;
484 	}
485 }
486 
487 static void
488 print_kernel_memlist(char *title, struct memlist *mp)
489 {
490 	prom_printf("MEMLIST: %s:\n", title);
491 	while (mp != NULL)  {
492 		prom_printf("\tAddress 0x%" PRIx64 ", size 0x%" PRIx64 "\n",
493 		    mp->address, mp->size);
494 		mp = mp->next;
495 	}
496 }
497 
498 /*
499  * XX64 need a comment here.. are these just default values, surely
500  * we read the "cpuid" type information to figure this out.
501  */
502 int	l2cache_sz = 0x80000;
503 int	l2cache_linesz = 0x40;
504 int	l2cache_assoc = 1;
505 
506 /*
507  * on 64 bit we use a predifined VA range for mapping devices in the kernel
508  * on 32 bit the mappings are intermixed in the heap, so we use a bit map
509  */
510 #ifdef __amd64
511 
512 vmem_t		*device_arena;
513 uintptr_t	toxic_addr = (uintptr_t)NULL;
514 size_t		toxic_size = 1 * 1024 * 1024 * 1024; /* Sparc uses 1 gig too */
515 
516 #else	/* __i386 */
517 
518 ulong_t		*toxic_bit_map;	/* one bit for each 4k of VA in heap_arena */
519 size_t		toxic_bit_map_len = 0;	/* in bits */
520 
521 #endif	/* __i386 */
522 
523 /*
524  * Simple boot time debug facilities
525  */
526 static char *prm_dbg_str[] = {
527 	"%s:%d: '%s' is 0x%x\n",
528 	"%s:%d: '%s' is 0x%llx\n"
529 };
530 
531 int prom_debug;
532 
533 #define	PRM_DEBUG(q)	if (prom_debug) 	\
534 	prom_printf(prm_dbg_str[sizeof (q) >> 3], "startup.c", __LINE__, #q, q);
535 #define	PRM_POINT(q)	if (prom_debug) 	\
536 	prom_printf("%s:%d: %s\n", "startup.c", __LINE__, q);
537 
538 /*
539  * This structure is used to keep track of the intial allocations
540  * done in startup_memlist(). The value of NUM_ALLOCATIONS needs to
541  * be >= the number of ADD_TO_ALLOCATIONS() executed in the code.
542  */
543 #define	NUM_ALLOCATIONS 7
544 int num_allocations = 0;
545 struct {
546 	void **al_ptr;
547 	size_t al_size;
548 } allocations[NUM_ALLOCATIONS];
549 size_t valloc_sz = 0;
550 uintptr_t valloc_base;
551 extern uintptr_t ptable_va;
552 extern size_t ptable_sz;
553 
554 #define	ADD_TO_ALLOCATIONS(ptr, size) {					\
555 		size = ROUND_UP_PAGE(size);		 		\
556 		if (num_allocations == NUM_ALLOCATIONS)			\
557 			panic("too many ADD_TO_ALLOCATIONS()");		\
558 		allocations[num_allocations].al_ptr = (void**)&ptr;	\
559 		allocations[num_allocations].al_size = size;		\
560 		valloc_sz += size;					\
561 		++num_allocations;				 	\
562 	}
563 
564 static void
565 perform_allocations(void)
566 {
567 	caddr_t mem;
568 	int i;
569 
570 	mem = BOP_ALLOC(bootops, (caddr_t)valloc_base, valloc_sz, BO_NO_ALIGN);
571 	if (mem != (caddr_t)valloc_base)
572 		panic("BOP_ALLOC() failed");
573 	bzero(mem, valloc_sz);
574 	for (i = 0; i < num_allocations; ++i) {
575 		*allocations[i].al_ptr = (void *)mem;
576 		mem += allocations[i].al_size;
577 	}
578 }
579 
580 /*
581  * Our world looks like this at startup time.
582  *
583  * In a 32-bit OS, boot loads the kernel text at 0xfe800000 and kernel data
584  * at 0xfec00000.  On a 64-bit OS, kernel text and data are loaded at
585  * 0xffffffff.fe800000 and 0xffffffff.fec00000 respectively.  Those
586  * addresses are fixed in the binary at link time.
587  *
588  * On the text page:
589  * unix/genunix/krtld/module text loads.
590  *
591  * On the data page:
592  * unix/genunix/krtld/module data loads and space for page_t's.
593  */
594 /*
595  * Machine-dependent startup code
596  */
597 void
598 startup(void)
599 {
600 	extern void startup_bios_disk(void);
601 	extern void startup_pci_bios(void);
602 	/*
603 	 * Make sure that nobody tries to use sekpm until we have
604 	 * initialized it properly.
605 	 */
606 #if defined(__amd64)
607 	kpm_desired = kpm_enable;
608 #endif
609 	kpm_enable = 0;
610 
611 	progressbar_init();
612 	startup_init();
613 	startup_memlist();
614 	startup_pci_bios();
615 	startup_modules();
616 	startup_bios_disk();
617 	startup_bop_gone();
618 	startup_vm();
619 	startup_end();
620 	progressbar_start();
621 }
622 
623 static void
624 startup_init()
625 {
626 	PRM_POINT("startup_init() starting...");
627 
628 	/*
629 	 * Complete the extraction of cpuid data
630 	 */
631 	cpuid_pass2(CPU);
632 
633 	(void) check_boot_version(BOP_GETVERSION(bootops));
634 
635 	/*
636 	 * Check for prom_debug in boot environment
637 	 */
638 	if (BOP_GETPROPLEN(bootops, "prom_debug") >= 0) {
639 		++prom_debug;
640 		PRM_POINT("prom_debug found in boot enviroment");
641 	}
642 
643 	/*
644 	 * Collect node, cpu and memory configuration information.
645 	 */
646 	get_system_configuration();
647 
648 	/*
649 	 * Halt if this is an unsupported processor.
650 	 */
651 	if (x86_type == X86_TYPE_486 || x86_type == X86_TYPE_CYRIX_486) {
652 		printf("\n486 processor (\"%s\") detected.\n",
653 		    CPU->cpu_brandstr);
654 		halt("This processor is not supported by this release "
655 		    "of Solaris.");
656 	}
657 
658 	PRM_POINT("startup_init() done");
659 }
660 
661 /*
662  * Callback for copy_memlist_filter() to filter nucleus, kadb/kmdb, (ie.
663  * everything mapped above KERNEL_TEXT) pages from phys_avail. Note it
664  * also filters out physical page zero.  There is some reliance on the
665  * boot loader allocating only a few contiguous physical memory chunks.
666  */
667 static void
668 avail_filter(uint64_t *addr, uint64_t *size)
669 {
670 	uintptr_t va;
671 	uintptr_t next_va;
672 	pfn_t pfn;
673 	uint64_t pfn_addr;
674 	uint64_t pfn_eaddr;
675 	uint_t prot;
676 	size_t len;
677 	uint_t change;
678 
679 	if (prom_debug)
680 		prom_printf("\tFilter: in: a=%" PRIx64 ", s=%" PRIx64 "\n",
681 		    *addr, *size);
682 
683 	/*
684 	 * page zero is required for BIOS.. never make it available
685 	 */
686 	if (*addr == 0) {
687 		*addr += MMU_PAGESIZE;
688 		*size -= MMU_PAGESIZE;
689 	}
690 
691 	/*
692 	 * First we trim from the front of the range. Since hat_boot_probe()
693 	 * walks ranges in virtual order, but addr/size are physical, we need
694 	 * to the list until no changes are seen.  This deals with the case
695 	 * where page "p" is mapped at v, page "p + PAGESIZE" is mapped at w
696 	 * but w < v.
697 	 */
698 	do {
699 		change = 0;
700 		for (va = KERNEL_TEXT;
701 		    *size > 0 && hat_boot_probe(&va, &len, &pfn, &prot) != 0;
702 		    va = next_va) {
703 
704 			next_va = va + len;
705 			pfn_addr = ptob((uint64_t)pfn);
706 			pfn_eaddr = pfn_addr + len;
707 
708 			if (pfn_addr <= *addr && pfn_eaddr > *addr) {
709 				change = 1;
710 				while (*size > 0 && len > 0) {
711 					*addr += MMU_PAGESIZE;
712 					*size -= MMU_PAGESIZE;
713 					len -= MMU_PAGESIZE;
714 				}
715 			}
716 		}
717 		if (change && prom_debug)
718 			prom_printf("\t\ttrim: a=%" PRIx64 ", s=%" PRIx64 "\n",
719 			    *addr, *size);
720 	} while (change);
721 
722 	/*
723 	 * Trim pages from the end of the range.
724 	 */
725 	for (va = KERNEL_TEXT;
726 	    *size > 0 && hat_boot_probe(&va, &len, &pfn, &prot) != 0;
727 	    va = next_va) {
728 
729 		next_va = va + len;
730 		pfn_addr = ptob((uint64_t)pfn);
731 
732 		if (pfn_addr >= *addr && pfn_addr < *addr + *size)
733 			*size = pfn_addr - *addr;
734 	}
735 
736 	if (prom_debug)
737 		prom_printf("\tFilter out: a=%" PRIx64 ", s=%" PRIx64 "\n",
738 		    *addr, *size);
739 }
740 
741 static void
742 kpm_init()
743 {
744 	struct segkpm_crargs b;
745 	uintptr_t start, end;
746 	struct memlist	*pmem;
747 
748 	/*
749 	 * These variables were all designed for sfmmu in which segkpm is
750 	 * mapped using a single pagesize - either 8KB or 4MB.  On x86, we
751 	 * might use 2+ page sizes on a single machine, so none of these
752 	 * variables have a single correct value.  They are set up as if we
753 	 * always use a 4KB pagesize, which should do no harm.  In the long
754 	 * run, we should get rid of KPM's assumption that only a single
755 	 * pagesize is used.
756 	 */
757 	kpm_pgshft = MMU_PAGESHIFT;
758 	kpm_pgsz =  MMU_PAGESIZE;
759 	kpm_pgoff = MMU_PAGEOFFSET;
760 	kpmp2pshft = 0;
761 	kpmpnpgs = 1;
762 	ASSERT(((uintptr_t)kpm_vbase & (kpm_pgsz - 1)) == 0);
763 
764 	PRM_POINT("about to create segkpm");
765 	rw_enter(&kas.a_lock, RW_WRITER);
766 
767 	if (seg_attach(&kas, kpm_vbase, kpm_size, segkpm) < 0)
768 		panic("cannot attach segkpm");
769 
770 	b.prot = PROT_READ | PROT_WRITE;
771 	b.nvcolors = 1;
772 
773 	if (segkpm_create(segkpm, (caddr_t)&b) != 0)
774 		panic("segkpm_create segkpm");
775 
776 	rw_exit(&kas.a_lock);
777 
778 	/*
779 	 * Map each of the memsegs into the kpm segment, coalesing adjacent
780 	 * memsegs to allow mapping with the largest possible pages.
781 	 */
782 	pmem = phys_install;
783 	start = pmem->address;
784 	end = start + pmem->size;
785 	for (;;) {
786 		if (pmem == NULL || pmem->address > end) {
787 			hat_devload(kas.a_hat, kpm_vbase + start,
788 			    end - start, mmu_btop(start),
789 			    PROT_READ | PROT_WRITE,
790 			    HAT_LOAD | HAT_LOAD_LOCK | HAT_LOAD_NOCONSIST);
791 			if (pmem == NULL)
792 				break;
793 			start = pmem->address;
794 		}
795 		end = pmem->address + pmem->size;
796 		pmem = pmem->next;
797 	}
798 }
799 
800 /*
801  * The purpose of startup memlist is to get the system to the
802  * point where it can use kmem_alloc()'s that operate correctly
803  * relying on BOP_ALLOC(). This includes allocating page_ts,
804  * page hash table, vmem initialized, etc.
805  *
806  * Boot's versions of physinstalled and physavail are insufficient for
807  * the kernel's purposes. Specifically we don't know which pages that
808  * are not in physavail can be reclaimed after boot is gone.
809  *
810  * This code solves the problem by dividing the address space
811  * into 3 regions as it takes over the MMU from the booter.
812  *
813  * 1) Any (non-nucleus) pages that are mapped at addresses above KERNEL_TEXT
814  * can not be used by the kernel.
815  *
816  * 2) Any free page that happens to be mapped below kernelbase
817  * is protected until the boot loader is released, but will then be reclaimed.
818  *
819  * 3) Boot shouldn't use any address in the remaining area between kernelbase
820  * and KERNEL_TEXT.
821  *
822  * In the case of multiple mappings to the same page, region 1 has precedence
823  * over region 2.
824  */
825 static void
826 startup_memlist(void)
827 {
828 	size_t memlist_sz;
829 	size_t memseg_sz;
830 	size_t pagehash_sz;
831 	size_t pp_sz;
832 	uintptr_t va;
833 	size_t len;
834 	uint_t prot;
835 	pfn_t pfn;
836 	int memblocks;
837 	caddr_t pagecolor_mem;
838 	size_t pagecolor_memsz;
839 	caddr_t page_ctrs_mem;
840 	size_t page_ctrs_size;
841 	struct memlist *current;
842 	pgcnt_t orig_npages = 0;
843 	extern void startup_build_mem_nodes(struct memlist *);
844 
845 	/* XX64 fix these - they should be in include files */
846 	extern ulong_t cr4_value;
847 	extern size_t page_coloring_init(uint_t, int, int);
848 	extern void page_coloring_setup(caddr_t);
849 
850 	PRM_POINT("startup_memlist() starting...");
851 
852 	/*
853 	 * Take the most current snapshot we can by calling mem-update.
854 	 * For this to work properly, we first have to ask boot for its
855 	 * end address.
856 	 */
857 	if (BOP_GETPROPLEN(bootops, "memory-update") == 0)
858 		(void) BOP_GETPROP(bootops, "memory-update", NULL);
859 
860 	/*
861 	 * find if the kernel is mapped on a large page
862 	 */
863 	va = KERNEL_TEXT;
864 	if (hat_boot_probe(&va, &len, &pfn, &prot) == 0)
865 		panic("Couldn't find kernel text boot mapping");
866 
867 	/*
868 	 * Use leftover large page nucleus text/data space for loadable modules.
869 	 * Use at most MODTEXT/MODDATA.
870 	 */
871 	if (len > MMU_PAGESIZE) {
872 
873 		moddata = (caddr_t)ROUND_UP_PAGE(e_data);
874 		e_moddata = (caddr_t)ROUND_UP_4MEG(e_data);
875 		if (e_moddata - moddata > MODDATA)
876 			e_moddata = moddata + MODDATA;
877 
878 		modtext = (caddr_t)ROUND_UP_PAGE(e_text);
879 		e_modtext = (caddr_t)ROUND_UP_4MEG(e_text);
880 		if (e_modtext - modtext > MODTEXT)
881 			e_modtext = modtext + MODTEXT;
882 
883 
884 	} else {
885 
886 		PRM_POINT("Kernel NOT loaded on Large Page!");
887 		e_moddata = moddata = (caddr_t)ROUND_UP_PAGE(e_data);
888 		e_modtext = modtext = (caddr_t)ROUND_UP_PAGE(e_text);
889 
890 	}
891 	econtig = e_moddata;
892 
893 	PRM_DEBUG(modtext);
894 	PRM_DEBUG(e_modtext);
895 	PRM_DEBUG(moddata);
896 	PRM_DEBUG(e_moddata);
897 	PRM_DEBUG(econtig);
898 
899 	/*
900 	 * For MP machines cr4_value must be set or the non-boot
901 	 * CPUs will not be able to start.
902 	 */
903 	if (x86_feature & X86_LARGEPAGE)
904 		cr4_value = getcr4();
905 	PRM_DEBUG(cr4_value);
906 
907 	/*
908 	 * Examine the boot loaders physical memory map to find out:
909 	 * - total memory in system - physinstalled
910 	 * - the max physical address - physmax
911 	 * - the number of segments the intsalled memory comes in
912 	 */
913 	if (prom_debug)
914 		print_boot_memlist("boot physinstalled",
915 		    bootops->boot_mem->physinstalled);
916 	installed_top_size(bootops->boot_mem->physinstalled, &physmax,
917 	    &physinstalled, &memblocks);
918 	PRM_DEBUG(physmax);
919 	PRM_DEBUG(physinstalled);
920 	PRM_DEBUG(memblocks);
921 
922 	if (prom_debug)
923 		print_boot_memlist("boot physavail",
924 		    bootops->boot_mem->physavail);
925 
926 	/*
927 	 * Initialize hat's mmu parameters.
928 	 * Check for enforce-prot-exec in boot environment. It's used to
929 	 * enable/disable support for the page table entry NX bit.
930 	 * The default is to enforce PROT_EXEC on processors that support NX.
931 	 * Boot seems to round up the "len", but 8 seems to be big enough.
932 	 */
933 	mmu_init();
934 
935 #ifdef	__i386
936 	/*
937 	 * physmax is lowered if there is more memory than can be
938 	 * physically addressed in 32 bit (PAE/non-PAE) modes.
939 	 */
940 	if (mmu.pae_hat) {
941 		if (PFN_ABOVE64G(physmax)) {
942 			physinstalled -= (physmax - (PFN_64G - 1));
943 			physmax = PFN_64G - 1;
944 		}
945 	} else {
946 		if (PFN_ABOVE4G(physmax)) {
947 			physinstalled -= (physmax - (PFN_4G - 1));
948 			physmax = PFN_4G - 1;
949 		}
950 	}
951 #endif
952 
953 	startup_build_mem_nodes(bootops->boot_mem->physinstalled);
954 
955 	if (BOP_GETPROPLEN(bootops, "enforce-prot-exec") >= 0) {
956 		int len = BOP_GETPROPLEN(bootops, "enforce-prot-exec");
957 		char value[8];
958 
959 		if (len < 8)
960 			(void) BOP_GETPROP(bootops, "enforce-prot-exec", value);
961 		else
962 			(void) strcpy(value, "");
963 		if (strcmp(value, "off") == 0)
964 			mmu.pt_nx = 0;
965 	}
966 	PRM_DEBUG(mmu.pt_nx);
967 
968 	/*
969 	 * We will need page_t's for every page in the system, except for
970 	 * memory mapped at or above above the start of the kernel text segment.
971 	 *
972 	 * pages above e_modtext are attributed to kernel debugger (obp_pages)
973 	 */
974 	npages = physinstalled - 1; /* avail_filter() skips page 0, so "- 1" */
975 	obp_pages = 0;
976 	va = KERNEL_TEXT;
977 	while (hat_boot_probe(&va, &len, &pfn, &prot) != 0) {
978 		npages -= len >> MMU_PAGESHIFT;
979 		if (va >= (uintptr_t)e_moddata)
980 			obp_pages += len >> MMU_PAGESHIFT;
981 		va += len;
982 	}
983 	PRM_DEBUG(npages);
984 	PRM_DEBUG(obp_pages);
985 
986 	/*
987 	 * If physmem is patched to be non-zero, use it instead of
988 	 * the computed value unless it is larger than the real
989 	 * amount of memory on hand.
990 	 */
991 	if (physmem == 0 || physmem > npages) {
992 		physmem = npages;
993 	} else if (physmem < npages) {
994 		orig_npages = npages;
995 		npages = physmem;
996 	}
997 	PRM_DEBUG(physmem);
998 
999 	/*
1000 	 * We now compute the sizes of all the  initial allocations for
1001 	 * structures the kernel needs in order do kmem_alloc(). These
1002 	 * include:
1003 	 *	memsegs
1004 	 *	memlists
1005 	 *	page hash table
1006 	 *	page_t's
1007 	 *	page coloring data structs
1008 	 */
1009 	memseg_sz = sizeof (struct memseg) * (memblocks + POSS_NEW_FRAGMENTS);
1010 	ADD_TO_ALLOCATIONS(memseg_base, memseg_sz);
1011 	PRM_DEBUG(memseg_sz);
1012 
1013 	/*
1014 	 * Reserve space for phys_avail/phys_install memlists.
1015 	 * There's no real good way to know exactly how much room we'll need,
1016 	 * but this should be a good upper bound.
1017 	 */
1018 	memlist_sz = ROUND_UP_PAGE(2 * sizeof (struct memlist) *
1019 	    (memblocks + POSS_NEW_FRAGMENTS));
1020 	ADD_TO_ALLOCATIONS(memlist, memlist_sz);
1021 	PRM_DEBUG(memlist_sz);
1022 
1023 	/*
1024 	 * The page structure hash table size is a power of 2
1025 	 * such that the average hash chain length is PAGE_HASHAVELEN.
1026 	 */
1027 	page_hashsz = npages / PAGE_HASHAVELEN;
1028 	page_hashsz = 1 << highbit(page_hashsz);
1029 	pagehash_sz = sizeof (struct page *) * page_hashsz;
1030 	ADD_TO_ALLOCATIONS(page_hash, pagehash_sz);
1031 	PRM_DEBUG(pagehash_sz);
1032 
1033 	/*
1034 	 * Set aside room for the page structures themselves.  Note: on
1035 	 * 64-bit systems we don't allocate page_t's for every page here.
1036 	 * We just allocate enough to map the lowest 4GB of physical
1037 	 * memory, minus those pages that are used for the "nucleus" kernel
1038 	 * text and data.  The remaining pages are allocated once we can
1039 	 * map around boot.
1040 	 *
1041 	 * boot_npages is used to allocate an area big enough for our
1042 	 * initial page_t's. kphym_init may use less than that.
1043 	 */
1044 	boot_npages = npages;
1045 #if defined(__amd64)
1046 	if (npages > mmu_btop(FOURGB - (econtig - s_text)))
1047 		boot_npages = mmu_btop(FOURGB - (econtig - s_text));
1048 #endif
1049 	PRM_DEBUG(boot_npages);
1050 	pp_sz = sizeof (struct page) * boot_npages;
1051 	ADD_TO_ALLOCATIONS(pp_base, pp_sz);
1052 	PRM_DEBUG(pp_sz);
1053 
1054 	/*
1055 	 * determine l2 cache info and memory size for page coloring
1056 	 */
1057 	(void) getl2cacheinfo(CPU,
1058 	    &l2cache_sz, &l2cache_linesz, &l2cache_assoc);
1059 	pagecolor_memsz =
1060 	    page_coloring_init(l2cache_sz, l2cache_linesz, l2cache_assoc);
1061 	ADD_TO_ALLOCATIONS(pagecolor_mem, pagecolor_memsz);
1062 	PRM_DEBUG(pagecolor_memsz);
1063 
1064 	page_ctrs_size = page_ctrs_sz();
1065 	ADD_TO_ALLOCATIONS(page_ctrs_mem, page_ctrs_size);
1066 	PRM_DEBUG(page_ctrs_size);
1067 
1068 	/*
1069 	 * valloc_base will be below kernel text
1070 	 * The extra pages are for the HAT and kmdb to map page tables.
1071 	 */
1072 	valloc_sz = ROUND_UP_LPAGE(valloc_sz);
1073 	valloc_base = KERNEL_TEXT - valloc_sz;
1074 	PRM_DEBUG(valloc_base);
1075 	ptable_va = valloc_base - ptable_sz;
1076 
1077 #if defined(__amd64)
1078 	if (eprom_kernelbase && eprom_kernelbase != KERNELBASE)
1079 		cmn_err(CE_NOTE, "!kernelbase cannot be changed on 64-bit "
1080 		    "systems.");
1081 	kernelbase = (uintptr_t)KERNELBASE;
1082 	core_base = (uintptr_t)COREHEAP_BASE;
1083 	core_size = ptable_va - core_base;
1084 #else	/* __i386 */
1085 	/*
1086 	 * We configure kernelbase based on:
1087 	 *
1088 	 * 1. user specified kernelbase via eeprom command. Value cannot exceed
1089 	 *    KERNELBASE_MAX. we large page align eprom_kernelbase
1090 	 *
1091 	 * 2. Default to KERNELBASE and adjust to 2X less the size for page_t.
1092 	 *    On large memory systems we must lower kernelbase to allow
1093 	 *    enough room for page_t's for all of memory.
1094 	 *
1095 	 * The value set here, might be changed a little later.
1096 	 */
1097 	if (eprom_kernelbase) {
1098 		kernelbase = eprom_kernelbase & mmu.level_mask[1];
1099 		if (kernelbase > KERNELBASE_MAX)
1100 			kernelbase = KERNELBASE_MAX;
1101 	} else {
1102 		kernelbase = (uintptr_t)KERNELBASE;
1103 		kernelbase -= ROUND_UP_4MEG(2 * valloc_sz);
1104 	}
1105 	ASSERT((kernelbase & mmu.level_offset[1]) == 0);
1106 	core_base = ptable_va;
1107 	core_size = 0;
1108 #endif
1109 
1110 	PRM_DEBUG(kernelbase);
1111 	PRM_DEBUG(core_base);
1112 	PRM_DEBUG(core_size);
1113 
1114 	/*
1115 	 * At this point, we can only use a portion of the kernelheap that
1116 	 * will be available after we boot.  Both 32-bit and 64-bit systems
1117 	 * have this limitation, although the reasons are completely
1118 	 * different.
1119 	 *
1120 	 * On 64-bit systems, the booter only supports allocations in the
1121 	 * upper 4GB of memory, so we have to work with a reduced kernel
1122 	 * heap until we take over all allocations.  The booter also sits
1123 	 * in the lower portion of that 4GB range, so we have to raise the
1124 	 * bottom of the heap even further.
1125 	 *
1126 	 * On 32-bit systems we have to leave room to place segmap below
1127 	 * the heap.  We don't yet know how large segmap will be, so we
1128 	 * have to be very conservative.
1129 	 */
1130 #if defined(__amd64)
1131 	/*
1132 	 * XX64: For now, we let boot have the lower 2GB of the top 4GB
1133 	 * address range.  In the long run, that should be fixed.  It's
1134 	 * insane for a booter to need 2 2GB address ranges.
1135 	 */
1136 	boot_kernelheap = (caddr_t)(BOOT_DOUBLEMAP_BASE + BOOT_DOUBLEMAP_SIZE);
1137 	segmap_reserved = 0;
1138 
1139 #else	/* __i386 */
1140 	segkp_fromheap = 1;
1141 	segmap_reserved = ROUND_UP_LPAGE(MAX(segmapsize, SEGMAPMAX));
1142 	boot_kernelheap = (caddr_t)(ROUND_UP_LPAGE(kernelbase) +
1143 	    segmap_reserved);
1144 #endif
1145 	PRM_DEBUG(boot_kernelheap);
1146 	kernelheap = boot_kernelheap;
1147 	ekernelheap = (char *)core_base;
1148 
1149 	/*
1150 	 * If segmap is too large we can push the bottom of the kernel heap
1151 	 * higher than the base.  Or worse, it could exceed the top of the
1152 	 * VA space entirely, causing it to wrap around.
1153 	 */
1154 	if (kernelheap >= ekernelheap || (uintptr_t)kernelheap < kernelbase)
1155 		panic("too little memory available for kernelheap,"
1156 			    " use a different kernelbase");
1157 
1158 	/*
1159 	 * Now that we know the real value of kernelbase,
1160 	 * update variables that were initialized with a value of
1161 	 * KERNELBASE (in common/conf/param.c).
1162 	 *
1163 	 * XXX	The problem with this sort of hackery is that the
1164 	 *	compiler just may feel like putting the const declarations
1165 	 *	(in param.c) into the .text section.  Perhaps they should
1166 	 *	just be declared as variables there?
1167 	 */
1168 
1169 #if defined(__amd64)
1170 	ASSERT(_kernelbase == KERNELBASE);
1171 	ASSERT(_userlimit == USERLIMIT);
1172 	/*
1173 	 * As one final sanity check, verify that the "red zone" between
1174 	 * kernel and userspace is exactly the size we expected.
1175 	 */
1176 	ASSERT(_kernelbase == (_userlimit + (2 * 1024 * 1024)));
1177 #else
1178 	*(uintptr_t *)&_kernelbase = kernelbase;
1179 	*(uintptr_t *)&_userlimit = kernelbase;
1180 	*(uintptr_t *)&_userlimit32 = _userlimit;
1181 #endif
1182 	PRM_DEBUG(_kernelbase);
1183 	PRM_DEBUG(_userlimit);
1184 	PRM_DEBUG(_userlimit32);
1185 
1186 	/*
1187 	 * do all the initial allocations
1188 	 */
1189 	perform_allocations();
1190 
1191 	/*
1192 	 * Initialize the kernel heap. Note 3rd argument must be > 1st.
1193 	 */
1194 	kernelheap_init(kernelheap, ekernelheap, kernelheap + MMU_PAGESIZE,
1195 	    (void *)core_base, (void *)ptable_va);
1196 
1197 	/*
1198 	 * Build phys_install and phys_avail in kernel memspace.
1199 	 * - phys_install should be all memory in the system.
1200 	 * - phys_avail is phys_install minus any memory mapped before this
1201 	 *    point above KERNEL_TEXT.
1202 	 */
1203 	current = phys_install = memlist;
1204 	copy_memlist_filter(bootops->boot_mem->physinstalled, &current, NULL);
1205 	if ((caddr_t)current > (caddr_t)memlist + memlist_sz)
1206 		panic("physinstalled was too big!");
1207 	if (prom_debug)
1208 		print_kernel_memlist("phys_install", phys_install);
1209 
1210 	phys_avail = current;
1211 	PRM_POINT("Building phys_avail:\n");
1212 	copy_memlist_filter(bootops->boot_mem->physinstalled, &current,
1213 	    avail_filter);
1214 	if ((caddr_t)current > (caddr_t)memlist + memlist_sz)
1215 		panic("physavail was too big!");
1216 	if (prom_debug)
1217 		print_kernel_memlist("phys_avail", phys_avail);
1218 
1219 	/*
1220 	 * setup page coloring
1221 	 */
1222 	page_coloring_setup(pagecolor_mem);
1223 	page_lock_init();	/* currently a no-op */
1224 
1225 	/*
1226 	 * free page list counters
1227 	 */
1228 	(void) page_ctrs_alloc(page_ctrs_mem);
1229 
1230 	/*
1231 	 * Initialize the page structures from the memory lists.
1232 	 */
1233 	availrmem_initial = availrmem = freemem = 0;
1234 	PRM_POINT("Calling kphysm_init()...");
1235 	boot_npages = kphysm_init(pp_base, memseg_base, 0, boot_npages);
1236 	PRM_POINT("kphysm_init() done");
1237 	PRM_DEBUG(boot_npages);
1238 
1239 	/*
1240 	 * Now that page_t's have been initialized, remove all the
1241 	 * initial allocation pages from the kernel free page lists.
1242 	 */
1243 	boot_mapin((caddr_t)valloc_base, valloc_sz);
1244 
1245 	/*
1246 	 * Initialize kernel memory allocator.
1247 	 */
1248 	kmem_init();
1249 
1250 	/*
1251 	 * print this out early so that we know what's going on
1252 	 */
1253 	cmn_err(CE_CONT, "?features: %b\n", x86_feature, FMT_X86_FEATURE);
1254 
1255 	/*
1256 	 * Initialize bp_mapin().
1257 	 */
1258 	bp_init(MMU_PAGESIZE, HAT_STORECACHING_OK);
1259 
1260 	/*
1261 	 * orig_npages is non-zero if physmem has been configured for less
1262 	 * than the available memory.
1263 	 */
1264 	if (orig_npages) {
1265 #ifdef __i386
1266 		/*
1267 		 * use npages for physmem in case it has been temporarily
1268 		 * modified via /etc/system in kmem_init/mod_read_system_file.
1269 		 */
1270 		if (npages == PHYSMEM32) {
1271 			cmn_err(CE_WARN, "!Due to 32 bit virtual"
1272 			    " address space limitations, limiting"
1273 			    " physmem to 0x%lx of 0x%lx available pages",
1274 			    npages, orig_npages);
1275 		} else {
1276 			cmn_err(CE_WARN, "!limiting physmem to 0x%lx of"
1277 			    " 0x%lx available pages", npages, orig_npages);
1278 		}
1279 #else
1280 		cmn_err(CE_WARN, "!limiting physmem to 0x%lx of"
1281 		    " 0x%lx available pages", npages, orig_npages);
1282 #endif
1283 	}
1284 #if defined(__i386)
1285 	if (eprom_kernelbase && (eprom_kernelbase != kernelbase))
1286 		cmn_err(CE_WARN, "kernelbase value, User specified 0x%lx, "
1287 		    "System using 0x%lx",
1288 		    (uintptr_t)eprom_kernelbase, (uintptr_t)kernelbase);
1289 #endif
1290 
1291 #ifdef	KERNELBASE_ABI_MIN
1292 	if (kernelbase < (uintptr_t)KERNELBASE_ABI_MIN) {
1293 		cmn_err(CE_NOTE, "!kernelbase set to 0x%lx, system is not "
1294 		    "i386 ABI compliant.", (uintptr_t)kernelbase);
1295 	}
1296 #endif
1297 
1298 	PRM_POINT("startup_memlist() done");
1299 }
1300 
1301 static void
1302 startup_modules(void)
1303 {
1304 	unsigned int i;
1305 	extern void prom_setup(void);
1306 
1307 	PRM_POINT("startup_modules() starting...");
1308 	/*
1309 	 * Initialize ten-micro second timer so that drivers will
1310 	 * not get short changed in their init phase. This was
1311 	 * not getting called until clkinit which, on fast cpu's
1312 	 * caused the drv_usecwait to be way too short.
1313 	 */
1314 	microfind();
1315 
1316 	/*
1317 	 * Read the GMT lag from /etc/rtc_config.
1318 	 */
1319 	gmt_lag = process_rtc_config_file();
1320 
1321 	/*
1322 	 * Calculate default settings of system parameters based upon
1323 	 * maxusers, yet allow to be overridden via the /etc/system file.
1324 	 */
1325 	param_calc(0);
1326 
1327 	mod_setup();
1328 
1329 	/*
1330 	 * Initialize system parameters.
1331 	 */
1332 	param_init();
1333 
1334 	/*
1335 	 * maxmem is the amount of physical memory we're playing with.
1336 	 */
1337 	maxmem = physmem;
1338 
1339 	/*
1340 	 * Initialize the hat layer.
1341 	 */
1342 	hat_init();
1343 
1344 	/*
1345 	 * Initialize segment management stuff.
1346 	 */
1347 	seg_init();
1348 
1349 	if (modload("fs", "specfs") == -1)
1350 		halt("Can't load specfs");
1351 
1352 	if (modload("fs", "devfs") == -1)
1353 		halt("Can't load devfs");
1354 
1355 	(void) modloadonly("sys", "lbl_edition");
1356 
1357 	dispinit();
1358 
1359 	/*
1360 	 * This is needed here to initialize hw_serial[] for cluster booting.
1361 	 */
1362 	if ((i = modload("misc", "sysinit")) != (unsigned int)-1)
1363 		(void) modunload(i);
1364 	else
1365 		cmn_err(CE_CONT, "sysinit load failed");
1366 
1367 	/* Read cluster configuration data. */
1368 	clconf_init();
1369 
1370 	/*
1371 	 * Create a kernel device tree. First, create rootnex and
1372 	 * then invoke bus specific code to probe devices.
1373 	 */
1374 	setup_ddi();
1375 
1376 	/*
1377 	 * Set up the CPU module subsystem.  Modifies the device tree, so it
1378 	 * must be done after setup_ddi().
1379 	 */
1380 	cmi_init();
1381 
1382 	/*
1383 	 * Initialize the MCA handlers
1384 	 */
1385 	if (x86_feature & X86_MCA)
1386 		cmi_mca_init();
1387 
1388 	/*
1389 	 * Fake a prom tree such that /dev/openprom continues to work
1390 	 */
1391 	prom_setup();
1392 
1393 	/*
1394 	 * Load all platform specific modules
1395 	 */
1396 	psm_modload();
1397 
1398 	PRM_POINT("startup_modules() done");
1399 }
1400 
1401 static void
1402 startup_bop_gone(void)
1403 {
1404 	PRM_POINT("startup_bop_gone() starting...");
1405 
1406 	/*
1407 	 * Do final allocations of HAT data structures that need to
1408 	 * be allocated before quiescing the boot loader.
1409 	 */
1410 	PRM_POINT("Calling hat_kern_alloc()...");
1411 	hat_kern_alloc();
1412 	PRM_POINT("hat_kern_alloc() done");
1413 
1414 	/*
1415 	 * Setup MTRR (Memory type range registers)
1416 	 */
1417 	setup_mtrr();
1418 	PRM_POINT("startup_bop_gone() done");
1419 }
1420 
1421 /*
1422  * Walk through the pagetables looking for pages mapped in by boot.  If the
1423  * setaside flag is set the pages are expected to be returned to the
1424  * kernel later in boot, so we add them to the bootpages list.
1425  */
1426 static void
1427 protect_boot_range(uintptr_t low, uintptr_t high, int setaside)
1428 {
1429 	uintptr_t va = low;
1430 	size_t len;
1431 	uint_t prot;
1432 	pfn_t pfn;
1433 	page_t *pp;
1434 	pgcnt_t boot_protect_cnt = 0;
1435 
1436 	while (hat_boot_probe(&va, &len, &pfn, &prot) != 0 && va < high) {
1437 		if (va + len >= high)
1438 			panic("0x%lx byte mapping at 0x%p exceeds boot's "
1439 			    "legal range.", len, (void *)va);
1440 
1441 		while (len > 0) {
1442 			pp = page_numtopp_alloc(pfn);
1443 			if (pp != NULL) {
1444 				if (setaside == 0)
1445 					panic("Unexpected mapping by boot.  "
1446 					    "addr=%p pfn=%lx\n",
1447 					    (void *)va, pfn);
1448 
1449 				pp->p_next = bootpages;
1450 				bootpages = pp;
1451 				++boot_protect_cnt;
1452 			}
1453 
1454 			++pfn;
1455 			len -= MMU_PAGESIZE;
1456 			va += MMU_PAGESIZE;
1457 		}
1458 	}
1459 	PRM_DEBUG(boot_protect_cnt);
1460 }
1461 
1462 static void
1463 startup_vm(void)
1464 {
1465 	struct segmap_crargs a;
1466 	extern void hat_kern_setup(void);
1467 	pgcnt_t pages_left;
1468 
1469 	extern int exec_lpg_disable, use_brk_lpg, use_stk_lpg, use_zmap_lpg;
1470 	extern pgcnt_t auto_lpg_min_physmem;
1471 
1472 	PRM_POINT("startup_vm() starting...");
1473 
1474 	/*
1475 	 * The next two loops are done in distinct steps in order
1476 	 * to be sure that any page that is doubly mapped (both above
1477 	 * KERNEL_TEXT and below kernelbase) is dealt with correctly.
1478 	 * Note this may never happen, but it might someday.
1479 	 */
1480 
1481 	bootpages = NULL;
1482 	PRM_POINT("Protecting boot pages");
1483 	/*
1484 	 * Protect any pages mapped above KERNEL_TEXT that somehow have
1485 	 * page_t's. This can only happen if something weird allocated
1486 	 * in this range (like kadb/kmdb).
1487 	 */
1488 	protect_boot_range(KERNEL_TEXT, (uintptr_t)-1, 0);
1489 
1490 	/*
1491 	 * Before we can take over memory allocation/mapping from the boot
1492 	 * loader we must remove from our free page lists any boot pages that
1493 	 * will stay mapped until release_bootstrap().
1494 	 */
1495 	protect_boot_range(0, kernelbase, 1);
1496 #if defined(__amd64)
1497 	protect_boot_range(BOOT_DOUBLEMAP_BASE,
1498 	    BOOT_DOUBLEMAP_BASE + BOOT_DOUBLEMAP_SIZE, 0);
1499 #endif
1500 
1501 	/*
1502 	 * Copy in boot's page tables, set up extra page tables for the kernel,
1503 	 * and switch to the kernel's context.
1504 	 */
1505 	PRM_POINT("Calling hat_kern_setup()...");
1506 	hat_kern_setup();
1507 
1508 	/*
1509 	 * It is no longer safe to call BOP_ALLOC(), so make sure we don't.
1510 	 */
1511 	bootops->bsys_alloc = NULL;
1512 	PRM_POINT("hat_kern_setup() done");
1513 
1514 	hat_cpu_online(CPU);
1515 
1516 	/*
1517 	 * Before we call kvm_init(), we need to establish the final size
1518 	 * of the kernel's heap.  So, we need to figure out how much space
1519 	 * to set aside for segkp, segkpm, and segmap.
1520 	 */
1521 	final_kernelheap = (caddr_t)ROUND_UP_LPAGE(kernelbase);
1522 #if defined(__amd64)
1523 	if (kpm_desired) {
1524 		/*
1525 		 * Segkpm appears at the bottom of the kernel's address
1526 		 * range.  To detect accidental overruns of the user
1527 		 * address space, we leave a "red zone" of unmapped memory
1528 		 * between kernelbase and the beginning of segkpm.
1529 		 */
1530 		kpm_vbase = final_kernelheap + KERNEL_REDZONE_SIZE;
1531 		kpm_size = mmu_ptob(physmax + 1);
1532 		PRM_DEBUG(kpm_vbase);
1533 		PRM_DEBUG(kpm_size);
1534 		final_kernelheap =
1535 		    (caddr_t)ROUND_UP_TOPLEVEL(kpm_vbase + kpm_size);
1536 	}
1537 
1538 	if (!segkp_fromheap) {
1539 		size_t sz = mmu_ptob(segkpsize);
1540 
1541 		/*
1542 		 * determine size of segkp and adjust the bottom of the
1543 		 * kernel's heap.
1544 		 */
1545 		if (sz < SEGKPMINSIZE || sz > SEGKPMAXSIZE) {
1546 			sz = SEGKPDEFSIZE;
1547 			cmn_err(CE_WARN, "!Illegal value for segkpsize. "
1548 			    "segkpsize has been reset to %ld pages",
1549 			    mmu_btop(sz));
1550 		}
1551 		sz = MIN(sz, MAX(SEGKPMINSIZE, mmu_ptob(physmem)));
1552 
1553 		segkpsize = mmu_btop(ROUND_UP_LPAGE(sz));
1554 		segkp_base = final_kernelheap;
1555 		PRM_DEBUG(segkpsize);
1556 		PRM_DEBUG(segkp_base);
1557 		final_kernelheap = segkp_base + mmu_ptob(segkpsize);
1558 		PRM_DEBUG(final_kernelheap);
1559 	}
1560 
1561 	/*
1562 	 * put the range of VA for device mappings next
1563 	 */
1564 	toxic_addr = (uintptr_t)final_kernelheap;
1565 	PRM_DEBUG(toxic_addr);
1566 	final_kernelheap = (char *)toxic_addr + toxic_size;
1567 #endif
1568 	PRM_DEBUG(final_kernelheap);
1569 	ASSERT(final_kernelheap < boot_kernelheap);
1570 
1571 	/*
1572 	 * Users can change segmapsize through eeprom or /etc/system.
1573 	 * If the variable is tuned through eeprom, there is no upper
1574 	 * bound on the size of segmap.  If it is tuned through
1575 	 * /etc/system on 32-bit systems, it must be no larger than we
1576 	 * planned for in startup_memlist().
1577 	 */
1578 	segmapsize = MAX(ROUND_UP_LPAGE(segmapsize), SEGMAPDEFAULT);
1579 	segkmap_start = ROUND_UP_LPAGE((uintptr_t)final_kernelheap);
1580 
1581 #if defined(__i386)
1582 	if (segmapsize > segmap_reserved) {
1583 		cmn_err(CE_NOTE, "!segmapsize may not be set > 0x%lx in "
1584 		    "/etc/system.  Use eeprom.", (long)SEGMAPMAX);
1585 		segmapsize = segmap_reserved;
1586 	}
1587 	/*
1588 	 * 32-bit systems don't have segkpm or segkp, so segmap appears at
1589 	 * the bottom of the kernel's address range.  Set aside space for a
1590 	 * red zone just below the start of segmap.
1591 	 */
1592 	segkmap_start += KERNEL_REDZONE_SIZE;
1593 	segmapsize -= KERNEL_REDZONE_SIZE;
1594 #endif
1595 	final_kernelheap = (char *)(segkmap_start + segmapsize);
1596 
1597 	PRM_DEBUG(segkmap_start);
1598 	PRM_DEBUG(segmapsize);
1599 	PRM_DEBUG(final_kernelheap);
1600 
1601 	/*
1602 	 * Initialize VM system
1603 	 */
1604 	PRM_POINT("Calling kvm_init()...");
1605 	kvm_init();
1606 	PRM_POINT("kvm_init() done");
1607 
1608 	/*
1609 	 * Tell kmdb that the VM system is now working
1610 	 */
1611 	if (boothowto & RB_DEBUG)
1612 		kdi_dvec_vmready();
1613 
1614 	/*
1615 	 * Mangle the brand string etc.
1616 	 */
1617 	cpuid_pass3(CPU);
1618 
1619 	PRM_DEBUG(final_kernelheap);
1620 
1621 	/*
1622 	 * Now that we can use memory outside the top 4GB (on 64-bit
1623 	 * systems) and we know the size of segmap, we can set the final
1624 	 * size of the kernel's heap.  Note: on 64-bit systems we still
1625 	 * can't touch anything in the bottom half of the top 4GB range
1626 	 * because boot still has pages mapped there.
1627 	 */
1628 	if (final_kernelheap < boot_kernelheap) {
1629 		kernelheap_extend(final_kernelheap, boot_kernelheap);
1630 #if defined(__amd64)
1631 		kmem_setaside = vmem_xalloc(heap_arena, BOOT_DOUBLEMAP_SIZE,
1632 		    MMU_PAGESIZE, 0, 0, (void *)(BOOT_DOUBLEMAP_BASE),
1633 		    (void *)(BOOT_DOUBLEMAP_BASE + BOOT_DOUBLEMAP_SIZE),
1634 		    VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
1635 		PRM_DEBUG(kmem_setaside);
1636 		if (kmem_setaside == NULL)
1637 			panic("Could not protect boot's memory");
1638 #endif
1639 	}
1640 	/*
1641 	 * Now that the kernel heap may have grown significantly, we need
1642 	 * to make all the remaining page_t's available to back that memory.
1643 	 *
1644 	 * XX64 this should probably wait till after release boot-strap too.
1645 	 */
1646 	pages_left = npages - boot_npages;
1647 	if (pages_left > 0) {
1648 		PRM_DEBUG(pages_left);
1649 		(void) kphysm_init(NULL, memseg_base, boot_npages, pages_left);
1650 	}
1651 
1652 #if defined(__amd64)
1653 
1654 	/*
1655 	 * Create the device arena for toxic (to dtrace/kmdb) mappings.
1656 	 */
1657 	device_arena = vmem_create("device", (void *)toxic_addr,
1658 	    toxic_size, MMU_PAGESIZE, NULL, NULL, NULL, 0, VM_SLEEP);
1659 
1660 #else	/* __i386 */
1661 
1662 	/*
1663 	 * allocate the bit map that tracks toxic pages
1664 	 */
1665 	toxic_bit_map_len = btop((ulong_t)(ptable_va - kernelbase));
1666 	PRM_DEBUG(toxic_bit_map_len);
1667 	toxic_bit_map =
1668 	    kmem_zalloc(BT_SIZEOFMAP(toxic_bit_map_len), KM_NOSLEEP);
1669 	ASSERT(toxic_bit_map != NULL);
1670 	PRM_DEBUG(toxic_bit_map);
1671 
1672 #endif	/* __i386 */
1673 
1674 
1675 	/*
1676 	 * Now that we've got more VA, as well as the ability to allocate from
1677 	 * it, tell the debugger.
1678 	 */
1679 	if (boothowto & RB_DEBUG)
1680 		kdi_dvec_memavail();
1681 
1682 	/*
1683 	 * The following code installs a special page fault handler (#pf)
1684 	 * to work around a pentium bug.
1685 	 */
1686 #if !defined(__amd64)
1687 	if (x86_type == X86_TYPE_P5) {
1688 		gate_desc_t *newidt;
1689 		desctbr_t    newidt_r;
1690 
1691 		if ((newidt = kmem_zalloc(MMU_PAGESIZE, KM_NOSLEEP)) == NULL)
1692 			panic("failed to install pentium_pftrap");
1693 
1694 		bcopy(idt0, newidt, sizeof (idt0));
1695 		set_gatesegd(&newidt[T_PGFLT], &pentium_pftrap,
1696 		    KCS_SEL, 0, SDT_SYSIGT, SEL_KPL);
1697 
1698 		(void) as_setprot(&kas, (caddr_t)newidt, MMU_PAGESIZE,
1699 		    PROT_READ|PROT_EXEC);
1700 
1701 		newidt_r.dtr_limit = sizeof (idt0) - 1;
1702 		newidt_r.dtr_base = (uintptr_t)newidt;
1703 		CPU->cpu_idt = newidt;
1704 		wr_idtr(&newidt_r);
1705 	}
1706 #endif	/* !__amd64 */
1707 
1708 	/*
1709 	 * Map page pfn=0 for drivers, such as kd, that need to pick up
1710 	 * parameters left there by controllers/BIOS.
1711 	 */
1712 	PRM_POINT("setup up p0_va");
1713 	p0_va = i86devmap(0, 1, PROT_READ);
1714 	PRM_DEBUG(p0_va);
1715 
1716 	cmn_err(CE_CONT, "?mem = %luK (0x%lx)\n",
1717 	    physinstalled << (MMU_PAGESHIFT - 10), ptob(physinstalled));
1718 
1719 	/*
1720 	 * disable automatic large pages for small memory systems or
1721 	 * when the disable flag is set.
1722 	 */
1723 	if (physmem < auto_lpg_min_physmem || auto_lpg_disable) {
1724 		exec_lpg_disable = 1;
1725 		use_brk_lpg = 0;
1726 		use_stk_lpg = 0;
1727 		use_zmap_lpg = 0;
1728 	}
1729 
1730 	PRM_POINT("Calling hat_init_finish()...");
1731 	hat_init_finish();
1732 	PRM_POINT("hat_init_finish() done");
1733 
1734 	/*
1735 	 * Initialize the segkp segment type.
1736 	 */
1737 	rw_enter(&kas.a_lock, RW_WRITER);
1738 	if (!segkp_fromheap) {
1739 		if (seg_attach(&kas, (caddr_t)segkp_base, mmu_ptob(segkpsize),
1740 		    segkp) < 0) {
1741 			panic("startup: cannot attach segkp");
1742 			/*NOTREACHED*/
1743 		}
1744 	} else {
1745 		/*
1746 		 * For 32 bit x86 systems, we will have segkp under the heap.
1747 		 * There will not be a segkp segment.  We do, however, need
1748 		 * to fill in the seg structure.
1749 		 */
1750 		segkp->s_as = &kas;
1751 	}
1752 	if (segkp_create(segkp) != 0) {
1753 		panic("startup: segkp_create failed");
1754 		/*NOTREACHED*/
1755 	}
1756 	PRM_DEBUG(segkp);
1757 	rw_exit(&kas.a_lock);
1758 
1759 	/*
1760 	 * kpm segment
1761 	 */
1762 	segmap_kpm = 0;
1763 	if (kpm_desired) {
1764 		kpm_init();
1765 		kpm_enable = 1;
1766 		vpm_enable = 1;
1767 	}
1768 
1769 	/*
1770 	 * Now create segmap segment.
1771 	 */
1772 	rw_enter(&kas.a_lock, RW_WRITER);
1773 	if (seg_attach(&kas, (caddr_t)segkmap_start, segmapsize, segkmap) < 0) {
1774 		panic("cannot attach segkmap");
1775 		/*NOTREACHED*/
1776 	}
1777 	PRM_DEBUG(segkmap);
1778 
1779 	/*
1780 	 * The 64 bit HAT permanently maps only segmap's page tables.
1781 	 * The 32 bit HAT maps the heap's page tables too.
1782 	 */
1783 #if defined(__amd64)
1784 	hat_kmap_init(segkmap_start, segmapsize);
1785 #else /* __i386 */
1786 	ASSERT(segkmap_start + segmapsize == (uintptr_t)final_kernelheap);
1787 	hat_kmap_init(segkmap_start, (uintptr_t)ekernelheap - segkmap_start);
1788 #endif /* __i386 */
1789 
1790 	a.prot = PROT_READ | PROT_WRITE;
1791 	a.shmsize = 0;
1792 	a.nfreelist = segmapfreelists;
1793 
1794 	if (segmap_create(segkmap, (caddr_t)&a) != 0)
1795 		panic("segmap_create segkmap");
1796 	rw_exit(&kas.a_lock);
1797 
1798 	setup_vaddr_for_ppcopy(CPU);
1799 
1800 	segdev_init();
1801 	pmem_init();
1802 	PRM_POINT("startup_vm() done");
1803 }
1804 
1805 static void
1806 startup_end(void)
1807 {
1808 	extern void setx86isalist(void);
1809 
1810 	PRM_POINT("startup_end() starting...");
1811 
1812 	/*
1813 	 * Perform tasks that get done after most of the VM
1814 	 * initialization has been done but before the clock
1815 	 * and other devices get started.
1816 	 */
1817 	kern_setup1();
1818 
1819 	/*
1820 	 * Perform CPC initialization for this CPU.
1821 	 */
1822 	kcpc_hw_init(CPU);
1823 
1824 #if defined(__amd64)
1825 	/*
1826 	 * Validate support for syscall/sysret
1827 	 * XX64 -- include SSE, SSE2, etc. here too?
1828 	 */
1829 	if ((x86_feature & X86_ASYSC) == 0) {
1830 		cmn_err(CE_WARN,
1831 		    "cpu%d does not support syscall/sysret", CPU->cpu_id);
1832 	}
1833 #endif
1834 
1835 #if defined(OPTERON_WORKAROUND_6323525)
1836 	if (opteron_workaround_6323525)
1837 		patch_workaround_6323525();
1838 #endif
1839 	/*
1840 	 * Configure the system.
1841 	 */
1842 	PRM_POINT("Calling configure()...");
1843 	configure();		/* set up devices */
1844 	PRM_POINT("configure() done");
1845 
1846 	/*
1847 	 * Set the isa_list string to the defined instruction sets we
1848 	 * support.
1849 	 */
1850 	setx86isalist();
1851 	cpu_intr_alloc(CPU, NINTR_THREADS);
1852 	psm_install();
1853 
1854 	/*
1855 	 * We're done with bootops.  We don't unmap the bootstrap yet because
1856 	 * we're still using bootsvcs.
1857 	 */
1858 	PRM_POINT("zeroing out bootops");
1859 	*bootopsp = (struct bootops *)0;
1860 	bootops = (struct bootops *)NULL;
1861 
1862 	PRM_POINT("Enabling interrupts");
1863 	(*picinitf)();
1864 	sti();
1865 
1866 	(void) add_avsoftintr((void *)&softlevel1_hdl, 1, softlevel1,
1867 		"softlevel1", NULL, NULL); /* XXX to be moved later */
1868 
1869 	PRM_POINT("startup_end() done");
1870 }
1871 
1872 extern char hw_serial[];
1873 char *_hs1107 = hw_serial;
1874 ulong_t  _bdhs34;
1875 
1876 void
1877 post_startup(void)
1878 {
1879 	/*
1880 	 * Set the system wide, processor-specific flags to be passed
1881 	 * to userland via the aux vector for performance hints and
1882 	 * instruction set extensions.
1883 	 */
1884 	bind_hwcap();
1885 
1886 	/*
1887 	 * Load the System Management BIOS into the global ksmbios handle,
1888 	 * if an SMBIOS is present on this system.
1889 	 */
1890 	ksmbios = smbios_open(NULL, SMB_VERSION, ksmbios_flags, NULL);
1891 
1892 	/*
1893 	 * Startup memory scrubber.
1894 	 */
1895 	memscrub_init();
1896 
1897 	/*
1898 	 * Complete CPU module initialization
1899 	 */
1900 	cmi_post_init();
1901 
1902 	/*
1903 	 * Perform forceloading tasks for /etc/system.
1904 	 */
1905 	(void) mod_sysctl(SYS_FORCELOAD, NULL);
1906 
1907 	/*
1908 	 * ON4.0: Force /proc module in until clock interrupt handle fixed
1909 	 * ON4.0: This must be fixed or restated in /etc/systems.
1910 	 */
1911 	(void) modload("fs", "procfs");
1912 
1913 #if defined(__i386)
1914 	/*
1915 	 * Check for required functional Floating Point hardware,
1916 	 * unless FP hardware explicitly disabled.
1917 	 */
1918 	if (fpu_exists && (fpu_pentium_fdivbug || fp_kind == FP_NO))
1919 		halt("No working FP hardware found");
1920 #endif
1921 
1922 	maxmem = freemem;
1923 
1924 	add_cpunode2devtree(CPU->cpu_id, CPU->cpu_m.mcpu_cpi);
1925 
1926 	/*
1927 	 * Perform the formal initialization of the boot chip,
1928 	 * and associate the boot cpu with it.
1929 	 * This must be done after the cpu node for CPU has been
1930 	 * added to the device tree, when the necessary probing to
1931 	 * know the chip type and chip "id" is performed.
1932 	 */
1933 	chip_cpu_init(CPU);
1934 	chip_cpu_assign(CPU);
1935 }
1936 
1937 static int
1938 pp_in_ramdisk(page_t *pp)
1939 {
1940 	extern uint64_t ramdisk_start, ramdisk_end;
1941 
1942 	return ((pp->p_pagenum >= btop(ramdisk_start)) &&
1943 	    (pp->p_pagenum < btopr(ramdisk_end)));
1944 }
1945 
1946 void
1947 release_bootstrap(void)
1948 {
1949 	int root_is_ramdisk;
1950 	pfn_t pfn;
1951 	page_t *pp;
1952 	extern void kobj_boot_unmountroot(void);
1953 	extern dev_t rootdev;
1954 
1955 	/* unmount boot ramdisk and release kmem usage */
1956 	kobj_boot_unmountroot();
1957 
1958 	/*
1959 	 * We're finished using the boot loader so free its pages.
1960 	 */
1961 	PRM_POINT("Unmapping lower boot pages");
1962 	clear_boot_mappings(0, kernelbase);
1963 #if defined(__amd64)
1964 	PRM_POINT("Unmapping upper boot pages");
1965 	clear_boot_mappings(BOOT_DOUBLEMAP_BASE,
1966 	    BOOT_DOUBLEMAP_BASE + BOOT_DOUBLEMAP_SIZE);
1967 #endif
1968 
1969 	/*
1970 	 * If root isn't on ramdisk, destroy the hardcoded
1971 	 * ramdisk node now and release the memory. Else,
1972 	 * ramdisk memory is kept in rd_pages.
1973 	 */
1974 	root_is_ramdisk = (getmajor(rootdev) == ddi_name_to_major("ramdisk"));
1975 	if (!root_is_ramdisk) {
1976 		dev_info_t *dip = ddi_find_devinfo("ramdisk", -1, 0);
1977 		ASSERT(dip && ddi_get_parent(dip) == ddi_root_node());
1978 		ndi_rele_devi(dip);	/* held from ddi_find_devinfo */
1979 		(void) ddi_remove_child(dip, 0);
1980 	}
1981 
1982 	PRM_POINT("Releasing boot pages");
1983 	while (bootpages) {
1984 		pp = bootpages;
1985 		bootpages = pp->p_next;
1986 		if (root_is_ramdisk && pp_in_ramdisk(pp)) {
1987 			pp->p_next = rd_pages;
1988 			rd_pages = pp;
1989 			continue;
1990 		}
1991 		pp->p_next = (struct page *)0;
1992 		page_free(pp, 1);
1993 	}
1994 
1995 	/*
1996 	 * Find 1 page below 1 MB so that other processors can boot up.
1997 	 * Make sure it has a kernel VA as well as a 1:1 mapping.
1998 	 * We should have just free'd one up.
1999 	 */
2000 	if (use_mp) {
2001 		for (pfn = 1; pfn < btop(1*1024*1024); pfn++) {
2002 			if (page_numtopp_alloc(pfn) == NULL)
2003 				continue;
2004 			rm_platter_va = i86devmap(pfn, 1,
2005 			    PROT_READ | PROT_WRITE | PROT_EXEC);
2006 			rm_platter_pa = ptob(pfn);
2007 			hat_devload(kas.a_hat,
2008 			    (caddr_t)(uintptr_t)rm_platter_pa, MMU_PAGESIZE,
2009 			    pfn, PROT_READ | PROT_WRITE | PROT_EXEC,
2010 			    HAT_LOAD_NOCONSIST);
2011 			break;
2012 		}
2013 		if (pfn == btop(1*1024*1024))
2014 			panic("No page available for starting "
2015 			    "other processors");
2016 	}
2017 
2018 #if defined(__amd64)
2019 	PRM_POINT("Returning boot's VA space to kernel heap");
2020 	if (kmem_setaside != NULL)
2021 		vmem_free(heap_arena, kmem_setaside, BOOT_DOUBLEMAP_SIZE);
2022 #endif
2023 }
2024 
2025 /*
2026  * Initialize the platform-specific parts of a page_t.
2027  */
2028 void
2029 add_physmem_cb(page_t *pp, pfn_t pnum)
2030 {
2031 	pp->p_pagenum = pnum;
2032 	pp->p_mapping = NULL;
2033 	pp->p_embed = 0;
2034 	pp->p_share = 0;
2035 	pp->p_mlentry = 0;
2036 }
2037 
2038 /*
2039  * kphysm_init() initializes physical memory.
2040  */
2041 static pgcnt_t
2042 kphysm_init(
2043 	page_t *inpp,
2044 	struct memseg *memsegp,
2045 	pgcnt_t start,
2046 	pgcnt_t npages)
2047 {
2048 	struct memlist	*pmem;
2049 	struct memseg	*cur_memseg;
2050 	struct memseg	**memsegpp;
2051 	pfn_t		base_pfn;
2052 	pgcnt_t		num;
2053 	pgcnt_t		total_skipped = 0;
2054 	pgcnt_t		skipping = 0;
2055 	pgcnt_t		pages_done = 0;
2056 	pgcnt_t		largepgcnt;
2057 	uint64_t	addr;
2058 	uint64_t	size;
2059 	page_t		*pp = inpp;
2060 	int		dobreak = 0;
2061 	extern pfn_t	ddiphysmin;
2062 
2063 	ASSERT(page_hash != NULL && page_hashsz != 0);
2064 
2065 	for (cur_memseg = memsegp; cur_memseg->pages != NULL; cur_memseg++);
2066 	ASSERT(cur_memseg == memsegp || start > 0);
2067 
2068 	for (pmem = phys_avail; pmem && npages; pmem = pmem->next) {
2069 		/*
2070 		 * In a 32 bit kernel can't use higher memory if we're
2071 		 * not booting in PAE mode. This check takes care of that.
2072 		 */
2073 		addr = pmem->address;
2074 		size = pmem->size;
2075 		if (btop(addr) > physmax)
2076 			continue;
2077 
2078 		/*
2079 		 * align addr and size - they may not be at page boundaries
2080 		 */
2081 		if ((addr & MMU_PAGEOFFSET) != 0) {
2082 			addr += MMU_PAGEOFFSET;
2083 			addr &= ~(uint64_t)MMU_PAGEOFFSET;
2084 			size -= addr - pmem->address;
2085 		}
2086 
2087 		/* only process pages below or equal to physmax */
2088 		if ((btop(addr + size) - 1) > physmax)
2089 			size = ptob(physmax - btop(addr) + 1);
2090 
2091 		num = btop(size);
2092 		if (num == 0)
2093 			continue;
2094 
2095 		if (total_skipped < start) {
2096 			if (start - total_skipped > num) {
2097 				total_skipped += num;
2098 				continue;
2099 			}
2100 			skipping = start - total_skipped;
2101 			num -= skipping;
2102 			addr += (MMU_PAGESIZE * skipping);
2103 			total_skipped = start;
2104 		}
2105 		if (num == 0)
2106 			continue;
2107 
2108 		if (num > npages)
2109 			num = npages;
2110 
2111 		npages -= num;
2112 		pages_done += num;
2113 		base_pfn = btop(addr);
2114 
2115 		/*
2116 		 * If the caller didn't provide space for the page
2117 		 * structures, carve them out of the memseg they will
2118 		 * represent.
2119 		 */
2120 		if (pp == NULL) {
2121 			pgcnt_t pp_pgs;
2122 
2123 			if (num <= 1)
2124 				continue;
2125 
2126 			/*
2127 			 * Compute how many of the pages we need to use for
2128 			 * page_ts
2129 			 */
2130 			pp_pgs = (num * sizeof (page_t)) / MMU_PAGESIZE + 1;
2131 			while (mmu_ptob(pp_pgs - 1) / sizeof (page_t) >=
2132 			    num - pp_pgs + 1)
2133 				--pp_pgs;
2134 			PRM_DEBUG(pp_pgs);
2135 
2136 			pp = vmem_alloc(heap_arena, mmu_ptob(pp_pgs),
2137 			    VM_NOSLEEP);
2138 			if (pp == NULL) {
2139 				cmn_err(CE_WARN, "Unable to add %ld pages to "
2140 				    "the system.", num);
2141 				continue;
2142 			}
2143 
2144 			hat_devload(kas.a_hat, (void *)pp, mmu_ptob(pp_pgs),
2145 			    base_pfn, PROT_READ | PROT_WRITE | HAT_UNORDERED_OK,
2146 			    HAT_LOAD | HAT_LOAD_LOCK | HAT_LOAD_NOCONSIST);
2147 			bzero(pp, mmu_ptob(pp_pgs));
2148 			num -= pp_pgs;
2149 			base_pfn += pp_pgs;
2150 		}
2151 
2152 		if (prom_debug)
2153 			prom_printf("MEMSEG addr=0x%" PRIx64
2154 			    " pgs=0x%lx pfn 0x%lx-0x%lx\n",
2155 			    addr, num, base_pfn, base_pfn + num);
2156 
2157 		/*
2158 		 * drop pages below ddiphysmin to simplify ddi memory
2159 		 * allocation with non-zero addr_lo requests.
2160 		 */
2161 		if (base_pfn < ddiphysmin) {
2162 			if (base_pfn + num <= ddiphysmin) {
2163 				/* drop entire range below ddiphysmin */
2164 				continue;
2165 			}
2166 			/* adjust range to ddiphysmin */
2167 			pp += (ddiphysmin - base_pfn);
2168 			num -= (ddiphysmin - base_pfn);
2169 			base_pfn = ddiphysmin;
2170 		}
2171 		/*
2172 		 * Build the memsegs entry
2173 		 */
2174 		cur_memseg->pages = pp;
2175 		cur_memseg->epages = pp + num;
2176 		cur_memseg->pages_base = base_pfn;
2177 		cur_memseg->pages_end = base_pfn + num;
2178 
2179 		/*
2180 		 * insert in memseg list in decreasing pfn range order.
2181 		 * Low memory is typically more fragmented such that this
2182 		 * ordering keeps the larger ranges at the front of the list
2183 		 * for code that searches memseg.
2184 		 */
2185 		memsegpp = &memsegs;
2186 		for (;;) {
2187 			if (*memsegpp == NULL) {
2188 				/* empty memsegs */
2189 				memsegs = cur_memseg;
2190 				break;
2191 			}
2192 			/* check for continuity with start of memsegpp */
2193 			if (cur_memseg->pages_end == (*memsegpp)->pages_base) {
2194 				if (cur_memseg->epages == (*memsegpp)->pages) {
2195 					/*
2196 					 * contiguous pfn and page_t's. Merge
2197 					 * cur_memseg into *memsegpp. Drop
2198 					 * cur_memseg
2199 					 */
2200 					(*memsegpp)->pages_base =
2201 					    cur_memseg->pages_base;
2202 					(*memsegpp)->pages =
2203 					    cur_memseg->pages;
2204 					/*
2205 					 * check if contiguous with the end of
2206 					 * the next memseg.
2207 					 */
2208 					if ((*memsegpp)->next &&
2209 					    ((*memsegpp)->pages_base ==
2210 					    (*memsegpp)->next->pages_end)) {
2211 						cur_memseg = *memsegpp;
2212 						memsegpp = &((*memsegpp)->next);
2213 						dobreak = 1;
2214 					} else {
2215 						break;
2216 					}
2217 				} else {
2218 					/*
2219 					 * contiguous pfn but not page_t's.
2220 					 * drop last pfn/page_t in cur_memseg
2221 					 * to prevent creation of large pages
2222 					 * with noncontiguous page_t's if not
2223 					 * aligned to largest page boundary.
2224 					 */
2225 					largepgcnt = page_get_pagecnt(
2226 					    page_num_pagesizes() - 1);
2227 
2228 					if (cur_memseg->pages_end &
2229 					    (largepgcnt - 1)) {
2230 						num--;
2231 						cur_memseg->epages--;
2232 						cur_memseg->pages_end--;
2233 					}
2234 				}
2235 			}
2236 
2237 			/* check for continuity with end of memsegpp */
2238 			if (cur_memseg->pages_base == (*memsegpp)->pages_end) {
2239 				if (cur_memseg->pages == (*memsegpp)->epages) {
2240 					/*
2241 					 * contiguous pfn and page_t's. Merge
2242 					 * cur_memseg into *memsegpp. Drop
2243 					 * cur_memseg.
2244 					 */
2245 					if (dobreak) {
2246 						/* merge previously done */
2247 						cur_memseg->pages =
2248 						    (*memsegpp)->pages;
2249 						cur_memseg->pages_base =
2250 						    (*memsegpp)->pages_base;
2251 						cur_memseg->next =
2252 						    (*memsegpp)->next;
2253 					} else {
2254 						(*memsegpp)->pages_end =
2255 						    cur_memseg->pages_end;
2256 						(*memsegpp)->epages =
2257 						    cur_memseg->epages;
2258 					}
2259 					break;
2260 				}
2261 				/*
2262 				 * contiguous pfn but not page_t's.
2263 				 * drop first pfn/page_t in cur_memseg
2264 				 * to prevent creation of large pages
2265 				 * with noncontiguous page_t's if not
2266 				 * aligned to largest page boundary.
2267 				 */
2268 				largepgcnt = page_get_pagecnt(
2269 				    page_num_pagesizes() - 1);
2270 				if (base_pfn & (largepgcnt - 1)) {
2271 					num--;
2272 					base_pfn++;
2273 					cur_memseg->pages++;
2274 					cur_memseg->pages_base++;
2275 					pp = cur_memseg->pages;
2276 				}
2277 				if (dobreak)
2278 					break;
2279 			}
2280 
2281 			if (cur_memseg->pages_base >=
2282 			    (*memsegpp)->pages_end) {
2283 				cur_memseg->next = *memsegpp;
2284 				*memsegpp = cur_memseg;
2285 				break;
2286 			}
2287 			if ((*memsegpp)->next == NULL) {
2288 				cur_memseg->next = NULL;
2289 				(*memsegpp)->next = cur_memseg;
2290 				break;
2291 			}
2292 			memsegpp = &((*memsegpp)->next);
2293 			ASSERT(*memsegpp != NULL);
2294 		}
2295 
2296 		/*
2297 		 * add_physmem() initializes the PSM part of the page
2298 		 * struct by calling the PSM back with add_physmem_cb().
2299 		 * In addition it coalesces pages into larger pages as
2300 		 * it initializes them.
2301 		 */
2302 		add_physmem(pp, num, base_pfn);
2303 		cur_memseg++;
2304 		availrmem_initial += num;
2305 		availrmem += num;
2306 
2307 		/*
2308 		 * If the caller provided the page frames to us, then
2309 		 * advance in that list.  Otherwise, prepare to allocate
2310 		 * our own page frames for the next memseg.
2311 		 */
2312 		pp = (inpp == NULL) ? NULL : pp + num;
2313 	}
2314 
2315 	PRM_DEBUG(availrmem_initial);
2316 	PRM_DEBUG(availrmem);
2317 	PRM_DEBUG(freemem);
2318 	build_pfn_hash();
2319 	return (pages_done);
2320 }
2321 
2322 /*
2323  * Kernel VM initialization.
2324  */
2325 static void
2326 kvm_init(void)
2327 {
2328 #ifdef DEBUG
2329 	extern void _start();
2330 
2331 	ASSERT((caddr_t)_start == s_text);
2332 #endif
2333 	ASSERT((((uintptr_t)s_text) & MMU_PAGEOFFSET) == 0);
2334 
2335 	/*
2336 	 * Put the kernel segments in kernel address space.
2337 	 */
2338 	rw_enter(&kas.a_lock, RW_WRITER);
2339 	as_avlinit(&kas);
2340 
2341 	(void) seg_attach(&kas, s_text, e_moddata - s_text, &ktextseg);
2342 	(void) segkmem_create(&ktextseg);
2343 
2344 	(void) seg_attach(&kas, (caddr_t)valloc_base, valloc_sz, &kvalloc);
2345 	(void) segkmem_create(&kvalloc);
2346 
2347 	/*
2348 	 * We're about to map out /boot.  This is the beginning of the
2349 	 * system resource management transition. We can no longer
2350 	 * call into /boot for I/O or memory allocations.
2351 	 *
2352 	 * XX64 - Is this still correct with kernelheap_extend() being called
2353 	 * later than this????
2354 	 */
2355 	(void) seg_attach(&kas, final_kernelheap,
2356 	    ekernelheap - final_kernelheap, &kvseg);
2357 	(void) segkmem_create(&kvseg);
2358 
2359 #if defined(__amd64)
2360 	(void) seg_attach(&kas, (caddr_t)core_base, core_size, &kvseg_core);
2361 	(void) segkmem_create(&kvseg_core);
2362 #endif
2363 
2364 	(void) seg_attach(&kas, (caddr_t)SEGDEBUGBASE, (size_t)SEGDEBUGSIZE,
2365 	    &kdebugseg);
2366 	(void) segkmem_create(&kdebugseg);
2367 
2368 	rw_exit(&kas.a_lock);
2369 
2370 	/*
2371 	 * Ensure that the red zone at kernelbase is never accessible.
2372 	 */
2373 	(void) as_setprot(&kas, (caddr_t)kernelbase, KERNEL_REDZONE_SIZE, 0);
2374 
2375 	/*
2376 	 * Make the text writable so that it can be hot patched by DTrace.
2377 	 */
2378 	(void) as_setprot(&kas, s_text, e_modtext - s_text,
2379 	    PROT_READ | PROT_WRITE | PROT_EXEC);
2380 
2381 	/*
2382 	 * Make data writable until end.
2383 	 */
2384 	(void) as_setprot(&kas, s_data, e_moddata - s_data,
2385 	    PROT_READ | PROT_WRITE | PROT_EXEC);
2386 }
2387 
2388 /*
2389  * These are MTTR registers supported by P6
2390  */
2391 static struct	mtrrvar	mtrrphys_arr[MAX_MTRRVAR];
2392 static uint64_t mtrr64k, mtrr16k1, mtrr16k2;
2393 static uint64_t mtrr4k1, mtrr4k2, mtrr4k3;
2394 static uint64_t mtrr4k4, mtrr4k5, mtrr4k6;
2395 static uint64_t mtrr4k7, mtrr4k8, mtrrcap;
2396 uint64_t mtrrdef, pat_attr_reg;
2397 
2398 /*
2399  * Disable reprogramming of MTRRs by default.
2400  */
2401 int	enable_relaxed_mtrr = 0;
2402 
2403 void
2404 setup_mtrr(void)
2405 {
2406 	int i, ecx;
2407 	int vcnt;
2408 	struct	mtrrvar	*mtrrphys;
2409 
2410 	if (!(x86_feature & X86_MTRR))
2411 		return;
2412 
2413 	mtrrcap = rdmsr(REG_MTRRCAP);
2414 	mtrrdef = rdmsr(REG_MTRRDEF);
2415 	if (mtrrcap & MTRRCAP_FIX) {
2416 		mtrr64k = rdmsr(REG_MTRR64K);
2417 		mtrr16k1 = rdmsr(REG_MTRR16K1);
2418 		mtrr16k2 = rdmsr(REG_MTRR16K2);
2419 		mtrr4k1 = rdmsr(REG_MTRR4K1);
2420 		mtrr4k2 = rdmsr(REG_MTRR4K2);
2421 		mtrr4k3 = rdmsr(REG_MTRR4K3);
2422 		mtrr4k4 = rdmsr(REG_MTRR4K4);
2423 		mtrr4k5 = rdmsr(REG_MTRR4K5);
2424 		mtrr4k6 = rdmsr(REG_MTRR4K6);
2425 		mtrr4k7 = rdmsr(REG_MTRR4K7);
2426 		mtrr4k8 = rdmsr(REG_MTRR4K8);
2427 	}
2428 	if ((vcnt = (mtrrcap & MTRRCAP_VCNTMASK)) > MAX_MTRRVAR)
2429 		vcnt = MAX_MTRRVAR;
2430 
2431 	for (i = 0, ecx = REG_MTRRPHYSBASE0, mtrrphys = mtrrphys_arr;
2432 		i <  vcnt - 1; i++, ecx += 2, mtrrphys++) {
2433 		mtrrphys->mtrrphys_base = rdmsr(ecx);
2434 		mtrrphys->mtrrphys_mask = rdmsr(ecx + 1);
2435 		if ((x86_feature & X86_PAT) && enable_relaxed_mtrr) {
2436 			mtrrphys->mtrrphys_mask &= ~MTRRPHYSMASK_V;
2437 		}
2438 	}
2439 	if (x86_feature & X86_PAT) {
2440 		if (enable_relaxed_mtrr)
2441 			mtrrdef = MTRR_TYPE_WB|MTRRDEF_FE|MTRRDEF_E;
2442 		pat_attr_reg = PAT_DEFAULT_ATTRIBUTE;
2443 	}
2444 
2445 	mtrr_sync();
2446 }
2447 
2448 /*
2449  * Sync current cpu mtrr with the incore copy of mtrr.
2450  * This function has to be invoked with interrupts disabled
2451  * Currently we do not capture other cpu's. This is invoked on cpu0
2452  * just after reading /etc/system.
2453  * On other cpu's its invoked from mp_startup().
2454  */
2455 void
2456 mtrr_sync(void)
2457 {
2458 	uint_t	crvalue, cr0_orig;
2459 	int	vcnt, i, ecx;
2460 	struct	mtrrvar	*mtrrphys;
2461 
2462 	cr0_orig = crvalue = getcr0();
2463 	crvalue |= CR0_CD;
2464 	crvalue &= ~CR0_NW;
2465 	setcr0(crvalue);
2466 	invalidate_cache();
2467 	setcr3(getcr3());
2468 
2469 	if (x86_feature & X86_PAT)
2470 		wrmsr(REG_MTRRPAT, pat_attr_reg);
2471 
2472 	wrmsr(REG_MTRRDEF, rdmsr(REG_MTRRDEF) &
2473 	    ~((uint64_t)(uintptr_t)MTRRDEF_E));
2474 
2475 	if (mtrrcap & MTRRCAP_FIX) {
2476 		wrmsr(REG_MTRR64K, mtrr64k);
2477 		wrmsr(REG_MTRR16K1, mtrr16k1);
2478 		wrmsr(REG_MTRR16K2, mtrr16k2);
2479 		wrmsr(REG_MTRR4K1, mtrr4k1);
2480 		wrmsr(REG_MTRR4K2, mtrr4k2);
2481 		wrmsr(REG_MTRR4K3, mtrr4k3);
2482 		wrmsr(REG_MTRR4K4, mtrr4k4);
2483 		wrmsr(REG_MTRR4K5, mtrr4k5);
2484 		wrmsr(REG_MTRR4K6, mtrr4k6);
2485 		wrmsr(REG_MTRR4K7, mtrr4k7);
2486 		wrmsr(REG_MTRR4K8, mtrr4k8);
2487 	}
2488 	if ((vcnt = (mtrrcap & MTRRCAP_VCNTMASK)) > MAX_MTRRVAR)
2489 		vcnt = MAX_MTRRVAR;
2490 	for (i = 0, ecx = REG_MTRRPHYSBASE0, mtrrphys = mtrrphys_arr;
2491 	    i <  vcnt - 1; i++, ecx += 2, mtrrphys++) {
2492 		wrmsr(ecx, mtrrphys->mtrrphys_base);
2493 		wrmsr(ecx + 1, mtrrphys->mtrrphys_mask);
2494 	}
2495 	wrmsr(REG_MTRRDEF, mtrrdef);
2496 	setcr3(getcr3());
2497 	invalidate_cache();
2498 	setcr0(cr0_orig);
2499 }
2500 
2501 /*
2502  * resync mtrr so that BIOS is happy. Called from mdboot
2503  */
2504 void
2505 mtrr_resync(void)
2506 {
2507 	if ((x86_feature & X86_PAT) && enable_relaxed_mtrr) {
2508 		/*
2509 		 * We could have changed the default mtrr definition.
2510 		 * Put it back to uncached which is what it is at power on
2511 		 */
2512 		mtrrdef = MTRR_TYPE_UC|MTRRDEF_FE|MTRRDEF_E;
2513 		mtrr_sync();
2514 	}
2515 }
2516 
2517 void
2518 get_system_configuration(void)
2519 {
2520 	char	prop[32];
2521 	u_longlong_t nodes_ll, cpus_pernode_ll, lvalue;
2522 
2523 	if (((BOP_GETPROPLEN(bootops, "nodes") > sizeof (prop)) ||
2524 		(BOP_GETPROP(bootops, "nodes", prop) < 0) 	||
2525 		(kobj_getvalue(prop, &nodes_ll) == -1) ||
2526 		(nodes_ll > MAXNODES))			   ||
2527 	    ((BOP_GETPROPLEN(bootops, "cpus_pernode") > sizeof (prop)) ||
2528 		(BOP_GETPROP(bootops, "cpus_pernode", prop) < 0) ||
2529 		(kobj_getvalue(prop, &cpus_pernode_ll) == -1))) {
2530 
2531 		system_hardware.hd_nodes = 1;
2532 		system_hardware.hd_cpus_per_node = 0;
2533 	} else {
2534 		system_hardware.hd_nodes = (int)nodes_ll;
2535 		system_hardware.hd_cpus_per_node = (int)cpus_pernode_ll;
2536 	}
2537 	if ((BOP_GETPROPLEN(bootops, "kernelbase") > sizeof (prop)) ||
2538 		(BOP_GETPROP(bootops, "kernelbase", prop) < 0) 	||
2539 		(kobj_getvalue(prop, &lvalue) == -1))
2540 			eprom_kernelbase = NULL;
2541 	else
2542 			eprom_kernelbase = (uintptr_t)lvalue;
2543 
2544 	if ((BOP_GETPROPLEN(bootops, "segmapsize") > sizeof (prop)) ||
2545 	    (BOP_GETPROP(bootops, "segmapsize", prop) < 0) ||
2546 	    (kobj_getvalue(prop, &lvalue) == -1)) {
2547 		segmapsize = SEGMAPDEFAULT;
2548 	} else {
2549 		segmapsize = (uintptr_t)lvalue;
2550 	}
2551 
2552 	if ((BOP_GETPROPLEN(bootops, "segmapfreelists") > sizeof (prop)) ||
2553 	    (BOP_GETPROP(bootops, "segmapfreelists", prop) < 0) ||
2554 	    (kobj_getvalue(prop, &lvalue) == -1)) {
2555 		segmapfreelists = 0;	/* use segmap driver default */
2556 	} else {
2557 		segmapfreelists = (int)lvalue;
2558 	}
2559 
2560 	if ((BOP_GETPROPLEN(bootops, "physmem") <= sizeof (prop)) &&
2561 	    (BOP_GETPROP(bootops, "physmem", prop) >= 0) &&
2562 	    (kobj_getvalue(prop, &lvalue) != -1)) {
2563 		physmem = (uintptr_t)lvalue;
2564 	}
2565 }
2566 
2567 /*
2568  * Add to a memory list.
2569  * start = start of new memory segment
2570  * len = length of new memory segment in bytes
2571  * new = pointer to a new struct memlist
2572  * memlistp = memory list to which to add segment.
2573  */
2574 static void
2575 memlist_add(
2576 	uint64_t start,
2577 	uint64_t len,
2578 	struct memlist *new,
2579 	struct memlist **memlistp)
2580 {
2581 	struct memlist *cur;
2582 	uint64_t end = start + len;
2583 
2584 	new->address = start;
2585 	new->size = len;
2586 
2587 	cur = *memlistp;
2588 
2589 	while (cur) {
2590 		if (cur->address >= end) {
2591 			new->next = cur;
2592 			*memlistp = new;
2593 			new->prev = cur->prev;
2594 			cur->prev = new;
2595 			return;
2596 		}
2597 		ASSERT(cur->address + cur->size <= start);
2598 		if (cur->next == NULL) {
2599 			cur->next = new;
2600 			new->prev = cur;
2601 			new->next = NULL;
2602 			return;
2603 		}
2604 		memlistp = &cur->next;
2605 		cur = cur->next;
2606 	}
2607 }
2608 
2609 void
2610 kobj_vmem_init(vmem_t **text_arena, vmem_t **data_arena)
2611 {
2612 	size_t tsize = e_modtext - modtext;
2613 	size_t dsize = e_moddata - moddata;
2614 
2615 	*text_arena = vmem_create("module_text", tsize ? modtext : NULL, tsize,
2616 	    1, segkmem_alloc, segkmem_free, heaptext_arena, 0, VM_SLEEP);
2617 	*data_arena = vmem_create("module_data", dsize ? moddata : NULL, dsize,
2618 	    1, segkmem_alloc, segkmem_free, heap32_arena, 0, VM_SLEEP);
2619 }
2620 
2621 caddr_t
2622 kobj_text_alloc(vmem_t *arena, size_t size)
2623 {
2624 	return (vmem_alloc(arena, size, VM_SLEEP | VM_BESTFIT));
2625 }
2626 
2627 /*ARGSUSED*/
2628 caddr_t
2629 kobj_texthole_alloc(caddr_t addr, size_t size)
2630 {
2631 	panic("unexpected call to kobj_texthole_alloc()");
2632 	/*NOTREACHED*/
2633 	return (0);
2634 }
2635 
2636 /*ARGSUSED*/
2637 void
2638 kobj_texthole_free(caddr_t addr, size_t size)
2639 {
2640 	panic("unexpected call to kobj_texthole_free()");
2641 }
2642 
2643 /*
2644  * This is called just after configure() in startup().
2645  *
2646  * The ISALIST concept is a bit hopeless on Intel, because
2647  * there's no guarantee of an ever-more-capable processor
2648  * given that various parts of the instruction set may appear
2649  * and disappear between different implementations.
2650  *
2651  * While it would be possible to correct it and even enhance
2652  * it somewhat, the explicit hardware capability bitmask allows
2653  * more flexibility.
2654  *
2655  * So, we just leave this alone.
2656  */
2657 void
2658 setx86isalist(void)
2659 {
2660 	char *tp;
2661 	size_t len;
2662 	extern char *isa_list;
2663 
2664 #define	TBUFSIZE	1024
2665 
2666 	tp = kmem_alloc(TBUFSIZE, KM_SLEEP);
2667 	*tp = '\0';
2668 
2669 #if defined(__amd64)
2670 	(void) strcpy(tp, "amd64 ");
2671 #endif
2672 
2673 	switch (x86_vendor) {
2674 	case X86_VENDOR_Intel:
2675 	case X86_VENDOR_AMD:
2676 	case X86_VENDOR_TM:
2677 		if (x86_feature & X86_CMOV) {
2678 			/*
2679 			 * Pentium Pro or later
2680 			 */
2681 			(void) strcat(tp, "pentium_pro");
2682 			(void) strcat(tp, x86_feature & X86_MMX ?
2683 			    "+mmx pentium_pro " : " ");
2684 		}
2685 		/*FALLTHROUGH*/
2686 	case X86_VENDOR_Cyrix:
2687 		/*
2688 		 * The Cyrix 6x86 does not have any Pentium features
2689 		 * accessible while not at privilege level 0.
2690 		 */
2691 		if (x86_feature & X86_CPUID) {
2692 			(void) strcat(tp, "pentium");
2693 			(void) strcat(tp, x86_feature & X86_MMX ?
2694 			    "+mmx pentium " : " ");
2695 		}
2696 		break;
2697 	default:
2698 		break;
2699 	}
2700 	(void) strcat(tp, "i486 i386 i86");
2701 	len = strlen(tp) + 1;   /* account for NULL at end of string */
2702 	isa_list = strcpy(kmem_alloc(len, KM_SLEEP), tp);
2703 	kmem_free(tp, TBUFSIZE);
2704 
2705 #undef TBUFSIZE
2706 }
2707 
2708 
2709 #ifdef __amd64
2710 
2711 void *
2712 device_arena_alloc(size_t size, int vm_flag)
2713 {
2714 	return (vmem_alloc(device_arena, size, vm_flag));
2715 }
2716 
2717 void
2718 device_arena_free(void *vaddr, size_t size)
2719 {
2720 	vmem_free(device_arena, vaddr, size);
2721 }
2722 
2723 #else
2724 
2725 void *
2726 device_arena_alloc(size_t size, int vm_flag)
2727 {
2728 	caddr_t	vaddr;
2729 	uintptr_t v;
2730 	size_t	start;
2731 	size_t	end;
2732 
2733 	vaddr = vmem_alloc(heap_arena, size, vm_flag);
2734 	if (vaddr == NULL)
2735 		return (NULL);
2736 
2737 	v = (uintptr_t)vaddr;
2738 	ASSERT(v >= kernelbase);
2739 	ASSERT(v + size <= ptable_va);
2740 
2741 	start = btop(v - kernelbase);
2742 	end = btop(v + size - 1 - kernelbase);
2743 	ASSERT(start < toxic_bit_map_len);
2744 	ASSERT(end < toxic_bit_map_len);
2745 
2746 	while (start <= end) {
2747 		BT_ATOMIC_SET(toxic_bit_map, start);
2748 		++start;
2749 	}
2750 	return (vaddr);
2751 }
2752 
2753 void
2754 device_arena_free(void *vaddr, size_t size)
2755 {
2756 	uintptr_t v = (uintptr_t)vaddr;
2757 	size_t	start;
2758 	size_t	end;
2759 
2760 	ASSERT(v >= kernelbase);
2761 	ASSERT(v + size <= ptable_va);
2762 
2763 	start = btop(v - kernelbase);
2764 	end = btop(v + size - 1 - kernelbase);
2765 	ASSERT(start < toxic_bit_map_len);
2766 	ASSERT(end < toxic_bit_map_len);
2767 
2768 	while (start <= end) {
2769 		ASSERT(BT_TEST(toxic_bit_map, start) != 0);
2770 		BT_ATOMIC_CLEAR(toxic_bit_map, start);
2771 		++start;
2772 	}
2773 	vmem_free(heap_arena, vaddr, size);
2774 }
2775 
2776 /*
2777  * returns 1st address in range that is in device arena, or NULL
2778  * if len is not NULL it returns the length of the toxic range
2779  */
2780 void *
2781 device_arena_contains(void *vaddr, size_t size, size_t *len)
2782 {
2783 	uintptr_t v = (uintptr_t)vaddr;
2784 	uintptr_t eaddr = v + size;
2785 	size_t start;
2786 	size_t end;
2787 
2788 	/*
2789 	 * if called very early by kmdb, just return NULL
2790 	 */
2791 	if (toxic_bit_map == NULL)
2792 		return (NULL);
2793 
2794 	/*
2795 	 * First check if we're completely outside the bitmap range.
2796 	 */
2797 	if (v >= ptable_va || eaddr < kernelbase)
2798 		return (NULL);
2799 
2800 	/*
2801 	 * Trim ends of search to look at only what the bitmap covers.
2802 	 */
2803 	if (v < kernelbase)
2804 		v = kernelbase;
2805 	start = btop(v - kernelbase);
2806 	end = btop(eaddr - kernelbase);
2807 	if (end >= toxic_bit_map_len)
2808 		end = toxic_bit_map_len;
2809 
2810 	if (bt_range(toxic_bit_map, &start, &end, end) == 0)
2811 		return (NULL);
2812 
2813 	v = kernelbase + ptob(start);
2814 	if (len != NULL)
2815 		*len = ptob(end - start);
2816 	return ((void *)v);
2817 }
2818 
2819 #endif
2820