xref: /illumos-gate/usr/src/uts/i86pc/os/startup.c (revision 567d55e140d6df8a327df15dac64a9a0c5b43c35)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #pragma ident	"%Z%%M%	%I%	%E% SMI"
27 
28 #include <sys/types.h>
29 #include <sys/t_lock.h>
30 #include <sys/param.h>
31 #include <sys/sysmacros.h>
32 #include <sys/signal.h>
33 #include <sys/systm.h>
34 #include <sys/user.h>
35 #include <sys/mman.h>
36 #include <sys/vm.h>
37 #include <sys/conf.h>
38 #include <sys/avintr.h>
39 #include <sys/autoconf.h>
40 #include <sys/disp.h>
41 #include <sys/class.h>
42 #include <sys/bitmap.h>
43 
44 #include <sys/privregs.h>
45 
46 #include <sys/proc.h>
47 #include <sys/buf.h>
48 #include <sys/kmem.h>
49 #include <sys/mem.h>
50 #include <sys/kstat.h>
51 
52 #include <sys/reboot.h>
53 
54 #include <sys/cred.h>
55 #include <sys/vnode.h>
56 #include <sys/file.h>
57 
58 #include <sys/procfs.h>
59 
60 #include <sys/vfs.h>
61 #include <sys/cmn_err.h>
62 #include <sys/utsname.h>
63 #include <sys/debug.h>
64 #include <sys/kdi.h>
65 
66 #include <sys/dumphdr.h>
67 #include <sys/bootconf.h>
68 #include <sys/varargs.h>
69 #include <sys/promif.h>
70 #include <sys/modctl.h>		/* for "procfs" hack */
71 
72 #include <sys/sunddi.h>
73 #include <sys/sunndi.h>
74 #include <sys/ndi_impldefs.h>
75 #include <sys/ddidmareq.h>
76 #include <sys/psw.h>
77 #include <sys/regset.h>
78 #include <sys/clock.h>
79 #include <sys/pte.h>
80 #include <sys/tss.h>
81 #include <sys/stack.h>
82 #include <sys/trap.h>
83 #include <sys/fp.h>
84 #include <vm/anon.h>
85 #include <vm/as.h>
86 #include <vm/page.h>
87 #include <vm/seg.h>
88 #include <vm/seg_dev.h>
89 #include <vm/seg_kmem.h>
90 #include <vm/seg_kpm.h>
91 #include <vm/seg_map.h>
92 #include <vm/seg_vn.h>
93 #include <vm/seg_kp.h>
94 #include <sys/memnode.h>
95 #include <vm/vm_dep.h>
96 #include <sys/thread.h>
97 #include <sys/sysconf.h>
98 #include <sys/vm_machparam.h>
99 #include <sys/archsystm.h>
100 #include <sys/machsystm.h>
101 #include <vm/hat.h>
102 #include <vm/hat_i86.h>
103 #include <sys/pmem.h>
104 #include <sys/smp_impldefs.h>
105 #include <sys/x86_archext.h>
106 #include <sys/segments.h>
107 #include <sys/clconf.h>
108 #include <sys/kobj.h>
109 #include <sys/kobj_lex.h>
110 #include <sys/cpc_impl.h>
111 #include <sys/pg.h>
112 #include <sys/x86_archext.h>
113 #include <sys/cpu_module.h>
114 #include <sys/smbios.h>
115 #include <sys/debug_info.h>
116 
117 
118 #include <sys/bootinfo.h>
119 #include <vm/kboot_mmu.h>
120 
121 extern void progressbar_init(void);
122 extern void progressbar_start(void);
123 extern void brand_init(void);
124 
125 /*
126  * XXX make declaration below "static" when drivers no longer use this
127  * interface.
128  */
129 extern caddr_t p0_va;	/* Virtual address for accessing physical page 0 */
130 
131 /*
132  * segkp
133  */
134 extern int segkp_fromheap;
135 
136 static void kvm_init(void);
137 static void startup_init(void);
138 static void startup_memlist(void);
139 static void startup_kmem(void);
140 static void startup_modules(void);
141 static void startup_vm(void);
142 static void startup_end(void);
143 
144 /*
145  * Declare these as initialized data so we can patch them.
146  */
147 #ifdef __i386
148 /*
149  * Due to virtual address space limitations running in 32 bit mode, restrict
150  * the amount of physical memory configured to a max of PHYSMEM32 pages (16g).
151  *
152  * If the physical max memory size of 64g were allowed to be configured, the
153  * size of user virtual address space will be less than 1g. A limited user
154  * address space greatly reduces the range of applications that can run.
155  *
156  * If more physical memory than PHYSMEM32 is required, users should preferably
157  * run in 64 bit mode which has no virtual address space limitation issues.
158  *
159  * If 64 bit mode is not available (as in IA32) and/or more physical memory
160  * than PHYSMEM32 is required in 32 bit mode, physmem can be set to the desired
161  * value or to 0 (to configure all available memory) via eeprom(1M). kernelbase
162  * should also be carefully tuned to balance out the need of the user
163  * application while minimizing the risk of kernel heap exhaustion due to
164  * kernelbase being set too high.
165  */
166 #define	PHYSMEM32	0x400000
167 
168 pgcnt_t physmem = PHYSMEM32;
169 #else
170 pgcnt_t physmem = 0;	/* memory size in pages, patch if you want less */
171 #endif
172 pgcnt_t obp_pages;	/* Memory used by PROM for its text and data */
173 
174 char *kobj_file_buf;
175 int kobj_file_bufsize;	/* set in /etc/system */
176 
177 /* Global variables for MP support. Used in mp_startup */
178 caddr_t	rm_platter_va;
179 uint32_t rm_platter_pa;
180 
181 int	auto_lpg_disable = 1;
182 
183 /*
184  * Some CPUs have holes in the middle of the 64-bit virtual address range.
185  */
186 uintptr_t hole_start, hole_end;
187 
188 /*
189  * kpm mapping window
190  */
191 caddr_t kpm_vbase;
192 size_t  kpm_size;
193 static int kpm_desired = 0;		/* Do we want to try to use segkpm? */
194 
195 /*
196  * Configuration parameters set at boot time.
197  */
198 
199 caddr_t econtig;		/* end of first block of contiguous kernel */
200 
201 struct bootops		*bootops = 0;	/* passed in from boot */
202 struct bootops		**bootopsp;
203 struct boot_syscalls	*sysp;		/* passed in from boot */
204 
205 char bootblock_fstype[16];
206 
207 char kern_bootargs[OBP_MAXPATHLEN];
208 
209 /*
210  * ZFS zio segment.  This allows us to exclude large portions of ZFS data that
211  * gets cached in kmem caches on the heap.  If this is set to zero, we allocate
212  * zio buffers from their own segment, otherwise they are allocated from the
213  * heap.  The optimization of allocating zio buffers from their own segment is
214  * only valid on 64-bit kernels.
215  */
216 #if defined(__amd64)
217 int segzio_fromheap = 0;
218 #else
219 int segzio_fromheap = 1;
220 #endif
221 
222 /*
223  * new memory fragmentations are possible in startup() due to BOP_ALLOCs. this
224  * depends on number of BOP_ALLOC calls made and requested size, memory size
225  * combination and whether boot.bin memory needs to be freed.
226  */
227 #define	POSS_NEW_FRAGMENTS	12
228 
229 /*
230  * VM data structures
231  */
232 long page_hashsz;		/* Size of page hash table (power of two) */
233 struct page *pp_base;		/* Base of initial system page struct array */
234 struct page **page_hash;	/* Page hash table */
235 struct seg ktextseg;		/* Segment used for kernel executable image */
236 struct seg kvalloc;		/* Segment used for "valloc" mapping */
237 struct seg kpseg;		/* Segment used for pageable kernel virt mem */
238 struct seg kmapseg;		/* Segment used for generic kernel mappings */
239 struct seg kdebugseg;		/* Segment used for the kernel debugger */
240 
241 struct seg *segkmap = &kmapseg;	/* Kernel generic mapping segment */
242 static struct seg *segmap = &kmapseg;	/* easier to use name for in here */
243 
244 struct seg *segkp = &kpseg;	/* Pageable kernel virtual memory segment */
245 
246 #if defined(__amd64)
247 struct seg kvseg_core;		/* Segment used for the core heap */
248 struct seg kpmseg;		/* Segment used for physical mapping */
249 struct seg *segkpm = &kpmseg;	/* 64bit kernel physical mapping segment */
250 #else
251 struct seg *segkpm = NULL;	/* Unused on IA32 */
252 #endif
253 
254 caddr_t segkp_base;		/* Base address of segkp */
255 caddr_t segzio_base;		/* Base address of segzio */
256 #if defined(__amd64)
257 pgcnt_t segkpsize = btop(SEGKPDEFSIZE);	/* size of segkp segment in pages */
258 #else
259 pgcnt_t segkpsize = 0;
260 #endif
261 pgcnt_t segziosize = 0;		/* size of zio segment in pages */
262 
263 /*
264  * VA range available to the debugger
265  */
266 const caddr_t kdi_segdebugbase = (const caddr_t)SEGDEBUGBASE;
267 const size_t kdi_segdebugsize = SEGDEBUGSIZE;
268 
269 struct memseg *memseg_base;
270 struct vnode unused_pages_vp;
271 
272 #define	FOURGB	0x100000000LL
273 
274 struct memlist *memlist;
275 
276 caddr_t s_text;		/* start of kernel text segment */
277 caddr_t e_text;		/* end of kernel text segment */
278 caddr_t s_data;		/* start of kernel data segment */
279 caddr_t e_data;		/* end of kernel data segment */
280 caddr_t modtext;	/* start of loadable module text reserved */
281 caddr_t e_modtext;	/* end of loadable module text reserved */
282 caddr_t moddata;	/* start of loadable module data reserved */
283 caddr_t e_moddata;	/* end of loadable module data reserved */
284 
285 struct memlist *phys_install;	/* Total installed physical memory */
286 struct memlist *phys_avail;	/* Total available physical memory */
287 
288 /*
289  * kphysm_init returns the number of pages that were processed
290  */
291 static pgcnt_t kphysm_init(page_t *, pgcnt_t);
292 
293 #define	IO_PROP_SIZE	64	/* device property size */
294 
295 /*
296  * a couple useful roundup macros
297  */
298 #define	ROUND_UP_PAGE(x)	\
299 	((uintptr_t)P2ROUNDUP((uintptr_t)(x), (uintptr_t)MMU_PAGESIZE))
300 #define	ROUND_UP_LPAGE(x)	\
301 	((uintptr_t)P2ROUNDUP((uintptr_t)(x), mmu.level_size[1]))
302 #define	ROUND_UP_4MEG(x)	\
303 	((uintptr_t)P2ROUNDUP((uintptr_t)(x), (uintptr_t)FOUR_MEG))
304 #define	ROUND_UP_TOPLEVEL(x)	\
305 	((uintptr_t)P2ROUNDUP((uintptr_t)(x), mmu.level_size[mmu.max_level]))
306 
307 /*
308  *	32-bit Kernel's Virtual memory layout.
309  *		+-----------------------+
310  *		|			|
311  * 0xFFC00000  -|-----------------------|- ARGSBASE
312  *		|	debugger	|
313  * 0xFF800000  -|-----------------------|- SEGDEBUGBASE
314  *		|      Kernel Data	|
315  * 0xFEC00000  -|-----------------------|
316  *              |      Kernel Text	|
317  * 0xFE800000  -|-----------------------|- KERNEL_TEXT
318  *		|---       GDT       ---|- GDT page (GDT_VA)
319  *		|---    debug info   ---|- debug info (DEBUG_INFO_VA)
320  *		|			|
321  * 		|   page_t structures	|
322  * 		|   memsegs, memlists, 	|
323  * 		|   page hash, etc.	|
324  * ---	       -|-----------------------|- ekernelheap, valloc_base (floating)
325  *		|			|  (segkp is just an arena in the heap)
326  *		|			|
327  *		|	kvseg		|
328  *		|			|
329  *		|			|
330  * ---         -|-----------------------|- kernelheap (floating)
331  * 		|        Segkmap	|
332  * 0xC3002000  -|-----------------------|- segmap_start (floating)
333  *		|	Red Zone	|
334  * 0xC3000000  -|-----------------------|- kernelbase / userlimit (floating)
335  *		|			|			||
336  *		|     Shared objects	|			\/
337  *		|			|
338  *		:			:
339  *		|	user data	|
340  *		|-----------------------|
341  *		|	user text	|
342  * 0x08048000  -|-----------------------|
343  *		|	user stack	|
344  *		:			:
345  *		|	invalid		|
346  * 0x00000000	+-----------------------+
347  *
348  *
349  *		64-bit Kernel's Virtual memory layout. (assuming 64 bit app)
350  *			+-----------------------+
351  *			|			|
352  * 0xFFFFFFFF.FFC00000  |-----------------------|- ARGSBASE
353  *			|	debugger (?)	|
354  * 0xFFFFFFFF.FF800000  |-----------------------|- SEGDEBUGBASE
355  *			|      unused    	|
356  *			+-----------------------+
357  *			|      Kernel Data	|
358  * 0xFFFFFFFF.FBC00000  |-----------------------|
359  *			|      Kernel Text	|
360  * 0xFFFFFFFF.FB800000  |-----------------------|- KERNEL_TEXT
361  *			|---       GDT       ---|- GDT page (GDT_VA)
362  *			|---    debug info   ---|- debug info (DEBUG_INFO_VA)
363  *			|			|
364  * 			|      Core heap	| (used for loadable modules)
365  * 0xFFFFFFFF.C0000000  |-----------------------|- core_base / ekernelheap
366  *			|	 Kernel		|
367  *			|	  heap		|
368  * 0xFFFFFXXX.XXX00000  |-----------------------|- kernelheap (floating)
369  *			|	 segmap		|
370  * 0xFFFFFXXX.XXX00000  |-----------------------|- segmap_start (floating)
371  *			|    device mappings	|
372  * 0xFFFFFXXX.XXX00000  |-----------------------|- toxic_addr (floating)
373  *			|	  segzio	|
374  * 0xFFFFFXXX.XXX00000  |-----------------------|- segzio_base (floating)
375  *			|	  segkp		|
376  * ---                  |-----------------------|- segkp_base (floating)
377  * 			|   page_t structures	|  valloc_base + valloc_sz
378  * 			|   memsegs, memlists, 	|
379  * 			|   page hash, etc.	|
380  * 0xFFFFFF00.00000000  |-----------------------|- valloc_base
381  *			|	 segkpm		|
382  * 0xFFFFFE00.00000000  |-----------------------|
383  *			|	Red Zone	|
384  * 0xFFFFFD80.00000000  |-----------------------|- KERNELBASE
385  *			|     User stack	|- User space memory
386  * 			|			|
387  * 			| shared objects, etc	|	(grows downwards)
388  *			:			:
389  * 			|			|
390  * 0xFFFF8000.00000000  |-----------------------|
391  * 			|			|
392  * 			| VA Hole / unused	|
393  * 			|			|
394  * 0x00008000.00000000  |-----------------------|
395  *			|			|
396  *			|			|
397  *			:			:
398  *			|	user heap	|	(grows upwards)
399  *			|			|
400  *			|	user data	|
401  *			|-----------------------|
402  *			|	user text	|
403  * 0x00000000.04000000  |-----------------------|
404  *			|	invalid		|
405  * 0x00000000.00000000	+-----------------------+
406  *
407  * A 32 bit app on the 64 bit kernel sees the same layout as on the 32 bit
408  * kernel, except that userlimit is raised to 0xfe000000
409  *
410  * Floating values:
411  *
412  * valloc_base: start of the kernel's memory management/tracking data
413  * structures.  This region contains page_t structures for
414  * physical memory, memsegs, memlists, and the page hash.
415  *
416  * core_base: start of the kernel's "core" heap area on 64-bit systems.
417  * This area is intended to be used for global data as well as for module
418  * text/data that does not fit into the nucleus pages.  The core heap is
419  * restricted to a 2GB range, allowing every address within it to be
420  * accessed using rip-relative addressing
421  *
422  * ekernelheap: end of kernelheap and start of segmap.
423  *
424  * kernelheap: start of kernel heap.  On 32-bit systems, this starts right
425  * above a red zone that separates the user's address space from the
426  * kernel's.  On 64-bit systems, it sits above segkp and segkpm.
427  *
428  * segmap_start: start of segmap. The length of segmap can be modified
429  * by changing segmapsize in /etc/system (preferred) or eeprom (deprecated).
430  * The default length is 16MB on 32-bit systems and 64MB on 64-bit systems.
431  *
432  * kernelbase: On a 32-bit kernel the default value of 0xd4000000 will be
433  * decreased by 2X the size required for page_t.  This allows the kernel
434  * heap to grow in size with physical memory.  With sizeof(page_t) == 80
435  * bytes, the following shows the values of kernelbase and kernel heap
436  * sizes for different memory configurations (assuming default segmap and
437  * segkp sizes).
438  *
439  *	mem	size for	kernelbase	kernel heap
440  *	size	page_t's			size
441  *	----	---------	----------	-----------
442  *	1gb	0x01400000	0xd1800000	684MB
443  *	2gb	0x02800000	0xcf000000	704MB
444  *	4gb	0x05000000	0xca000000	744MB
445  *	6gb	0x07800000	0xc5000000	784MB
446  *	8gb	0x0a000000	0xc0000000	824MB
447  *	16gb	0x14000000	0xac000000	984MB
448  *	32gb	0x28000000	0x84000000	1304MB
449  *	64gb	0x50000000	0x34000000	1944MB (*)
450  *
451  * kernelbase is less than the abi minimum of 0xc0000000 for memory
452  * configurations above 8gb.
453  *
454  * (*) support for memory configurations above 32gb will require manual tuning
455  * of kernelbase to balance out the need of user applications.
456  */
457 
458 /* real-time-clock initialization parameters */
459 extern time_t process_rtc_config_file(void);
460 
461 char		*final_kernelheap;
462 char		*boot_kernelheap;
463 uintptr_t	kernelbase;
464 uintptr_t	postbootkernelbase;	/* not set till boot loader is gone */
465 uintptr_t	eprom_kernelbase;
466 size_t		segmapsize;
467 static uintptr_t segmap_reserved;
468 uintptr_t	segmap_start;
469 int		segmapfreelists;
470 pgcnt_t		npages;
471 pgcnt_t		orig_npages;
472 size_t		core_size;		/* size of "core" heap */
473 uintptr_t	core_base;		/* base address of "core" heap */
474 
475 /*
476  * List of bootstrap pages. We mark these as allocated in startup.
477  * release_bootstrap() will free them when we're completely done with
478  * the bootstrap.
479  */
480 static page_t *bootpages;
481 
482 /*
483  * boot time pages that have a vnode from the ramdisk will keep that forever.
484  */
485 static page_t *rd_pages;
486 
487 struct system_hardware system_hardware;
488 
489 /*
490  * Enable some debugging messages concerning memory usage...
491  */
492 static void
493 print_memlist(char *title, struct memlist *mp)
494 {
495 	prom_printf("MEMLIST: %s:\n", title);
496 	while (mp != NULL)  {
497 		prom_printf("\tAddress 0x%" PRIx64 ", size 0x%" PRIx64 "\n",
498 		    mp->address, mp->size);
499 		mp = mp->next;
500 	}
501 }
502 
503 /*
504  * XX64 need a comment here.. are these just default values, surely
505  * we read the "cpuid" type information to figure this out.
506  */
507 int	l2cache_sz = 0x80000;
508 int	l2cache_linesz = 0x40;
509 int	l2cache_assoc = 1;
510 
511 static size_t	textrepl_min_gb = 10;
512 
513 /*
514  * on 64 bit we use a predifined VA range for mapping devices in the kernel
515  * on 32 bit the mappings are intermixed in the heap, so we use a bit map
516  */
517 #ifdef __amd64
518 
519 vmem_t		*device_arena;
520 uintptr_t	toxic_addr = (uintptr_t)NULL;
521 size_t		toxic_size = 1024 * 1024 * 1024; /* Sparc uses 1 gig too */
522 
523 #else	/* __i386 */
524 
525 ulong_t		*toxic_bit_map;	/* one bit for each 4k of VA in heap_arena */
526 size_t		toxic_bit_map_len = 0;	/* in bits */
527 
528 #endif	/* __i386 */
529 
530 /*
531  * Simple boot time debug facilities
532  */
533 static char *prm_dbg_str[] = {
534 	"%s:%d: '%s' is 0x%x\n",
535 	"%s:%d: '%s' is 0x%llx\n"
536 };
537 
538 int prom_debug;
539 
540 #define	PRM_DEBUG(q)	if (prom_debug) 	\
541 	prom_printf(prm_dbg_str[sizeof (q) >> 3], "startup.c", __LINE__, #q, q);
542 #define	PRM_POINT(q)	if (prom_debug) 	\
543 	prom_printf("%s:%d: %s\n", "startup.c", __LINE__, q);
544 
545 /*
546  * This structure is used to keep track of the intial allocations
547  * done in startup_memlist(). The value of NUM_ALLOCATIONS needs to
548  * be >= the number of ADD_TO_ALLOCATIONS() executed in the code.
549  */
550 #define	NUM_ALLOCATIONS 7
551 int num_allocations = 0;
552 struct {
553 	void **al_ptr;
554 	size_t al_size;
555 } allocations[NUM_ALLOCATIONS];
556 size_t valloc_sz = 0;
557 uintptr_t valloc_base;
558 
559 #define	ADD_TO_ALLOCATIONS(ptr, size) {					\
560 		size = ROUND_UP_PAGE(size);		 		\
561 		if (num_allocations == NUM_ALLOCATIONS)			\
562 			panic("too many ADD_TO_ALLOCATIONS()");		\
563 		allocations[num_allocations].al_ptr = (void**)&ptr;	\
564 		allocations[num_allocations].al_size = size;		\
565 		valloc_sz += size;					\
566 		++num_allocations;				 	\
567 	}
568 
569 /*
570  * Allocate all the initial memory needed by the page allocator.
571  */
572 static void
573 perform_allocations(void)
574 {
575 	caddr_t mem;
576 	int i;
577 	int valloc_align;
578 
579 	PRM_DEBUG(valloc_base);
580 	PRM_DEBUG(valloc_sz);
581 	valloc_align = mmu.level_size[mmu.max_page_level > 0];
582 	mem = BOP_ALLOC(bootops, (caddr_t)valloc_base, valloc_sz, valloc_align);
583 	if (mem != (caddr_t)valloc_base)
584 		panic("BOP_ALLOC() failed");
585 	bzero(mem, valloc_sz);
586 	for (i = 0; i < num_allocations; ++i) {
587 		*allocations[i].al_ptr = (void *)mem;
588 		mem += allocations[i].al_size;
589 	}
590 }
591 
592 /*
593  * Our world looks like this at startup time.
594  *
595  * In a 32-bit OS, boot loads the kernel text at 0xfe800000 and kernel data
596  * at 0xfec00000.  On a 64-bit OS, kernel text and data are loaded at
597  * 0xffffffff.fe800000 and 0xffffffff.fec00000 respectively.  Those
598  * addresses are fixed in the binary at link time.
599  *
600  * On the text page:
601  * unix/genunix/krtld/module text loads.
602  *
603  * On the data page:
604  * unix/genunix/krtld/module data loads.
605  *
606  * Machine-dependent startup code
607  */
608 void
609 startup(void)
610 {
611 	extern void startup_bios_disk(void);
612 	extern void startup_pci_bios(void);
613 	/*
614 	 * Make sure that nobody tries to use sekpm until we have
615 	 * initialized it properly.
616 	 */
617 #if defined(__amd64)
618 	kpm_desired = kpm_enable;
619 #endif
620 	kpm_enable = 0;
621 
622 	progressbar_init();
623 	startup_init();
624 	startup_memlist();
625 	startup_kmem();
626 	startup_pci_bios();
627 	startup_modules();
628 	startup_bios_disk();
629 	startup_vm();
630 	startup_end();
631 	progressbar_start();
632 }
633 
634 static void
635 startup_init()
636 {
637 	PRM_POINT("startup_init() starting...");
638 
639 	/*
640 	 * Complete the extraction of cpuid data
641 	 */
642 	cpuid_pass2(CPU);
643 
644 	(void) check_boot_version(BOP_GETVERSION(bootops));
645 
646 	/*
647 	 * Check for prom_debug in boot environment
648 	 */
649 	if (BOP_GETPROPLEN(bootops, "prom_debug") >= 0) {
650 		++prom_debug;
651 		PRM_POINT("prom_debug found in boot enviroment");
652 	}
653 
654 	/*
655 	 * Collect node, cpu and memory configuration information.
656 	 */
657 	get_system_configuration();
658 
659 	/*
660 	 * Halt if this is an unsupported processor.
661 	 */
662 	if (x86_type == X86_TYPE_486 || x86_type == X86_TYPE_CYRIX_486) {
663 		printf("\n486 processor (\"%s\") detected.\n",
664 		    CPU->cpu_brandstr);
665 		halt("This processor is not supported by this release "
666 		    "of Solaris.");
667 	}
668 
669 	PRM_POINT("startup_init() done");
670 }
671 
672 /*
673  * Callback for copy_memlist_filter() to filter nucleus, kadb/kmdb, (ie.
674  * everything mapped above KERNEL_TEXT) pages from phys_avail. Note it
675  * also filters out physical page zero.  There is some reliance on the
676  * boot loader allocating only a few contiguous physical memory chunks.
677  */
678 static void
679 avail_filter(uint64_t *addr, uint64_t *size)
680 {
681 	uintptr_t va;
682 	uintptr_t next_va;
683 	pfn_t pfn;
684 	uint64_t pfn_addr;
685 	uint64_t pfn_eaddr;
686 	uint_t prot;
687 	size_t len;
688 	uint_t change;
689 
690 	if (prom_debug)
691 		prom_printf("\tFilter: in: a=%" PRIx64 ", s=%" PRIx64 "\n",
692 		    *addr, *size);
693 
694 	/*
695 	 * page zero is required for BIOS.. never make it available
696 	 */
697 	if (*addr == 0) {
698 		*addr += MMU_PAGESIZE;
699 		*size -= MMU_PAGESIZE;
700 	}
701 
702 	/*
703 	 * First we trim from the front of the range. Since kbm_probe()
704 	 * walks ranges in virtual order, but addr/size are physical, we need
705 	 * to the list until no changes are seen.  This deals with the case
706 	 * where page "p" is mapped at v, page "p + PAGESIZE" is mapped at w
707 	 * but w < v.
708 	 */
709 	do {
710 		change = 0;
711 		for (va = KERNEL_TEXT;
712 		    *size > 0 && kbm_probe(&va, &len, &pfn, &prot) != 0;
713 		    va = next_va) {
714 
715 			next_va = va + len;
716 			pfn_addr = pfn_to_pa(pfn);
717 			pfn_eaddr = pfn_addr + len;
718 
719 			if (pfn_addr <= *addr && pfn_eaddr > *addr) {
720 				change = 1;
721 				while (*size > 0 && len > 0) {
722 					*addr += MMU_PAGESIZE;
723 					*size -= MMU_PAGESIZE;
724 					len -= MMU_PAGESIZE;
725 				}
726 			}
727 		}
728 		if (change && prom_debug)
729 			prom_printf("\t\ttrim: a=%" PRIx64 ", s=%" PRIx64 "\n",
730 			    *addr, *size);
731 	} while (change);
732 
733 	/*
734 	 * Trim pages from the end of the range.
735 	 */
736 	for (va = KERNEL_TEXT;
737 	    *size > 0 && kbm_probe(&va, &len, &pfn, &prot) != 0;
738 	    va = next_va) {
739 
740 		next_va = va + len;
741 		pfn_addr = pfn_to_pa(pfn);
742 
743 		if (pfn_addr >= *addr && pfn_addr < *addr + *size)
744 			*size = pfn_addr - *addr;
745 	}
746 
747 	if (prom_debug)
748 		prom_printf("\tFilter out: a=%" PRIx64 ", s=%" PRIx64 "\n",
749 		    *addr, *size);
750 }
751 
752 static void
753 kpm_init()
754 {
755 	struct segkpm_crargs b;
756 
757 	/*
758 	 * These variables were all designed for sfmmu in which segkpm is
759 	 * mapped using a single pagesize - either 8KB or 4MB.  On x86, we
760 	 * might use 2+ page sizes on a single machine, so none of these
761 	 * variables have a single correct value.  They are set up as if we
762 	 * always use a 4KB pagesize, which should do no harm.  In the long
763 	 * run, we should get rid of KPM's assumption that only a single
764 	 * pagesize is used.
765 	 */
766 	kpm_pgshft = MMU_PAGESHIFT;
767 	kpm_pgsz =  MMU_PAGESIZE;
768 	kpm_pgoff = MMU_PAGEOFFSET;
769 	kpmp2pshft = 0;
770 	kpmpnpgs = 1;
771 	ASSERT(((uintptr_t)kpm_vbase & (kpm_pgsz - 1)) == 0);
772 
773 	PRM_POINT("about to create segkpm");
774 	rw_enter(&kas.a_lock, RW_WRITER);
775 
776 	if (seg_attach(&kas, kpm_vbase, kpm_size, segkpm) < 0)
777 		panic("cannot attach segkpm");
778 
779 	b.prot = PROT_READ | PROT_WRITE;
780 	b.nvcolors = 1;
781 
782 	if (segkpm_create(segkpm, (caddr_t)&b) != 0)
783 		panic("segkpm_create segkpm");
784 
785 	rw_exit(&kas.a_lock);
786 }
787 
788 /*
789  * The debug info page provides enough information to allow external
790  * inspectors (e.g. when running under a hypervisor) to bootstrap
791  * themselves into allowing full-blown kernel debugging.
792  */
793 static void
794 init_debug_info(void)
795 {
796 	caddr_t mem;
797 	debug_info_t *di;
798 
799 #ifndef __lint
800 	ASSERT(sizeof (debug_info_t) < MMU_PAGESIZE);
801 #endif
802 
803 	mem = BOP_ALLOC(bootops, (caddr_t)DEBUG_INFO_VA, MMU_PAGESIZE,
804 	    MMU_PAGESIZE);
805 
806 	if (mem != (caddr_t)DEBUG_INFO_VA)
807 		panic("BOP_ALLOC() failed");
808 	bzero(mem, MMU_PAGESIZE);
809 
810 	di = (debug_info_t *)mem;
811 
812 	di->di_magic = DEBUG_INFO_MAGIC;
813 	di->di_version = DEBUG_INFO_VERSION;
814 }
815 
816 /*
817  * Build the memlists and other kernel essential memory system data structures.
818  * This is everything at valloc_base.
819  */
820 static void
821 startup_memlist(void)
822 {
823 	size_t memlist_sz;
824 	size_t memseg_sz;
825 	size_t pagehash_sz;
826 	size_t pp_sz;
827 	uintptr_t va;
828 	size_t len;
829 	uint_t prot;
830 	pfn_t pfn;
831 	int memblocks;
832 	caddr_t pagecolor_mem;
833 	size_t pagecolor_memsz;
834 	caddr_t page_ctrs_mem;
835 	size_t page_ctrs_size;
836 	struct memlist *current;
837 	extern void startup_build_mem_nodes(struct memlist *);
838 
839 	/* XX64 fix these - they should be in include files */
840 	extern size_t page_coloring_init(uint_t, int, int);
841 	extern void page_coloring_setup(caddr_t);
842 
843 	PRM_POINT("startup_memlist() starting...");
844 
845 	/*
846 	 * Use leftover large page nucleus text/data space for loadable modules.
847 	 * Use at most MODTEXT/MODDATA.
848 	 */
849 	len = kbm_nucleus_size;
850 	ASSERT(len > MMU_PAGESIZE);
851 
852 	moddata = (caddr_t)ROUND_UP_PAGE(e_data);
853 	e_moddata = (caddr_t)P2ROUNDUP((uintptr_t)e_data, (uintptr_t)len);
854 	if (e_moddata - moddata > MODDATA)
855 		e_moddata = moddata + MODDATA;
856 
857 	modtext = (caddr_t)ROUND_UP_PAGE(e_text);
858 	e_modtext = (caddr_t)P2ROUNDUP((uintptr_t)e_text, (uintptr_t)len);
859 	if (e_modtext - modtext > MODTEXT)
860 		e_modtext = modtext + MODTEXT;
861 
862 	econtig = e_moddata;
863 
864 	PRM_DEBUG(modtext);
865 	PRM_DEBUG(e_modtext);
866 	PRM_DEBUG(moddata);
867 	PRM_DEBUG(e_moddata);
868 	PRM_DEBUG(econtig);
869 
870 	/*
871 	 * Examine the boot loader physical memory map to find out:
872 	 * - total memory in system - physinstalled
873 	 * - the max physical address - physmax
874 	 * - the number of discontiguous segments of memory.
875 	 */
876 	if (prom_debug)
877 		print_memlist("boot physinstalled",
878 		    bootops->boot_mem->physinstalled);
879 	installed_top_size(bootops->boot_mem->physinstalled, &physmax,
880 	    &physinstalled, &memblocks);
881 	PRM_DEBUG(physmax);
882 	PRM_DEBUG(physinstalled);
883 	PRM_DEBUG(memblocks);
884 
885 	/*
886 	 * Initialize hat's mmu parameters.
887 	 * Check for enforce-prot-exec in boot environment. It's used to
888 	 * enable/disable support for the page table entry NX bit.
889 	 * The default is to enforce PROT_EXEC on processors that support NX.
890 	 * Boot seems to round up the "len", but 8 seems to be big enough.
891 	 */
892 	mmu_init();
893 
894 #ifdef	__i386
895 	/*
896 	 * physmax is lowered if there is more memory than can be
897 	 * physically addressed in 32 bit (PAE/non-PAE) modes.
898 	 */
899 	if (mmu.pae_hat) {
900 		if (PFN_ABOVE64G(physmax)) {
901 			physinstalled -= (physmax - (PFN_64G - 1));
902 			physmax = PFN_64G - 1;
903 		}
904 	} else {
905 		if (PFN_ABOVE4G(physmax)) {
906 			physinstalled -= (physmax - (PFN_4G - 1));
907 			physmax = PFN_4G - 1;
908 		}
909 	}
910 #endif
911 
912 	startup_build_mem_nodes(bootops->boot_mem->physinstalled);
913 
914 	if (BOP_GETPROPLEN(bootops, "enforce-prot-exec") >= 0) {
915 		int len = BOP_GETPROPLEN(bootops, "enforce-prot-exec");
916 		char value[8];
917 
918 		if (len < 8)
919 			(void) BOP_GETPROP(bootops, "enforce-prot-exec", value);
920 		else
921 			(void) strcpy(value, "");
922 		if (strcmp(value, "off") == 0)
923 			mmu.pt_nx = 0;
924 	}
925 	PRM_DEBUG(mmu.pt_nx);
926 
927 	/*
928 	 * We will need page_t's for every page in the system, except for
929 	 * memory mapped at or above above the start of the kernel text segment.
930 	 *
931 	 * pages above e_modtext are attributed to kernel debugger (obp_pages)
932 	 */
933 	npages = physinstalled - 1; /* avail_filter() skips page 0, so "- 1" */
934 	obp_pages = 0;
935 	va = KERNEL_TEXT;
936 	while (kbm_probe(&va, &len, &pfn, &prot) != 0) {
937 		npages -= len >> MMU_PAGESHIFT;
938 		if (va >= (uintptr_t)e_moddata)
939 			obp_pages += len >> MMU_PAGESHIFT;
940 		va += len;
941 	}
942 	PRM_DEBUG(npages);
943 	PRM_DEBUG(obp_pages);
944 
945 	/*
946 	 * If physmem is patched to be non-zero, use it instead of
947 	 * the computed value unless it is larger than the real
948 	 * amount of memory on hand.
949 	 */
950 	if (physmem == 0 || physmem > npages) {
951 		physmem = npages;
952 	} else if (physmem < npages) {
953 		orig_npages = npages;
954 		npages = physmem;
955 	}
956 	PRM_DEBUG(physmem);
957 
958 	/*
959 	 * We now compute the sizes of all the  initial allocations for
960 	 * structures the kernel needs in order do kmem_alloc(). These
961 	 * include:
962 	 *	memsegs
963 	 *	memlists
964 	 *	page hash table
965 	 *	page_t's
966 	 *	page coloring data structs
967 	 */
968 	memseg_sz = sizeof (struct memseg) * (memblocks + POSS_NEW_FRAGMENTS);
969 	ADD_TO_ALLOCATIONS(memseg_base, memseg_sz);
970 	PRM_DEBUG(memseg_sz);
971 
972 	/*
973 	 * Reserve space for memlists. There's no real good way to know exactly
974 	 * how much room we'll need, but this should be a good upper bound.
975 	 */
976 	memlist_sz = ROUND_UP_PAGE(2 * sizeof (struct memlist) *
977 	    (memblocks + POSS_NEW_FRAGMENTS));
978 	ADD_TO_ALLOCATIONS(memlist, memlist_sz);
979 	PRM_DEBUG(memlist_sz);
980 
981 	/*
982 	 * The page structure hash table size is a power of 2
983 	 * such that the average hash chain length is PAGE_HASHAVELEN.
984 	 */
985 	page_hashsz = npages / PAGE_HASHAVELEN;
986 	page_hashsz = 1 << highbit(page_hashsz);
987 	pagehash_sz = sizeof (struct page *) * page_hashsz;
988 	ADD_TO_ALLOCATIONS(page_hash, pagehash_sz);
989 	PRM_DEBUG(pagehash_sz);
990 
991 	/*
992 	 * Set aside room for the page structures themselves.
993 	 */
994 	PRM_DEBUG(npages);
995 	pp_sz = sizeof (struct page) * npages;
996 	ADD_TO_ALLOCATIONS(pp_base, pp_sz);
997 	PRM_DEBUG(pp_sz);
998 
999 	/*
1000 	 * determine l2 cache info and memory size for page coloring
1001 	 */
1002 	(void) getl2cacheinfo(CPU,
1003 	    &l2cache_sz, &l2cache_linesz, &l2cache_assoc);
1004 	pagecolor_memsz =
1005 	    page_coloring_init(l2cache_sz, l2cache_linesz, l2cache_assoc);
1006 	ADD_TO_ALLOCATIONS(pagecolor_mem, pagecolor_memsz);
1007 	PRM_DEBUG(pagecolor_memsz);
1008 
1009 	page_ctrs_size = page_ctrs_sz();
1010 	ADD_TO_ALLOCATIONS(page_ctrs_mem, page_ctrs_size);
1011 	PRM_DEBUG(page_ctrs_size);
1012 
1013 #if defined(__amd64)
1014 	valloc_sz = ROUND_UP_LPAGE(valloc_sz);
1015 	valloc_base = VALLOC_BASE;
1016 #else	/* __i386 */
1017 	valloc_base = (uintptr_t)(MISC_VA_BASE - valloc_sz);
1018 	valloc_base = P2ALIGN(valloc_base, mmu.level_size[1]);
1019 #endif	/* __i386 */
1020 	PRM_DEBUG(valloc_base);
1021 
1022 	/*
1023 	 * do all the initial allocations
1024 	 */
1025 	perform_allocations();
1026 
1027 	/*
1028 	 * Build phys_install and phys_avail in kernel memspace.
1029 	 * - phys_install should be all memory in the system.
1030 	 * - phys_avail is phys_install minus any memory mapped before this
1031 	 *    point above KERNEL_TEXT.
1032 	 */
1033 	current = phys_install = memlist;
1034 	copy_memlist_filter(bootops->boot_mem->physinstalled, &current, NULL);
1035 	if ((caddr_t)current > (caddr_t)memlist + memlist_sz)
1036 		panic("physinstalled was too big!");
1037 	if (prom_debug)
1038 		print_memlist("phys_install", phys_install);
1039 
1040 	phys_avail = current;
1041 	PRM_POINT("Building phys_avail:\n");
1042 	copy_memlist_filter(bootops->boot_mem->physinstalled, &current,
1043 	    avail_filter);
1044 	if ((caddr_t)current > (caddr_t)memlist + memlist_sz)
1045 		panic("physavail was too big!");
1046 	if (prom_debug)
1047 		print_memlist("phys_avail", phys_avail);
1048 
1049 	/*
1050 	 * setup page coloring
1051 	 */
1052 	page_coloring_setup(pagecolor_mem);
1053 	page_lock_init();	/* currently a no-op */
1054 
1055 	/*
1056 	 * free page list counters
1057 	 */
1058 	(void) page_ctrs_alloc(page_ctrs_mem);
1059 
1060 	/*
1061 	 * Initialize the page structures from the memory lists.
1062 	 */
1063 	availrmem_initial = availrmem = freemem = 0;
1064 	PRM_POINT("Calling kphysm_init()...");
1065 	npages = kphysm_init(pp_base, npages);
1066 	PRM_POINT("kphysm_init() done");
1067 	PRM_DEBUG(npages);
1068 
1069 	init_debug_info();
1070 
1071 	/*
1072 	 * Now that page_t's have been initialized, remove all the
1073 	 * initial allocation pages from the kernel free page lists.
1074 	 */
1075 	boot_mapin((caddr_t)valloc_base, valloc_sz);
1076 	boot_mapin((caddr_t)GDT_VA, MMU_PAGESIZE);
1077 	boot_mapin((caddr_t)DEBUG_INFO_VA, MMU_PAGESIZE);
1078 	PRM_POINT("startup_memlist() done");
1079 
1080 	PRM_DEBUG(valloc_sz);
1081 
1082 #if defined(__amd64)
1083 	if ((availrmem >> (30 - MMU_PAGESHIFT)) >=
1084 	    textrepl_min_gb && l2cache_sz <= 2 << 20) {
1085 		extern size_t textrepl_size_thresh;
1086 		textrepl_size_thresh = (16 << 20) - 1;
1087 	}
1088 #endif
1089 }
1090 
1091 /*
1092  * Layout the kernel's part of address space and initialize kmem allocator.
1093  */
1094 static void
1095 startup_kmem(void)
1096 {
1097 	extern void page_set_colorequiv_arr(void);
1098 
1099 	PRM_POINT("startup_kmem() starting...");
1100 
1101 #if defined(__amd64)
1102 	if (eprom_kernelbase && eprom_kernelbase != KERNELBASE)
1103 		cmn_err(CE_NOTE, "!kernelbase cannot be changed on 64-bit "
1104 		    "systems.");
1105 	kernelbase = (uintptr_t)KERNELBASE;
1106 	core_base = (uintptr_t)COREHEAP_BASE;
1107 	core_size = (size_t)MISC_VA_BASE - COREHEAP_BASE;
1108 #else	/* __i386 */
1109 	/*
1110 	 * We configure kernelbase based on:
1111 	 *
1112 	 * 1. user specified kernelbase via eeprom command. Value cannot exceed
1113 	 *    KERNELBASE_MAX. we large page align eprom_kernelbase
1114 	 *
1115 	 * 2. Default to KERNELBASE and adjust to 2X less the size for page_t.
1116 	 *    On large memory systems we must lower kernelbase to allow
1117 	 *    enough room for page_t's for all of memory.
1118 	 *
1119 	 * The value set here, might be changed a little later.
1120 	 */
1121 	if (eprom_kernelbase) {
1122 		kernelbase = eprom_kernelbase & mmu.level_mask[1];
1123 		if (kernelbase > KERNELBASE_MAX)
1124 			kernelbase = KERNELBASE_MAX;
1125 	} else {
1126 		kernelbase = (uintptr_t)KERNELBASE;
1127 		kernelbase -= ROUND_UP_4MEG(2 * valloc_sz);
1128 	}
1129 	ASSERT((kernelbase & mmu.level_offset[1]) == 0);
1130 	core_base = valloc_base;
1131 	core_size = 0;
1132 #endif	/* __i386 */
1133 
1134 	PRM_DEBUG(core_base);
1135 	PRM_DEBUG(core_size);
1136 	PRM_DEBUG(kernelbase);
1137 
1138 	/*
1139 	 * At this point, we can only use a portion of the kernelheap that
1140 	 * will be available after we boot.  32-bit systems have this
1141 	 * limitation.
1142 	 *
1143 	 * On 32-bit systems we have to leave room to place segmap below
1144 	 * the heap.  We don't yet know how large segmap will be, so we
1145 	 * have to be very conservative.
1146 	 *
1147 	 * On 64 bit systems there should be LOTS of room so just use
1148 	 * the next 4Gig below core_base.
1149 	 */
1150 #if defined(__amd64)
1151 
1152 	boot_kernelheap = (caddr_t)core_base  - FOURGB;
1153 	segmap_reserved = 0;
1154 
1155 #else	/* __i386 */
1156 
1157 	segkp_fromheap = 1;
1158 	segmap_reserved = ROUND_UP_LPAGE(MAX(segmapsize, SEGMAPMAX));
1159 	boot_kernelheap =
1160 	    (caddr_t)ROUND_UP_LPAGE(kernelbase) + segmap_reserved;
1161 
1162 #endif	/* __i386 */
1163 	PRM_DEBUG(boot_kernelheap);
1164 	ekernelheap = (char *)core_base;
1165 	PRM_DEBUG(ekernelheap);
1166 	kernelheap = boot_kernelheap;
1167 
1168 	/*
1169 	 * If segmap is too large we can push the bottom of the kernel heap
1170 	 * higher than the base.  Or worse, it could exceed the top of the
1171 	 * VA space entirely, causing it to wrap around.
1172 	 */
1173 	if (kernelheap >= ekernelheap || (uintptr_t)kernelheap < kernelbase)
1174 		panic("too little memory available for kernelheap,"
1175 			    " use a different kernelbase");
1176 
1177 	/*
1178 	 * Now that we know the real value of kernelbase,
1179 	 * update variables that were initialized with a value of
1180 	 * KERNELBASE (in common/conf/param.c).
1181 	 *
1182 	 * XXX	The problem with this sort of hackery is that the
1183 	 *	compiler just may feel like putting the const declarations
1184 	 *	(in param.c) into the .text section.  Perhaps they should
1185 	 *	just be declared as variables there?
1186 	 */
1187 
1188 #if defined(__amd64)
1189 	ASSERT(_kernelbase == KERNELBASE);
1190 	ASSERT(_userlimit == USERLIMIT);
1191 #else
1192 	*(uintptr_t *)&_kernelbase = kernelbase;
1193 	*(uintptr_t *)&_userlimit = kernelbase;
1194 	*(uintptr_t *)&_userlimit32 = _userlimit;
1195 #endif
1196 	PRM_DEBUG(_kernelbase);
1197 	PRM_DEBUG(_userlimit);
1198 	PRM_DEBUG(_userlimit32);
1199 
1200 	/*
1201 	 * Initialize the kernel heap. Note 3rd argument must be > 1st.
1202 	 */
1203 	kernelheap_init(boot_kernelheap, ekernelheap,
1204 	    boot_kernelheap + MMU_PAGESIZE,
1205 	    (void *)core_base, (void *)(core_base + core_size));
1206 
1207 	/*
1208 	 * Initialize kernel memory allocator.
1209 	 */
1210 	kmem_init();
1211 
1212 	/*
1213 	 * Factor in colorequiv to check additional 'equivalent' bins
1214 	 */
1215 	page_set_colorequiv_arr();
1216 
1217 	/*
1218 	 * print this out early so that we know what's going on
1219 	 */
1220 	cmn_err(CE_CONT, "?features: %b\n", x86_feature, FMT_X86_FEATURE);
1221 
1222 	/*
1223 	 * Initialize bp_mapin().
1224 	 */
1225 	bp_init(MMU_PAGESIZE, HAT_STORECACHING_OK);
1226 
1227 	/*
1228 	 * orig_npages is non-zero if physmem has been configured for less
1229 	 * than the available memory.
1230 	 */
1231 	if (orig_npages) {
1232 #ifdef __i386
1233 		/*
1234 		 * use npages for physmem in case it has been temporarily
1235 		 * modified via /etc/system in kmem_init/mod_read_system_file.
1236 		 */
1237 		if (npages == PHYSMEM32) {
1238 			cmn_err(CE_WARN, "!Due to 32-bit virtual"
1239 			    " address space limitations, limiting"
1240 			    " physmem to 0x%lx of 0x%lx available pages",
1241 			    npages, orig_npages);
1242 		} else {
1243 			cmn_err(CE_WARN, "!limiting physmem to 0x%lx of"
1244 			    " 0x%lx available pages", npages, orig_npages);
1245 		}
1246 #else
1247 		cmn_err(CE_WARN, "!limiting physmem to 0x%lx of"
1248 		    " 0x%lx available pages", npages, orig_npages);
1249 #endif
1250 	}
1251 #if defined(__i386)
1252 	if (eprom_kernelbase && (eprom_kernelbase != kernelbase))
1253 		cmn_err(CE_WARN, "kernelbase value, User specified 0x%lx, "
1254 		    "System using 0x%lx",
1255 		    (uintptr_t)eprom_kernelbase, (uintptr_t)kernelbase);
1256 #endif
1257 
1258 #ifdef	KERNELBASE_ABI_MIN
1259 	if (kernelbase < (uintptr_t)KERNELBASE_ABI_MIN) {
1260 		cmn_err(CE_NOTE, "!kernelbase set to 0x%lx, system is not "
1261 		    "i386 ABI compliant.", (uintptr_t)kernelbase);
1262 	}
1263 #endif
1264 
1265 	PRM_POINT("startup_kmem() done");
1266 }
1267 
1268 static void
1269 startup_modules(void)
1270 {
1271 	unsigned int i;
1272 	extern void prom_setup(void);
1273 
1274 	PRM_POINT("startup_modules() starting...");
1275 	/*
1276 	 * Initialize ten-micro second timer so that drivers will
1277 	 * not get short changed in their init phase. This was
1278 	 * not getting called until clkinit which, on fast cpu's
1279 	 * caused the drv_usecwait to be way too short.
1280 	 */
1281 	microfind();
1282 
1283 	/*
1284 	 * Read the GMT lag from /etc/rtc_config.
1285 	 */
1286 	sgmtl(process_rtc_config_file());
1287 
1288 	/*
1289 	 * Calculate default settings of system parameters based upon
1290 	 * maxusers, yet allow to be overridden via the /etc/system file.
1291 	 */
1292 	param_calc(0);
1293 
1294 	mod_setup();
1295 
1296 	/*
1297 	 * Initialize system parameters.
1298 	 */
1299 	param_init();
1300 
1301 	/*
1302 	 * Initialize the default brands
1303 	 */
1304 	brand_init();
1305 
1306 	/*
1307 	 * maxmem is the amount of physical memory we're playing with.
1308 	 */
1309 	maxmem = physmem;
1310 
1311 	/*
1312 	 * Initialize the hat layer.
1313 	 */
1314 	hat_init();
1315 
1316 	/*
1317 	 * Initialize segment management stuff.
1318 	 */
1319 	seg_init();
1320 
1321 	if (modload("fs", "specfs") == -1)
1322 		halt("Can't load specfs");
1323 
1324 	if (modload("fs", "devfs") == -1)
1325 		halt("Can't load devfs");
1326 
1327 	if (modload("fs", "dev") == -1)
1328 		halt("Can't load dev");
1329 
1330 	(void) modloadonly("sys", "lbl_edition");
1331 
1332 	dispinit();
1333 
1334 	/*
1335 	 * This is needed here to initialize hw_serial[] for cluster booting.
1336 	 */
1337 	if ((i = modload("misc", "sysinit")) != (unsigned int)-1)
1338 		(void) modunload(i);
1339 	else
1340 		cmn_err(CE_CONT, "sysinit load failed");
1341 
1342 	/* Read cluster configuration data. */
1343 	clconf_init();
1344 
1345 	/*
1346 	 * Create a kernel device tree. First, create rootnex and
1347 	 * then invoke bus specific code to probe devices.
1348 	 */
1349 	setup_ddi();
1350 
1351 	/*
1352 	 * Set up the CPU module subsystem.  Modifies the device tree, so it
1353 	 * must be done after setup_ddi().
1354 	 */
1355 	cmi_init();
1356 
1357 	/*
1358 	 * Initialize the MCA handlers
1359 	 */
1360 	if (x86_feature & X86_MCA)
1361 		cmi_mca_init();
1362 
1363 	/*
1364 	 * Fake a prom tree such that /dev/openprom continues to work
1365 	 */
1366 	PRM_POINT("startup_modules: calling prom_setup...");
1367 	prom_setup();
1368 	PRM_POINT("startup_modules: done");
1369 
1370 	/*
1371 	 * Load all platform specific modules
1372 	 */
1373 	PRM_POINT("startup_modules: calling psm_modload...");
1374 	psm_modload();
1375 
1376 	PRM_POINT("startup_modules() done");
1377 }
1378 
1379 /*
1380  * claim a "setaside" boot page for use in the kernel
1381  */
1382 page_t *
1383 boot_claim_page(pfn_t pfn)
1384 {
1385 	page_t *pp;
1386 
1387 	pp = page_numtopp_nolock(pfn);
1388 	ASSERT(pp != NULL);
1389 
1390 	if (PP_ISBOOTPAGES(pp)) {
1391 		if (pp->p_next != NULL)
1392 			pp->p_next->p_prev = pp->p_prev;
1393 		if (pp->p_prev == NULL)
1394 			bootpages = pp->p_next;
1395 		else
1396 			pp->p_prev->p_next = pp->p_next;
1397 	} else {
1398 		/*
1399 		 * htable_attach() expects a base pagesize page
1400 		 */
1401 		if (pp->p_szc != 0)
1402 			page_boot_demote(pp);
1403 		pp = page_numtopp(pfn, SE_EXCL);
1404 	}
1405 	return (pp);
1406 }
1407 
1408 /*
1409  * Walk through the pagetables looking for pages mapped in by boot.  If the
1410  * setaside flag is set the pages are expected to be returned to the
1411  * kernel later in boot, so we add them to the bootpages list.
1412  */
1413 static void
1414 protect_boot_range(uintptr_t low, uintptr_t high, int setaside)
1415 {
1416 	uintptr_t va = low;
1417 	size_t len;
1418 	uint_t prot;
1419 	pfn_t pfn;
1420 	page_t *pp;
1421 	pgcnt_t boot_protect_cnt = 0;
1422 
1423 	while (kbm_probe(&va, &len, &pfn, &prot) != 0 && va < high) {
1424 		if (va + len >= high)
1425 			panic("0x%lx byte mapping at 0x%p exceeds boot's "
1426 			    "legal range.", len, (void *)va);
1427 
1428 		while (len > 0) {
1429 			pp = page_numtopp_alloc(pfn);
1430 			if (pp != NULL) {
1431 				if (setaside == 0)
1432 					panic("Unexpected mapping by boot.  "
1433 					    "addr=%p pfn=%lx\n",
1434 					    (void *)va, pfn);
1435 
1436 				pp->p_next = bootpages;
1437 				pp->p_prev = NULL;
1438 				PP_SETBOOTPAGES(pp);
1439 				if (bootpages != NULL) {
1440 					bootpages->p_prev = pp;
1441 				}
1442 				bootpages = pp;
1443 				++boot_protect_cnt;
1444 			}
1445 
1446 			++pfn;
1447 			len -= MMU_PAGESIZE;
1448 			va += MMU_PAGESIZE;
1449 		}
1450 	}
1451 	PRM_DEBUG(boot_protect_cnt);
1452 }
1453 
1454 /*
1455  * Finish initializing the VM system, now that we are no longer
1456  * relying on the boot time memory allocators.
1457  */
1458 static void
1459 startup_vm(void)
1460 {
1461 	struct segmap_crargs a;
1462 
1463 	extern int use_brk_lpg, use_stk_lpg;
1464 
1465 	PRM_POINT("startup_vm() starting...");
1466 
1467 	/*
1468 	 * Establish the final size of the kernel's heap, size of segmap,
1469 	 * segkp, etc.
1470 	 */
1471 
1472 #if defined(__amd64)
1473 
1474 	/*
1475 	 * Check if there is enough virtual address space in KPM region to
1476 	 * map physmax.
1477 	 */
1478 	kpm_vbase = (caddr_t)(uintptr_t)SEGKPM_BASE;
1479 	kpm_size = 0;
1480 	if (kpm_desired) {
1481 		kpm_size = ROUND_UP_LPAGE(mmu_ptob(physmax + 1));
1482 		if ((uintptr_t)kpm_vbase + kpm_size > (uintptr_t)VALLOC_BASE) {
1483 			kpm_size = 0;
1484 			kpm_desired = 0;
1485 		}
1486 	}
1487 
1488 	PRM_DEBUG(kpm_size);
1489 	PRM_DEBUG(kpm_vbase);
1490 
1491 	/*
1492 	 * By default we create a seg_kp in 64 bit kernels, it's a little
1493 	 * faster to access than embedding it in the heap.
1494 	 */
1495 	segkp_base = (caddr_t)valloc_base + valloc_sz;
1496 	if (!segkp_fromheap) {
1497 		size_t sz = mmu_ptob(segkpsize);
1498 
1499 		/*
1500 		 * determine size of segkp
1501 		 */
1502 		if (sz < SEGKPMINSIZE || sz > SEGKPMAXSIZE) {
1503 			sz = SEGKPDEFSIZE;
1504 			cmn_err(CE_WARN, "!Illegal value for segkpsize. "
1505 			    "segkpsize has been reset to %ld pages",
1506 			    mmu_btop(sz));
1507 		}
1508 		sz = MIN(sz, MAX(SEGKPMINSIZE, mmu_ptob(physmem)));
1509 
1510 		segkpsize = mmu_btop(ROUND_UP_LPAGE(sz));
1511 	}
1512 	PRM_DEBUG(segkp_base);
1513 	PRM_DEBUG(segkpsize);
1514 
1515 	segzio_base = segkp_base + mmu_ptob(segkpsize);
1516 	if (segzio_fromheap) {
1517 		segziosize = 0;
1518 	} else {
1519 		size_t size;
1520 		size_t physmem_b = mmu_ptob(physmem);
1521 
1522 		/* size is in bytes, segziosize is in pages */
1523 		if (segziosize == 0) {
1524 			size = physmem_b;
1525 		} else {
1526 			size = mmu_ptob(segziosize);
1527 		}
1528 
1529 		if (size < SEGZIOMINSIZE) {
1530 			size = SEGZIOMINSIZE;
1531 		} else if (size > SEGZIOMAXSIZE) {
1532 			size = SEGZIOMAXSIZE;
1533 			/*
1534 			 * SEGZIOMAXSIZE is capped at 512gb so that segzio
1535 			 * doesn't consume all of KVA.  However, if we have a
1536 			 * system that has more thant 512gb of physical memory,
1537 			 * we can actually consume about half of the difference
1538 			 * between 512gb and the rest of the available physical
1539 			 * memory.
1540 			 */
1541 			if (physmem_b > SEGZIOMAXSIZE) {
1542 				size += (physmem_b - SEGZIOMAXSIZE) / 2;
1543 			}
1544 		}
1545 		segziosize = mmu_btop(ROUND_UP_LPAGE(size));
1546 	}
1547 	PRM_DEBUG(segziosize);
1548 	PRM_DEBUG(segzio_base);
1549 
1550 	/*
1551 	 * Put the range of VA for device mappings next, kmdb knows to not
1552 	 * grep in this range of addresses.
1553 	 */
1554 	toxic_addr =
1555 	    ROUND_UP_LPAGE((uintptr_t)segzio_base + mmu_ptob(segziosize));
1556 	PRM_DEBUG(toxic_addr);
1557 	segmap_start = ROUND_UP_LPAGE(toxic_addr + toxic_size);
1558 #else /* __i386 */
1559 	segmap_start = ROUND_UP_LPAGE(kernelbase);
1560 #endif /* __i386 */
1561 	PRM_DEBUG(segmap_start);
1562 	ASSERT((caddr_t)segmap_start < boot_kernelheap);
1563 
1564 	/*
1565 	 * Users can change segmapsize through eeprom or /etc/system.
1566 	 * If the variable is tuned through eeprom, there is no upper
1567 	 * bound on the size of segmap.  If it is tuned through
1568 	 * /etc/system on 32-bit systems, it must be no larger than we
1569 	 * planned for in startup_memlist().
1570 	 */
1571 	segmapsize = MAX(ROUND_UP_LPAGE(segmapsize), SEGMAPDEFAULT);
1572 
1573 #if defined(__i386)
1574 	if (segmapsize > segmap_reserved) {
1575 		cmn_err(CE_NOTE, "!segmapsize may not be set > 0x%lx in "
1576 		    "/etc/system.  Use eeprom.", (long)SEGMAPMAX);
1577 		segmapsize = segmap_reserved;
1578 	}
1579 	/*
1580 	 * 32-bit systems don't have segkpm or segkp, so segmap appears at
1581 	 * the bottom of the kernel's address range.  Set aside space for a
1582 	 * small red zone just below the start of segmap.
1583 	 */
1584 	segmap_start += KERNEL_REDZONE_SIZE;
1585 	segmapsize -= KERNEL_REDZONE_SIZE;
1586 #endif
1587 
1588 	PRM_DEBUG(segmap_start);
1589 	PRM_DEBUG(segmapsize);
1590 	final_kernelheap = (caddr_t)ROUND_UP_LPAGE(segmap_start + segmapsize);
1591 	PRM_DEBUG(final_kernelheap);
1592 
1593 	/*
1594 	 * Do final allocations of HAT data structures that need to
1595 	 * be allocated before quiescing the boot loader.
1596 	 */
1597 	PRM_POINT("Calling hat_kern_alloc()...");
1598 	hat_kern_alloc((caddr_t)segmap_start, segmapsize, ekernelheap);
1599 	PRM_POINT("hat_kern_alloc() done");
1600 
1601 	/*
1602 	 * Setup MTRR (Memory type range registers)
1603 	 */
1604 	setup_mtrr();
1605 
1606 	/*
1607 	 * The next two loops are done in distinct steps in order
1608 	 * to be sure that any page that is doubly mapped (both above
1609 	 * KERNEL_TEXT and below kernelbase) is dealt with correctly.
1610 	 * Note this may never happen, but it might someday.
1611 	 */
1612 	bootpages = NULL;
1613 	PRM_POINT("Protecting boot pages");
1614 
1615 	/*
1616 	 * Protect any pages mapped above KERNEL_TEXT that somehow have
1617 	 * page_t's. This can only happen if something weird allocated
1618 	 * in this range (like kadb/kmdb).
1619 	 */
1620 	protect_boot_range(KERNEL_TEXT, (uintptr_t)-1, 0);
1621 
1622 	/*
1623 	 * Before we can take over memory allocation/mapping from the boot
1624 	 * loader we must remove from our free page lists any boot allocated
1625 	 * pages that stay mapped until release_bootstrap().
1626 	 */
1627 	protect_boot_range(0, kernelbase, 1);
1628 
1629 	/*
1630 	 * Switch to running on regular HAT (not boot_mmu)
1631 	 */
1632 	PRM_POINT("Calling hat_kern_setup()...");
1633 	hat_kern_setup();
1634 
1635 	/*
1636 	 * It is no longer safe to call BOP_ALLOC(), so make sure we don't.
1637 	 */
1638 	bop_no_more_mem();
1639 
1640 	PRM_POINT("hat_kern_setup() done");
1641 
1642 	hat_cpu_online(CPU);
1643 
1644 	/*
1645 	 * Initialize VM system
1646 	 */
1647 	PRM_POINT("Calling kvm_init()...");
1648 	kvm_init();
1649 	PRM_POINT("kvm_init() done");
1650 
1651 	/*
1652 	 * Tell kmdb that the VM system is now working
1653 	 */
1654 	if (boothowto & RB_DEBUG)
1655 		kdi_dvec_vmready();
1656 
1657 	/*
1658 	 * Mangle the brand string etc.
1659 	 */
1660 	cpuid_pass3(CPU);
1661 
1662 	/*
1663 	 * Now that we can use memory outside the top 4GB (on 64-bit
1664 	 * systems) and we know the size of segmap, we can set the final
1665 	 * size of the kernel's heap.
1666 	 */
1667 	if (final_kernelheap < boot_kernelheap) {
1668 		PRM_POINT("kernelheap_extend()");
1669 		PRM_DEBUG(boot_kernelheap);
1670 		PRM_DEBUG(final_kernelheap);
1671 		kernelheap_extend(final_kernelheap, boot_kernelheap);
1672 	}
1673 
1674 #if defined(__amd64)
1675 
1676 	/*
1677 	 * Create the device arena for toxic (to dtrace/kmdb) mappings.
1678 	 */
1679 	device_arena = vmem_create("device", (void *)toxic_addr,
1680 	    toxic_size, MMU_PAGESIZE, NULL, NULL, NULL, 0, VM_SLEEP);
1681 
1682 #else	/* __i386 */
1683 
1684 	/*
1685 	 * allocate the bit map that tracks toxic pages
1686 	 */
1687 	toxic_bit_map_len = btop((ulong_t)(valloc_base - kernelbase));
1688 	PRM_DEBUG(toxic_bit_map_len);
1689 	toxic_bit_map =
1690 	    kmem_zalloc(BT_SIZEOFMAP(toxic_bit_map_len), KM_NOSLEEP);
1691 	ASSERT(toxic_bit_map != NULL);
1692 	PRM_DEBUG(toxic_bit_map);
1693 
1694 #endif	/* __i386 */
1695 
1696 
1697 	/*
1698 	 * Now that we've got more VA, as well as the ability to allocate from
1699 	 * it, tell the debugger.
1700 	 */
1701 	if (boothowto & RB_DEBUG)
1702 		kdi_dvec_memavail();
1703 
1704 	/*
1705 	 * The following code installs a special page fault handler (#pf)
1706 	 * to work around a pentium bug.
1707 	 */
1708 #if !defined(__amd64)
1709 	if (x86_type == X86_TYPE_P5) {
1710 		desctbr_t idtr;
1711 		gate_desc_t *newidt;
1712 		struct machcpu *mcpu = &CPU->cpu_m;
1713 
1714 		if ((newidt = kmem_zalloc(MMU_PAGESIZE, KM_NOSLEEP)) == NULL)
1715 			panic("failed to install pentium_pftrap");
1716 
1717 		bcopy(idt0, newidt, sizeof (idt0));
1718 		set_gatesegd(&newidt[T_PGFLT], &pentium_pftrap,
1719 		    KCS_SEL, SDT_SYSIGT, SEL_KPL);
1720 
1721 		(void) as_setprot(&kas, (caddr_t)newidt, MMU_PAGESIZE,
1722 		    PROT_READ|PROT_EXEC);
1723 
1724 		mcpu->mcpu_idt = newidt;
1725 		idtr.dtr_base = (uintptr_t)mcpu->mcpu_idt;
1726 		idtr.dtr_limit = sizeof (idt0) - 1;
1727 		wr_idtr(&idtr);
1728 	}
1729 #endif	/* !__amd64 */
1730 
1731 	/*
1732 	 * Map page pfn=0 for drivers, such as kd, that need to pick up
1733 	 * parameters left there by controllers/BIOS.
1734 	 */
1735 	PRM_POINT("setup up p0_va");
1736 	p0_va = i86devmap(0, 1, PROT_READ);
1737 	PRM_DEBUG(p0_va);
1738 
1739 	cmn_err(CE_CONT, "?mem = %luK (0x%lx)\n",
1740 	    physinstalled << (MMU_PAGESHIFT - 10), ptob(physinstalled));
1741 
1742 	/*
1743 	 * disable automatic large pages for small memory systems or
1744 	 * when the disable flag is set.
1745 	 */
1746 	if (!auto_lpg_disable && mmu.max_page_level > 0) {
1747 		max_uheap_lpsize = LEVEL_SIZE(1);
1748 		max_ustack_lpsize = LEVEL_SIZE(1);
1749 		max_privmap_lpsize = LEVEL_SIZE(1);
1750 		max_uidata_lpsize = LEVEL_SIZE(1);
1751 		max_utext_lpsize = LEVEL_SIZE(1);
1752 		max_shm_lpsize = LEVEL_SIZE(1);
1753 	}
1754 	if (physmem < privm_lpg_min_physmem || mmu.max_page_level == 0 ||
1755 	    auto_lpg_disable) {
1756 		use_brk_lpg = 0;
1757 		use_stk_lpg = 0;
1758 	}
1759 	if (mmu.max_page_level > 0) {
1760 		mcntl0_lpsize = LEVEL_SIZE(1);
1761 	}
1762 
1763 	PRM_POINT("Calling hat_init_finish()...");
1764 	hat_init_finish();
1765 	PRM_POINT("hat_init_finish() done");
1766 
1767 	/*
1768 	 * Initialize the segkp segment type.
1769 	 */
1770 	rw_enter(&kas.a_lock, RW_WRITER);
1771 	PRM_POINT("Attaching segkp");
1772 	if (segkp_fromheap) {
1773 		segkp->s_as = &kas;
1774 	} else if (seg_attach(&kas, (caddr_t)segkp_base, mmu_ptob(segkpsize),
1775 	    segkp) < 0) {
1776 		panic("startup: cannot attach segkp");
1777 		/*NOTREACHED*/
1778 	}
1779 	PRM_POINT("Doing segkp_create()");
1780 	if (segkp_create(segkp) != 0) {
1781 		panic("startup: segkp_create failed");
1782 		/*NOTREACHED*/
1783 	}
1784 	PRM_DEBUG(segkp);
1785 	rw_exit(&kas.a_lock);
1786 
1787 	/*
1788 	 * kpm segment
1789 	 */
1790 	segmap_kpm = 0;
1791 	if (kpm_desired) {
1792 		kpm_init();
1793 		kpm_enable = 1;
1794 		vpm_enable = 1;
1795 	}
1796 
1797 	/*
1798 	 * Now create segmap segment.
1799 	 */
1800 	rw_enter(&kas.a_lock, RW_WRITER);
1801 	if (seg_attach(&kas, (caddr_t)segmap_start, segmapsize, segmap) < 0) {
1802 		panic("cannot attach segmap");
1803 		/*NOTREACHED*/
1804 	}
1805 	PRM_DEBUG(segmap);
1806 
1807 	a.prot = PROT_READ | PROT_WRITE;
1808 	a.shmsize = 0;
1809 	a.nfreelist = segmapfreelists;
1810 
1811 	if (segmap_create(segmap, (caddr_t)&a) != 0)
1812 		panic("segmap_create segmap");
1813 	rw_exit(&kas.a_lock);
1814 
1815 	setup_vaddr_for_ppcopy(CPU);
1816 
1817 	segdev_init();
1818 	pmem_init();
1819 
1820 	PRM_POINT("startup_vm() done");
1821 }
1822 
1823 /*
1824  * Load a tod module for the non-standard tod part found on this system.
1825  */
1826 static void
1827 load_tod_module(char *todmod)
1828 {
1829 	if (modload("tod", todmod) == -1)
1830 		halt("Can't load TOD module");
1831 }
1832 
1833 static void
1834 startup_end(void)
1835 {
1836 	extern void setx86isalist(void);
1837 
1838 	PRM_POINT("startup_end() starting...");
1839 
1840 	/*
1841 	 * Perform tasks that get done after most of the VM
1842 	 * initialization has been done but before the clock
1843 	 * and other devices get started.
1844 	 */
1845 	kern_setup1();
1846 
1847 	/*
1848 	 * Perform CPC initialization for this CPU.
1849 	 */
1850 	kcpc_hw_init(CPU);
1851 
1852 #if defined(OPTERON_WORKAROUND_6323525)
1853 	if (opteron_workaround_6323525)
1854 		patch_workaround_6323525();
1855 #endif
1856 	/*
1857 	 * If needed, load TOD module now so that ddi_get_time(9F) etc. work
1858 	 * (For now, "needed" is defined as set tod_module_name in /etc/system)
1859 	 */
1860 	if (tod_module_name != NULL) {
1861 		PRM_POINT("load_tod_module()");
1862 		load_tod_module(tod_module_name);
1863 	}
1864 
1865 	/*
1866 	 * Configure the system.
1867 	 */
1868 	PRM_POINT("Calling configure()...");
1869 	configure();		/* set up devices */
1870 	PRM_POINT("configure() done");
1871 
1872 	/*
1873 	 * Set the isa_list string to the defined instruction sets we
1874 	 * support.
1875 	 */
1876 	setx86isalist();
1877 	cpu_intr_alloc(CPU, NINTR_THREADS);
1878 	psm_install();
1879 
1880 	/*
1881 	 * We're done with bootops.  We don't unmap the bootstrap yet because
1882 	 * we're still using bootsvcs.
1883 	 */
1884 	PRM_POINT("NULLing out bootops");
1885 	*bootopsp = (struct bootops *)NULL;
1886 	bootops = (struct bootops *)NULL;
1887 
1888 	PRM_POINT("Enabling interrupts");
1889 	(*picinitf)();
1890 	sti();
1891 
1892 	(void) add_avsoftintr((void *)&softlevel1_hdl, 1, softlevel1,
1893 		"softlevel1", NULL, NULL); /* XXX to be moved later */
1894 
1895 	PRM_POINT("startup_end() done");
1896 }
1897 
1898 extern char hw_serial[];
1899 char *_hs1107 = hw_serial;
1900 ulong_t  _bdhs34;
1901 
1902 void
1903 post_startup(void)
1904 {
1905 	/*
1906 	 * Set the system wide, processor-specific flags to be passed
1907 	 * to userland via the aux vector for performance hints and
1908 	 * instruction set extensions.
1909 	 */
1910 	bind_hwcap();
1911 
1912 	/*
1913 	 * Load the System Management BIOS into the global ksmbios
1914 	 * handle, if an SMBIOS is present on this system.
1915 	 */
1916 	ksmbios = smbios_open(NULL, SMB_VERSION, ksmbios_flags, NULL);
1917 
1918 	/*
1919 	 * Startup the memory scrubber.
1920 	 */
1921 	memscrub_init();
1922 
1923 	/*
1924 	 * Complete CPU module initialization
1925 	 */
1926 	cmi_post_init();
1927 
1928 	/*
1929 	 * Perform forceloading tasks for /etc/system.
1930 	 */
1931 	(void) mod_sysctl(SYS_FORCELOAD, NULL);
1932 
1933 	/*
1934 	 * ON4.0: Force /proc module in until clock interrupt handle fixed
1935 	 * ON4.0: This must be fixed or restated in /etc/systems.
1936 	 */
1937 	(void) modload("fs", "procfs");
1938 
1939 #if defined(__i386)
1940 	/*
1941 	 * Check for required functional Floating Point hardware,
1942 	 * unless FP hardware explicitly disabled.
1943 	 */
1944 	if (fpu_exists && (fpu_pentium_fdivbug || fp_kind == FP_NO))
1945 		halt("No working FP hardware found");
1946 #endif
1947 
1948 	maxmem = freemem;
1949 
1950 	add_cpunode2devtree(CPU->cpu_id, CPU->cpu_m.mcpu_cpi);
1951 }
1952 
1953 static int
1954 pp_in_ramdisk(page_t *pp)
1955 {
1956 	extern uint64_t ramdisk_start, ramdisk_end;
1957 
1958 	return ((pp->p_pagenum >= btop(ramdisk_start)) &&
1959 	    (pp->p_pagenum < btopr(ramdisk_end)));
1960 }
1961 
1962 void
1963 release_bootstrap(void)
1964 {
1965 	int root_is_ramdisk;
1966 	page_t *pp;
1967 	extern void kobj_boot_unmountroot(void);
1968 	extern dev_t rootdev;
1969 
1970 	/* unmount boot ramdisk and release kmem usage */
1971 	kobj_boot_unmountroot();
1972 
1973 	/*
1974 	 * We're finished using the boot loader so free its pages.
1975 	 */
1976 	PRM_POINT("Unmapping lower boot pages");
1977 	clear_boot_mappings(0, _userlimit);
1978 	postbootkernelbase = kernelbase;
1979 
1980 	/*
1981 	 * If root isn't on ramdisk, destroy the hardcoded
1982 	 * ramdisk node now and release the memory. Else,
1983 	 * ramdisk memory is kept in rd_pages.
1984 	 */
1985 	root_is_ramdisk = (getmajor(rootdev) == ddi_name_to_major("ramdisk"));
1986 	if (!root_is_ramdisk) {
1987 		dev_info_t *dip = ddi_find_devinfo("ramdisk", -1, 0);
1988 		ASSERT(dip && ddi_get_parent(dip) == ddi_root_node());
1989 		ndi_rele_devi(dip);	/* held from ddi_find_devinfo */
1990 		(void) ddi_remove_child(dip, 0);
1991 	}
1992 
1993 	PRM_POINT("Releasing boot pages");
1994 	while (bootpages) {
1995 		pp = bootpages;
1996 		bootpages = pp->p_next;
1997 		if (root_is_ramdisk && pp_in_ramdisk(pp)) {
1998 			pp->p_next = rd_pages;
1999 			rd_pages = pp;
2000 			continue;
2001 		}
2002 		pp->p_next = (struct page *)0;
2003 		pp->p_prev = (struct page *)0;
2004 		PP_CLRBOOTPAGES(pp);
2005 		page_free(pp, 1);
2006 	}
2007 	PRM_POINT("Boot pages released");
2008 
2009 	/*
2010 	 * Find 1 page below 1 MB so that other processors can boot up.
2011 	 * Make sure it has a kernel VA as well as a 1:1 mapping.
2012 	 * We should have just free'd one up.
2013 	 */
2014 	if (use_mp) {
2015 		pfn_t pfn;
2016 
2017 		for (pfn = 1; pfn < btop(1*1024*1024); pfn++) {
2018 			if (page_numtopp_alloc(pfn) == NULL)
2019 				continue;
2020 			rm_platter_va = i86devmap(pfn, 1,
2021 			    PROT_READ | PROT_WRITE | PROT_EXEC);
2022 			rm_platter_pa = ptob(pfn);
2023 			hat_devload(kas.a_hat,
2024 			    (caddr_t)(uintptr_t)rm_platter_pa, MMU_PAGESIZE,
2025 			    pfn, PROT_READ | PROT_WRITE | PROT_EXEC,
2026 			    HAT_LOAD_NOCONSIST);
2027 			break;
2028 		}
2029 		if (pfn == btop(1*1024*1024))
2030 			panic("No page available for starting "
2031 			    "other processors");
2032 	}
2033 
2034 }
2035 
2036 /*
2037  * Initialize the platform-specific parts of a page_t.
2038  */
2039 void
2040 add_physmem_cb(page_t *pp, pfn_t pnum)
2041 {
2042 	pp->p_pagenum = pnum;
2043 	pp->p_mapping = NULL;
2044 	pp->p_embed = 0;
2045 	pp->p_share = 0;
2046 	pp->p_mlentry = 0;
2047 }
2048 
2049 /*
2050  * kphysm_init() initializes physical memory.
2051  */
2052 static pgcnt_t
2053 kphysm_init(
2054 	page_t *pp,
2055 	pgcnt_t npages)
2056 {
2057 	struct memlist	*pmem;
2058 	struct memseg	*cur_memseg;
2059 	pfn_t		base_pfn;
2060 	pgcnt_t		num;
2061 	pgcnt_t		pages_done = 0;
2062 	uint64_t	addr;
2063 	uint64_t	size;
2064 	extern pfn_t	ddiphysmin;
2065 
2066 	ASSERT(page_hash != NULL && page_hashsz != 0);
2067 
2068 	cur_memseg = memseg_base;
2069 	for (pmem = phys_avail; pmem && npages; pmem = pmem->next) {
2070 		/*
2071 		 * In a 32 bit kernel can't use higher memory if we're
2072 		 * not booting in PAE mode. This check takes care of that.
2073 		 */
2074 		addr = pmem->address;
2075 		size = pmem->size;
2076 		if (btop(addr) > physmax)
2077 			continue;
2078 
2079 		/*
2080 		 * align addr and size - they may not be at page boundaries
2081 		 */
2082 		if ((addr & MMU_PAGEOFFSET) != 0) {
2083 			addr += MMU_PAGEOFFSET;
2084 			addr &= ~(uint64_t)MMU_PAGEOFFSET;
2085 			size -= addr - pmem->address;
2086 		}
2087 
2088 		/* only process pages below or equal to physmax */
2089 		if ((btop(addr + size) - 1) > physmax)
2090 			size = ptob(physmax - btop(addr) + 1);
2091 
2092 		num = btop(size);
2093 		if (num == 0)
2094 			continue;
2095 
2096 		if (num > npages)
2097 			num = npages;
2098 
2099 		npages -= num;
2100 		pages_done += num;
2101 		base_pfn = btop(addr);
2102 
2103 		if (prom_debug)
2104 			prom_printf("MEMSEG addr=0x%" PRIx64
2105 			    " pgs=0x%lx pfn 0x%lx-0x%lx\n",
2106 			    addr, num, base_pfn, base_pfn + num);
2107 
2108 		/*
2109 		 * Ignore pages below ddiphysmin to simplify ddi memory
2110 		 * allocation with non-zero addr_lo requests.
2111 		 */
2112 		if (base_pfn < ddiphysmin) {
2113 			if (base_pfn + num <= ddiphysmin)
2114 				continue;
2115 			pp += (ddiphysmin - base_pfn);
2116 			num -= (ddiphysmin - base_pfn);
2117 			base_pfn = ddiphysmin;
2118 		}
2119 
2120 		/*
2121 		 * Build the memsegs entry
2122 		 */
2123 		cur_memseg->pages = pp;
2124 		cur_memseg->epages = pp + num;
2125 		cur_memseg->pages_base = base_pfn;
2126 		cur_memseg->pages_end = base_pfn + num;
2127 
2128 		/*
2129 		 * Insert into memseg list in decreasing pfn range order.
2130 		 * Low memory is typically more fragmented such that this
2131 		 * ordering keeps the larger ranges at the front of the list
2132 		 * for code that searches memseg.
2133 		 * This ASSERTS that the memsegs coming in from boot are in
2134 		 * increasing physical address order and not contiguous.
2135 		 */
2136 		if (memsegs != NULL) {
2137 			ASSERT(cur_memseg->pages_base >= memsegs->pages_end);
2138 			cur_memseg->next = memsegs;
2139 		}
2140 		memsegs = cur_memseg;
2141 
2142 		/*
2143 		 * add_physmem() initializes the PSM part of the page
2144 		 * struct by calling the PSM back with add_physmem_cb().
2145 		 * In addition it coalesces pages into larger pages as
2146 		 * it initializes them.
2147 		 */
2148 		add_physmem(pp, num, base_pfn);
2149 		cur_memseg++;
2150 		availrmem_initial += num;
2151 		availrmem += num;
2152 
2153 		pp += num;
2154 	}
2155 
2156 	PRM_DEBUG(availrmem_initial);
2157 	PRM_DEBUG(availrmem);
2158 	PRM_DEBUG(freemem);
2159 	build_pfn_hash();
2160 	return (pages_done);
2161 }
2162 
2163 /*
2164  * Kernel VM initialization.
2165  */
2166 static void
2167 kvm_init(void)
2168 {
2169 	ASSERT((((uintptr_t)s_text) & MMU_PAGEOFFSET) == 0);
2170 
2171 	/*
2172 	 * Put the kernel segments in kernel address space.
2173 	 */
2174 	rw_enter(&kas.a_lock, RW_WRITER);
2175 	as_avlinit(&kas);
2176 
2177 	(void) seg_attach(&kas, s_text, e_moddata - s_text, &ktextseg);
2178 	(void) segkmem_create(&ktextseg);
2179 
2180 	(void) seg_attach(&kas, (caddr_t)valloc_base, valloc_sz, &kvalloc);
2181 	(void) segkmem_create(&kvalloc);
2182 
2183 	/*
2184 	 * We're about to map out /boot.  This is the beginning of the
2185 	 * system resource management transition. We can no longer
2186 	 * call into /boot for I/O or memory allocations.
2187 	 */
2188 	(void) seg_attach(&kas, final_kernelheap,
2189 	    ekernelheap - final_kernelheap, &kvseg);
2190 	(void) segkmem_create(&kvseg);
2191 
2192 	if (core_size > 0) {
2193 		PRM_POINT("attaching kvseg_core");
2194 		(void) seg_attach(&kas, (caddr_t)core_base, core_size,
2195 		    &kvseg_core);
2196 		(void) segkmem_create(&kvseg_core);
2197 	}
2198 
2199 	if (segziosize > 0) {
2200 		PRM_POINT("attaching segzio");
2201 		(void) seg_attach(&kas, segzio_base, mmu_ptob(segziosize),
2202 		    &kzioseg);
2203 		(void) segkmem_zio_create(&kzioseg);
2204 
2205 		/* create zio area covering new segment */
2206 		segkmem_zio_init(segzio_base, mmu_ptob(segziosize));
2207 	}
2208 
2209 	(void) seg_attach(&kas, kdi_segdebugbase, kdi_segdebugsize, &kdebugseg);
2210 	(void) segkmem_create(&kdebugseg);
2211 
2212 	rw_exit(&kas.a_lock);
2213 
2214 	/*
2215 	 * Ensure that the red zone at kernelbase is never accessible.
2216 	 */
2217 	PRM_POINT("protecting redzone");
2218 	(void) as_setprot(&kas, (caddr_t)kernelbase, KERNEL_REDZONE_SIZE, 0);
2219 
2220 	/*
2221 	 * Make the text writable so that it can be hot patched by DTrace.
2222 	 */
2223 	(void) as_setprot(&kas, s_text, e_modtext - s_text,
2224 	    PROT_READ | PROT_WRITE | PROT_EXEC);
2225 
2226 	/*
2227 	 * Make data writable until end.
2228 	 */
2229 	(void) as_setprot(&kas, s_data, e_moddata - s_data,
2230 	    PROT_READ | PROT_WRITE | PROT_EXEC);
2231 }
2232 
2233 /*
2234  * These are MTTR registers supported by P6
2235  */
2236 static struct	mtrrvar	mtrrphys_arr[MAX_MTRRVAR];
2237 static uint64_t mtrr64k, mtrr16k1, mtrr16k2;
2238 static uint64_t mtrr4k1, mtrr4k2, mtrr4k3;
2239 static uint64_t mtrr4k4, mtrr4k5, mtrr4k6;
2240 static uint64_t mtrr4k7, mtrr4k8, mtrrcap;
2241 uint64_t mtrrdef, pat_attr_reg;
2242 
2243 /*
2244  * Disable reprogramming of MTRRs by default.
2245  */
2246 int	enable_relaxed_mtrr = 0;
2247 
2248 void
2249 setup_mtrr(void)
2250 {
2251 	int i, ecx;
2252 	int vcnt;
2253 	struct	mtrrvar	*mtrrphys;
2254 
2255 	if (!(x86_feature & X86_MTRR))
2256 		return;
2257 
2258 	mtrrcap = rdmsr(REG_MTRRCAP);
2259 	mtrrdef = rdmsr(REG_MTRRDEF);
2260 	if (mtrrcap & MTRRCAP_FIX) {
2261 		mtrr64k = rdmsr(REG_MTRR64K);
2262 		mtrr16k1 = rdmsr(REG_MTRR16K1);
2263 		mtrr16k2 = rdmsr(REG_MTRR16K2);
2264 		mtrr4k1 = rdmsr(REG_MTRR4K1);
2265 		mtrr4k2 = rdmsr(REG_MTRR4K2);
2266 		mtrr4k3 = rdmsr(REG_MTRR4K3);
2267 		mtrr4k4 = rdmsr(REG_MTRR4K4);
2268 		mtrr4k5 = rdmsr(REG_MTRR4K5);
2269 		mtrr4k6 = rdmsr(REG_MTRR4K6);
2270 		mtrr4k7 = rdmsr(REG_MTRR4K7);
2271 		mtrr4k8 = rdmsr(REG_MTRR4K8);
2272 	}
2273 	if ((vcnt = (mtrrcap & MTRRCAP_VCNTMASK)) > MAX_MTRRVAR)
2274 		vcnt = MAX_MTRRVAR;
2275 
2276 	for (i = 0, ecx = REG_MTRRPHYSBASE0, mtrrphys = mtrrphys_arr;
2277 		i <  vcnt - 1; i++, ecx += 2, mtrrphys++) {
2278 		mtrrphys->mtrrphys_base = rdmsr(ecx);
2279 		mtrrphys->mtrrphys_mask = rdmsr(ecx + 1);
2280 		if ((x86_feature & X86_PAT) && enable_relaxed_mtrr) {
2281 			mtrrphys->mtrrphys_mask &= ~MTRRPHYSMASK_V;
2282 		}
2283 	}
2284 	if (x86_feature & X86_PAT) {
2285 		if (enable_relaxed_mtrr)
2286 			mtrrdef = MTRR_TYPE_WB|MTRRDEF_FE|MTRRDEF_E;
2287 		pat_attr_reg = PAT_DEFAULT_ATTRIBUTE;
2288 	}
2289 
2290 	mtrr_sync();
2291 }
2292 
2293 /*
2294  * Sync current cpu mtrr with the incore copy of mtrr.
2295  * This function has to be invoked with interrupts disabled
2296  * Currently we do not capture other cpu's. This is invoked on cpu0
2297  * just after reading /etc/system.
2298  * On other cpu's its invoked from mp_startup().
2299  */
2300 void
2301 mtrr_sync(void)
2302 {
2303 	uint_t	crvalue, cr0_orig;
2304 	int	vcnt, i, ecx;
2305 	struct	mtrrvar	*mtrrphys;
2306 
2307 	cr0_orig = crvalue = getcr0();
2308 	crvalue |= CR0_CD;
2309 	crvalue &= ~CR0_NW;
2310 	setcr0(crvalue);
2311 	invalidate_cache();
2312 
2313 	reload_cr3();
2314 	if (x86_feature & X86_PAT)
2315 		wrmsr(REG_MTRRPAT, pat_attr_reg);
2316 
2317 	wrmsr(REG_MTRRDEF, rdmsr(REG_MTRRDEF) &
2318 	    ~((uint64_t)(uintptr_t)MTRRDEF_E));
2319 
2320 	if (mtrrcap & MTRRCAP_FIX) {
2321 		wrmsr(REG_MTRR64K, mtrr64k);
2322 		wrmsr(REG_MTRR16K1, mtrr16k1);
2323 		wrmsr(REG_MTRR16K2, mtrr16k2);
2324 		wrmsr(REG_MTRR4K1, mtrr4k1);
2325 		wrmsr(REG_MTRR4K2, mtrr4k2);
2326 		wrmsr(REG_MTRR4K3, mtrr4k3);
2327 		wrmsr(REG_MTRR4K4, mtrr4k4);
2328 		wrmsr(REG_MTRR4K5, mtrr4k5);
2329 		wrmsr(REG_MTRR4K6, mtrr4k6);
2330 		wrmsr(REG_MTRR4K7, mtrr4k7);
2331 		wrmsr(REG_MTRR4K8, mtrr4k8);
2332 	}
2333 	if ((vcnt = (mtrrcap & MTRRCAP_VCNTMASK)) > MAX_MTRRVAR)
2334 		vcnt = MAX_MTRRVAR;
2335 	for (i = 0, ecx = REG_MTRRPHYSBASE0, mtrrphys = mtrrphys_arr;
2336 	    i <  vcnt - 1; i++, ecx += 2, mtrrphys++) {
2337 		wrmsr(ecx, mtrrphys->mtrrphys_base);
2338 		wrmsr(ecx + 1, mtrrphys->mtrrphys_mask);
2339 	}
2340 	wrmsr(REG_MTRRDEF, mtrrdef);
2341 
2342 	reload_cr3();
2343 	invalidate_cache();
2344 	setcr0(cr0_orig);
2345 }
2346 
2347 /*
2348  * resync mtrr so that BIOS is happy. Called from mdboot
2349  */
2350 void
2351 mtrr_resync(void)
2352 {
2353 	if ((x86_feature & X86_PAT) && enable_relaxed_mtrr) {
2354 		/*
2355 		 * We could have changed the default mtrr definition.
2356 		 * Put it back to uncached which is what it is at power on
2357 		 */
2358 		mtrrdef = MTRR_TYPE_UC|MTRRDEF_FE|MTRRDEF_E;
2359 		mtrr_sync();
2360 	}
2361 }
2362 
2363 void
2364 get_system_configuration(void)
2365 {
2366 	char	prop[32];
2367 	u_longlong_t nodes_ll, cpus_pernode_ll, lvalue;
2368 
2369 	if (((BOP_GETPROPLEN(bootops, "nodes") > sizeof (prop)) ||
2370 		(BOP_GETPROP(bootops, "nodes", prop) < 0) 	||
2371 		(kobj_getvalue(prop, &nodes_ll) == -1) ||
2372 		(nodes_ll > MAXNODES))			   ||
2373 	    ((BOP_GETPROPLEN(bootops, "cpus_pernode") > sizeof (prop)) ||
2374 		(BOP_GETPROP(bootops, "cpus_pernode", prop) < 0) ||
2375 		(kobj_getvalue(prop, &cpus_pernode_ll) == -1))) {
2376 
2377 		system_hardware.hd_nodes = 1;
2378 		system_hardware.hd_cpus_per_node = 0;
2379 	} else {
2380 		system_hardware.hd_nodes = (int)nodes_ll;
2381 		system_hardware.hd_cpus_per_node = (int)cpus_pernode_ll;
2382 	}
2383 	if ((BOP_GETPROPLEN(bootops, "kernelbase") > sizeof (prop)) ||
2384 		(BOP_GETPROP(bootops, "kernelbase", prop) < 0) 	||
2385 		(kobj_getvalue(prop, &lvalue) == -1))
2386 			eprom_kernelbase = NULL;
2387 	else
2388 			eprom_kernelbase = (uintptr_t)lvalue;
2389 
2390 	if ((BOP_GETPROPLEN(bootops, "segmapsize") > sizeof (prop)) ||
2391 	    (BOP_GETPROP(bootops, "segmapsize", prop) < 0) ||
2392 	    (kobj_getvalue(prop, &lvalue) == -1)) {
2393 		segmapsize = SEGMAPDEFAULT;
2394 	} else {
2395 		segmapsize = (uintptr_t)lvalue;
2396 	}
2397 
2398 	if ((BOP_GETPROPLEN(bootops, "segmapfreelists") > sizeof (prop)) ||
2399 	    (BOP_GETPROP(bootops, "segmapfreelists", prop) < 0) ||
2400 	    (kobj_getvalue(prop, &lvalue) == -1)) {
2401 		segmapfreelists = 0;	/* use segmap driver default */
2402 	} else {
2403 		segmapfreelists = (int)lvalue;
2404 	}
2405 
2406 	/* physmem used to be here, but moved much earlier to fakebop.c */
2407 }
2408 
2409 /*
2410  * Add to a memory list.
2411  * start = start of new memory segment
2412  * len = length of new memory segment in bytes
2413  * new = pointer to a new struct memlist
2414  * memlistp = memory list to which to add segment.
2415  */
2416 void
2417 memlist_add(
2418 	uint64_t start,
2419 	uint64_t len,
2420 	struct memlist *new,
2421 	struct memlist **memlistp)
2422 {
2423 	struct memlist *cur;
2424 	uint64_t end = start + len;
2425 
2426 	new->address = start;
2427 	new->size = len;
2428 
2429 	cur = *memlistp;
2430 
2431 	while (cur) {
2432 		if (cur->address >= end) {
2433 			new->next = cur;
2434 			*memlistp = new;
2435 			new->prev = cur->prev;
2436 			cur->prev = new;
2437 			return;
2438 		}
2439 		ASSERT(cur->address + cur->size <= start);
2440 		if (cur->next == NULL) {
2441 			cur->next = new;
2442 			new->prev = cur;
2443 			new->next = NULL;
2444 			return;
2445 		}
2446 		memlistp = &cur->next;
2447 		cur = cur->next;
2448 	}
2449 }
2450 
2451 void
2452 kobj_vmem_init(vmem_t **text_arena, vmem_t **data_arena)
2453 {
2454 	size_t tsize = e_modtext - modtext;
2455 	size_t dsize = e_moddata - moddata;
2456 
2457 	*text_arena = vmem_create("module_text", tsize ? modtext : NULL, tsize,
2458 	    1, segkmem_alloc, segkmem_free, heaptext_arena, 0, VM_SLEEP);
2459 	*data_arena = vmem_create("module_data", dsize ? moddata : NULL, dsize,
2460 	    1, segkmem_alloc, segkmem_free, heap32_arena, 0, VM_SLEEP);
2461 }
2462 
2463 caddr_t
2464 kobj_text_alloc(vmem_t *arena, size_t size)
2465 {
2466 	return (vmem_alloc(arena, size, VM_SLEEP | VM_BESTFIT));
2467 }
2468 
2469 /*ARGSUSED*/
2470 caddr_t
2471 kobj_texthole_alloc(caddr_t addr, size_t size)
2472 {
2473 	panic("unexpected call to kobj_texthole_alloc()");
2474 	/*NOTREACHED*/
2475 	return (0);
2476 }
2477 
2478 /*ARGSUSED*/
2479 void
2480 kobj_texthole_free(caddr_t addr, size_t size)
2481 {
2482 	panic("unexpected call to kobj_texthole_free()");
2483 }
2484 
2485 /*
2486  * This is called just after configure() in startup().
2487  *
2488  * The ISALIST concept is a bit hopeless on Intel, because
2489  * there's no guarantee of an ever-more-capable processor
2490  * given that various parts of the instruction set may appear
2491  * and disappear between different implementations.
2492  *
2493  * While it would be possible to correct it and even enhance
2494  * it somewhat, the explicit hardware capability bitmask allows
2495  * more flexibility.
2496  *
2497  * So, we just leave this alone.
2498  */
2499 void
2500 setx86isalist(void)
2501 {
2502 	char *tp;
2503 	size_t len;
2504 	extern char *isa_list;
2505 
2506 #define	TBUFSIZE	1024
2507 
2508 	tp = kmem_alloc(TBUFSIZE, KM_SLEEP);
2509 	*tp = '\0';
2510 
2511 #if defined(__amd64)
2512 	(void) strcpy(tp, "amd64 ");
2513 #endif
2514 
2515 	switch (x86_vendor) {
2516 	case X86_VENDOR_Intel:
2517 	case X86_VENDOR_AMD:
2518 	case X86_VENDOR_TM:
2519 		if (x86_feature & X86_CMOV) {
2520 			/*
2521 			 * Pentium Pro or later
2522 			 */
2523 			(void) strcat(tp, "pentium_pro");
2524 			(void) strcat(tp, x86_feature & X86_MMX ?
2525 			    "+mmx pentium_pro " : " ");
2526 		}
2527 		/*FALLTHROUGH*/
2528 	case X86_VENDOR_Cyrix:
2529 		/*
2530 		 * The Cyrix 6x86 does not have any Pentium features
2531 		 * accessible while not at privilege level 0.
2532 		 */
2533 		if (x86_feature & X86_CPUID) {
2534 			(void) strcat(tp, "pentium");
2535 			(void) strcat(tp, x86_feature & X86_MMX ?
2536 			    "+mmx pentium " : " ");
2537 		}
2538 		break;
2539 	default:
2540 		break;
2541 	}
2542 	(void) strcat(tp, "i486 i386 i86");
2543 	len = strlen(tp) + 1;   /* account for NULL at end of string */
2544 	isa_list = strcpy(kmem_alloc(len, KM_SLEEP), tp);
2545 	kmem_free(tp, TBUFSIZE);
2546 
2547 #undef TBUFSIZE
2548 }
2549 
2550 
2551 #ifdef __amd64
2552 
2553 void *
2554 device_arena_alloc(size_t size, int vm_flag)
2555 {
2556 	return (vmem_alloc(device_arena, size, vm_flag));
2557 }
2558 
2559 void
2560 device_arena_free(void *vaddr, size_t size)
2561 {
2562 	vmem_free(device_arena, vaddr, size);
2563 }
2564 
2565 #else /* __i386 */
2566 
2567 void *
2568 device_arena_alloc(size_t size, int vm_flag)
2569 {
2570 	caddr_t	vaddr;
2571 	uintptr_t v;
2572 	size_t	start;
2573 	size_t	end;
2574 
2575 	vaddr = vmem_alloc(heap_arena, size, vm_flag);
2576 	if (vaddr == NULL)
2577 		return (NULL);
2578 
2579 	v = (uintptr_t)vaddr;
2580 	ASSERT(v >= kernelbase);
2581 	ASSERT(v + size <= valloc_base);
2582 
2583 	start = btop(v - kernelbase);
2584 	end = btop(v + size - 1 - kernelbase);
2585 	ASSERT(start < toxic_bit_map_len);
2586 	ASSERT(end < toxic_bit_map_len);
2587 
2588 	while (start <= end) {
2589 		BT_ATOMIC_SET(toxic_bit_map, start);
2590 		++start;
2591 	}
2592 	return (vaddr);
2593 }
2594 
2595 void
2596 device_arena_free(void *vaddr, size_t size)
2597 {
2598 	uintptr_t v = (uintptr_t)vaddr;
2599 	size_t	start;
2600 	size_t	end;
2601 
2602 	ASSERT(v >= kernelbase);
2603 	ASSERT(v + size <= valloc_base);
2604 
2605 	start = btop(v - kernelbase);
2606 	end = btop(v + size - 1 - kernelbase);
2607 	ASSERT(start < toxic_bit_map_len);
2608 	ASSERT(end < toxic_bit_map_len);
2609 
2610 	while (start <= end) {
2611 		ASSERT(BT_TEST(toxic_bit_map, start) != 0);
2612 		BT_ATOMIC_CLEAR(toxic_bit_map, start);
2613 		++start;
2614 	}
2615 	vmem_free(heap_arena, vaddr, size);
2616 }
2617 
2618 /*
2619  * returns 1st address in range that is in device arena, or NULL
2620  * if len is not NULL it returns the length of the toxic range
2621  */
2622 void *
2623 device_arena_contains(void *vaddr, size_t size, size_t *len)
2624 {
2625 	uintptr_t v = (uintptr_t)vaddr;
2626 	uintptr_t eaddr = v + size;
2627 	size_t start;
2628 	size_t end;
2629 
2630 	/*
2631 	 * if called very early by kmdb, just return NULL
2632 	 */
2633 	if (toxic_bit_map == NULL)
2634 		return (NULL);
2635 
2636 	/*
2637 	 * First check if we're completely outside the bitmap range.
2638 	 */
2639 	if (v >= valloc_base || eaddr < kernelbase)
2640 		return (NULL);
2641 
2642 	/*
2643 	 * Trim ends of search to look at only what the bitmap covers.
2644 	 */
2645 	if (v < kernelbase)
2646 		v = kernelbase;
2647 	start = btop(v - kernelbase);
2648 	end = btop(eaddr - kernelbase);
2649 	if (end >= toxic_bit_map_len)
2650 		end = toxic_bit_map_len;
2651 
2652 	if (bt_range(toxic_bit_map, &start, &end, end) == 0)
2653 		return (NULL);
2654 
2655 	v = kernelbase + ptob(start);
2656 	if (len != NULL)
2657 		*len = ptob(end - start);
2658 	return ((void *)v);
2659 }
2660 
2661 #endif	/* __i386 */
2662