1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22/*
23 * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright (c) 2016 by Delphix. All rights reserved.
25 * Copyright 2018 Joyent, Inc.
26 * Copyright 2019 Peter Tribble.
27 */
28
29#include <sys/machsystm.h>
30#include <sys/archsystm.h>
31#include <sys/vm.h>
32#include <sys/cpu.h>
33#include <sys/atomic.h>
34#include <sys/reboot.h>
35#include <sys/kdi.h>
36#include <sys/bootconf.h>
37#include <sys/memlist_plat.h>
38#include <sys/memlist_impl.h>
39#include <sys/prom_plat.h>
40#include <sys/prom_isa.h>
41#include <sys/autoconf.h>
42#include <sys/ivintr.h>
43#include <sys/fpu/fpusystm.h>
44#include <sys/iommutsb.h>
45#include <vm/vm_dep.h>
46#include <vm/seg_dev.h>
47#include <vm/seg_kmem.h>
48#include <vm/seg_kpm.h>
49#include <vm/seg_map.h>
50#include <vm/seg_kp.h>
51#include <sys/sysconf.h>
52#include <vm/hat_sfmmu.h>
53#include <sys/kobj.h>
54#include <sys/sun4asi.h>
55#include <sys/clconf.h>
56#include <sys/platform_module.h>
57#include <sys/panic.h>
58#include <sys/cpu_sgnblk_defs.h>
59#include <sys/clock.h>
60#include <sys/cmn_err.h>
61#include <sys/dumphdr.h>
62#include <sys/promif.h>
63#include <sys/prom_debug.h>
64#include <sys/traptrace.h>
65#include <sys/memnode.h>
66#include <sys/mem_cage.h>
67#include <sys/mmu.h>
68#include <sys/swap.h>
69
70extern void setup_trap_table(void);
71extern int cpu_intrq_setup(struct cpu *);
72extern void cpu_intrq_register(struct cpu *);
73extern void contig_mem_init(void);
74extern caddr_t contig_mem_prealloc(caddr_t, pgcnt_t);
75extern void mach_dump_buffer_init(void);
76extern void mach_descrip_init(void);
77extern void mach_descrip_startup_fini(void);
78extern void mach_memscrub(void);
79extern void mach_fpras(void);
80extern void mach_cpu_halt_idle(void);
81extern void mach_hw_copy_limit(void);
82extern void load_mach_drivers(void);
83extern void load_tod_module(void);
84#pragma weak load_tod_module
85
86extern int ndata_alloc_mmfsa(struct memlist *ndata);
87#pragma weak ndata_alloc_mmfsa
88
89extern void cif_init(void);
90#pragma weak cif_init
91
92extern void parse_idprom(void);
93extern void add_vx_handler(char *, int, void (*)(cell_t *));
94extern void mem_config_init(void);
95extern void memseg_remap_init(void);
96
97extern void mach_kpm_init(void);
98extern void pcf_init();
99extern int size_pse_array(pgcnt_t, int);
100extern void pg_init();
101
102/*
103 * External Data:
104 */
105extern int vac_size;	/* cache size in bytes */
106extern uint_t vac_mask;	/* VAC alignment consistency mask */
107extern uint_t vac_colors;
108
109/*
110 * Global Data Definitions:
111 */
112
113/*
114 * XXX - Don't port this to new architectures
115 * A 3rd party volume manager driver (vxdm) depends on the symbol romp.
116 * 'romp' has no use with a prom with an IEEE 1275 client interface.
117 * The driver doesn't use the value, but it depends on the symbol.
118 */
119void *romp;		/* veritas driver won't load without romp 4154976 */
120/*
121 * Declare these as initialized data so we can patch them.
122 */
123pgcnt_t physmem = 0;	/* memory size in pages, patch if you want less */
124pgcnt_t segkpsize =
125    btop(SEGKPDEFSIZE);	/* size of segkp segment in pages */
126uint_t segmap_percent = 6; /* Size of segmap segment */
127
128int use_cache = 1;		/* cache not reliable (605 bugs) with MP */
129int vac_copyback = 1;
130char *cache_mode = NULL;
131int use_mix = 1;
132int prom_debug = 0;
133
134caddr_t boot_tba;		/* %tba at boot - used by kmdb */
135uint_t	tba_taken_over = 0;
136
137caddr_t s_text;			/* start of kernel text segment */
138caddr_t e_text;			/* end of kernel text segment */
139caddr_t s_data;			/* start of kernel data segment */
140caddr_t e_data;			/* end of kernel data segment */
141
142caddr_t modtext;		/* beginning of module text */
143size_t	modtext_sz;		/* size of module text */
144caddr_t moddata;		/* beginning of module data reserve */
145caddr_t e_moddata;		/* end of module data reserve */
146
147/*
148 * End of first block of contiguous kernel in 32-bit virtual address space
149 */
150caddr_t		econtig32;	/* end of first blk of contiguous kernel */
151
152caddr_t		ncbase;		/* beginning of non-cached segment */
153caddr_t		ncend;		/* end of non-cached segment */
154
155size_t	ndata_remain_sz;	/* bytes from end of data to 4MB boundary */
156caddr_t	nalloc_base;		/* beginning of nucleus allocation */
157caddr_t nalloc_end;		/* end of nucleus allocatable memory */
158caddr_t valloc_base;		/* beginning of kvalloc segment	*/
159
160caddr_t kmem64_base;		/* base of kernel mem segment in 64-bit space */
161caddr_t kmem64_end;		/* end of kernel mem segment in 64-bit space */
162size_t	kmem64_sz;		/* bytes in kernel mem segment, 64-bit space */
163caddr_t kmem64_aligned_end;	/* end of large page, overmaps 64-bit space */
164int	kmem64_szc;		/* page size code */
165uint64_t kmem64_pabase = (uint64_t)-1;	/* physical address of kmem64_base */
166
167uintptr_t shm_alignment;	/* VAC address consistency modulus */
168struct memlist *phys_install;	/* Total installed physical memory */
169struct memlist *phys_avail;	/* Available (unreserved) physical memory */
170struct memlist *virt_avail;	/* Available (unmapped?) virtual memory */
171struct memlist *nopp_list;	/* pages with no backing page structs */
172struct memlist ndata;		/* memlist of nucleus allocatable memory */
173int memexp_flag;		/* memory expansion card flag */
174uint64_t ecache_flushaddr;	/* physical address used for flushing E$ */
175pgcnt_t obp_pages;		/* Physical pages used by OBP */
176
177/*
178 * VM data structures
179 */
180long page_hashsz;		/* Size of page hash table (power of two) */
181unsigned int page_hashsz_shift;	/* log2(page_hashsz) */
182struct page *pp_base;		/* Base of system page struct array */
183size_t pp_sz;			/* Size in bytes of page struct array */
184struct page **page_hash;	/* Page hash table */
185pad_mutex_t *pse_mutex;		/* Locks protecting pp->p_selock */
186size_t pse_table_size;		/* Number of mutexes in pse_mutex[] */
187int pse_shift;			/* log2(pse_table_size) */
188struct seg ktextseg;		/* Segment used for kernel executable image */
189struct seg kvalloc;		/* Segment used for "valloc" mapping */
190struct seg kpseg;		/* Segment used for pageable kernel virt mem */
191struct seg ktexthole;		/* Segment used for nucleus text hole */
192struct seg kmapseg;		/* Segment used for generic kernel mappings */
193struct seg kpmseg;		/* Segment used for physical mapping */
194struct seg kdebugseg;		/* Segment used for the kernel debugger */
195
196void *kpm_pp_base;		/* Base of system kpm_page array */
197size_t	kpm_pp_sz;		/* Size of system kpm_page array */
198pgcnt_t	kpm_npages;		/* How many kpm pages are managed */
199
200struct seg *segkp = &kpseg;	/* Pageable kernel virtual memory segment */
201struct seg *segkmap = &kmapseg;	/* Kernel generic mapping segment */
202struct seg *segkpm = &kpmseg;	/* 64bit kernel physical mapping segment */
203
204int segzio_fromheap = 0;	/* zio allocations occur from heap */
205caddr_t segzio_base;		/* Base address of segzio */
206pgcnt_t segziosize = 0;		/* size of zio segment in pages */
207
208/*
209 * A static DR page_t VA map is reserved that can map the page structures
210 * for a domain's entire RA space. The pages that backs this space are
211 * dynamically allocated and need not be physically contiguous.  The DR
212 * map size is derived from KPM size.
213 */
214int ppvm_enable = 0;		/* Static virtual map for page structs */
215page_t *ppvm_base;		/* Base of page struct map */
216pgcnt_t ppvm_size = 0;		/* Size of page struct map */
217
218/*
219 * debugger pages (if allocated)
220 */
221struct vnode kdebugvp;
222
223/*
224 * VA range available to the debugger
225 */
226const caddr_t kdi_segdebugbase = (const caddr_t)SEGDEBUGBASE;
227const size_t kdi_segdebugsize = SEGDEBUGSIZE;
228
229/*
230 * Segment for relocated kernel structures in 64-bit large RAM kernels
231 */
232struct seg kmem64;
233
234struct memseg *memseg_free;
235
236struct vnode unused_pages_vp;
237
238/*
239 * VM data structures allocated early during boot.
240 */
241size_t pagehash_sz;
242uint64_t memlist_sz;
243
244char tbr_wr_addr_inited = 0;
245
246caddr_t	mpo_heap32_buf = NULL;
247size_t	mpo_heap32_bufsz = 0;
248
249/*
250 * Static Routines:
251 */
252static int ndata_alloc_memseg(struct memlist *, size_t);
253static void memlist_new(uint64_t, uint64_t, struct memlist **);
254static void memlist_add(uint64_t, uint64_t,
255	struct memlist **, struct memlist **);
256static void kphysm_init(void);
257static void kvm_init(void);
258static void install_kmem64_tte(void);
259
260static void startup_init(void);
261static void startup_memlist(void);
262static void startup_modules(void);
263static void startup_bop_gone(void);
264static void startup_vm(void);
265static void startup_end(void);
266static void setup_cage_params(void);
267static void startup_create_io_node(void);
268
269static pgcnt_t npages;
270static struct memlist *memlist;
271void *memlist_end;
272
273static pgcnt_t bop_alloc_pages;
274static caddr_t hblk_base;
275uint_t hblk_alloc_dynamic = 0;
276uint_t hblk1_min = H1MIN;
277
278
279/*
280 * After receiving a thermal interrupt, this is the number of seconds
281 * to delay before shutting off the system, assuming
282 * shutdown fails.  Use /etc/system to change the delay if this isn't
283 * large enough.
284 */
285int thermal_powerdown_delay = 1200;
286
287/*
288 * Used to hold off page relocations into the cage until OBP has completed
289 * its boot-time handoff of its resources to the kernel.
290 */
291int page_relocate_ready = 0;
292
293/*
294 * Indicate if kmem64 allocation was done in small chunks
295 */
296int kmem64_smchunks = 0;
297
298/*
299 * Enable some debugging messages concerning memory usage...
300 */
301#ifdef  DEBUGGING_MEM
302static int debugging_mem;
303static void
304printmemlist(char *title, struct memlist *list)
305{
306	if (!debugging_mem)
307		return;
308
309	printf("%s\n", title);
310
311	while (list) {
312		prom_printf("\taddr = 0x%x %8x, size = 0x%x %8x\n",
313		    (uint32_t)(list->ml_address >> 32),
314		    (uint32_t)list->ml_address,
315		    (uint32_t)(list->ml_size >> 32),
316		    (uint32_t)(list->ml_size));
317		list = list->ml_next;
318	}
319}
320
321void
322printmemseg(struct memseg *memseg)
323{
324	if (!debugging_mem)
325		return;
326
327	printf("memseg\n");
328
329	while (memseg) {
330		prom_printf("\tpage = 0x%p, epage = 0x%p, "
331		    "pfn = 0x%x, epfn = 0x%x\n",
332		    memseg->pages, memseg->epages,
333		    memseg->pages_base, memseg->pages_end);
334		memseg = memseg->next;
335	}
336}
337
338#define	debug_pause(str)	halt((str))
339#define	MPRINTF(str)		if (debugging_mem) prom_printf((str))
340#define	MPRINTF1(str, a)	if (debugging_mem) prom_printf((str), (a))
341#define	MPRINTF2(str, a, b)	if (debugging_mem) prom_printf((str), (a), (b))
342#define	MPRINTF3(str, a, b, c) \
343	if (debugging_mem) prom_printf((str), (a), (b), (c))
344#else	/* DEBUGGING_MEM */
345#define	MPRINTF(str)
346#define	MPRINTF1(str, a)
347#define	MPRINTF2(str, a, b)
348#define	MPRINTF3(str, a, b, c)
349#endif	/* DEBUGGING_MEM */
350
351
352/*
353 *
354 *                    Kernel's Virtual Memory Layout.
355 *                       /-----------------------\
356 * 0xFFFFFFFF.FFFFFFFF  -|                       |-
357 *                       |   OBP's virtual page  |
358 *                       |        tables         |
359 * 0xFFFFFFFC.00000000  -|-----------------------|-
360 *                       :                       :
361 *                       :                       :
362 *                      -|-----------------------|-
363 *                       |       segzio          | (base and size vary)
364 * 0xFFFFFE00.00000000  -|-----------------------|-
365 *                       |                       |  Ultrasparc I/II support
366 *                       |    segkpm segment     |  up to 2TB of physical
367 *                       | (64-bit kernel ONLY)  |  memory, VAC has 2 colors
368 *                       |                       |
369 * 0xFFFFFA00.00000000  -|-----------------------|- 2TB segkpm alignment
370 *                       :                       :
371 *                       :                       :
372 * 0xFFFFF810.00000000  -|-----------------------|- hole_end
373 *                       |                       |      ^
374 *                       |  UltraSPARC I/II call |      |
375 *                       | bug requires an extra |      |
376 *                       | 4 GB of space between |      |
377 *                       |   hole and used RAM   |	|
378 *                       |                       |      |
379 * 0xFFFFF800.00000000  -|-----------------------|-     |
380 *                       |                       |      |
381 *                       | Virtual Address Hole  |   UltraSPARC
382 *                       |  on UltraSPARC I/II   |  I/II * ONLY *
383 *                       |                       |      |
384 * 0x00000800.00000000  -|-----------------------|-     |
385 *                       |                       |      |
386 *                       |  UltraSPARC I/II call |      |
387 *                       | bug requires an extra |      |
388 *                       | 4 GB of space between |      |
389 *                       |   hole and used RAM   |      |
390 *                       |                       |      v
391 * 0x000007FF.00000000  -|-----------------------|- hole_start -----
392 *                       :                       :		   ^
393 *                       :                       :		   |
394 *                       |-----------------------|                 |
395 *                       |                       |                 |
396 *                       |  ecache flush area    |                 |
397 *                       |  (twice largest e$)   |                 |
398 *                       |                       |                 |
399 * 0x00000XXX.XXX00000  -|-----------------------|- kmem64_	   |
400 *                       | overmapped area       |   alignend_end  |
401 *                       | (kmem64_alignsize     |		   |
402 *                       |  boundary)            |		   |
403 * 0x00000XXX.XXXXXXXX  -|-----------------------|- kmem64_end	   |
404 *                       |                       |		   |
405 *                       |   64-bit kernel ONLY  |		   |
406 *                       |                       |		   |
407 *                       |    kmem64 segment     |		   |
408 *                       |                       |		   |
409 *                       | (Relocated extra HME  |	     Approximately
410 *                       |   block allocations,  |	    1 TB of virtual
411 *                       |   memnode freelists,  |	     address space
412 *                       |    HME hash buckets,  |		   |
413 *                       | mml_table, kpmp_table,|		   |
414 *                       |  page_t array and     |		   |
415 *                       |  hashblock pool to    |		   |
416 *                       |   avoid hard-coded    |		   |
417 *                       |     32-bit vaddr      |		   |
418 *                       |     limitations)      |		   |
419 *                       |                       |		   v
420 * 0x00000700.00000000  -|-----------------------|- SYSLIMIT (kmem64_base)
421 *                       |                       |
422 *                       |  segkmem segment      | (SYSLIMIT - SYSBASE = 4TB)
423 *                       |                       |
424 * 0x00000300.00000000  -|-----------------------|- SYSBASE
425 *                       :                       :
426 *                       :                       :
427 *                      -|-----------------------|-
428 *                       |                       |
429 *                       |  segmap segment       |   SEGMAPSIZE (1/8th physmem,
430 *                       |                       |               256G MAX)
431 * 0x000002a7.50000000  -|-----------------------|- SEGMAPBASE
432 *                       :                       :
433 *                       :                       :
434 *                      -|-----------------------|-
435 *                       |                       |
436 *                       |       segkp           |    SEGKPSIZE (2GB)
437 *                       |                       |
438 *                       |                       |
439 * 0x000002a1.00000000  -|-----------------------|- SEGKPBASE
440 *                       |                       |
441 * 0x000002a0.00000000  -|-----------------------|- MEMSCRUBBASE
442 *                       |                       |       (SEGKPBASE - 0x400000)
443 * 0x0000029F.FFE00000  -|-----------------------|- ARGSBASE
444 *                       |                       |       (MEMSCRUBBASE - NCARGS)
445 * 0x0000029F.FFD80000  -|-----------------------|- PPMAPBASE
446 *                       |                       |       (ARGSBASE - PPMAPSIZE)
447 * 0x0000029F.FFD00000  -|-----------------------|- PPMAP_FAST_BASE
448 *                       |                       |
449 * 0x0000029F.FF980000  -|-----------------------|- PIOMAPBASE
450 *                       |                       |
451 * 0x0000029F.FF580000  -|-----------------------|- NARG_BASE
452 *                       :                       :
453 *                       :                       :
454 * 0x00000000.FFFFFFFF  -|-----------------------|- OFW_END_ADDR
455 *                       |                       |
456 *                       |         OBP           |
457 *                       |                       |
458 * 0x00000000.F0000000  -|-----------------------|- OFW_START_ADDR
459 *                       |         kmdb          |
460 * 0x00000000.EDD00000  -|-----------------------|- SEGDEBUGBASE
461 *                       :                       :
462 *                       :                       :
463 * 0x00000000.7c000000  -|-----------------------|- SYSLIMIT32
464 *                       |                       |
465 *                       |  segkmem32 segment    | (SYSLIMIT32 - SYSBASE32 =
466 *                       |                       |    ~64MB)
467 *			-|-----------------------|
468 *			 |	IVSIZE		 |
469 * 0x00000000.70004000  -|-----------------------|
470 *                       |     panicbuf          |
471 * 0x00000000.70002000	-|-----------------------|
472 *			 |	PAGESIZE	 |
473 * 0x00000000.70000000  -|-----------------------|- SYSBASE32
474 *                       |       boot-time       |
475 *                       |    temporary space    |
476 * 0x00000000.4C000000  -|-----------------------|- BOOTTMPBASE
477 *                       :                       :
478 *                       :                       :
479 *                       |                       |
480 *                       |-----------------------|- econtig32
481 *                       |    vm structures      |
482 * 0x00000000.01C00000   |-----------------------|- nalloc_end
483 *                       |         TSBs          |
484 *                       |-----------------------|- end/nalloc_base
485 *                       |   kernel data & bss   |
486 * 0x00000000.01800000  -|-----------------------|
487 *                       :   nucleus text hole   :
488 * 0x00000000.01400000  -|-----------------------|
489 *                       :                       :
490 *                       |-----------------------|
491 *                       |      module text      |
492 *                       |-----------------------|- e_text/modtext
493 *                       |      kernel text      |
494 *                       |-----------------------|
495 *                       |    trap table (48k)   |
496 * 0x00000000.01000000  -|-----------------------|- KERNELBASE
497 *                       | reserved for trapstat |} TSTAT_TOTAL_SIZE
498 *                       |-----------------------|
499 *                       |                       |
500 *                       |        invalid        |
501 *                       |                       |
502 * 0x00000000.00000000  _|_______________________|
503 *
504 *
505 *
506 *                   32-bit User Virtual Memory Layout.
507 *                       /-----------------------\
508 *                       |                       |
509 *                       |        invalid        |
510 *                       |                       |
511 *          0xFFC00000  -|-----------------------|- USERLIMIT
512 *                       |       user stack      |
513 *                       :                       :
514 *                       :                       :
515 *                       :                       :
516 *                       |       user data       |
517 *                      -|-----------------------|-
518 *                       |       user text       |
519 *          0x00002000  -|-----------------------|-
520 *                       |       invalid         |
521 *          0x00000000  _|_______________________|
522 *
523 *
524 *
525 *                   64-bit User Virtual Memory Layout.
526 *                       /-----------------------\
527 *                       |                       |
528 *                       |        invalid        |
529 *                       |                       |
530 *  0xFFFFFFFF.80000000 -|-----------------------|- USERLIMIT
531 *                       |       user stack      |
532 *                       :                       :
533 *                       :                       :
534 *                       :                       :
535 *                       |       user data       |
536 *                      -|-----------------------|-
537 *                       |       user text       |
538 *  0x00000000.01000000 -|-----------------------|-
539 *                       |       invalid         |
540 *  0x00000000.00000000 _|_______________________|
541 */
542
543extern caddr_t ecache_init_scrub_flush_area(caddr_t alloc_base);
544extern uint64_t ecache_flush_address(void);
545
546#pragma weak load_platform_modules
547#pragma weak plat_startup_memlist
548#pragma weak ecache_init_scrub_flush_area
549#pragma weak ecache_flush_address
550
551
552/*
553 * By default the DR Cage is enabled for maximum OS
554 * MPSS performance.  Users needing to disable the cage mechanism
555 * can set this variable to zero via /etc/system.
556 * Disabling the cage on systems supporting Dynamic Reconfiguration (DR)
557 * will result in loss of DR functionality.
558 * Platforms wishing to disable kernel Cage by default
559 * should do so in their set_platform_defaults() routine.
560 */
561int	kernel_cage_enable = 1;
562
563static void
564setup_cage_params(void)
565{
566	void (*func)(void);
567
568	func = (void (*)(void))kobj_getsymvalue("set_platform_cage_params", 0);
569	if (func != NULL) {
570		(*func)();
571		return;
572	}
573
574	if (kernel_cage_enable == 0) {
575		return;
576	}
577	kcage_range_init(phys_avail, KCAGE_DOWN, total_pages / 256);
578
579	if (kcage_on) {
580		cmn_err(CE_NOTE, "!Kernel Cage is ENABLED");
581	} else {
582		cmn_err(CE_NOTE, "!Kernel Cage is DISABLED");
583	}
584
585}
586
587/*
588 * Machine-dependent startup code
589 */
590void
591startup(void)
592{
593	startup_init();
594	if (&startup_platform)
595		startup_platform();
596	startup_memlist();
597	startup_modules();
598	setup_cage_params();
599	startup_bop_gone();
600	startup_vm();
601	startup_end();
602}
603
604struct regs sync_reg_buf;
605uint64_t sync_tt;
606
607void
608sync_handler(void)
609{
610	struct  panic_trap_info	ti;
611	int i;
612
613	/*
614	 * Prevent trying to talk to the other CPUs since they are
615	 * sitting in the prom and won't reply.
616	 */
617	for (i = 0; i < NCPU; i++) {
618		if ((i != CPU->cpu_id) && CPU_XCALL_READY(i)) {
619			cpu[i]->cpu_flags &= ~CPU_READY;
620			cpu[i]->cpu_flags |= CPU_QUIESCED;
621			CPUSET_DEL(cpu_ready_set, cpu[i]->cpu_id);
622		}
623	}
624
625	/*
626	 * Force a serial dump, since there are no CPUs to help.
627	 */
628	dump_plat_mincpu = 0;
629
630	/*
631	 * We've managed to get here without going through the
632	 * normal panic code path. Try and save some useful
633	 * information.
634	 */
635	if (!panicstr && (curthread->t_panic_trap == NULL)) {
636		ti.trap_type = sync_tt;
637		ti.trap_regs = &sync_reg_buf;
638		ti.trap_addr = NULL;
639		ti.trap_mmu_fsr = 0x0;
640
641		curthread->t_panic_trap = &ti;
642	}
643
644	/*
645	 * If we're re-entering the panic path, update the signature
646	 * block so that the SC knows we're in the second part of panic.
647	 */
648	if (panicstr)
649		CPU_SIGNATURE(OS_SIG, SIGST_EXIT, SIGSUBST_DUMP, -1);
650
651	nopanicdebug = 1; /* do not perform debug_enter() prior to dump */
652	panic("sync initiated");
653}
654
655
656static void
657startup_init(void)
658{
659	/*
660	 * We want to save the registers while we're still in OBP
661	 * so that we know they haven't been fiddled with since.
662	 * (In principle, OBP can't change them just because it
663	 * makes a callback, but we'd rather not depend on that
664	 * behavior.)
665	 */
666	char		sync_str[] =
667	    "warning @ warning off : sync "
668	    "%%tl-c %%tstate h# %p x! "
669	    "%%g1 h# %p x! %%g2 h# %p x! %%g3 h# %p x! "
670	    "%%g4 h# %p x! %%g5 h# %p x! %%g6 h# %p x! "
671	    "%%g7 h# %p x! %%o0 h# %p x! %%o1 h# %p x! "
672	    "%%o2 h# %p x! %%o3 h# %p x! %%o4 h# %p x! "
673	    "%%o5 h# %p x! %%o6 h# %p x! %%o7 h# %p x! "
674	    "%%tl-c %%tpc h# %p x! %%tl-c %%tnpc h# %p x! "
675	    "%%y h# %p l! %%tl-c %%tt h# %p x! "
676	    "sync ; warning !";
677
678	/*
679	 * 20 == num of %p substrings
680	 * 16 == max num of chars %p will expand to.
681	 */
682	char		bp[sizeof (sync_str) + 16 * 20];
683
684	/*
685	 * Initialize ptl1 stack for the 1st CPU.
686	 */
687	ptl1_init_cpu(&cpu0);
688
689	/*
690	 * Initialize the address map for cache consistent mappings
691	 * to random pages; must be done after vac_size is set.
692	 */
693	ppmapinit();
694
695	/*
696	 * Initialize the PROM callback handler.
697	 */
698	init_vx_handler();
699
700	/*
701	 * have prom call sync_callback() to handle the sync and
702	 * save some useful information which will be stored in the
703	 * core file later.
704	 */
705	(void) sprintf((char *)bp, sync_str,
706	    (void *)&sync_reg_buf.r_tstate, (void *)&sync_reg_buf.r_g1,
707	    (void *)&sync_reg_buf.r_g2, (void *)&sync_reg_buf.r_g3,
708	    (void *)&sync_reg_buf.r_g4, (void *)&sync_reg_buf.r_g5,
709	    (void *)&sync_reg_buf.r_g6, (void *)&sync_reg_buf.r_g7,
710	    (void *)&sync_reg_buf.r_o0, (void *)&sync_reg_buf.r_o1,
711	    (void *)&sync_reg_buf.r_o2, (void *)&sync_reg_buf.r_o3,
712	    (void *)&sync_reg_buf.r_o4, (void *)&sync_reg_buf.r_o5,
713	    (void *)&sync_reg_buf.r_o6, (void *)&sync_reg_buf.r_o7,
714	    (void *)&sync_reg_buf.r_pc, (void *)&sync_reg_buf.r_npc,
715	    (void *)&sync_reg_buf.r_y, (void *)&sync_tt);
716	prom_interpret(bp, 0, 0, 0, 0, 0);
717	add_vx_handler("sync", 1, (void (*)(cell_t *))sync_handler);
718}
719
720
721size_t
722calc_pp_sz(pgcnt_t npages)
723{
724
725	return (npages * sizeof (struct page));
726}
727
728size_t
729calc_kpmpp_sz(pgcnt_t npages)
730{
731
732	kpm_pgshft = (kpm_smallpages == 0) ? MMU_PAGESHIFT4M : MMU_PAGESHIFT;
733	kpm_pgsz = 1ull << kpm_pgshft;
734	kpm_pgoff = kpm_pgsz - 1;
735	kpmp2pshft = kpm_pgshft - PAGESHIFT;
736	kpmpnpgs = 1 << kpmp2pshft;
737
738	if (kpm_smallpages == 0) {
739		/*
740		 * Avoid fragmentation problems in kphysm_init()
741		 * by allocating for all of physical memory
742		 */
743		kpm_npages = ptokpmpr(physinstalled);
744		return (kpm_npages * sizeof (kpm_page_t));
745	} else {
746		kpm_npages = npages;
747		return (kpm_npages * sizeof (kpm_spage_t));
748	}
749}
750
751size_t
752calc_pagehash_sz(pgcnt_t npages)
753{
754	/* LINTED */
755	ASSERT(P2SAMEHIGHBIT((1 << PP_SHIFT), (sizeof (struct page))));
756	/*
757	 * The page structure hash table size is a power of 2
758	 * such that the average hash chain length is PAGE_HASHAVELEN.
759	 */
760	page_hashsz = npages / PAGE_HASHAVELEN;
761	page_hashsz_shift = MAX((AN_VPSHIFT + VNODE_ALIGN_LOG2 + 1),
762	    highbit(page_hashsz));
763	page_hashsz = 1 << page_hashsz_shift;
764	return (page_hashsz * sizeof (struct page *));
765}
766
767int testkmem64_smchunks = 0;
768
769int
770alloc_kmem64(caddr_t base, caddr_t end)
771{
772	int i;
773	caddr_t aligned_end = NULL;
774
775	if (testkmem64_smchunks)
776		return (1);
777
778	/*
779	 * Make one large memory alloc after figuring out the 64-bit size. This
780	 * will enable use of the largest page size appropriate for the system
781	 * architecture.
782	 */
783	ASSERT(mmu_exported_pagesize_mask & (1 << TTE8K));
784	ASSERT(IS_P2ALIGNED(base, TTEBYTES(max_bootlp_tteszc)));
785	for (i = max_bootlp_tteszc; i >= TTE8K; i--) {
786		size_t alloc_size, alignsize;
787#if !defined(C_OBP)
788		unsigned long long pa;
789#endif	/* !C_OBP */
790
791		if ((mmu_exported_pagesize_mask & (1 << i)) == 0)
792			continue;
793		alignsize = TTEBYTES(i);
794		kmem64_szc = i;
795
796		/* limit page size for small memory */
797		if (mmu_btop(alignsize) > (npages >> 2))
798			continue;
799
800		aligned_end = (caddr_t)roundup((uintptr_t)end, alignsize);
801		alloc_size = aligned_end - base;
802#if !defined(C_OBP)
803		if (prom_allocate_phys(alloc_size, alignsize, &pa) == 0) {
804			if (prom_claim_virt(alloc_size, base) != (caddr_t)-1) {
805				kmem64_pabase = pa;
806				kmem64_aligned_end = aligned_end;
807				install_kmem64_tte();
808				break;
809			} else {
810				prom_free_phys(alloc_size, pa);
811			}
812		}
813#else	/* !C_OBP */
814		if (prom_alloc(base, alloc_size, alignsize) == base) {
815			kmem64_pabase = va_to_pa(kmem64_base);
816			kmem64_aligned_end = aligned_end;
817			break;
818		}
819#endif	/* !C_OBP */
820		if (i == TTE8K) {
821#ifdef sun4v
822			/* return failure to try small allocations */
823			return (1);
824#else
825			prom_panic("kmem64 allocation failure");
826#endif
827		}
828	}
829	ASSERT(aligned_end != NULL);
830	return (0);
831}
832
833static prom_memlist_t *boot_physinstalled, *boot_physavail, *boot_virtavail;
834static size_t boot_physinstalled_len, boot_physavail_len, boot_virtavail_len;
835
836#if !defined(C_OBP)
837/*
838 * Install a temporary tte handler in OBP for kmem64 area.
839 *
840 * We map kmem64 area with large pages before the trap table is taken
841 * over. Since OBP makes 8K mappings, it can create 8K tlb entries in
842 * the same area. Duplicate tlb entries with different page sizes
843 * cause unpredicatble behavior.  To avoid this, we don't create
844 * kmem64 mappings via BOP_ALLOC (ends up as prom_alloc() call to
845 * OBP).  Instead, we manage translations with a temporary va>tte-data
846 * handler (kmem64-tte).  This handler is replaced by unix-tte when
847 * the trap table is taken over.
848 *
849 * The temporary handler knows the physical address of the kmem64
850 * area. It uses the prom's pgmap@ Forth word for other addresses.
851 *
852 * We have to use BOP_ALLOC() method for C-OBP platforms because
853 * pgmap@ is not defined in C-OBP. C-OBP is only used on serengeti
854 * sun4u platforms. On sun4u we flush tlb after trap table is taken
855 * over if we use large pages for kernel heap and kmem64. Since sun4u
856 * prom (unlike sun4v) calls va>tte-data first for client address
857 * translation prom's ttes for kmem64 can't get into TLB even if we
858 * later switch to prom's trap table again. C-OBP uses 4M pages for
859 * client mappings when possible so on all platforms we get the
860 * benefit from large mappings for kmem64 area immediately during
861 * boot.
862 *
863 * pseudo code:
864 * if (context != 0) {
865 *	return false
866 * } else if (miss_va in range[kmem64_base, kmem64_end)) {
867 *	tte = tte_template +
868 *		(((miss_va & pagemask) - kmem64_base));
869 *	return tte, true
870 * } else {
871 *	return pgmap@ result
872 * }
873 */
874char kmem64_obp_str[] =
875	"h# %lx constant kmem64-base "
876	"h# %lx constant kmem64-end "
877	"h# %lx constant kmem64-pagemask "
878	"h# %lx constant kmem64-template "
879
880	": kmem64-tte ( addr cnum -- false | tte-data true ) "
881	"    if                                       ( addr ) "
882	"       drop false exit then                  ( false ) "
883	"    dup  kmem64-base kmem64-end  within  if  ( addr ) "
884	"	kmem64-pagemask and                   ( addr' ) "
885	"	kmem64-base -                         ( addr' ) "
886	"	kmem64-template +                     ( tte ) "
887	"	true                                  ( tte true ) "
888	"    else                                     ( addr ) "
889	"	pgmap@                                ( tte ) "
890	"       dup 0< if true else drop false then   ( tte true  |  false ) "
891	"    then                                     ( tte true  |  false ) "
892	"; "
893
894	"' kmem64-tte is va>tte-data "
895;
896
897static void
898install_kmem64_tte()
899{
900	char b[sizeof (kmem64_obp_str) + (4 * 16)];
901	tte_t tte;
902
903	PRM_DEBUG(kmem64_pabase);
904	PRM_DEBUG(kmem64_szc);
905	sfmmu_memtte(&tte, kmem64_pabase >> MMU_PAGESHIFT,
906	    PROC_DATA | HAT_NOSYNC, kmem64_szc);
907	PRM_DEBUG(tte.ll);
908	(void) sprintf(b, kmem64_obp_str,
909	    kmem64_base, kmem64_end, TTE_PAGEMASK(kmem64_szc), tte.ll);
910	ASSERT(strlen(b) < sizeof (b));
911	prom_interpret(b, 0, 0, 0, 0, 0);
912}
913#endif	/* !C_OBP */
914
915/*
916 * As OBP takes up some RAM when the system boots, pages will already be "lost"
917 * to the system and reflected in npages by the time we see it.
918 *
919 * We only want to allocate kernel structures in the 64-bit virtual address
920 * space on systems with enough RAM to make the overhead of keeping track of
921 * an extra kernel memory segment worthwhile.
922 *
923 * Since OBP has already performed its memory allocations by this point, if we
924 * have more than MINMOVE_RAM_MB MB of RAM left free, go ahead and map
925 * memory in the 64-bit virtual address space; otherwise keep allocations
926 * contiguous with we've mapped so far in the 32-bit virtual address space.
927 */
928#define	MINMOVE_RAM_MB	((size_t)1900)
929#define	MB_TO_BYTES(mb)	((mb) * 1048576ul)
930#define	BYTES_TO_MB(b) ((b) / 1048576ul)
931
932pgcnt_t	tune_npages = (pgcnt_t)
933	(MB_TO_BYTES(MINMOVE_RAM_MB)/ (size_t)MMU_PAGESIZE);
934
935#pragma weak page_set_colorequiv_arr_cpu
936extern void page_set_colorequiv_arr_cpu(void);
937extern void page_set_colorequiv_arr(void);
938
939static pgcnt_t ramdisk_npages;
940static struct memlist *old_phys_avail;
941
942kcage_dir_t kcage_startup_dir = KCAGE_DOWN;
943
944static void
945startup_memlist(void)
946{
947	size_t hmehash_sz, pagelist_sz, tt_sz;
948	size_t psetable_sz;
949	caddr_t alloc_base;
950	caddr_t memspace;
951	struct memlist *cur;
952	size_t syslimit = (size_t)SYSLIMIT;
953	size_t sysbase = (size_t)SYSBASE;
954
955	/*
956	 * Initialize enough of the system to allow kmem_alloc to work by
957	 * calling boot to allocate its memory until the time that
958	 * kvm_init is completed.  The page structs are allocated after
959	 * rounding up end to the nearest page boundary; the memsegs are
960	 * initialized and the space they use comes from the kernel heap.
961	 * With appropriate initialization, they can be reallocated later
962	 * to a size appropriate for the machine's configuration.
963	 *
964	 * At this point, memory is allocated for things that will never
965	 * need to be freed, this used to be "valloced".  This allows a
966	 * savings as the pages don't need page structures to describe
967	 * them because them will not be managed by the vm system.
968	 */
969
970	/*
971	 * We're loaded by boot with the following configuration (as
972	 * specified in the sun4u/conf/Mapfile):
973	 *
974	 *	text:		4 MB chunk aligned on a 4MB boundary
975	 *	data & bss:	4 MB chunk aligned on a 4MB boundary
976	 *
977	 * These two chunks will eventually be mapped by 2 locked 4MB
978	 * ttes and will represent the nucleus of the kernel.  This gives
979	 * us some free space that is already allocated, some or all of
980	 * which is made available to kernel module text.
981	 *
982	 * The free space in the data-bss chunk is used for nucleus
983	 * allocatable data structures and we reserve it using the
984	 * nalloc_base and nalloc_end variables.  This space is currently
985	 * being used for hat data structures required for tlb miss
986	 * handling operations.  We align nalloc_base to a l2 cache
987	 * linesize because this is the line size the hardware uses to
988	 * maintain cache coherency.
989	 * 512K is carved out for module data.
990	 */
991
992	moddata = (caddr_t)roundup((uintptr_t)e_data, MMU_PAGESIZE);
993	e_moddata = moddata + MODDATA;
994	nalloc_base = e_moddata;
995
996	nalloc_end = (caddr_t)roundup((uintptr_t)nalloc_base, MMU_PAGESIZE4M);
997	valloc_base = nalloc_base;
998
999	/*
1000	 * Calculate the start of the data segment.
1001	 */
1002	if (((uintptr_t)e_moddata & MMU_PAGEMASK4M) != (uintptr_t)s_data)
1003		prom_panic("nucleus data overflow");
1004
1005	PRM_DEBUG(moddata);
1006	PRM_DEBUG(nalloc_base);
1007	PRM_DEBUG(nalloc_end);
1008
1009	/*
1010	 * Remember any slop after e_text so we can give it to the modules.
1011	 */
1012	PRM_DEBUG(e_text);
1013	modtext = (caddr_t)roundup((uintptr_t)e_text, MMU_PAGESIZE);
1014	if (((uintptr_t)e_text & MMU_PAGEMASK4M) != (uintptr_t)s_text)
1015		prom_panic("nucleus text overflow");
1016	modtext_sz = (caddr_t)roundup((uintptr_t)modtext, MMU_PAGESIZE4M) -
1017	    modtext;
1018	PRM_DEBUG(modtext);
1019	PRM_DEBUG(modtext_sz);
1020
1021	init_boot_memlists();
1022	copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,
1023	    &boot_physavail, &boot_physavail_len,
1024	    &boot_virtavail, &boot_virtavail_len);
1025
1026	/*
1027	 * Remember what the physically available highest page is
1028	 * so that dumpsys works properly, and find out how much
1029	 * memory is installed.
1030	 */
1031	installed_top_size_memlist_array(boot_physinstalled,
1032	    boot_physinstalled_len, &physmax, &physinstalled);
1033	PRM_DEBUG(physinstalled);
1034	PRM_DEBUG(physmax);
1035
1036	/* Fill out memory nodes config structure */
1037	startup_build_mem_nodes(boot_physinstalled, boot_physinstalled_len);
1038
1039	/*
1040	 * npages is the maximum of available physical memory possible.
1041	 * (ie. it will never be more than this)
1042	 *
1043	 * When we boot from a ramdisk, the ramdisk memory isn't free, so
1044	 * using phys_avail will underestimate what will end up being freed.
1045	 * A better initial guess is just total memory minus the kernel text
1046	 */
1047	npages = physinstalled - btop(MMU_PAGESIZE4M);
1048
1049	/*
1050	 * First allocate things that can go in the nucleus data page
1051	 * (fault status, TSBs, dmv, CPUs)
1052	 */
1053	ndata_alloc_init(&ndata, (uintptr_t)nalloc_base, (uintptr_t)nalloc_end);
1054
1055	if ((&ndata_alloc_mmfsa != NULL) && (ndata_alloc_mmfsa(&ndata) != 0))
1056		cmn_err(CE_PANIC, "no more nucleus memory after mfsa alloc");
1057
1058	if (ndata_alloc_tsbs(&ndata, npages) != 0)
1059		cmn_err(CE_PANIC, "no more nucleus memory after tsbs alloc");
1060
1061	if (ndata_alloc_dmv(&ndata) != 0)
1062		cmn_err(CE_PANIC, "no more nucleus memory after dmv alloc");
1063
1064	if (ndata_alloc_page_mutexs(&ndata) != 0)
1065		cmn_err(CE_PANIC,
1066		    "no more nucleus memory after page free lists alloc");
1067
1068	if (ndata_alloc_hat(&ndata) != 0)
1069		cmn_err(CE_PANIC, "no more nucleus memory after hat alloc");
1070
1071	if (ndata_alloc_memseg(&ndata, boot_physavail_len) != 0)
1072		cmn_err(CE_PANIC, "no more nucleus memory after memseg alloc");
1073
1074	/*
1075	 * WARNING WARNING WARNING WARNING WARNING WARNING WARNING
1076	 *
1077	 * There are comments all over the SFMMU code warning of dire
1078	 * consequences if the TSBs are moved out of 32-bit space.  This
1079	 * is largely because the asm code uses "sethi %hi(addr)"-type
1080	 * instructions which will not provide the expected result if the
1081	 * address is a 64-bit one.
1082	 *
1083	 * WARNING WARNING WARNING WARNING WARNING WARNING WARNING
1084	 */
1085	alloc_base = (caddr_t)roundup((uintptr_t)nalloc_end, MMU_PAGESIZE);
1086	PRM_DEBUG(alloc_base);
1087
1088	alloc_base = sfmmu_ktsb_alloc(alloc_base);
1089	alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1090	PRM_DEBUG(alloc_base);
1091
1092	/*
1093	 * Allocate IOMMU TSB array.  We do this here so that the physical
1094	 * memory gets deducted from the PROM's physical memory list.
1095	 */
1096	alloc_base = iommu_tsb_init(alloc_base);
1097	alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1098	PRM_DEBUG(alloc_base);
1099
1100	/*
1101	 * Allow for an early allocation of physically contiguous memory.
1102	 */
1103	alloc_base = contig_mem_prealloc(alloc_base, npages);
1104
1105	/*
1106	 * Platforms like Starcat and OPL need special structures assigned in
1107	 * 32-bit virtual address space because their probing routines execute
1108	 * FCode, and FCode can't handle 64-bit virtual addresses...
1109	 */
1110	if (&plat_startup_memlist) {
1111		alloc_base = plat_startup_memlist(alloc_base);
1112		alloc_base = (caddr_t)roundup((uintptr_t)alloc_base,
1113		    ecache_alignsize);
1114		PRM_DEBUG(alloc_base);
1115	}
1116
1117	/*
1118	 * Save off where the contiguous allocations to date have ended
1119	 * in econtig32.
1120	 */
1121	econtig32 = alloc_base;
1122	PRM_DEBUG(econtig32);
1123	if (econtig32 > (caddr_t)KERNEL_LIMIT32)
1124		cmn_err(CE_PANIC, "econtig32 too big");
1125
1126	pp_sz = calc_pp_sz(npages);
1127	PRM_DEBUG(pp_sz);
1128	if (kpm_enable) {
1129		kpm_pp_sz = calc_kpmpp_sz(npages);
1130		PRM_DEBUG(kpm_pp_sz);
1131	}
1132
1133	hmehash_sz = calc_hmehash_sz(npages);
1134	PRM_DEBUG(hmehash_sz);
1135
1136	pagehash_sz = calc_pagehash_sz(npages);
1137	PRM_DEBUG(pagehash_sz);
1138
1139	pagelist_sz = calc_free_pagelist_sz();
1140	PRM_DEBUG(pagelist_sz);
1141
1142#ifdef	TRAPTRACE
1143	tt_sz = calc_traptrace_sz();
1144	PRM_DEBUG(tt_sz);
1145#else
1146	tt_sz = 0;
1147#endif	/* TRAPTRACE */
1148
1149	/*
1150	 * Place the array that protects pp->p_selock in the kmem64 wad.
1151	 */
1152	pse_shift = size_pse_array(npages, max_ncpus);
1153	PRM_DEBUG(pse_shift);
1154	pse_table_size = 1 << pse_shift;
1155	PRM_DEBUG(pse_table_size);
1156	psetable_sz = roundup(
1157	    pse_table_size * sizeof (pad_mutex_t), ecache_alignsize);
1158	PRM_DEBUG(psetable_sz);
1159
1160	/*
1161	 * Now allocate the whole wad
1162	 */
1163	kmem64_sz = pp_sz + kpm_pp_sz + hmehash_sz + pagehash_sz +
1164	    pagelist_sz + tt_sz + psetable_sz;
1165	kmem64_sz = roundup(kmem64_sz, PAGESIZE);
1166	kmem64_base = (caddr_t)syslimit;
1167	kmem64_end = kmem64_base + kmem64_sz;
1168	if (alloc_kmem64(kmem64_base, kmem64_end)) {
1169		/*
1170		 * Attempt for kmem64 to allocate one big
1171		 * contiguous chunk of memory failed.
1172		 * We get here because we are sun4v.
1173		 * We will proceed by breaking up
1174		 * the allocation into two attempts.
1175		 * First, we allocate kpm_pp_sz, hmehash_sz,
1176		 * pagehash_sz, pagelist_sz, tt_sz & psetable_sz as
1177		 * one contiguous chunk. This is a much smaller
1178		 * chunk and we should get it, if not we panic.
1179		 * Note that hmehash and tt need to be physically
1180		 * (in the real address sense) contiguous.
1181		 * Next, we use bop_alloc_chunk() to
1182		 * to allocate the page_t structures.
1183		 * This will allow the page_t to be allocated
1184		 * in multiple smaller chunks.
1185		 * In doing so, the assumption that page_t is
1186		 * physically contiguous no longer hold, this is ok
1187		 * for sun4v but not for sun4u.
1188		 */
1189		size_t  tmp_size;
1190		caddr_t tmp_base;
1191
1192		pp_sz  = roundup(pp_sz, PAGESIZE);
1193
1194		/*
1195		 * Allocate kpm_pp_sz, hmehash_sz,
1196		 * pagehash_sz, pagelist_sz, tt_sz & psetable_sz
1197		 */
1198		tmp_base = kmem64_base + pp_sz;
1199		tmp_size = roundup(kpm_pp_sz + hmehash_sz + pagehash_sz +
1200		    pagelist_sz + tt_sz + psetable_sz, PAGESIZE);
1201		if (prom_alloc(tmp_base, tmp_size, PAGESIZE) == 0)
1202			prom_panic("kmem64 prom_alloc contig failed");
1203		PRM_DEBUG(tmp_base);
1204		PRM_DEBUG(tmp_size);
1205
1206		/*
1207		 * Allocate the page_ts
1208		 */
1209		if (bop_alloc_chunk(kmem64_base, pp_sz, PAGESIZE) == 0)
1210			prom_panic("kmem64 bop_alloc_chunk page_t failed");
1211		PRM_DEBUG(kmem64_base);
1212		PRM_DEBUG(pp_sz);
1213
1214		kmem64_aligned_end = kmem64_base + pp_sz + tmp_size;
1215		ASSERT(kmem64_aligned_end >= kmem64_end);
1216
1217		kmem64_smchunks = 1;
1218	} else {
1219
1220		/*
1221		 * We need to adjust pp_sz for the normal
1222		 * case where kmem64 can allocate one large chunk
1223		 */
1224		if (kpm_smallpages == 0) {
1225			npages -= kmem64_sz / (PAGESIZE + sizeof (struct page));
1226		} else {
1227			npages -= kmem64_sz / (PAGESIZE + sizeof (struct page) +
1228			    sizeof (kpm_spage_t));
1229		}
1230		pp_sz = npages * sizeof (struct page);
1231	}
1232
1233	if (kmem64_aligned_end > (hole_start ? hole_start : kpm_vbase))
1234		cmn_err(CE_PANIC, "not enough kmem64 space");
1235	PRM_DEBUG(kmem64_base);
1236	PRM_DEBUG(kmem64_end);
1237	PRM_DEBUG(kmem64_aligned_end);
1238
1239	/*
1240	 * ... and divy it up
1241	 */
1242	alloc_base = kmem64_base;
1243
1244	pp_base = (page_t *)alloc_base;
1245	alloc_base += pp_sz;
1246	alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1247	PRM_DEBUG(pp_base);
1248	PRM_DEBUG(npages);
1249
1250	if (kpm_enable) {
1251		kpm_pp_base = alloc_base;
1252		if (kpm_smallpages == 0) {
1253			/* kpm_npages based on physinstalled, don't reset */
1254			kpm_pp_sz = kpm_npages * sizeof (kpm_page_t);
1255		} else {
1256			kpm_npages = ptokpmpr(npages);
1257			kpm_pp_sz = kpm_npages * sizeof (kpm_spage_t);
1258		}
1259		alloc_base += kpm_pp_sz;
1260		alloc_base =
1261		    (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1262		PRM_DEBUG(kpm_pp_base);
1263	}
1264
1265	alloc_base = alloc_hmehash(alloc_base);
1266	alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1267	PRM_DEBUG(alloc_base);
1268
1269	page_hash = (page_t **)alloc_base;
1270	alloc_base += pagehash_sz;
1271	alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1272	PRM_DEBUG(page_hash);
1273
1274	alloc_base = alloc_page_freelists(alloc_base);
1275	alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1276	PRM_DEBUG(alloc_base);
1277
1278#ifdef	TRAPTRACE
1279	ttrace_buf = alloc_base;
1280	alloc_base += tt_sz;
1281	alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1282	PRM_DEBUG(alloc_base);
1283#endif	/* TRAPTRACE */
1284
1285	pse_mutex = (pad_mutex_t *)alloc_base;
1286	alloc_base += psetable_sz;
1287	alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize);
1288	PRM_DEBUG(alloc_base);
1289
1290	/*
1291	 * Note that if we use small chunk allocations for
1292	 * kmem64, we need to ensure kmem64_end is the same as
1293	 * kmem64_aligned_end to prevent subsequent logic from
1294	 * trying to reuse the overmapping.
1295	 * Otherwise we adjust kmem64_end to what we really allocated.
1296	 */
1297	if (kmem64_smchunks) {
1298		kmem64_end = kmem64_aligned_end;
1299	} else {
1300		kmem64_end = (caddr_t)roundup((uintptr_t)alloc_base, PAGESIZE);
1301	}
1302	kmem64_sz = kmem64_end - kmem64_base;
1303
1304	if (&ecache_init_scrub_flush_area) {
1305		alloc_base = ecache_init_scrub_flush_area(kmem64_aligned_end);
1306		ASSERT(alloc_base <= (hole_start ? hole_start : kpm_vbase));
1307	}
1308
1309	/*
1310	 * If physmem is patched to be non-zero, use it instead of
1311	 * the monitor value unless physmem is larger than the total
1312	 * amount of memory on hand.
1313	 */
1314	if (physmem == 0 || physmem > npages)
1315		physmem = npages;
1316
1317	/*
1318	 * root_is_ramdisk is set via /etc/system when the ramdisk miniroot
1319	 * is mounted as root. This memory is held down by OBP and unlike
1320	 * the stub boot_archive is never released.
1321	 *
1322	 * In order to get things sized correctly on lower memory
1323	 * machines (where the memory used by the ramdisk represents
1324	 * a significant portion of memory), physmem is adjusted.
1325	 *
1326	 * This is done by subtracting the ramdisk_size which is set
1327	 * to the size of the ramdisk (in Kb) in /etc/system at the
1328	 * time the miniroot archive is constructed.
1329	 */
1330	if (root_is_ramdisk == B_TRUE) {
1331		ramdisk_npages = (ramdisk_size * 1024) / PAGESIZE;
1332		physmem -= ramdisk_npages;
1333	}
1334
1335	if (kpm_enable && (ndata_alloc_kpm(&ndata, kpm_npages) != 0))
1336		cmn_err(CE_PANIC, "no more nucleus memory after kpm alloc");
1337
1338	/*
1339	 * Allocate space for the interrupt vector table.
1340	 */
1341	memspace = prom_alloc((caddr_t)intr_vec_table, IVSIZE, MMU_PAGESIZE);
1342	if (memspace != (caddr_t)intr_vec_table)
1343		prom_panic("interrupt vector table allocation failure");
1344
1345	/*
1346	 * Between now and when we finish copying in the memory lists,
1347	 * allocations happen so the space gets fragmented and the
1348	 * lists longer.  Leave enough space for lists twice as
1349	 * long as we have now; then roundup to a pagesize.
1350	 */
1351	memlist_sz = sizeof (struct memlist) * (prom_phys_installed_len() +
1352	    prom_phys_avail_len() + prom_virt_avail_len());
1353	memlist_sz *= 2;
1354	memlist_sz = roundup(memlist_sz, PAGESIZE);
1355	memspace = ndata_alloc(&ndata, memlist_sz, ecache_alignsize);
1356	if (memspace == NULL)
1357		cmn_err(CE_PANIC, "no more nucleus memory after memlist alloc");
1358
1359	memlist = (struct memlist *)memspace;
1360	memlist_end = (char *)memspace + memlist_sz;
1361	PRM_DEBUG(memlist);
1362	PRM_DEBUG(memlist_end);
1363
1364	PRM_DEBUG(sysbase);
1365	PRM_DEBUG(syslimit);
1366	kernelheap_init((void *)sysbase, (void *)syslimit,
1367	    (caddr_t)sysbase + PAGESIZE, NULL, NULL);
1368
1369	/*
1370	 * Take the most current snapshot we can by calling mem-update.
1371	 */
1372	copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,
1373	    &boot_physavail, &boot_physavail_len,
1374	    &boot_virtavail, &boot_virtavail_len);
1375
1376	/*
1377	 * Remove the space used by prom_alloc from the kernel heap
1378	 * plus the area actually used by the OBP (if any)
1379	 * ignoring virtual addresses in virt_avail, above syslimit.
1380	 */
1381	virt_avail = memlist;
1382	copy_memlist(boot_virtavail, boot_virtavail_len, &memlist);
1383
1384	for (cur = virt_avail; cur->ml_next; cur = cur->ml_next) {
1385		uint64_t range_base, range_size;
1386
1387		if ((range_base = cur->ml_address + cur->ml_size) <
1388		    (uint64_t)sysbase)
1389			continue;
1390		if (range_base >= (uint64_t)syslimit)
1391			break;
1392		/*
1393		 * Limit the range to end at syslimit.
1394		 */
1395		range_size = MIN(cur->ml_next->ml_address,
1396		    (uint64_t)syslimit) - range_base;
1397		(void) vmem_xalloc(heap_arena, (size_t)range_size, PAGESIZE,
1398		    0, 0, (void *)range_base, (void *)(range_base + range_size),
1399		    VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
1400	}
1401
1402	phys_avail = memlist;
1403	copy_memlist(boot_physavail, boot_physavail_len, &memlist);
1404
1405	/*
1406	 * Add any extra memory at the end of the ndata region if there's at
1407	 * least a page to add.  There might be a few more pages available in
1408	 * the middle of the ndata region, but for now they are ignored.
1409	 */
1410	nalloc_base = ndata_extra_base(&ndata, MMU_PAGESIZE, nalloc_end);
1411	if (nalloc_base == NULL)
1412		nalloc_base = nalloc_end;
1413	ndata_remain_sz = nalloc_end - nalloc_base;
1414
1415	/*
1416	 * Copy physinstalled list into kernel space.
1417	 */
1418	phys_install = memlist;
1419	copy_memlist(boot_physinstalled, boot_physinstalled_len, &memlist);
1420
1421	/*
1422	 * Create list of physical addrs we don't need pp's for:
1423	 * kernel text 4M page
1424	 * kernel data 4M page - ndata_remain_sz
1425	 * kmem64 pages
1426	 *
1427	 * NB if adding any pages here, make sure no kpm page
1428	 * overlaps can occur (see ASSERTs in kphysm_memsegs)
1429	 */
1430	nopp_list = memlist;
1431	memlist_new(va_to_pa(s_text), MMU_PAGESIZE4M, &memlist);
1432	memlist_add(va_to_pa(s_data), MMU_PAGESIZE4M - ndata_remain_sz,
1433	    &memlist, &nopp_list);
1434
1435	/* Don't add to nopp_list if kmem64 was allocated in smchunks */
1436	if (!kmem64_smchunks)
1437		memlist_add(kmem64_pabase, kmem64_sz, &memlist, &nopp_list);
1438
1439	if ((caddr_t)memlist > (memspace + memlist_sz))
1440		prom_panic("memlist overflow");
1441
1442	/*
1443	 * Size the pcf array based on the number of cpus in the box at
1444	 * boot time.
1445	 */
1446	pcf_init();
1447
1448	/*
1449	 * Initialize the page structures from the memory lists.
1450	 */
1451	kphysm_init();
1452
1453	availrmem_initial = availrmem = freemem;
1454	PRM_DEBUG(availrmem);
1455
1456	/*
1457	 * Some of the locks depend on page_hashsz being set!
1458	 * kmem_init() depends on this; so, keep it here.
1459	 */
1460	page_lock_init();
1461
1462	/*
1463	 * Initialize kernel memory allocator.
1464	 */
1465	kmem_init();
1466
1467	/*
1468	 * Factor in colorequiv to check additional 'equivalent' bins
1469	 */
1470	if (&page_set_colorequiv_arr_cpu != NULL)
1471		page_set_colorequiv_arr_cpu();
1472	else
1473		page_set_colorequiv_arr();
1474
1475	/*
1476	 * Initialize bp_mapin().
1477	 */
1478	bp_init(shm_alignment, HAT_STRICTORDER);
1479
1480	/*
1481	 * Reserve space for MPO mblock structs from the 32-bit heap.
1482	 */
1483
1484	if (mpo_heap32_bufsz > (size_t)0) {
1485		(void) vmem_xalloc(heap32_arena, mpo_heap32_bufsz,
1486		    PAGESIZE, 0, 0, mpo_heap32_buf,
1487		    mpo_heap32_buf + mpo_heap32_bufsz,
1488		    VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
1489	}
1490	mem_config_init();
1491}
1492
1493static void
1494startup_modules(void)
1495{
1496	int nhblk1, nhblk8;
1497	size_t  nhblksz;
1498	pgcnt_t pages_per_hblk;
1499	size_t hme8blk_sz, hme1blk_sz;
1500
1501	/*
1502	 * The system file /etc/system was read already under startup_memlist.
1503	 */
1504	if (&set_platform_defaults)
1505		set_platform_defaults();
1506
1507	/*
1508	 * Calculate default settings of system parameters based upon
1509	 * maxusers, yet allow to be overridden via the /etc/system file.
1510	 */
1511	param_calc(0);
1512
1513	mod_setup();
1514
1515	/*
1516	 * Initialize system parameters
1517	 */
1518	param_init();
1519
1520	/*
1521	 * maxmem is the amount of physical memory we're playing with.
1522	 */
1523	maxmem = physmem;
1524
1525	/* Set segkp limits. */
1526	ncbase = kdi_segdebugbase;
1527	ncend = kdi_segdebugbase;
1528
1529	/*
1530	 * Initialize the hat layer.
1531	 */
1532	hat_init();
1533
1534	/*
1535	 * Initialize segment management stuff.
1536	 */
1537	seg_init();
1538
1539	/*
1540	 * Create the va>tte handler, so the prom can understand
1541	 * kernel translations.  The handler is installed later, just
1542	 * as we are about to take over the trap table from the prom.
1543	 */
1544	create_va_to_tte();
1545
1546	/*
1547	 * Load the forthdebugger (optional)
1548	 */
1549	forthdebug_init();
1550
1551	/*
1552	 * Create OBP node for console input callbacks
1553	 * if it is needed.
1554	 */
1555	startup_create_io_node();
1556
1557	if (modloadonly("fs", "specfs") == -1)
1558		halt("Can't load specfs");
1559
1560	if (modloadonly("fs", "devfs") == -1)
1561		halt("Can't load devfs");
1562
1563	if (modloadonly("fs", "procfs") == -1)
1564		halt("Can't load procfs");
1565
1566	if (modloadonly("misc", "swapgeneric") == -1)
1567		halt("Can't load swapgeneric");
1568
1569	(void) modloadonly("sys", "lbl_edition");
1570
1571	dispinit();
1572
1573	/*
1574	 * Infer meanings to the members of the idprom buffer.
1575	 */
1576	parse_idprom();
1577
1578	/* Read cluster configuration data. */
1579	clconf_init();
1580
1581	setup_ddi();
1582
1583	/*
1584	 * Lets take this opportunity to load the root device.
1585	 */
1586	if (loadrootmodules() != 0)
1587		debug_enter("Can't load the root filesystem");
1588
1589	/*
1590	 * Load tod driver module for the tod part found on this system.
1591	 * Recompute the cpu frequency/delays based on tod as tod part
1592	 * tends to keep time more accurately.
1593	 */
1594	if (&load_tod_module)
1595		load_tod_module();
1596
1597	/*
1598	 * Allow platforms to load modules which might
1599	 * be needed after bootops are gone.
1600	 */
1601	if (&load_platform_modules)
1602		load_platform_modules();
1603
1604	setcpudelay();
1605
1606	copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,
1607	    &boot_physavail, &boot_physavail_len,
1608	    &boot_virtavail, &boot_virtavail_len);
1609
1610	/*
1611	 * Calculation and allocation of hmeblks needed to remap
1612	 * the memory allocated by PROM till now.
1613	 * Overestimate the number of hblk1 elements by assuming
1614	 * worst case of TTE64K mappings.
1615	 * sfmmu_hblk_alloc will panic if this calculation is wrong.
1616	 */
1617	bop_alloc_pages = btopr(kmem64_end - kmem64_base);
1618	pages_per_hblk = btop(HMEBLK_SPAN(TTE64K));
1619	bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk);
1620	nhblk1 = bop_alloc_pages / pages_per_hblk + hblk1_min;
1621
1622	bop_alloc_pages = size_virtalloc(boot_virtavail, boot_virtavail_len);
1623
1624	/* sfmmu_init_nucleus_hblks expects properly aligned data structures */
1625	hme8blk_sz = roundup(HME8BLK_SZ, sizeof (int64_t));
1626	hme1blk_sz = roundup(HME1BLK_SZ, sizeof (int64_t));
1627
1628	bop_alloc_pages += btopr(nhblk1 * hme1blk_sz);
1629
1630	pages_per_hblk = btop(HMEBLK_SPAN(TTE8K));
1631	nhblk8 = 0;
1632	while (bop_alloc_pages > 1) {
1633		bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk);
1634		nhblk8 += bop_alloc_pages /= pages_per_hblk;
1635		bop_alloc_pages *= hme8blk_sz;
1636		bop_alloc_pages = btopr(bop_alloc_pages);
1637	}
1638	nhblk8 += 2;
1639
1640	/*
1641	 * Since hblk8's can hold up to 64k of mappings aligned on a 64k
1642	 * boundary, the number of hblk8's needed to map the entries in the
1643	 * boot_virtavail list needs to be adjusted to take this into
1644	 * consideration.  Thus, we need to add additional hblk8's since it
1645	 * is possible that an hblk8 will not have all 8 slots used due to
1646	 * alignment constraints.  Since there were boot_virtavail_len entries
1647	 * in that list, we need to add that many hblk8's to the number
1648	 * already calculated to make sure we don't underestimate.
1649	 */
1650	nhblk8 += boot_virtavail_len;
1651	nhblksz = nhblk8 * hme8blk_sz + nhblk1 * hme1blk_sz;
1652
1653	/* Allocate in pagesize chunks */
1654	nhblksz = roundup(nhblksz, MMU_PAGESIZE);
1655	hblk_base = kmem_zalloc(nhblksz, KM_SLEEP);
1656	sfmmu_init_nucleus_hblks(hblk_base, nhblksz, nhblk8, nhblk1);
1657}
1658
1659static void
1660startup_bop_gone(void)
1661{
1662
1663	/*
1664	 * Destroy the MD initialized at startup
1665	 * The startup initializes the MD framework
1666	 * using prom and BOP alloc free it now.
1667	 */
1668	mach_descrip_startup_fini();
1669
1670	/*
1671	 * We're done with prom allocations.
1672	 */
1673	bop_fini();
1674
1675	copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,
1676	    &boot_physavail, &boot_physavail_len,
1677	    &boot_virtavail, &boot_virtavail_len);
1678
1679	/*
1680	 * setup physically contiguous area twice as large as the ecache.
1681	 * this is used while doing displacement flush of ecaches
1682	 */
1683	if (&ecache_flush_address) {
1684		ecache_flushaddr = ecache_flush_address();
1685		if (ecache_flushaddr == (uint64_t)-1) {
1686			cmn_err(CE_PANIC,
1687			    "startup: no memory to set ecache_flushaddr");
1688		}
1689	}
1690
1691	/*
1692	 * Virtual available next.
1693	 */
1694	ASSERT(virt_avail != NULL);
1695	memlist_free_list(virt_avail);
1696	virt_avail = memlist;
1697	copy_memlist(boot_virtavail, boot_virtavail_len, &memlist);
1698
1699}
1700
1701
1702/*
1703 * startup_fixup_physavail - called from mach_sfmmu.c after the final
1704 * allocations have been performed.  We can't call it in startup_bop_gone
1705 * since later operations can cause obp to allocate more memory.
1706 */
1707void
1708startup_fixup_physavail(void)
1709{
1710	struct memlist *cur;
1711	size_t kmem64_overmap_size = kmem64_aligned_end - kmem64_end;
1712
1713	PRM_DEBUG(kmem64_overmap_size);
1714
1715	/*
1716	 * take the most current snapshot we can by calling mem-update
1717	 */
1718	copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len,
1719	    &boot_physavail, &boot_physavail_len,
1720	    &boot_virtavail, &boot_virtavail_len);
1721
1722	/*
1723	 * Copy phys_avail list, again.
1724	 * Both the kernel/boot and the prom have been allocating
1725	 * from the original list we copied earlier.
1726	 */
1727	cur = memlist;
1728	copy_memlist(boot_physavail, boot_physavail_len, &memlist);
1729
1730	/*
1731	 * Add any unused kmem64 memory from overmapped page
1732	 * (Note: va_to_pa does not work for kmem64_end)
1733	 */
1734	if (kmem64_overmap_size) {
1735		memlist_add(kmem64_pabase + (kmem64_end - kmem64_base),
1736		    kmem64_overmap_size, &memlist, &cur);
1737	}
1738
1739	/*
1740	 * Add any extra memory after e_data we added to the phys_avail list
1741	 * back to the old list.
1742	 */
1743	if (ndata_remain_sz >= MMU_PAGESIZE)
1744		memlist_add(va_to_pa(nalloc_base),
1745		    (uint64_t)ndata_remain_sz, &memlist, &cur);
1746
1747	/*
1748	 * There isn't any bounds checking on the memlist area
1749	 * so ensure it hasn't overgrown.
1750	 */
1751	if ((caddr_t)memlist > (caddr_t)memlist_end)
1752		cmn_err(CE_PANIC, "startup: memlist size exceeded");
1753
1754	/*
1755	 * The kernel removes the pages that were allocated for it from
1756	 * the freelist, but we now have to find any -extra- pages that
1757	 * the prom has allocated for it's own book-keeping, and remove
1758	 * them from the freelist too. sigh.
1759	 */
1760	sync_memlists(phys_avail, cur);
1761
1762	ASSERT(phys_avail != NULL);
1763
1764	old_phys_avail = phys_avail;
1765	phys_avail = cur;
1766}
1767
1768void
1769update_kcage_ranges(uint64_t addr, uint64_t len)
1770{
1771	pfn_t base = btop(addr);
1772	pgcnt_t num = btop(len);
1773	int rv;
1774
1775	rv = kcage_range_add(base, num, kcage_startup_dir);
1776
1777	if (rv == ENOMEM) {
1778		cmn_err(CE_WARN, "%ld megabytes not available to kernel cage",
1779		    (len == 0 ? 0 : BYTES_TO_MB(len)));
1780	} else if (rv != 0) {
1781		/* catch this in debug kernels */
1782		ASSERT(0);
1783
1784		cmn_err(CE_WARN, "unexpected kcage_range_add"
1785		    " return value %d", rv);
1786	}
1787}
1788
1789static void
1790startup_vm(void)
1791{
1792	size_t	i;
1793	struct segmap_crargs a;
1794	struct segkpm_crargs b;
1795
1796	uint64_t avmem;
1797	caddr_t va;
1798	pgcnt_t	max_phys_segkp;
1799	int	mnode;
1800
1801	extern int use_brk_lpg, use_stk_lpg;
1802
1803	/*
1804	 * get prom's mappings, create hments for them and switch
1805	 * to the kernel context.
1806	 */
1807	hat_kern_setup();
1808
1809	/*
1810	 * Take over trap table
1811	 */
1812	setup_trap_table();
1813
1814	/*
1815	 * Install the va>tte handler, so that the prom can handle
1816	 * misses and understand the kernel table layout in case
1817	 * we need call into the prom.
1818	 */
1819	install_va_to_tte();
1820
1821	/*
1822	 * Set a flag to indicate that the tba has been taken over.
1823	 */
1824	tba_taken_over = 1;
1825
1826	/* initialize MMU primary context register */
1827	mmu_init_kcontext();
1828
1829	/*
1830	 * The boot cpu can now take interrupts, x-calls, x-traps
1831	 */
1832	CPUSET_ADD(cpu_ready_set, CPU->cpu_id);
1833	CPU->cpu_flags |= (CPU_READY | CPU_ENABLE | CPU_EXISTS);
1834
1835	/*
1836	 * Set a flag to tell write_scb_int() that it can access V_TBR_WR_ADDR.
1837	 */
1838	tbr_wr_addr_inited = 1;
1839
1840	/*
1841	 * Initialize VM system, and map kernel address space.
1842	 */
1843	kvm_init();
1844
1845	ASSERT(old_phys_avail != NULL && phys_avail != NULL);
1846	if (kernel_cage_enable) {
1847		diff_memlists(phys_avail, old_phys_avail, update_kcage_ranges);
1848	}
1849	memlist_free_list(old_phys_avail);
1850
1851	/*
1852	 * If the following is true, someone has patched
1853	 * phsymem to be less than the number of pages that
1854	 * the system actually has.  Remove pages until system
1855	 * memory is limited to the requested amount.  Since we
1856	 * have allocated page structures for all pages, we
1857	 * correct the amount of memory we want to remove
1858	 * by the size of the memory used to hold page structures
1859	 * for the non-used pages.
1860	 */
1861	if (physmem + ramdisk_npages < npages) {
1862		pgcnt_t diff, off;
1863		struct page *pp;
1864		struct seg kseg;
1865
1866		cmn_err(CE_WARN, "limiting physmem to %ld pages", physmem);
1867
1868		off = 0;
1869		diff = npages - (physmem + ramdisk_npages);
1870		diff -= mmu_btopr(diff * sizeof (struct page));
1871		kseg.s_as = &kas;
1872		while (diff--) {
1873			pp = page_create_va(&unused_pages_vp, (offset_t)off,
1874			    MMU_PAGESIZE, PG_WAIT | PG_EXCL,
1875			    &kseg, (caddr_t)off);
1876			if (pp == NULL)
1877				cmn_err(CE_PANIC, "limited physmem too much!");
1878			page_io_unlock(pp);
1879			page_downgrade(pp);
1880			availrmem--;
1881			off += MMU_PAGESIZE;
1882		}
1883	}
1884
1885	/*
1886	 * When printing memory, show the total as physmem less
1887	 * that stolen by a debugger.
1888	 */
1889	cmn_err(CE_CONT, "?mem = %ldK (0x%lx000)\n",
1890	    (ulong_t)(physinstalled) << (PAGESHIFT - 10),
1891	    (ulong_t)(physinstalled) << (PAGESHIFT - 12));
1892
1893	avmem = (uint64_t)freemem << PAGESHIFT;
1894	cmn_err(CE_CONT, "?avail mem = %lld\n", (unsigned long long)avmem);
1895
1896	/*
1897	 * For small memory systems disable automatic large pages.
1898	 */
1899	if (physmem < privm_lpg_min_physmem) {
1900		use_brk_lpg = 0;
1901		use_stk_lpg = 0;
1902	}
1903
1904	/*
1905	 * Perform platform specific freelist processing
1906	 */
1907	if (&plat_freelist_process) {
1908		for (mnode = 0; mnode < max_mem_nodes; mnode++)
1909			if (mem_node_config[mnode].exists)
1910				plat_freelist_process(mnode);
1911	}
1912
1913	/*
1914	 * Initialize the segkp segment type.  We position it
1915	 * after the configured tables and buffers (whose end
1916	 * is given by econtig) and before V_WKBASE_ADDR.
1917	 * Also in this area is segkmap (size SEGMAPSIZE).
1918	 */
1919
1920	/* XXX - cache alignment? */
1921	va = (caddr_t)SEGKPBASE;
1922	ASSERT(((uintptr_t)va & PAGEOFFSET) == 0);
1923
1924	max_phys_segkp = (physmem * 2);
1925
1926	if (segkpsize < btop(SEGKPMINSIZE) || segkpsize > btop(SEGKPMAXSIZE)) {
1927		segkpsize = btop(SEGKPDEFSIZE);
1928		cmn_err(CE_WARN, "Illegal value for segkpsize. "
1929		    "segkpsize has been reset to %ld pages", segkpsize);
1930	}
1931
1932	i = ptob(MIN(segkpsize, max_phys_segkp));
1933
1934	rw_enter(&kas.a_lock, RW_WRITER);
1935	if (seg_attach(&kas, va, i, segkp) < 0)
1936		cmn_err(CE_PANIC, "startup: cannot attach segkp");
1937	if (segkp_create(segkp) != 0)
1938		cmn_err(CE_PANIC, "startup: segkp_create failed");
1939	rw_exit(&kas.a_lock);
1940
1941	/*
1942	 * kpm segment
1943	 */
1944	segmap_kpm = kpm_enable &&
1945	    segmap_kpm && PAGESIZE == MAXBSIZE;
1946
1947	if (kpm_enable) {
1948		rw_enter(&kas.a_lock, RW_WRITER);
1949
1950		/*
1951		 * The segkpm virtual range range is larger than the
1952		 * actual physical memory size and also covers gaps in
1953		 * the physical address range for the following reasons:
1954		 * . keep conversion between segkpm and physical addresses
1955		 *   simple, cheap and unambiguous.
1956		 * . avoid extension/shrink of the the segkpm in case of DR.
1957		 * . avoid complexity for handling of virtual addressed
1958		 *   caches, segkpm and the regular mapping scheme must be
1959		 *   kept in sync wrt. the virtual color of mapped pages.
1960		 * Any accesses to virtual segkpm ranges not backed by
1961		 * physical memory will fall through the memseg pfn hash
1962		 * and will be handled in segkpm_fault.
1963		 * Additional kpm_size spaces needed for vac alias prevention.
1964		 */
1965		if (seg_attach(&kas, kpm_vbase, kpm_size * vac_colors,
1966		    segkpm) < 0)
1967			cmn_err(CE_PANIC, "cannot attach segkpm");
1968
1969		b.prot = PROT_READ | PROT_WRITE;
1970		b.nvcolors = shm_alignment >> MMU_PAGESHIFT;
1971
1972		if (segkpm_create(segkpm, (caddr_t)&b) != 0)
1973			panic("segkpm_create segkpm");
1974
1975		rw_exit(&kas.a_lock);
1976
1977		mach_kpm_init();
1978	}
1979
1980	va = kpm_vbase + (kpm_size * vac_colors);
1981
1982	if (!segzio_fromheap) {
1983		size_t size;
1984		size_t physmem_b = mmu_ptob(physmem);
1985
1986		/* size is in bytes, segziosize is in pages */
1987		if (segziosize == 0) {
1988			size = physmem_b;
1989		} else {
1990			size = mmu_ptob(segziosize);
1991		}
1992
1993		if (size < SEGZIOMINSIZE) {
1994			size = SEGZIOMINSIZE;
1995		} else if (size > SEGZIOMAXSIZE) {
1996			size = SEGZIOMAXSIZE;
1997			/*
1998			 * On 64-bit x86, we only have 2TB of KVA.  This exists
1999			 * for parity with x86.
2000			 *
2001			 * SEGZIOMAXSIZE is capped at 512gb so that segzio
2002			 * doesn't consume all of KVA.  However, if we have a
2003			 * system that has more thant 512gb of physical memory,
2004			 * we can actually consume about half of the difference
2005			 * between 512gb and the rest of the available physical
2006			 * memory.
2007			 */
2008			if (physmem_b > SEGZIOMAXSIZE) {
2009				size += (physmem_b - SEGZIOMAXSIZE) / 2;
2010		}
2011		}
2012		segziosize = mmu_btop(roundup(size, MMU_PAGESIZE));
2013		/* put the base of the ZIO segment after the kpm segment */
2014		segzio_base = va;
2015		va += mmu_ptob(segziosize);
2016		PRM_DEBUG(segziosize);
2017		PRM_DEBUG(segzio_base);
2018
2019		/*
2020		 * On some platforms, kvm_init is called after the kpm
2021		 * sizes have been determined.  On SPARC, kvm_init is called
2022		 * before, so we have to attach the kzioseg after kvm is
2023		 * initialized, otherwise we'll try to allocate from the boot
2024		 * area since the kernel heap hasn't yet been configured.
2025		 */
2026		rw_enter(&kas.a_lock, RW_WRITER);
2027
2028		(void) seg_attach(&kas, segzio_base, mmu_ptob(segziosize),
2029		    &kzioseg);
2030		(void) segkmem_create(&kzioseg);
2031
2032		/* create zio area covering new segment */
2033		segkmem_zio_init(segzio_base, mmu_ptob(segziosize));
2034
2035		rw_exit(&kas.a_lock);
2036	}
2037
2038	if (ppvm_enable) {
2039		caddr_t ppvm_max;
2040
2041		/*
2042		 * ppvm refers to the static VA space used to map
2043		 * the page_t's for dynamically added memory.
2044		 *
2045		 * ppvm_base should not cross a potential VA hole.
2046		 *
2047		 * ppvm_size should be large enough to map the
2048		 * page_t's needed to manage all of KPM range.
2049		 */
2050		ppvm_size =
2051		    roundup(mmu_btop(kpm_size * vac_colors) * sizeof (page_t),
2052		    MMU_PAGESIZE);
2053		ppvm_max = (caddr_t)(0ull - ppvm_size);
2054		ppvm_base = (page_t *)va;
2055
2056		if ((caddr_t)ppvm_base <= hole_end) {
2057			cmn_err(CE_WARN,
2058			    "Memory DR disabled: invalid DR map base: 0x%p\n",
2059			    (void *)ppvm_base);
2060			ppvm_enable = 0;
2061		} else if ((caddr_t)ppvm_base > ppvm_max) {
2062			uint64_t diff = (caddr_t)ppvm_base - ppvm_max;
2063
2064			cmn_err(CE_WARN,
2065			    "Memory DR disabled: insufficient DR map size:"
2066			    " 0x%lx (needed 0x%lx)\n",
2067			    ppvm_size - diff, ppvm_size);
2068			ppvm_enable = 0;
2069		}
2070		PRM_DEBUG(ppvm_size);
2071		PRM_DEBUG(ppvm_base);
2072	}
2073
2074	/*
2075	 * Now create generic mapping segment.  This mapping
2076	 * goes SEGMAPSIZE beyond SEGMAPBASE.  But if the total
2077	 * virtual address is greater than the amount of free
2078	 * memory that is available, then we trim back the
2079	 * segment size to that amount
2080	 */
2081	va = (caddr_t)SEGMAPBASE;
2082
2083	/*
2084	 * 1201049: segkmap base address must be MAXBSIZE aligned
2085	 */
2086	ASSERT(((uintptr_t)va & MAXBOFFSET) == 0);
2087
2088	/*
2089	 * Set size of segmap to percentage of freemem at boot,
2090	 * but stay within the allowable range
2091	 * Note we take percentage  before converting from pages
2092	 * to bytes to avoid an overflow on 32-bit kernels.
2093	 */
2094	i = mmu_ptob((freemem * segmap_percent) / 100);
2095
2096	if (i < MINMAPSIZE)
2097		i = MINMAPSIZE;
2098
2099	if (i > MIN(SEGMAPSIZE, mmu_ptob(freemem)))
2100		i = MIN(SEGMAPSIZE, mmu_ptob(freemem));
2101
2102	i &= MAXBMASK;	/* 1201049: segkmap size must be MAXBSIZE aligned */
2103
2104	rw_enter(&kas.a_lock, RW_WRITER);
2105	if (seg_attach(&kas, va, i, segkmap) < 0)
2106		cmn_err(CE_PANIC, "cannot attach segkmap");
2107
2108	a.prot = PROT_READ | PROT_WRITE;
2109	a.shmsize = shm_alignment;
2110	a.nfreelist = 0;	/* use segmap driver defaults */
2111
2112	if (segmap_create(segkmap, (caddr_t)&a) != 0)
2113		panic("segmap_create segkmap");
2114	rw_exit(&kas.a_lock);
2115
2116	segdev_init();
2117}
2118
2119static void
2120startup_end(void)
2121{
2122	if ((caddr_t)memlist > (caddr_t)memlist_end)
2123		panic("memlist overflow 2");
2124	memlist_free_block((caddr_t)memlist,
2125	    ((caddr_t)memlist_end - (caddr_t)memlist));
2126	memlist = NULL;
2127
2128	/* enable page_relocation since OBP is now done */
2129	page_relocate_ready = 1;
2130
2131	/*
2132	 * Perform tasks that get done after most of the VM
2133	 * initialization has been done but before the clock
2134	 * and other devices get started.
2135	 */
2136	kern_setup1();
2137
2138	/*
2139	 * Perform CPC initialization for this CPU.
2140	 */
2141	kcpc_hw_init();
2142
2143	/*
2144	 * Intialize the VM arenas for allocating physically
2145	 * contiguus memory chunk for interrupt queues snd
2146	 * allocate/register boot cpu's queues, if any and
2147	 * allocate dump buffer for sun4v systems to store
2148	 * extra crash information during crash dump
2149	 */
2150	contig_mem_init();
2151	mach_descrip_init();
2152
2153	if (cpu_intrq_setup(CPU)) {
2154		cmn_err(CE_PANIC, "cpu%d: setup failed", CPU->cpu_id);
2155	}
2156	cpu_intrq_register(CPU);
2157	mach_htraptrace_setup(CPU->cpu_id);
2158	mach_htraptrace_configure(CPU->cpu_id);
2159	mach_dump_buffer_init();
2160
2161	/*
2162	 * Initialize interrupt related stuff
2163	 */
2164	cpu_intr_alloc(CPU, NINTR_THREADS);
2165
2166	(void) splzs();			/* allow hi clock ints but not zs */
2167
2168	/*
2169	 * Initialize errors.
2170	 */
2171	error_init();
2172
2173	/*
2174	 * Note that we may have already used kernel bcopy before this
2175	 * point - but if you really care about this, adb the use_hw_*
2176	 * variables to 0 before rebooting.
2177	 */
2178	mach_hw_copy_limit();
2179
2180	/*
2181	 * Install the "real" preemption guards before DDI services
2182	 * are available.
2183	 */
2184	(void) prom_set_preprom(kern_preprom);
2185	(void) prom_set_postprom(kern_postprom);
2186	CPU->cpu_m.mutex_ready = 1;
2187
2188	/*
2189	 * Initialize segnf (kernel support for non-faulting loads).
2190	 */
2191	segnf_init();
2192
2193	/*
2194	 * Configure the root devinfo node.
2195	 */
2196	configure();		/* set up devices */
2197	mach_cpu_halt_idle();
2198}
2199
2200
2201void
2202post_startup(void)
2203{
2204#ifdef	PTL1_PANIC_DEBUG
2205	extern void init_ptl1_thread(void);
2206#endif	/* PTL1_PANIC_DEBUG */
2207	extern void abort_sequence_init(void);
2208
2209	/*
2210	 * Set the system wide, processor-specific flags to be passed
2211	 * to userland via the aux vector for performance hints and
2212	 * instruction set extensions.
2213	 */
2214	bind_hwcap();
2215
2216	/*
2217	 * Startup memory scrubber (if any)
2218	 */
2219	mach_memscrub();
2220
2221	/*
2222	 * Allocate soft interrupt to handle abort sequence.
2223	 */
2224	abort_sequence_init();
2225
2226	/*
2227	 * Configure the rest of the system.
2228	 * Perform forceloading tasks for /etc/system.
2229	 */
2230	(void) mod_sysctl(SYS_FORCELOAD, NULL);
2231	/*
2232	 * ON4.0: Force /proc module in until clock interrupt handle fixed
2233	 * ON4.0: This must be fixed or restated in /etc/systems.
2234	 */
2235	(void) modload("fs", "procfs");
2236
2237	/* load machine class specific drivers */
2238	load_mach_drivers();
2239
2240	/* load platform specific drivers */
2241	if (&load_platform_drivers)
2242		load_platform_drivers();
2243
2244	/* load vis simulation module, if we are running w/fpu off */
2245	if (!fpu_exists) {
2246		if (modload("misc", "vis") == -1)
2247			halt("Can't load vis");
2248	}
2249
2250	mach_fpras();
2251
2252	maxmem = freemem;
2253
2254	pg_init();
2255
2256#ifdef	PTL1_PANIC_DEBUG
2257	init_ptl1_thread();
2258#endif	/* PTL1_PANIC_DEBUG */
2259}
2260
2261#ifdef	PTL1_PANIC_DEBUG
2262int		ptl1_panic_test = 0;
2263int		ptl1_panic_xc_one_test = 0;
2264int		ptl1_panic_xc_all_test = 0;
2265int		ptl1_panic_xt_one_test = 0;
2266int		ptl1_panic_xt_all_test = 0;
2267kthread_id_t	ptl1_thread_p = NULL;
2268kcondvar_t	ptl1_cv;
2269kmutex_t	ptl1_mutex;
2270int		ptl1_recurse_count_threshold = 0x40;
2271int		ptl1_recurse_trap_threshold = 0x3d;
2272extern void	ptl1_recurse(int, int);
2273extern void	ptl1_panic_xt(int, int);
2274
2275/*
2276 * Called once per second by timeout() to wake up
2277 * the ptl1_panic thread to see if it should cause
2278 * a trap to the ptl1_panic() code.
2279 */
2280/* ARGSUSED */
2281static void
2282ptl1_wakeup(void *arg)
2283{
2284	mutex_enter(&ptl1_mutex);
2285	cv_signal(&ptl1_cv);
2286	mutex_exit(&ptl1_mutex);
2287}
2288
2289/*
2290 * ptl1_panic cross call function:
2291 *     Needed because xc_one() and xc_some() can pass
2292 *	64 bit args but ptl1_recurse() expects ints.
2293 */
2294static void
2295ptl1_panic_xc(void)
2296{
2297	ptl1_recurse(ptl1_recurse_count_threshold,
2298	    ptl1_recurse_trap_threshold);
2299}
2300
2301/*
2302 * The ptl1 thread waits for a global flag to be set
2303 * and uses the recurse thresholds to set the stack depth
2304 * to cause a ptl1_panic() directly via a call to ptl1_recurse
2305 * or indirectly via the cross call and cross trap functions.
2306 *
2307 * This is useful testing stack overflows and normal
2308 * ptl1_panic() states with a know stack frame.
2309 *
2310 * ptl1_recurse() is an asm function in ptl1_panic.s that
2311 * sets the {In, Local, Out, and Global} registers to a
2312 * know state on the stack and just prior to causing a
2313 * test ptl1_panic trap.
2314 */
2315static void
2316ptl1_thread(void)
2317{
2318	mutex_enter(&ptl1_mutex);
2319	while (ptl1_thread_p) {
2320		cpuset_t	other_cpus;
2321		int		cpu_id;
2322		int		my_cpu_id;
2323		int		target_cpu_id;
2324		int		target_found;
2325
2326		if (ptl1_panic_test) {
2327			ptl1_recurse(ptl1_recurse_count_threshold,
2328			    ptl1_recurse_trap_threshold);
2329		}
2330
2331		/*
2332		 * Find potential targets for x-call and x-trap,
2333		 * if any exist while preempt is disabled we
2334		 * start a ptl1_panic if requested via a
2335		 * globals.
2336		 */
2337		kpreempt_disable();
2338		my_cpu_id = CPU->cpu_id;
2339		other_cpus = cpu_ready_set;
2340		CPUSET_DEL(other_cpus, CPU->cpu_id);
2341		target_found = 0;
2342		if (!CPUSET_ISNULL(other_cpus)) {
2343			/*
2344			 * Pick the first one
2345			 */
2346			for (cpu_id = 0; cpu_id < NCPU; cpu_id++) {
2347				if (cpu_id == my_cpu_id)
2348					continue;
2349
2350				if (CPU_XCALL_READY(cpu_id)) {
2351					target_cpu_id = cpu_id;
2352					target_found = 1;
2353					break;
2354				}
2355			}
2356			ASSERT(target_found);
2357
2358			if (ptl1_panic_xc_one_test) {
2359				xc_one(target_cpu_id,
2360				    (xcfunc_t *)ptl1_panic_xc, 0, 0);
2361			}
2362			if (ptl1_panic_xc_all_test) {
2363				xc_some(other_cpus,
2364				    (xcfunc_t *)ptl1_panic_xc, 0, 0);
2365			}
2366			if (ptl1_panic_xt_one_test) {
2367				xt_one(target_cpu_id,
2368				    (xcfunc_t *)ptl1_panic_xt, 0, 0);
2369			}
2370			if (ptl1_panic_xt_all_test) {
2371				xt_some(other_cpus,
2372				    (xcfunc_t *)ptl1_panic_xt, 0, 0);
2373			}
2374		}
2375		kpreempt_enable();
2376		(void) timeout(ptl1_wakeup, NULL, hz);
2377		(void) cv_wait(&ptl1_cv, &ptl1_mutex);
2378	}
2379	mutex_exit(&ptl1_mutex);
2380}
2381
2382/*
2383 * Called during early startup to create the ptl1_thread
2384 */
2385void
2386init_ptl1_thread(void)
2387{
2388	ptl1_thread_p = thread_create(NULL, 0, ptl1_thread, NULL, 0,
2389	    &p0, TS_RUN, 0);
2390}
2391#endif	/* PTL1_PANIC_DEBUG */
2392
2393
2394static void
2395memlist_new(uint64_t start, uint64_t len, struct memlist **memlistp)
2396{
2397	struct memlist *new;
2398
2399	new = *memlistp;
2400	new->ml_address = start;
2401	new->ml_size = len;
2402	*memlistp = new + 1;
2403}
2404
2405/*
2406 * Add to a memory list.
2407 * start = start of new memory segment
2408 * len = length of new memory segment in bytes
2409 * memlistp = pointer to array of available memory segment structures
2410 * curmemlistp = memory list to which to add segment.
2411 */
2412static void
2413memlist_add(uint64_t start, uint64_t len, struct memlist **memlistp,
2414    struct memlist **curmemlistp)
2415{
2416	struct memlist *new = *memlistp;
2417
2418	memlist_new(start, len, memlistp);
2419	memlist_insert(new, curmemlistp);
2420}
2421
2422static int
2423ndata_alloc_memseg(struct memlist *ndata, size_t avail)
2424{
2425	int nseg;
2426	size_t memseg_sz;
2427	struct memseg *msp;
2428
2429	/*
2430	 * The memseg list is for the chunks of physical memory that
2431	 * will be managed by the vm system.  The number calculated is
2432	 * a guess as boot may fragment it more when memory allocations
2433	 * are made before kphysm_init().
2434	 */
2435	memseg_sz = (avail + 10) * sizeof (struct memseg);
2436	memseg_sz = roundup(memseg_sz, PAGESIZE);
2437	nseg = memseg_sz / sizeof (struct memseg);
2438	msp = ndata_alloc(ndata, memseg_sz, ecache_alignsize);
2439	if (msp == NULL)
2440		return (1);
2441	PRM_DEBUG(memseg_free);
2442
2443	while (nseg--) {
2444		msp->next = memseg_free;
2445		memseg_free = msp;
2446		msp++;
2447	}
2448	return (0);
2449}
2450
2451/*
2452 * In the case of architectures that support dynamic addition of
2453 * memory at run-time there are two cases where memsegs need to
2454 * be initialized and added to the memseg list.
2455 * 1) memsegs that are constructed at startup.
2456 * 2) memsegs that are constructed at run-time on
2457 *    hot-plug capable architectures.
2458 * This code was originally part of the function kphysm_init().
2459 */
2460
2461static void
2462memseg_list_add(struct memseg *memsegp)
2463{
2464	struct memseg **prev_memsegp;
2465	pgcnt_t num;
2466
2467	/* insert in memseg list, decreasing number of pages order */
2468
2469	num = MSEG_NPAGES(memsegp);
2470
2471	for (prev_memsegp = &memsegs; *prev_memsegp;
2472	    prev_memsegp = &((*prev_memsegp)->next)) {
2473		if (num > MSEG_NPAGES(*prev_memsegp))
2474			break;
2475	}
2476
2477	memsegp->next = *prev_memsegp;
2478	*prev_memsegp = memsegp;
2479
2480	if (kpm_enable) {
2481		memsegp->nextpa = (memsegp->next) ?
2482		    va_to_pa(memsegp->next) : MSEG_NULLPTR_PA;
2483
2484		if (prev_memsegp != &memsegs) {
2485			struct memseg *msp;
2486			msp = (struct memseg *)((caddr_t)prev_memsegp -
2487			    offsetof(struct memseg, next));
2488			msp->nextpa = va_to_pa(memsegp);
2489		} else {
2490			memsegspa = va_to_pa(memsegs);
2491		}
2492	}
2493}
2494
2495/*
2496 * PSM add_physmem_cb(). US-II and newer processors have some
2497 * flavor of the prefetch capability implemented. We exploit
2498 * this capability for optimum performance.
2499 */
2500#define	PREFETCH_BYTES	64
2501
2502void
2503add_physmem_cb(page_t *pp, pfn_t pnum)
2504{
2505	extern void	 prefetch_page_w(void *);
2506
2507	pp->p_pagenum = pnum;
2508
2509	/*
2510	 * Prefetch one more page_t into E$. To prevent future
2511	 * mishaps with the sizeof(page_t) changing on us, we
2512	 * catch this on debug kernels if we can't bring in the
2513	 * entire hpage with 2 PREFETCH_BYTES reads. See
2514	 * also, sun4u/cpu/cpu_module.c
2515	 */
2516	/*LINTED*/
2517	ASSERT(sizeof (page_t) <= 2*PREFETCH_BYTES);
2518	prefetch_page_w((char *)pp);
2519}
2520
2521/*
2522 * Find memseg with given pfn
2523 */
2524static struct memseg *
2525memseg_find(pfn_t base, pfn_t *next)
2526{
2527	struct memseg *seg;
2528
2529	if (next != NULL)
2530		*next = LONG_MAX;
2531	for (seg = memsegs; seg != NULL; seg = seg->next) {
2532		if (base >= seg->pages_base && base < seg->pages_end)
2533			return (seg);
2534		if (next != NULL && seg->pages_base > base &&
2535		    seg->pages_base < *next)
2536			*next = seg->pages_base;
2537	}
2538	return (NULL);
2539}
2540
2541/*
2542 * Put page allocated by OBP on prom_ppages
2543 */
2544static void
2545kphysm_erase(uint64_t addr, uint64_t len)
2546{
2547	struct page *pp;
2548	struct memseg *seg;
2549	pfn_t base = btop(addr), next;
2550	pgcnt_t num = btop(len);
2551
2552	while (num != 0) {
2553		pgcnt_t off, left;
2554
2555		seg = memseg_find(base, &next);
2556		if (seg == NULL) {
2557			if (next == LONG_MAX)
2558				break;
2559			left = MIN(next - base, num);
2560			base += left, num -= left;
2561			continue;
2562		}
2563		off = base - seg->pages_base;
2564		pp = seg->pages + off;
2565		left = num - MIN(num, (seg->pages_end - seg->pages_base) - off);
2566		while (num != left) {
2567			/*
2568			 * init it, lock it, and hashin on prom_pages vp.
2569			 *
2570			 * Mark it as NONRELOC to let DR know the page
2571			 * is locked long term, otherwise DR hangs when
2572			 * trying to remove those pages.
2573			 *
2574			 * XXX	vnode offsets on the prom_ppages vnode
2575			 *	are page numbers (gack) for >32 bit
2576			 *	physical memory machines.
2577			 */
2578			PP_SETNORELOC(pp);
2579			add_physmem_cb(pp, base);
2580			if (page_trylock(pp, SE_EXCL) == 0)
2581				cmn_err(CE_PANIC, "prom page locked");
2582			(void) page_hashin(pp, &promvp,
2583			    (offset_t)base, NULL);
2584			(void) page_pp_lock(pp, 0, 1);
2585			pp++, base++, num--;
2586		}
2587	}
2588}
2589
2590static page_t *ppnext;
2591static pgcnt_t ppleft;
2592
2593static void *kpm_ppnext;
2594static pgcnt_t kpm_ppleft;
2595
2596/*
2597 * Create a memseg
2598 */
2599static void
2600kphysm_memseg(uint64_t addr, uint64_t len)
2601{
2602	pfn_t base = btop(addr);
2603	pgcnt_t num = btop(len);
2604	struct memseg *seg;
2605
2606	seg = memseg_free;
2607	memseg_free = seg->next;
2608	ASSERT(seg != NULL);
2609
2610	seg->pages = ppnext;
2611	seg->epages = ppnext + num;
2612	seg->pages_base = base;
2613	seg->pages_end = base + num;
2614	ppnext += num;
2615	ppleft -= num;
2616
2617	if (kpm_enable) {
2618		pgcnt_t kpnum = ptokpmpr(num);
2619
2620		if (kpnum > kpm_ppleft)
2621			panic("kphysm_memseg: kpm_pp overflow");
2622		seg->pagespa = va_to_pa(seg->pages);
2623		seg->epagespa = va_to_pa(seg->epages);
2624		seg->kpm_pbase = kpmptop(ptokpmp(base));
2625		seg->kpm_nkpmpgs = kpnum;
2626		/*
2627		 * In the kpm_smallpage case, the kpm array
2628		 * is 1-1 wrt the page array
2629		 */
2630		if (kpm_smallpages) {
2631			kpm_spage_t *kpm_pp = kpm_ppnext;
2632
2633			kpm_ppnext = kpm_pp + kpnum;
2634			seg->kpm_spages = kpm_pp;
2635			seg->kpm_pagespa = va_to_pa(seg->kpm_spages);
2636		} else {
2637			kpm_page_t *kpm_pp = kpm_ppnext;
2638
2639			kpm_ppnext = kpm_pp + kpnum;
2640			seg->kpm_pages = kpm_pp;
2641			seg->kpm_pagespa = va_to_pa(seg->kpm_pages);
2642			/* ASSERT no kpm overlaps */
2643			ASSERT(
2644			    memseg_find(base - pmodkpmp(base), NULL) == NULL);
2645			ASSERT(memseg_find(
2646			    roundup(base + num, kpmpnpgs) - 1, NULL) == NULL);
2647		}
2648		kpm_ppleft -= kpnum;
2649	}
2650
2651	memseg_list_add(seg);
2652}
2653
2654/*
2655 * Add range to free list
2656 */
2657void
2658kphysm_add(uint64_t addr, uint64_t len, int reclaim)
2659{
2660	struct page *pp;
2661	struct memseg *seg;
2662	pfn_t base = btop(addr);
2663	pgcnt_t num = btop(len);
2664
2665	seg = memseg_find(base, NULL);
2666	ASSERT(seg != NULL);
2667	pp = seg->pages + (base - seg->pages_base);
2668
2669	if (reclaim) {
2670		struct page *rpp = pp;
2671		struct page *lpp = pp + num;
2672
2673		/*
2674		 * page should be locked on prom_ppages
2675		 * unhash and unlock it
2676		 */
2677		while (rpp < lpp) {
2678			ASSERT(PAGE_EXCL(rpp) && rpp->p_vnode == &promvp);
2679			ASSERT(PP_ISNORELOC(rpp));
2680			PP_CLRNORELOC(rpp);
2681			page_pp_unlock(rpp, 0, 1);
2682			page_hashout(rpp, NULL);
2683			page_unlock(rpp);
2684			rpp++;
2685		}
2686	}
2687
2688	/*
2689	 * add_physmem() initializes the PSM part of the page
2690	 * struct by calling the PSM back with add_physmem_cb().
2691	 * In addition it coalesces pages into larger pages as
2692	 * it initializes them.
2693	 */
2694	add_physmem(pp, num, base);
2695}
2696
2697/*
2698 * kphysm_init() tackles the problem of initializing physical memory.
2699 */
2700static void
2701kphysm_init(void)
2702{
2703	struct memlist *pmem;
2704
2705	ASSERT(page_hash != NULL && page_hashsz != 0);
2706
2707	ppnext = pp_base;
2708	ppleft = npages;
2709	kpm_ppnext = kpm_pp_base;
2710	kpm_ppleft = kpm_npages;
2711
2712	/*
2713	 * installed pages not on nopp_memlist go in memseg list
2714	 */
2715	diff_memlists(phys_install, nopp_list, kphysm_memseg);
2716
2717	/*
2718	 * Free the avail list
2719	 */
2720	for (pmem = phys_avail; pmem != NULL; pmem = pmem->ml_next)
2721		kphysm_add(pmem->ml_address, pmem->ml_size, 0);
2722
2723	/*
2724	 * Erase pages that aren't available
2725	 */
2726	diff_memlists(phys_install, phys_avail, kphysm_erase);
2727
2728	build_pfn_hash();
2729}
2730
2731/*
2732 * Kernel VM initialization.
2733 * Assumptions about kernel address space ordering:
2734 *	(1) gap (user space)
2735 *	(2) kernel text
2736 *	(3) kernel data/bss
2737 *	(4) gap
2738 *	(5) kernel data structures
2739 *	(6) gap
2740 *	(7) debugger (optional)
2741 *	(8) monitor
2742 *	(9) gap (possibly null)
2743 *	(10) dvma
2744 *	(11) devices
2745 */
2746static void
2747kvm_init(void)
2748{
2749	/*
2750	 * Put the kernel segments in kernel address space.
2751	 */
2752	rw_enter(&kas.a_lock, RW_WRITER);
2753	as_avlinit(&kas);
2754
2755	(void) seg_attach(&kas, (caddr_t)KERNELBASE,
2756	    (size_t)(e_moddata - KERNELBASE), &ktextseg);
2757	(void) segkmem_create(&ktextseg);
2758
2759	(void) seg_attach(&kas, (caddr_t)(KERNELBASE + MMU_PAGESIZE4M),
2760	    (size_t)(MMU_PAGESIZE4M), &ktexthole);
2761	(void) segkmem_create(&ktexthole);
2762
2763	(void) seg_attach(&kas, (caddr_t)valloc_base,
2764	    (size_t)(econtig32 - valloc_base), &kvalloc);
2765	(void) segkmem_create(&kvalloc);
2766
2767	if (kmem64_base) {
2768		(void) seg_attach(&kas, (caddr_t)kmem64_base,
2769		    (size_t)(kmem64_end - kmem64_base), &kmem64);
2770		(void) segkmem_create(&kmem64);
2771	}
2772
2773	/*
2774	 * We're about to map out /boot.  This is the beginning of the
2775	 * system resource management transition. We can no longer
2776	 * call into /boot for I/O or memory allocations.
2777	 */
2778	(void) seg_attach(&kas, kernelheap, ekernelheap - kernelheap, &kvseg);
2779	(void) segkmem_create(&kvseg);
2780	hblk_alloc_dynamic = 1;
2781
2782	/*
2783	 * we need to preallocate pages for DR operations before enabling large
2784	 * page kernel heap because of memseg_remap_init() hat_unload() hack.
2785	 */
2786	memseg_remap_init();
2787
2788	/* at this point we are ready to use large page heap */
2789	segkmem_heap_lp_init();
2790
2791	(void) seg_attach(&kas, (caddr_t)SYSBASE32, SYSLIMIT32 - SYSBASE32,
2792	    &kvseg32);
2793	(void) segkmem_create(&kvseg32);
2794
2795	/*
2796	 * Create a segment for the debugger.
2797	 */
2798	(void) seg_attach(&kas, kdi_segdebugbase, kdi_segdebugsize, &kdebugseg);
2799	(void) segkmem_create(&kdebugseg);
2800
2801	rw_exit(&kas.a_lock);
2802}
2803
2804char obp_tte_str[] =
2805	"h# %x constant MMU_PAGESHIFT "
2806	"h# %x constant TTE8K "
2807	"h# %x constant SFHME_SIZE "
2808	"h# %x constant SFHME_TTE "
2809	"h# %x constant HMEBLK_TAG "
2810	"h# %x constant HMEBLK_NEXT "
2811	"h# %x constant HMEBLK_MISC "
2812	"h# %x constant HMEBLK_HME1 "
2813	"h# %x constant NHMENTS "
2814	"h# %x constant HBLK_SZMASK "
2815	"h# %x constant HBLK_RANGE_SHIFT "
2816	"h# %x constant HMEBP_HBLK "
2817	"h# %x constant HMEBLK_ENDPA "
2818	"h# %x constant HMEBUCKET_SIZE "
2819	"h# %x constant HTAG_SFMMUPSZ "
2820	"h# %x constant HTAG_BSPAGE_SHIFT "
2821	"h# %x constant HTAG_REHASH_SHIFT "
2822	"h# %x constant SFMMU_INVALID_SHMERID "
2823	"h# %x constant mmu_hashcnt "
2824	"h# %p constant uhme_hash "
2825	"h# %p constant khme_hash "
2826	"h# %x constant UHMEHASH_SZ "
2827	"h# %x constant KHMEHASH_SZ "
2828	"h# %p constant KCONTEXT "
2829	"h# %p constant KHATID "
2830	"h# %x constant ASI_MEM "
2831
2832	": PHYS-X@ ( phys -- data ) "
2833	"   ASI_MEM spacex@ "
2834	"; "
2835
2836	": PHYS-W@ ( phys -- data ) "
2837	"   ASI_MEM spacew@ "
2838	"; "
2839
2840	": PHYS-L@ ( phys -- data ) "
2841	"   ASI_MEM spaceL@ "
2842	"; "
2843
2844	": TTE_PAGE_SHIFT ( ttesz -- hmeshift ) "
2845	"   3 * MMU_PAGESHIFT + "
2846	"; "
2847
2848	": TTE_IS_VALID ( ttep -- flag ) "
2849	"   PHYS-X@ 0< "
2850	"; "
2851
2852	": HME_HASH_SHIFT ( ttesz -- hmeshift ) "
2853	"   dup TTE8K =  if "
2854	"      drop HBLK_RANGE_SHIFT "
2855	"   else "
2856	"      TTE_PAGE_SHIFT "
2857	"   then "
2858	"; "
2859
2860	": HME_HASH_BSPAGE ( addr hmeshift -- bspage ) "
2861	"   tuck >> swap MMU_PAGESHIFT - << "
2862	"; "
2863
2864	": HME_HASH_FUNCTION ( sfmmup addr hmeshift -- hmebp ) "
2865	"   >> over xor swap                    ( hash sfmmup ) "
2866	"   KHATID <>  if                       ( hash ) "
2867	"      UHMEHASH_SZ and                  ( bucket ) "
2868	"      HMEBUCKET_SIZE * uhme_hash +     ( hmebp ) "
2869	"   else                                ( hash ) "
2870	"      KHMEHASH_SZ and                  ( bucket ) "
2871	"      HMEBUCKET_SIZE * khme_hash +     ( hmebp ) "
2872	"   then                                ( hmebp ) "
2873	"; "
2874
2875	": HME_HASH_TABLE_SEARCH "
2876	"       ( sfmmup hmebp hblktag --  sfmmup null | sfmmup hmeblkp ) "
2877	"   >r hmebp_hblk + phys-x@ begin ( sfmmup hmeblkp ) ( r: hblktag ) "
2878	"      dup HMEBLK_ENDPA <> if     ( sfmmup hmeblkp ) ( r: hblktag ) "
2879	"         dup hmeblk_tag + phys-x@ r@ = if ( sfmmup hmeblkp )	  "
2880	"	     dup hmeblk_tag + 8 + phys-x@ 2 pick = if		  "
2881	"		  true	( sfmmup hmeblkp true ) ( r: hblktag )	  "
2882	"	     else						  "
2883	"		  hmeblk_next + phys-x@ false			  "
2884	"			( sfmmup hmeblkp false ) ( r: hblktag )   "
2885	"	     then						  "
2886	"	  else							  "
2887	"	     hmeblk_next + phys-x@ false			  "
2888	"			( sfmmup hmeblkp false ) ( r: hblktag )   "
2889	"	  then							  "
2890	"      else							  "
2891	"         drop 0 true						  "
2892	"      then							  "
2893	"   until r> drop						  "
2894	"; "
2895
2896	": HME_HASH_TAG ( sfmmup rehash addr -- hblktag ) "
2897	"   over HME_HASH_SHIFT HME_HASH_BSPAGE  ( sfmmup rehash bspage ) "
2898	"   HTAG_BSPAGE_SHIFT <<		 ( sfmmup rehash htag-bspage )"
2899	"   swap HTAG_REHASH_SHIFT << or	 ( sfmmup htag-bspage-rehash )"
2900	"   SFMMU_INVALID_SHMERID or nip	 ( hblktag ) "
2901	"; "
2902
2903	": HBLK_TO_TTEP ( hmeblkp addr -- ttep ) "
2904	"   over HMEBLK_MISC + PHYS-L@ HBLK_SZMASK and  ( hmeblkp addr ttesz ) "
2905	"   TTE8K =  if                            ( hmeblkp addr ) "
2906	"      MMU_PAGESHIFT >> NHMENTS 1- and     ( hmeblkp hme-index ) "
2907	"   else                                   ( hmeblkp addr ) "
2908	"      drop 0                              ( hmeblkp 0 ) "
2909	"   then                                   ( hmeblkp hme-index ) "
2910	"   SFHME_SIZE * + HMEBLK_HME1 +           ( hmep ) "
2911	"   SFHME_TTE +                            ( ttep ) "
2912	"; "
2913
2914	": unix-tte ( addr cnum -- false | tte-data true ) "
2915	"    KCONTEXT = if                   ( addr ) "
2916	"	KHATID                       ( addr khatid ) "
2917	"    else                            ( addr ) "
2918	"       drop false exit              ( false ) "
2919	"    then "
2920	"      ( addr khatid ) "
2921	"      mmu_hashcnt 1+ 1  do           ( addr sfmmup ) "
2922	"         2dup swap i HME_HASH_SHIFT  "
2923					"( addr sfmmup sfmmup addr hmeshift ) "
2924	"         HME_HASH_FUNCTION           ( addr sfmmup hmebp ) "
2925	"         over i 4 pick               "
2926				"( addr sfmmup hmebp sfmmup rehash addr ) "
2927	"         HME_HASH_TAG                ( addr sfmmup hmebp hblktag ) "
2928	"         HME_HASH_TABLE_SEARCH       "
2929					"( addr sfmmup { null | hmeblkp } ) "
2930	"         ?dup  if                    ( addr sfmmup hmeblkp ) "
2931	"            nip swap HBLK_TO_TTEP    ( ttep ) "
2932	"            dup TTE_IS_VALID  if     ( valid-ttep ) "
2933	"               PHYS-X@ true          ( tte-data true ) "
2934	"            else                     ( invalid-tte ) "
2935	"               drop false            ( false ) "
2936	"            then                     ( false | tte-data true ) "
2937	"            unloop exit              ( false | tte-data true ) "
2938	"         then                        ( addr sfmmup ) "
2939	"      loop                           ( addr sfmmup ) "
2940	"      2drop false                    ( false ) "
2941	"; "
2942;
2943
2944void
2945create_va_to_tte(void)
2946{
2947	char *bp;
2948	extern int khmehash_num, uhmehash_num;
2949	extern struct hmehash_bucket *khme_hash, *uhme_hash;
2950
2951#define	OFFSET(type, field)	((uintptr_t)(&((type *)0)->field))
2952
2953	bp = (char *)kobj_zalloc(MMU_PAGESIZE, KM_SLEEP);
2954
2955	/*
2956	 * Teach obp how to parse our sw ttes.
2957	 */
2958	(void) sprintf(bp, obp_tte_str,
2959	    MMU_PAGESHIFT,
2960	    TTE8K,
2961	    sizeof (struct sf_hment),
2962	    OFFSET(struct sf_hment, hme_tte),
2963	    OFFSET(struct hme_blk, hblk_tag),
2964	    OFFSET(struct hme_blk, hblk_nextpa),
2965	    OFFSET(struct hme_blk, hblk_misc),
2966	    OFFSET(struct hme_blk, hblk_hme),
2967	    NHMENTS,
2968	    HBLK_SZMASK,
2969	    HBLK_RANGE_SHIFT,
2970	    OFFSET(struct hmehash_bucket, hmeh_nextpa),
2971	    HMEBLK_ENDPA,
2972	    sizeof (struct hmehash_bucket),
2973	    HTAG_SFMMUPSZ,
2974	    HTAG_BSPAGE_SHIFT,
2975	    HTAG_REHASH_SHIFT,
2976	    SFMMU_INVALID_SHMERID,
2977	    mmu_hashcnt,
2978	    (caddr_t)va_to_pa((caddr_t)uhme_hash),
2979	    (caddr_t)va_to_pa((caddr_t)khme_hash),
2980	    UHMEHASH_SZ,
2981	    KHMEHASH_SZ,
2982	    KCONTEXT,
2983	    KHATID,
2984	    ASI_MEM);
2985	prom_interpret(bp, 0, 0, 0, 0, 0);
2986
2987	kobj_free(bp, MMU_PAGESIZE);
2988}
2989
2990void
2991install_va_to_tte(void)
2992{
2993	/*
2994	 * advise prom that it can use unix-tte
2995	 */
2996	prom_interpret("' unix-tte is va>tte-data", 0, 0, 0, 0, 0);
2997}
2998
2999/*
3000 * Here we add "device-type=console" for /os-io node, for currently
3001 * our kernel console output only supports displaying text and
3002 * performing cursor-positioning operations (through kernel framebuffer
3003 * driver) and it doesn't support other functionalities required for a
3004 * standard "display" device as specified in 1275 spec. The main missing
3005 * interface defined by the 1275 spec is "draw-logo".
3006 * also see the comments above prom_stdout_is_framebuffer().
3007 */
3008static char *create_node =
3009	"\" /\" find-device "
3010	"new-device "
3011	"\" os-io\" device-name "
3012	"\" "OBP_DISPLAY_CONSOLE"\" device-type "
3013	": cb-r/w  ( adr,len method$ -- #read/#written ) "
3014	"   2>r swap 2 2r> ['] $callback  catch  if "
3015	"      2drop 3drop 0 "
3016	"   then "
3017	"; "
3018	": read ( adr,len -- #read ) "
3019	"       \" read\" ['] cb-r/w catch  if  2drop 2drop -2 exit then "
3020	"       ( retN ... ret1 N ) "
3021	"       ?dup  if "
3022	"               swap >r 1-  0  ?do  drop  loop  r> "
3023	"       else "
3024	"               -2 "
3025	"       then "
3026	";    "
3027	": write ( adr,len -- #written ) "
3028	"       \" write\" ['] cb-r/w catch  if  2drop 2drop 0 exit  then "
3029	"       ( retN ... ret1 N ) "
3030	"       ?dup  if "
3031	"               swap >r 1-  0  ?do  drop  loop  r> "
3032	"        else "
3033	"               0 "
3034	"       then "
3035	"; "
3036	": poll-tty ( -- ) ; "
3037	": install-abort  ( -- )  ['] poll-tty d# 10 alarm ; "
3038	": remove-abort ( -- )  ['] poll-tty 0 alarm ; "
3039	": cb-give/take ( $method -- ) "
3040	"       0 -rot ['] $callback catch  ?dup  if "
3041	"               >r 2drop 2drop r> throw "
3042	"       else "
3043	"               0  ?do  drop  loop "
3044	"       then "
3045	"; "
3046	": give ( -- )  \" exit-input\" cb-give/take ; "
3047	": take ( -- )  \" enter-input\" cb-give/take ; "
3048	": open ( -- ok? )  true ; "
3049	": close ( -- ) ; "
3050	"finish-device "
3051	"device-end ";
3052
3053/*
3054 * Create the OBP input/output node (FCode serial driver).
3055 * It is needed for both USB console keyboard and for
3056 * the kernel terminal emulator.  It is too early to check for a
3057 * kernel console compatible framebuffer now, so we create this
3058 * so that we're ready if we need to enable kernel terminal emulation.
3059 *
3060 * When the USB software takes over the input device at the time
3061 * consconfig runs, OBP's stdin is redirected to this node.
3062 * Whenever the FORTH user interface is used after this switch,
3063 * the node will call back into the kernel for console input.
3064 * If a serial device such as ttya or a UART with a Type 5 keyboard
3065 * attached is used, OBP takes over the serial device when the system
3066 * goes to the debugger after the system is booted.  This sharing
3067 * of the relatively simple serial device is difficult but possible.
3068 * Sharing the USB host controller is impossible due its complexity.
3069 *
3070 * Similarly to USB keyboard input redirection, after consconfig_dacf
3071 * configures a kernel console framebuffer as the standard output
3072 * device, OBP's stdout is switched to to vector through the
3073 * /os-io node into the kernel terminal emulator.
3074 */
3075static void
3076startup_create_io_node(void)
3077{
3078	prom_interpret(create_node, 0, 0, 0, 0, 0);
3079}
3080
3081
3082/*
3083 * Must be defined in platform dependent code.
3084 */
3085extern caddr_t modtext;
3086extern size_t modtext_sz;
3087extern caddr_t moddata;
3088
3089#define	HEAPTEXT_ARENA(addr)	\
3090	((uintptr_t)(addr) < KERNELBASE + 2 * MMU_PAGESIZE4M ? 0 : \
3091	(((uintptr_t)(addr) - HEAPTEXT_BASE) / \
3092	(HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) + 1))
3093
3094#define	HEAPTEXT_OVERSIZED(addr)	\
3095	((uintptr_t)(addr) >= HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE)
3096
3097#define	HEAPTEXT_IN_NUCLEUSDATA(addr) \
3098	(((uintptr_t)(addr) >= KERNELBASE + 2 * MMU_PAGESIZE4M) && \
3099	((uintptr_t)(addr) < KERNELBASE + 3 * MMU_PAGESIZE4M))
3100
3101vmem_t *texthole_source[HEAPTEXT_NARENAS];
3102vmem_t *texthole_arena[HEAPTEXT_NARENAS];
3103kmutex_t texthole_lock;
3104
3105char kern_bootargs[OBP_MAXPATHLEN];
3106char kern_bootfile[OBP_MAXPATHLEN];
3107
3108void
3109kobj_vmem_init(vmem_t **text_arena, vmem_t **data_arena)
3110{
3111	uintptr_t addr, limit;
3112
3113	addr = HEAPTEXT_BASE;
3114	limit = addr + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE;
3115
3116	/*
3117	 * Before we initialize the text_arena, we want to punch holes in the
3118	 * underlying heaptext_arena.  This guarantees that for any text
3119	 * address we can find a text hole less than HEAPTEXT_MAPPED away.
3120	 */
3121	for (; addr + HEAPTEXT_UNMAPPED <= limit;
3122	    addr += HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) {
3123		(void) vmem_xalloc(heaptext_arena, HEAPTEXT_UNMAPPED, PAGESIZE,
3124		    0, 0, (void *)addr, (void *)(addr + HEAPTEXT_UNMAPPED),
3125		    VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
3126	}
3127
3128	/*
3129	 * Allocate one page at the oversize to break up the text region
3130	 * from the oversized region.
3131	 */
3132	(void) vmem_xalloc(heaptext_arena, PAGESIZE, PAGESIZE, 0, 0,
3133	    (void *)limit, (void *)(limit + PAGESIZE),
3134	    VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
3135
3136	*text_arena = vmem_create("module_text", modtext_sz ? modtext : NULL,
3137	    modtext_sz, sizeof (uintptr_t), segkmem_alloc, segkmem_free,
3138	    heaptext_arena, 0, VM_SLEEP);
3139	*data_arena = vmem_create("module_data", moddata, MODDATA, 1,
3140	    segkmem_alloc, segkmem_free, heap32_arena, 0, VM_SLEEP);
3141}
3142
3143caddr_t
3144kobj_text_alloc(vmem_t *arena, size_t size)
3145{
3146	caddr_t rval, better;
3147
3148	/*
3149	 * First, try a sleeping allocation.
3150	 */
3151	rval = vmem_alloc(arena, size, VM_SLEEP | VM_BESTFIT);
3152
3153	if (size >= HEAPTEXT_MAPPED || !HEAPTEXT_OVERSIZED(rval))
3154		return (rval);
3155
3156	/*
3157	 * We didn't get the area that we wanted.  We're going to try to do an
3158	 * allocation with explicit constraints.
3159	 */
3160	better = vmem_xalloc(arena, size, sizeof (uintptr_t), 0, 0, NULL,
3161	    (void *)(HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE),
3162	    VM_NOSLEEP | VM_BESTFIT);
3163
3164	if (better != NULL) {
3165		/*
3166		 * That worked.  Free our first attempt and return.
3167		 */
3168		vmem_free(arena, rval, size);
3169		return (better);
3170	}
3171
3172	/*
3173	 * That didn't work; we'll have to return our first attempt.
3174	 */
3175	return (rval);
3176}
3177
3178caddr_t
3179kobj_texthole_alloc(caddr_t addr, size_t size)
3180{
3181	int arena = HEAPTEXT_ARENA(addr);
3182	char c[30];
3183	uintptr_t base;
3184
3185	if (HEAPTEXT_OVERSIZED(addr) || HEAPTEXT_IN_NUCLEUSDATA(addr)) {
3186		/*
3187		 * If this is an oversized allocation or it is allocated in
3188		 * the nucleus data page, there is no text hole available for
3189		 * it; return NULL.
3190		 */
3191		return (NULL);
3192	}
3193
3194	mutex_enter(&texthole_lock);
3195
3196	if (texthole_arena[arena] == NULL) {
3197		ASSERT(texthole_source[arena] == NULL);
3198
3199		if (arena == 0) {
3200			texthole_source[0] = vmem_create("module_text_holesrc",
3201			    (void *)(KERNELBASE + MMU_PAGESIZE4M),
3202			    MMU_PAGESIZE4M, PAGESIZE, NULL, NULL, NULL,
3203			    0, VM_SLEEP);
3204		} else {
3205			base = HEAPTEXT_BASE +
3206			    (arena - 1) * (HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED);
3207
3208			(void) snprintf(c, sizeof (c),
3209			    "heaptext_holesrc_%d", arena);
3210
3211			texthole_source[arena] = vmem_create(c, (void *)base,
3212			    HEAPTEXT_UNMAPPED, PAGESIZE, NULL, NULL, NULL,
3213			    0, VM_SLEEP);
3214		}
3215
3216		(void) snprintf(c, sizeof (c), "heaptext_hole_%d", arena);
3217
3218		texthole_arena[arena] = vmem_create(c, NULL, 0,
3219		    sizeof (uint32_t), segkmem_alloc_permanent, segkmem_free,
3220		    texthole_source[arena], 0, VM_SLEEP);
3221	}
3222
3223	mutex_exit(&texthole_lock);
3224
3225	ASSERT(texthole_arena[arena] != NULL);
3226	ASSERT(arena >= 0 && arena < HEAPTEXT_NARENAS);
3227	return (vmem_alloc(texthole_arena[arena], size,
3228	    VM_BESTFIT | VM_NOSLEEP));
3229}
3230
3231void
3232kobj_texthole_free(caddr_t addr, size_t size)
3233{
3234	int arena = HEAPTEXT_ARENA(addr);
3235
3236	ASSERT(arena >= 0 && arena < HEAPTEXT_NARENAS);
3237	ASSERT(texthole_arena[arena] != NULL);
3238	vmem_free(texthole_arena[arena], addr, size);
3239}
3240
3241void
3242release_bootstrap(void)
3243{
3244	if (&cif_init)
3245		cif_init();
3246}
3247