1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. 24 * Copyright (c) 2016 by Delphix. All rights reserved. 25 * Copyright 2019 Peter Tribble. 26 */ 27 28 #include <sys/machsystm.h> 29 #include <sys/archsystm.h> 30 #include <sys/vm.h> 31 #include <sys/cpu.h> 32 #include <sys/atomic.h> 33 #include <sys/reboot.h> 34 #include <sys/kdi.h> 35 #include <sys/bootconf.h> 36 #include <sys/memlist_plat.h> 37 #include <sys/memlist_impl.h> 38 #include <sys/prom_plat.h> 39 #include <sys/prom_isa.h> 40 #include <sys/autoconf.h> 41 #include <sys/ivintr.h> 42 #include <sys/fpu/fpusystm.h> 43 #include <sys/iommutsb.h> 44 #include <vm/vm_dep.h> 45 #include <vm/seg_dev.h> 46 #include <vm/seg_kmem.h> 47 #include <vm/seg_kpm.h> 48 #include <vm/seg_map.h> 49 #include <vm/seg_kp.h> 50 #include <sys/sysconf.h> 51 #include <vm/hat_sfmmu.h> 52 #include <sys/kobj.h> 53 #include <sys/sun4asi.h> 54 #include <sys/clconf.h> 55 #include <sys/platform_module.h> 56 #include <sys/panic.h> 57 #include <sys/cpu_sgnblk_defs.h> 58 #include <sys/clock.h> 59 #include <sys/cmn_err.h> 60 #include <sys/dumphdr.h> 61 #include <sys/promif.h> 62 #include <sys/prom_debug.h> 63 #include <sys/traptrace.h> 64 #include <sys/memnode.h> 65 #include <sys/mem_cage.h> 66 #include <sys/mmu.h> 67 #include <sys/swap.h> 68 69 extern void setup_trap_table(void); 70 extern int cpu_intrq_setup(struct cpu *); 71 extern void cpu_intrq_register(struct cpu *); 72 extern void contig_mem_init(void); 73 extern caddr_t contig_mem_prealloc(caddr_t, pgcnt_t); 74 extern void mach_dump_buffer_init(void); 75 extern void mach_descrip_init(void); 76 extern void mach_descrip_startup_fini(void); 77 extern void mach_memscrub(void); 78 extern void mach_fpras(void); 79 extern void mach_cpu_halt_idle(void); 80 extern void mach_hw_copy_limit(void); 81 extern void load_mach_drivers(void); 82 extern void load_tod_module(void); 83 #pragma weak load_tod_module 84 85 extern int ndata_alloc_mmfsa(struct memlist *ndata); 86 #pragma weak ndata_alloc_mmfsa 87 88 extern void cif_init(void); 89 #pragma weak cif_init 90 91 extern void parse_idprom(void); 92 extern void add_vx_handler(char *, int, void (*)(cell_t *)); 93 extern void mem_config_init(void); 94 extern void memseg_remap_init(void); 95 96 extern void mach_kpm_init(void); 97 extern void pcf_init(); 98 extern int size_pse_array(pgcnt_t, int); 99 extern void pg_init(); 100 101 /* 102 * External Data: 103 */ 104 extern int vac_size; /* cache size in bytes */ 105 extern uint_t vac_mask; /* VAC alignment consistency mask */ 106 extern uint_t vac_colors; 107 108 /* 109 * Global Data Definitions: 110 */ 111 112 /* 113 * XXX - Don't port this to new architectures 114 * A 3rd party volume manager driver (vxdm) depends on the symbol romp. 115 * 'romp' has no use with a prom with an IEEE 1275 client interface. 116 * The driver doesn't use the value, but it depends on the symbol. 117 */ 118 void *romp; /* veritas driver won't load without romp 4154976 */ 119 /* 120 * Declare these as initialized data so we can patch them. 121 */ 122 pgcnt_t physmem = 0; /* memory size in pages, patch if you want less */ 123 pgcnt_t segkpsize = 124 btop(SEGKPDEFSIZE); /* size of segkp segment in pages */ 125 uint_t segmap_percent = 6; /* Size of segmap segment */ 126 127 int use_cache = 1; /* cache not reliable (605 bugs) with MP */ 128 int vac_copyback = 1; 129 char *cache_mode = NULL; 130 int use_mix = 1; 131 int prom_debug = 0; 132 133 caddr_t boot_tba; /* %tba at boot - used by kmdb */ 134 uint_t tba_taken_over = 0; 135 136 caddr_t s_text; /* start of kernel text segment */ 137 caddr_t e_text; /* end of kernel text segment */ 138 caddr_t s_data; /* start of kernel data segment */ 139 caddr_t e_data; /* end of kernel data segment */ 140 141 caddr_t modtext; /* beginning of module text */ 142 size_t modtext_sz; /* size of module text */ 143 caddr_t moddata; /* beginning of module data reserve */ 144 caddr_t e_moddata; /* end of module data reserve */ 145 146 /* 147 * End of first block of contiguous kernel in 32-bit virtual address space 148 */ 149 caddr_t econtig32; /* end of first blk of contiguous kernel */ 150 151 caddr_t ncbase; /* beginning of non-cached segment */ 152 caddr_t ncend; /* end of non-cached segment */ 153 154 size_t ndata_remain_sz; /* bytes from end of data to 4MB boundary */ 155 caddr_t nalloc_base; /* beginning of nucleus allocation */ 156 caddr_t nalloc_end; /* end of nucleus allocatable memory */ 157 caddr_t valloc_base; /* beginning of kvalloc segment */ 158 159 caddr_t kmem64_base; /* base of kernel mem segment in 64-bit space */ 160 caddr_t kmem64_end; /* end of kernel mem segment in 64-bit space */ 161 size_t kmem64_sz; /* bytes in kernel mem segment, 64-bit space */ 162 caddr_t kmem64_aligned_end; /* end of large page, overmaps 64-bit space */ 163 int kmem64_szc; /* page size code */ 164 uint64_t kmem64_pabase = (uint64_t)-1; /* physical address of kmem64_base */ 165 166 uintptr_t shm_alignment; /* VAC address consistency modulus */ 167 struct memlist *phys_install; /* Total installed physical memory */ 168 struct memlist *phys_avail; /* Available (unreserved) physical memory */ 169 struct memlist *virt_avail; /* Available (unmapped?) virtual memory */ 170 struct memlist *nopp_list; /* pages with no backing page structs */ 171 struct memlist ndata; /* memlist of nucleus allocatable memory */ 172 int memexp_flag; /* memory expansion card flag */ 173 uint64_t ecache_flushaddr; /* physical address used for flushing E$ */ 174 pgcnt_t obp_pages; /* Physical pages used by OBP */ 175 176 /* 177 * VM data structures 178 */ 179 long page_hashsz; /* Size of page hash table (power of two) */ 180 unsigned int page_hashsz_shift; /* log2(page_hashsz) */ 181 struct page *pp_base; /* Base of system page struct array */ 182 size_t pp_sz; /* Size in bytes of page struct array */ 183 struct page **page_hash; /* Page hash table */ 184 pad_mutex_t *pse_mutex; /* Locks protecting pp->p_selock */ 185 size_t pse_table_size; /* Number of mutexes in pse_mutex[] */ 186 int pse_shift; /* log2(pse_table_size) */ 187 struct seg ktextseg; /* Segment used for kernel executable image */ 188 struct seg kvalloc; /* Segment used for "valloc" mapping */ 189 struct seg kpseg; /* Segment used for pageable kernel virt mem */ 190 struct seg ktexthole; /* Segment used for nucleus text hole */ 191 struct seg kmapseg; /* Segment used for generic kernel mappings */ 192 struct seg kpmseg; /* Segment used for physical mapping */ 193 struct seg kdebugseg; /* Segment used for the kernel debugger */ 194 195 void *kpm_pp_base; /* Base of system kpm_page array */ 196 size_t kpm_pp_sz; /* Size of system kpm_page array */ 197 pgcnt_t kpm_npages; /* How many kpm pages are managed */ 198 199 struct seg *segkp = &kpseg; /* Pageable kernel virtual memory segment */ 200 struct seg *segkmap = &kmapseg; /* Kernel generic mapping segment */ 201 struct seg *segkpm = &kpmseg; /* 64bit kernel physical mapping segment */ 202 203 int segzio_fromheap = 0; /* zio allocations occur from heap */ 204 caddr_t segzio_base; /* Base address of segzio */ 205 pgcnt_t segziosize = 0; /* size of zio segment in pages */ 206 207 /* 208 * A static DR page_t VA map is reserved that can map the page structures 209 * for a domain's entire RA space. The pages that backs this space are 210 * dynamically allocated and need not be physically contiguous. The DR 211 * map size is derived from KPM size. 212 */ 213 int ppvm_enable = 0; /* Static virtual map for page structs */ 214 page_t *ppvm_base; /* Base of page struct map */ 215 pgcnt_t ppvm_size = 0; /* Size of page struct map */ 216 217 /* 218 * debugger pages (if allocated) 219 */ 220 struct vnode kdebugvp; 221 222 /* 223 * VA range available to the debugger 224 */ 225 const caddr_t kdi_segdebugbase = (const caddr_t)SEGDEBUGBASE; 226 const size_t kdi_segdebugsize = SEGDEBUGSIZE; 227 228 /* 229 * Segment for relocated kernel structures in 64-bit large RAM kernels 230 */ 231 struct seg kmem64; 232 233 struct memseg *memseg_free; 234 235 struct vnode unused_pages_vp; 236 237 /* 238 * VM data structures allocated early during boot. 239 */ 240 size_t pagehash_sz; 241 uint64_t memlist_sz; 242 243 char tbr_wr_addr_inited = 0; 244 245 caddr_t mpo_heap32_buf = NULL; 246 size_t mpo_heap32_bufsz = 0; 247 248 /* 249 * Static Routines: 250 */ 251 static int ndata_alloc_memseg(struct memlist *, size_t); 252 static void memlist_new(uint64_t, uint64_t, struct memlist **); 253 static void memlist_add(uint64_t, uint64_t, 254 struct memlist **, struct memlist **); 255 static void kphysm_init(void); 256 static void kvm_init(void); 257 static void install_kmem64_tte(void); 258 259 static void startup_init(void); 260 static void startup_memlist(void); 261 static void startup_modules(void); 262 static void startup_bop_gone(void); 263 static void startup_vm(void); 264 static void startup_end(void); 265 static void setup_cage_params(void); 266 static void startup_create_io_node(void); 267 268 static pgcnt_t npages; 269 static struct memlist *memlist; 270 void *memlist_end; 271 272 static pgcnt_t bop_alloc_pages; 273 static caddr_t hblk_base; 274 uint_t hblk_alloc_dynamic = 0; 275 uint_t hblk1_min = H1MIN; 276 277 278 /* 279 * Hook for down-rev firmware 280 */ 281 static void do_prom_version_check(void); 282 283 /* 284 * After receiving a thermal interrupt, this is the number of seconds 285 * to delay before shutting off the system, assuming 286 * shutdown fails. Use /etc/system to change the delay if this isn't 287 * large enough. 288 */ 289 int thermal_powerdown_delay = 1200; 290 291 /* 292 * Used to hold off page relocations into the cage until OBP has completed 293 * its boot-time handoff of its resources to the kernel. 294 */ 295 int page_relocate_ready = 0; 296 297 /* 298 * Indicate if kmem64 allocation was done in small chunks 299 */ 300 int kmem64_smchunks = 0; 301 302 /* 303 * Enable some debugging messages concerning memory usage... 304 */ 305 #ifdef DEBUGGING_MEM 306 static int debugging_mem; 307 static void 308 printmemlist(char *title, struct memlist *list) 309 { 310 if (!debugging_mem) 311 return; 312 313 printf("%s\n", title); 314 315 while (list) { 316 prom_printf("\taddr = 0x%x %8x, size = 0x%x %8x\n", 317 (uint32_t)(list->ml_address >> 32), 318 (uint32_t)list->ml_address, 319 (uint32_t)(list->ml_size >> 32), 320 (uint32_t)(list->ml_size)); 321 list = list->ml_next; 322 } 323 } 324 325 void 326 printmemseg(struct memseg *memseg) 327 { 328 if (!debugging_mem) 329 return; 330 331 printf("memseg\n"); 332 333 while (memseg) { 334 prom_printf("\tpage = 0x%p, epage = 0x%p, " 335 "pfn = 0x%x, epfn = 0x%x\n", 336 memseg->pages, memseg->epages, 337 memseg->pages_base, memseg->pages_end); 338 memseg = memseg->next; 339 } 340 } 341 342 #define debug_pause(str) halt((str)) 343 #define MPRINTF(str) if (debugging_mem) prom_printf((str)) 344 #define MPRINTF1(str, a) if (debugging_mem) prom_printf((str), (a)) 345 #define MPRINTF2(str, a, b) if (debugging_mem) prom_printf((str), (a), (b)) 346 #define MPRINTF3(str, a, b, c) \ 347 if (debugging_mem) prom_printf((str), (a), (b), (c)) 348 #else /* DEBUGGING_MEM */ 349 #define MPRINTF(str) 350 #define MPRINTF1(str, a) 351 #define MPRINTF2(str, a, b) 352 #define MPRINTF3(str, a, b, c) 353 #endif /* DEBUGGING_MEM */ 354 355 356 /* 357 * 358 * Kernel's Virtual Memory Layout. 359 * /-----------------------\ 360 * 0xFFFFFFFF.FFFFFFFF -| |- 361 * | OBP's virtual page | 362 * | tables | 363 * 0xFFFFFFFC.00000000 -|-----------------------|- 364 * : : 365 * : : 366 * -|-----------------------|- 367 * | segzio | (base and size vary) 368 * 0xFFFFFE00.00000000 -|-----------------------|- 369 * | | Ultrasparc I/II support 370 * | segkpm segment | up to 2TB of physical 371 * | (64-bit kernel ONLY) | memory, VAC has 2 colors 372 * | | 373 * 0xFFFFFA00.00000000 -|-----------------------|- 2TB segkpm alignment 374 * : : 375 * : : 376 * 0xFFFFF810.00000000 -|-----------------------|- hole_end 377 * | | ^ 378 * | UltraSPARC I/II call | | 379 * | bug requires an extra | | 380 * | 4 GB of space between | | 381 * | hole and used RAM | | 382 * | | | 383 * 0xFFFFF800.00000000 -|-----------------------|- | 384 * | | | 385 * | Virtual Address Hole | UltraSPARC 386 * | on UltraSPARC I/II | I/II * ONLY * 387 * | | | 388 * 0x00000800.00000000 -|-----------------------|- | 389 * | | | 390 * | UltraSPARC I/II call | | 391 * | bug requires an extra | | 392 * | 4 GB of space between | | 393 * | hole and used RAM | | 394 * | | v 395 * 0x000007FF.00000000 -|-----------------------|- hole_start ----- 396 * : : ^ 397 * : : | 398 * |-----------------------| | 399 * | | | 400 * | ecache flush area | | 401 * | (twice largest e$) | | 402 * | | | 403 * 0x00000XXX.XXX00000 -|-----------------------|- kmem64_ | 404 * | overmapped area | alignend_end | 405 * | (kmem64_alignsize | | 406 * | boundary) | | 407 * 0x00000XXX.XXXXXXXX -|-----------------------|- kmem64_end | 408 * | | | 409 * | 64-bit kernel ONLY | | 410 * | | | 411 * | kmem64 segment | | 412 * | | | 413 * | (Relocated extra HME | Approximately 414 * | block allocations, | 1 TB of virtual 415 * | memnode freelists, | address space 416 * | HME hash buckets, | | 417 * | mml_table, kpmp_table,| | 418 * | page_t array and | | 419 * | hashblock pool to | | 420 * | avoid hard-coded | | 421 * | 32-bit vaddr | | 422 * | limitations) | | 423 * | | v 424 * 0x00000700.00000000 -|-----------------------|- SYSLIMIT (kmem64_base) 425 * | | 426 * | segkmem segment | (SYSLIMIT - SYSBASE = 4TB) 427 * | | 428 * 0x00000300.00000000 -|-----------------------|- SYSBASE 429 * : : 430 * : : 431 * -|-----------------------|- 432 * | | 433 * | segmap segment | SEGMAPSIZE (1/8th physmem, 434 * | | 256G MAX) 435 * 0x000002a7.50000000 -|-----------------------|- SEGMAPBASE 436 * : : 437 * : : 438 * -|-----------------------|- 439 * | | 440 * | segkp | SEGKPSIZE (2GB) 441 * | | 442 * | | 443 * 0x000002a1.00000000 -|-----------------------|- SEGKPBASE 444 * | | 445 * 0x000002a0.00000000 -|-----------------------|- MEMSCRUBBASE 446 * | | (SEGKPBASE - 0x400000) 447 * 0x0000029F.FFE00000 -|-----------------------|- ARGSBASE 448 * | | (MEMSCRUBBASE - NCARGS) 449 * 0x0000029F.FFD80000 -|-----------------------|- PPMAPBASE 450 * | | (ARGSBASE - PPMAPSIZE) 451 * 0x0000029F.FFD00000 -|-----------------------|- PPMAP_FAST_BASE 452 * | | 453 * 0x0000029F.FF980000 -|-----------------------|- PIOMAPBASE 454 * | | 455 * 0x0000029F.FF580000 -|-----------------------|- NARG_BASE 456 * : : 457 * : : 458 * 0x00000000.FFFFFFFF -|-----------------------|- OFW_END_ADDR 459 * | | 460 * | OBP | 461 * | | 462 * 0x00000000.F0000000 -|-----------------------|- OFW_START_ADDR 463 * | kmdb | 464 * 0x00000000.EDD00000 -|-----------------------|- SEGDEBUGBASE 465 * : : 466 * : : 467 * 0x00000000.7c000000 -|-----------------------|- SYSLIMIT32 468 * | | 469 * | segkmem32 segment | (SYSLIMIT32 - SYSBASE32 = 470 * | | ~64MB) 471 * -|-----------------------| 472 * | IVSIZE | 473 * 0x00000000.70004000 -|-----------------------| 474 * | panicbuf | 475 * 0x00000000.70002000 -|-----------------------| 476 * | PAGESIZE | 477 * 0x00000000.70000000 -|-----------------------|- SYSBASE32 478 * | boot-time | 479 * | temporary space | 480 * 0x00000000.4C000000 -|-----------------------|- BOOTTMPBASE 481 * : : 482 * : : 483 * | | 484 * |-----------------------|- econtig32 485 * | vm structures | 486 * 0x00000000.01C00000 |-----------------------|- nalloc_end 487 * | TSBs | 488 * |-----------------------|- end/nalloc_base 489 * | kernel data & bss | 490 * 0x00000000.01800000 -|-----------------------| 491 * : nucleus text hole : 492 * 0x00000000.01400000 -|-----------------------| 493 * : : 494 * |-----------------------| 495 * | module text | 496 * |-----------------------|- e_text/modtext 497 * | kernel text | 498 * |-----------------------| 499 * | trap table (48k) | 500 * 0x00000000.01000000 -|-----------------------|- KERNELBASE 501 * | reserved for trapstat |} TSTAT_TOTAL_SIZE 502 * |-----------------------| 503 * | | 504 * | invalid | 505 * | | 506 * 0x00000000.00000000 _|_______________________| 507 * 508 * 509 * 510 * 32-bit User Virtual Memory Layout. 511 * /-----------------------\ 512 * | | 513 * | invalid | 514 * | | 515 * 0xFFC00000 -|-----------------------|- USERLIMIT 516 * | user stack | 517 * : : 518 * : : 519 * : : 520 * | user data | 521 * -|-----------------------|- 522 * | user text | 523 * 0x00002000 -|-----------------------|- 524 * | invalid | 525 * 0x00000000 _|_______________________| 526 * 527 * 528 * 529 * 64-bit User Virtual Memory Layout. 530 * /-----------------------\ 531 * | | 532 * | invalid | 533 * | | 534 * 0xFFFFFFFF.80000000 -|-----------------------|- USERLIMIT 535 * | user stack | 536 * : : 537 * : : 538 * : : 539 * | user data | 540 * -|-----------------------|- 541 * | user text | 542 * 0x00000000.01000000 -|-----------------------|- 543 * | invalid | 544 * 0x00000000.00000000 _|_______________________| 545 */ 546 547 extern caddr_t ecache_init_scrub_flush_area(caddr_t alloc_base); 548 extern uint64_t ecache_flush_address(void); 549 550 #pragma weak load_platform_modules 551 #pragma weak plat_startup_memlist 552 #pragma weak ecache_init_scrub_flush_area 553 #pragma weak ecache_flush_address 554 555 556 /* 557 * By default the DR Cage is enabled for maximum OS 558 * MPSS performance. Users needing to disable the cage mechanism 559 * can set this variable to zero via /etc/system. 560 * Disabling the cage on systems supporting Dynamic Reconfiguration (DR) 561 * will result in loss of DR functionality. 562 * Platforms wishing to disable kernel Cage by default 563 * should do so in their set_platform_defaults() routine. 564 */ 565 int kernel_cage_enable = 1; 566 567 static void 568 setup_cage_params(void) 569 { 570 void (*func)(void); 571 572 func = (void (*)(void))kobj_getsymvalue("set_platform_cage_params", 0); 573 if (func != NULL) { 574 (*func)(); 575 return; 576 } 577 578 if (kernel_cage_enable == 0) { 579 return; 580 } 581 kcage_range_init(phys_avail, KCAGE_DOWN, total_pages / 256); 582 583 if (kcage_on) { 584 cmn_err(CE_NOTE, "!Kernel Cage is ENABLED"); 585 } else { 586 cmn_err(CE_NOTE, "!Kernel Cage is DISABLED"); 587 } 588 589 } 590 591 /* 592 * Machine-dependent startup code 593 */ 594 void 595 startup(void) 596 { 597 startup_init(); 598 if (&startup_platform) 599 startup_platform(); 600 startup_memlist(); 601 startup_modules(); 602 setup_cage_params(); 603 startup_bop_gone(); 604 startup_vm(); 605 startup_end(); 606 } 607 608 struct regs sync_reg_buf; 609 uint64_t sync_tt; 610 611 void 612 sync_handler(void) 613 { 614 struct panic_trap_info ti; 615 int i; 616 617 /* 618 * Prevent trying to talk to the other CPUs since they are 619 * sitting in the prom and won't reply. 620 */ 621 for (i = 0; i < NCPU; i++) { 622 if ((i != CPU->cpu_id) && CPU_XCALL_READY(i)) { 623 cpu[i]->cpu_flags &= ~CPU_READY; 624 cpu[i]->cpu_flags |= CPU_QUIESCED; 625 CPUSET_DEL(cpu_ready_set, cpu[i]->cpu_id); 626 } 627 } 628 629 /* 630 * Force a serial dump, since there are no CPUs to help. 631 */ 632 dump_plat_mincpu = 0; 633 634 /* 635 * We've managed to get here without going through the 636 * normal panic code path. Try and save some useful 637 * information. 638 */ 639 if (!panicstr && (curthread->t_panic_trap == NULL)) { 640 ti.trap_type = sync_tt; 641 ti.trap_regs = &sync_reg_buf; 642 ti.trap_addr = NULL; 643 ti.trap_mmu_fsr = 0x0; 644 645 curthread->t_panic_trap = &ti; 646 } 647 648 /* 649 * If we're re-entering the panic path, update the signature 650 * block so that the SC knows we're in the second part of panic. 651 */ 652 if (panicstr) 653 CPU_SIGNATURE(OS_SIG, SIGST_EXIT, SIGSUBST_DUMP, -1); 654 655 nopanicdebug = 1; /* do not perform debug_enter() prior to dump */ 656 panic("sync initiated"); 657 } 658 659 660 static void 661 startup_init(void) 662 { 663 /* 664 * We want to save the registers while we're still in OBP 665 * so that we know they haven't been fiddled with since. 666 * (In principle, OBP can't change them just because it 667 * makes a callback, but we'd rather not depend on that 668 * behavior.) 669 */ 670 char sync_str[] = 671 "warning @ warning off : sync " 672 "%%tl-c %%tstate h# %p x! " 673 "%%g1 h# %p x! %%g2 h# %p x! %%g3 h# %p x! " 674 "%%g4 h# %p x! %%g5 h# %p x! %%g6 h# %p x! " 675 "%%g7 h# %p x! %%o0 h# %p x! %%o1 h# %p x! " 676 "%%o2 h# %p x! %%o3 h# %p x! %%o4 h# %p x! " 677 "%%o5 h# %p x! %%o6 h# %p x! %%o7 h# %p x! " 678 "%%tl-c %%tpc h# %p x! %%tl-c %%tnpc h# %p x! " 679 "%%y h# %p l! %%tl-c %%tt h# %p x! " 680 "sync ; warning !"; 681 682 /* 683 * 20 == num of %p substrings 684 * 16 == max num of chars %p will expand to. 685 */ 686 char bp[sizeof (sync_str) + 16 * 20]; 687 688 /* 689 * Initialize ptl1 stack for the 1st CPU. 690 */ 691 ptl1_init_cpu(&cpu0); 692 693 /* 694 * Initialize the address map for cache consistent mappings 695 * to random pages; must be done after vac_size is set. 696 */ 697 ppmapinit(); 698 699 /* 700 * Initialize the PROM callback handler. 701 */ 702 init_vx_handler(); 703 704 /* 705 * have prom call sync_callback() to handle the sync and 706 * save some useful information which will be stored in the 707 * core file later. 708 */ 709 (void) sprintf((char *)bp, sync_str, 710 (void *)&sync_reg_buf.r_tstate, (void *)&sync_reg_buf.r_g1, 711 (void *)&sync_reg_buf.r_g2, (void *)&sync_reg_buf.r_g3, 712 (void *)&sync_reg_buf.r_g4, (void *)&sync_reg_buf.r_g5, 713 (void *)&sync_reg_buf.r_g6, (void *)&sync_reg_buf.r_g7, 714 (void *)&sync_reg_buf.r_o0, (void *)&sync_reg_buf.r_o1, 715 (void *)&sync_reg_buf.r_o2, (void *)&sync_reg_buf.r_o3, 716 (void *)&sync_reg_buf.r_o4, (void *)&sync_reg_buf.r_o5, 717 (void *)&sync_reg_buf.r_o6, (void *)&sync_reg_buf.r_o7, 718 (void *)&sync_reg_buf.r_pc, (void *)&sync_reg_buf.r_npc, 719 (void *)&sync_reg_buf.r_y, (void *)&sync_tt); 720 prom_interpret(bp, 0, 0, 0, 0, 0); 721 add_vx_handler("sync", 1, (void (*)(cell_t *))sync_handler); 722 } 723 724 725 size_t 726 calc_pp_sz(pgcnt_t npages) 727 { 728 729 return (npages * sizeof (struct page)); 730 } 731 732 size_t 733 calc_kpmpp_sz(pgcnt_t npages) 734 { 735 736 kpm_pgshft = (kpm_smallpages == 0) ? MMU_PAGESHIFT4M : MMU_PAGESHIFT; 737 kpm_pgsz = 1ull << kpm_pgshft; 738 kpm_pgoff = kpm_pgsz - 1; 739 kpmp2pshft = kpm_pgshft - PAGESHIFT; 740 kpmpnpgs = 1 << kpmp2pshft; 741 742 if (kpm_smallpages == 0) { 743 /* 744 * Avoid fragmentation problems in kphysm_init() 745 * by allocating for all of physical memory 746 */ 747 kpm_npages = ptokpmpr(physinstalled); 748 return (kpm_npages * sizeof (kpm_page_t)); 749 } else { 750 kpm_npages = npages; 751 return (kpm_npages * sizeof (kpm_spage_t)); 752 } 753 } 754 755 size_t 756 calc_pagehash_sz(pgcnt_t npages) 757 { 758 /* LINTED */ 759 ASSERT(P2SAMEHIGHBIT((1 << PP_SHIFT), (sizeof (struct page)))); 760 /* 761 * The page structure hash table size is a power of 2 762 * such that the average hash chain length is PAGE_HASHAVELEN. 763 */ 764 page_hashsz = npages / PAGE_HASHAVELEN; 765 page_hashsz_shift = MAX((AN_VPSHIFT + VNODE_ALIGN_LOG2 + 1), 766 highbit(page_hashsz)); 767 page_hashsz = 1 << page_hashsz_shift; 768 return (page_hashsz * sizeof (struct page *)); 769 } 770 771 int testkmem64_smchunks = 0; 772 773 int 774 alloc_kmem64(caddr_t base, caddr_t end) 775 { 776 int i; 777 caddr_t aligned_end = NULL; 778 779 if (testkmem64_smchunks) 780 return (1); 781 782 /* 783 * Make one large memory alloc after figuring out the 64-bit size. This 784 * will enable use of the largest page size appropriate for the system 785 * architecture. 786 */ 787 ASSERT(mmu_exported_pagesize_mask & (1 << TTE8K)); 788 ASSERT(IS_P2ALIGNED(base, TTEBYTES(max_bootlp_tteszc))); 789 for (i = max_bootlp_tteszc; i >= TTE8K; i--) { 790 size_t alloc_size, alignsize; 791 #if !defined(C_OBP) 792 unsigned long long pa; 793 #endif /* !C_OBP */ 794 795 if ((mmu_exported_pagesize_mask & (1 << i)) == 0) 796 continue; 797 alignsize = TTEBYTES(i); 798 kmem64_szc = i; 799 800 /* limit page size for small memory */ 801 if (mmu_btop(alignsize) > (npages >> 2)) 802 continue; 803 804 aligned_end = (caddr_t)roundup((uintptr_t)end, alignsize); 805 alloc_size = aligned_end - base; 806 #if !defined(C_OBP) 807 if (prom_allocate_phys(alloc_size, alignsize, &pa) == 0) { 808 if (prom_claim_virt(alloc_size, base) != (caddr_t)-1) { 809 kmem64_pabase = pa; 810 kmem64_aligned_end = aligned_end; 811 install_kmem64_tte(); 812 break; 813 } else { 814 prom_free_phys(alloc_size, pa); 815 } 816 } 817 #else /* !C_OBP */ 818 if (prom_alloc(base, alloc_size, alignsize) == base) { 819 kmem64_pabase = va_to_pa(kmem64_base); 820 kmem64_aligned_end = aligned_end; 821 break; 822 } 823 #endif /* !C_OBP */ 824 if (i == TTE8K) { 825 #ifdef sun4v 826 /* return failure to try small allocations */ 827 return (1); 828 #else 829 prom_panic("kmem64 allocation failure"); 830 #endif 831 } 832 } 833 ASSERT(aligned_end != NULL); 834 return (0); 835 } 836 837 static prom_memlist_t *boot_physinstalled, *boot_physavail, *boot_virtavail; 838 static size_t boot_physinstalled_len, boot_physavail_len, boot_virtavail_len; 839 840 #if !defined(C_OBP) 841 /* 842 * Install a temporary tte handler in OBP for kmem64 area. 843 * 844 * We map kmem64 area with large pages before the trap table is taken 845 * over. Since OBP makes 8K mappings, it can create 8K tlb entries in 846 * the same area. Duplicate tlb entries with different page sizes 847 * cause unpredicatble behavior. To avoid this, we don't create 848 * kmem64 mappings via BOP_ALLOC (ends up as prom_alloc() call to 849 * OBP). Instead, we manage translations with a temporary va>tte-data 850 * handler (kmem64-tte). This handler is replaced by unix-tte when 851 * the trap table is taken over. 852 * 853 * The temporary handler knows the physical address of the kmem64 854 * area. It uses the prom's pgmap@ Forth word for other addresses. 855 * 856 * We have to use BOP_ALLOC() method for C-OBP platforms because 857 * pgmap@ is not defined in C-OBP. C-OBP is only used on serengeti 858 * sun4u platforms. On sun4u we flush tlb after trap table is taken 859 * over if we use large pages for kernel heap and kmem64. Since sun4u 860 * prom (unlike sun4v) calls va>tte-data first for client address 861 * translation prom's ttes for kmem64 can't get into TLB even if we 862 * later switch to prom's trap table again. C-OBP uses 4M pages for 863 * client mappings when possible so on all platforms we get the 864 * benefit from large mappings for kmem64 area immediately during 865 * boot. 866 * 867 * pseudo code: 868 * if (context != 0) { 869 * return false 870 * } else if (miss_va in range[kmem64_base, kmem64_end)) { 871 * tte = tte_template + 872 * (((miss_va & pagemask) - kmem64_base)); 873 * return tte, true 874 * } else { 875 * return pgmap@ result 876 * } 877 */ 878 char kmem64_obp_str[] = 879 "h# %lx constant kmem64-base " 880 "h# %lx constant kmem64-end " 881 "h# %lx constant kmem64-pagemask " 882 "h# %lx constant kmem64-template " 883 884 ": kmem64-tte ( addr cnum -- false | tte-data true ) " 885 " if ( addr ) " 886 " drop false exit then ( false ) " 887 " dup kmem64-base kmem64-end within if ( addr ) " 888 " kmem64-pagemask and ( addr' ) " 889 " kmem64-base - ( addr' ) " 890 " kmem64-template + ( tte ) " 891 " true ( tte true ) " 892 " else ( addr ) " 893 " pgmap@ ( tte ) " 894 " dup 0< if true else drop false then ( tte true | false ) " 895 " then ( tte true | false ) " 896 "; " 897 898 "' kmem64-tte is va>tte-data " 899 ; 900 901 static void 902 install_kmem64_tte() 903 { 904 char b[sizeof (kmem64_obp_str) + (4 * 16)]; 905 tte_t tte; 906 907 PRM_DEBUG(kmem64_pabase); 908 PRM_DEBUG(kmem64_szc); 909 sfmmu_memtte(&tte, kmem64_pabase >> MMU_PAGESHIFT, 910 PROC_DATA | HAT_NOSYNC, kmem64_szc); 911 PRM_DEBUG(tte.ll); 912 (void) sprintf(b, kmem64_obp_str, 913 kmem64_base, kmem64_end, TTE_PAGEMASK(kmem64_szc), tte.ll); 914 ASSERT(strlen(b) < sizeof (b)); 915 prom_interpret(b, 0, 0, 0, 0, 0); 916 } 917 #endif /* !C_OBP */ 918 919 /* 920 * As OBP takes up some RAM when the system boots, pages will already be "lost" 921 * to the system and reflected in npages by the time we see it. 922 * 923 * We only want to allocate kernel structures in the 64-bit virtual address 924 * space on systems with enough RAM to make the overhead of keeping track of 925 * an extra kernel memory segment worthwhile. 926 * 927 * Since OBP has already performed its memory allocations by this point, if we 928 * have more than MINMOVE_RAM_MB MB of RAM left free, go ahead and map 929 * memory in the 64-bit virtual address space; otherwise keep allocations 930 * contiguous with we've mapped so far in the 32-bit virtual address space. 931 */ 932 #define MINMOVE_RAM_MB ((size_t)1900) 933 #define MB_TO_BYTES(mb) ((mb) * 1048576ul) 934 #define BYTES_TO_MB(b) ((b) / 1048576ul) 935 936 pgcnt_t tune_npages = (pgcnt_t) 937 (MB_TO_BYTES(MINMOVE_RAM_MB)/ (size_t)MMU_PAGESIZE); 938 939 #pragma weak page_set_colorequiv_arr_cpu 940 extern void page_set_colorequiv_arr_cpu(void); 941 extern void page_set_colorequiv_arr(void); 942 943 static pgcnt_t ramdisk_npages; 944 static struct memlist *old_phys_avail; 945 946 kcage_dir_t kcage_startup_dir = KCAGE_DOWN; 947 948 static void 949 startup_memlist(void) 950 { 951 size_t hmehash_sz, pagelist_sz, tt_sz; 952 size_t psetable_sz; 953 caddr_t alloc_base; 954 caddr_t memspace; 955 struct memlist *cur; 956 size_t syslimit = (size_t)SYSLIMIT; 957 size_t sysbase = (size_t)SYSBASE; 958 959 /* 960 * Initialize enough of the system to allow kmem_alloc to work by 961 * calling boot to allocate its memory until the time that 962 * kvm_init is completed. The page structs are allocated after 963 * rounding up end to the nearest page boundary; the memsegs are 964 * initialized and the space they use comes from the kernel heap. 965 * With appropriate initialization, they can be reallocated later 966 * to a size appropriate for the machine's configuration. 967 * 968 * At this point, memory is allocated for things that will never 969 * need to be freed, this used to be "valloced". This allows a 970 * savings as the pages don't need page structures to describe 971 * them because them will not be managed by the vm system. 972 */ 973 974 /* 975 * We're loaded by boot with the following configuration (as 976 * specified in the sun4u/conf/Mapfile): 977 * 978 * text: 4 MB chunk aligned on a 4MB boundary 979 * data & bss: 4 MB chunk aligned on a 4MB boundary 980 * 981 * These two chunks will eventually be mapped by 2 locked 4MB 982 * ttes and will represent the nucleus of the kernel. This gives 983 * us some free space that is already allocated, some or all of 984 * which is made available to kernel module text. 985 * 986 * The free space in the data-bss chunk is used for nucleus 987 * allocatable data structures and we reserve it using the 988 * nalloc_base and nalloc_end variables. This space is currently 989 * being used for hat data structures required for tlb miss 990 * handling operations. We align nalloc_base to a l2 cache 991 * linesize because this is the line size the hardware uses to 992 * maintain cache coherency. 993 * 512K is carved out for module data. 994 */ 995 996 moddata = (caddr_t)roundup((uintptr_t)e_data, MMU_PAGESIZE); 997 e_moddata = moddata + MODDATA; 998 nalloc_base = e_moddata; 999 1000 nalloc_end = (caddr_t)roundup((uintptr_t)nalloc_base, MMU_PAGESIZE4M); 1001 valloc_base = nalloc_base; 1002 1003 /* 1004 * Calculate the start of the data segment. 1005 */ 1006 if (((uintptr_t)e_moddata & MMU_PAGEMASK4M) != (uintptr_t)s_data) 1007 prom_panic("nucleus data overflow"); 1008 1009 PRM_DEBUG(moddata); 1010 PRM_DEBUG(nalloc_base); 1011 PRM_DEBUG(nalloc_end); 1012 1013 /* 1014 * Remember any slop after e_text so we can give it to the modules. 1015 */ 1016 PRM_DEBUG(e_text); 1017 modtext = (caddr_t)roundup((uintptr_t)e_text, MMU_PAGESIZE); 1018 if (((uintptr_t)e_text & MMU_PAGEMASK4M) != (uintptr_t)s_text) 1019 prom_panic("nucleus text overflow"); 1020 modtext_sz = (caddr_t)roundup((uintptr_t)modtext, MMU_PAGESIZE4M) - 1021 modtext; 1022 PRM_DEBUG(modtext); 1023 PRM_DEBUG(modtext_sz); 1024 1025 init_boot_memlists(); 1026 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1027 &boot_physavail, &boot_physavail_len, 1028 &boot_virtavail, &boot_virtavail_len); 1029 1030 /* 1031 * Remember what the physically available highest page is 1032 * so that dumpsys works properly, and find out how much 1033 * memory is installed. 1034 */ 1035 installed_top_size_memlist_array(boot_physinstalled, 1036 boot_physinstalled_len, &physmax, &physinstalled); 1037 PRM_DEBUG(physinstalled); 1038 PRM_DEBUG(physmax); 1039 1040 /* Fill out memory nodes config structure */ 1041 startup_build_mem_nodes(boot_physinstalled, boot_physinstalled_len); 1042 1043 /* 1044 * npages is the maximum of available physical memory possible. 1045 * (ie. it will never be more than this) 1046 * 1047 * When we boot from a ramdisk, the ramdisk memory isn't free, so 1048 * using phys_avail will underestimate what will end up being freed. 1049 * A better initial guess is just total memory minus the kernel text 1050 */ 1051 npages = physinstalled - btop(MMU_PAGESIZE4M); 1052 1053 /* 1054 * First allocate things that can go in the nucleus data page 1055 * (fault status, TSBs, dmv, CPUs) 1056 */ 1057 ndata_alloc_init(&ndata, (uintptr_t)nalloc_base, (uintptr_t)nalloc_end); 1058 1059 if ((&ndata_alloc_mmfsa != NULL) && (ndata_alloc_mmfsa(&ndata) != 0)) 1060 cmn_err(CE_PANIC, "no more nucleus memory after mfsa alloc"); 1061 1062 if (ndata_alloc_tsbs(&ndata, npages) != 0) 1063 cmn_err(CE_PANIC, "no more nucleus memory after tsbs alloc"); 1064 1065 if (ndata_alloc_dmv(&ndata) != 0) 1066 cmn_err(CE_PANIC, "no more nucleus memory after dmv alloc"); 1067 1068 if (ndata_alloc_page_mutexs(&ndata) != 0) 1069 cmn_err(CE_PANIC, 1070 "no more nucleus memory after page free lists alloc"); 1071 1072 if (ndata_alloc_hat(&ndata) != 0) 1073 cmn_err(CE_PANIC, "no more nucleus memory after hat alloc"); 1074 1075 if (ndata_alloc_memseg(&ndata, boot_physavail_len) != 0) 1076 cmn_err(CE_PANIC, "no more nucleus memory after memseg alloc"); 1077 1078 /* 1079 * WARNING WARNING WARNING WARNING WARNING WARNING WARNING 1080 * 1081 * There are comments all over the SFMMU code warning of dire 1082 * consequences if the TSBs are moved out of 32-bit space. This 1083 * is largely because the asm code uses "sethi %hi(addr)"-type 1084 * instructions which will not provide the expected result if the 1085 * address is a 64-bit one. 1086 * 1087 * WARNING WARNING WARNING WARNING WARNING WARNING WARNING 1088 */ 1089 alloc_base = (caddr_t)roundup((uintptr_t)nalloc_end, MMU_PAGESIZE); 1090 PRM_DEBUG(alloc_base); 1091 1092 alloc_base = sfmmu_ktsb_alloc(alloc_base); 1093 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1094 PRM_DEBUG(alloc_base); 1095 1096 /* 1097 * Allocate IOMMU TSB array. We do this here so that the physical 1098 * memory gets deducted from the PROM's physical memory list. 1099 */ 1100 alloc_base = iommu_tsb_init(alloc_base); 1101 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1102 PRM_DEBUG(alloc_base); 1103 1104 /* 1105 * Allow for an early allocation of physically contiguous memory. 1106 */ 1107 alloc_base = contig_mem_prealloc(alloc_base, npages); 1108 1109 /* 1110 * Platforms like Starcat and OPL need special structures assigned in 1111 * 32-bit virtual address space because their probing routines execute 1112 * FCode, and FCode can't handle 64-bit virtual addresses... 1113 */ 1114 if (&plat_startup_memlist) { 1115 alloc_base = plat_startup_memlist(alloc_base); 1116 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, 1117 ecache_alignsize); 1118 PRM_DEBUG(alloc_base); 1119 } 1120 1121 /* 1122 * Save off where the contiguous allocations to date have ended 1123 * in econtig32. 1124 */ 1125 econtig32 = alloc_base; 1126 PRM_DEBUG(econtig32); 1127 if (econtig32 > (caddr_t)KERNEL_LIMIT32) 1128 cmn_err(CE_PANIC, "econtig32 too big"); 1129 1130 pp_sz = calc_pp_sz(npages); 1131 PRM_DEBUG(pp_sz); 1132 if (kpm_enable) { 1133 kpm_pp_sz = calc_kpmpp_sz(npages); 1134 PRM_DEBUG(kpm_pp_sz); 1135 } 1136 1137 hmehash_sz = calc_hmehash_sz(npages); 1138 PRM_DEBUG(hmehash_sz); 1139 1140 pagehash_sz = calc_pagehash_sz(npages); 1141 PRM_DEBUG(pagehash_sz); 1142 1143 pagelist_sz = calc_free_pagelist_sz(); 1144 PRM_DEBUG(pagelist_sz); 1145 1146 #ifdef TRAPTRACE 1147 tt_sz = calc_traptrace_sz(); 1148 PRM_DEBUG(tt_sz); 1149 #else 1150 tt_sz = 0; 1151 #endif /* TRAPTRACE */ 1152 1153 /* 1154 * Place the array that protects pp->p_selock in the kmem64 wad. 1155 */ 1156 pse_shift = size_pse_array(npages, max_ncpus); 1157 PRM_DEBUG(pse_shift); 1158 pse_table_size = 1 << pse_shift; 1159 PRM_DEBUG(pse_table_size); 1160 psetable_sz = roundup( 1161 pse_table_size * sizeof (pad_mutex_t), ecache_alignsize); 1162 PRM_DEBUG(psetable_sz); 1163 1164 /* 1165 * Now allocate the whole wad 1166 */ 1167 kmem64_sz = pp_sz + kpm_pp_sz + hmehash_sz + pagehash_sz + 1168 pagelist_sz + tt_sz + psetable_sz; 1169 kmem64_sz = roundup(kmem64_sz, PAGESIZE); 1170 kmem64_base = (caddr_t)syslimit; 1171 kmem64_end = kmem64_base + kmem64_sz; 1172 if (alloc_kmem64(kmem64_base, kmem64_end)) { 1173 /* 1174 * Attempt for kmem64 to allocate one big 1175 * contiguous chunk of memory failed. 1176 * We get here because we are sun4v. 1177 * We will proceed by breaking up 1178 * the allocation into two attempts. 1179 * First, we allocate kpm_pp_sz, hmehash_sz, 1180 * pagehash_sz, pagelist_sz, tt_sz & psetable_sz as 1181 * one contiguous chunk. This is a much smaller 1182 * chunk and we should get it, if not we panic. 1183 * Note that hmehash and tt need to be physically 1184 * (in the real address sense) contiguous. 1185 * Next, we use bop_alloc_chunk() to 1186 * to allocate the page_t structures. 1187 * This will allow the page_t to be allocated 1188 * in multiple smaller chunks. 1189 * In doing so, the assumption that page_t is 1190 * physically contiguous no longer hold, this is ok 1191 * for sun4v but not for sun4u. 1192 */ 1193 size_t tmp_size; 1194 caddr_t tmp_base; 1195 1196 pp_sz = roundup(pp_sz, PAGESIZE); 1197 1198 /* 1199 * Allocate kpm_pp_sz, hmehash_sz, 1200 * pagehash_sz, pagelist_sz, tt_sz & psetable_sz 1201 */ 1202 tmp_base = kmem64_base + pp_sz; 1203 tmp_size = roundup(kpm_pp_sz + hmehash_sz + pagehash_sz + 1204 pagelist_sz + tt_sz + psetable_sz, PAGESIZE); 1205 if (prom_alloc(tmp_base, tmp_size, PAGESIZE) == 0) 1206 prom_panic("kmem64 prom_alloc contig failed"); 1207 PRM_DEBUG(tmp_base); 1208 PRM_DEBUG(tmp_size); 1209 1210 /* 1211 * Allocate the page_ts 1212 */ 1213 if (bop_alloc_chunk(kmem64_base, pp_sz, PAGESIZE) == 0) 1214 prom_panic("kmem64 bop_alloc_chunk page_t failed"); 1215 PRM_DEBUG(kmem64_base); 1216 PRM_DEBUG(pp_sz); 1217 1218 kmem64_aligned_end = kmem64_base + pp_sz + tmp_size; 1219 ASSERT(kmem64_aligned_end >= kmem64_end); 1220 1221 kmem64_smchunks = 1; 1222 } else { 1223 1224 /* 1225 * We need to adjust pp_sz for the normal 1226 * case where kmem64 can allocate one large chunk 1227 */ 1228 if (kpm_smallpages == 0) { 1229 npages -= kmem64_sz / (PAGESIZE + sizeof (struct page)); 1230 } else { 1231 npages -= kmem64_sz / (PAGESIZE + sizeof (struct page) + 1232 sizeof (kpm_spage_t)); 1233 } 1234 pp_sz = npages * sizeof (struct page); 1235 } 1236 1237 if (kmem64_aligned_end > (hole_start ? hole_start : kpm_vbase)) 1238 cmn_err(CE_PANIC, "not enough kmem64 space"); 1239 PRM_DEBUG(kmem64_base); 1240 PRM_DEBUG(kmem64_end); 1241 PRM_DEBUG(kmem64_aligned_end); 1242 1243 /* 1244 * ... and divy it up 1245 */ 1246 alloc_base = kmem64_base; 1247 1248 pp_base = (page_t *)alloc_base; 1249 alloc_base += pp_sz; 1250 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1251 PRM_DEBUG(pp_base); 1252 PRM_DEBUG(npages); 1253 1254 if (kpm_enable) { 1255 kpm_pp_base = alloc_base; 1256 if (kpm_smallpages == 0) { 1257 /* kpm_npages based on physinstalled, don't reset */ 1258 kpm_pp_sz = kpm_npages * sizeof (kpm_page_t); 1259 } else { 1260 kpm_npages = ptokpmpr(npages); 1261 kpm_pp_sz = kpm_npages * sizeof (kpm_spage_t); 1262 } 1263 alloc_base += kpm_pp_sz; 1264 alloc_base = 1265 (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1266 PRM_DEBUG(kpm_pp_base); 1267 } 1268 1269 alloc_base = alloc_hmehash(alloc_base); 1270 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1271 PRM_DEBUG(alloc_base); 1272 1273 page_hash = (page_t **)alloc_base; 1274 alloc_base += pagehash_sz; 1275 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1276 PRM_DEBUG(page_hash); 1277 1278 alloc_base = alloc_page_freelists(alloc_base); 1279 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1280 PRM_DEBUG(alloc_base); 1281 1282 #ifdef TRAPTRACE 1283 ttrace_buf = alloc_base; 1284 alloc_base += tt_sz; 1285 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1286 PRM_DEBUG(alloc_base); 1287 #endif /* TRAPTRACE */ 1288 1289 pse_mutex = (pad_mutex_t *)alloc_base; 1290 alloc_base += psetable_sz; 1291 alloc_base = (caddr_t)roundup((uintptr_t)alloc_base, ecache_alignsize); 1292 PRM_DEBUG(alloc_base); 1293 1294 /* 1295 * Note that if we use small chunk allocations for 1296 * kmem64, we need to ensure kmem64_end is the same as 1297 * kmem64_aligned_end to prevent subsequent logic from 1298 * trying to reuse the overmapping. 1299 * Otherwise we adjust kmem64_end to what we really allocated. 1300 */ 1301 if (kmem64_smchunks) { 1302 kmem64_end = kmem64_aligned_end; 1303 } else { 1304 kmem64_end = (caddr_t)roundup((uintptr_t)alloc_base, PAGESIZE); 1305 } 1306 kmem64_sz = kmem64_end - kmem64_base; 1307 1308 if (&ecache_init_scrub_flush_area) { 1309 alloc_base = ecache_init_scrub_flush_area(kmem64_aligned_end); 1310 ASSERT(alloc_base <= (hole_start ? hole_start : kpm_vbase)); 1311 } 1312 1313 /* 1314 * If physmem is patched to be non-zero, use it instead of 1315 * the monitor value unless physmem is larger than the total 1316 * amount of memory on hand. 1317 */ 1318 if (physmem == 0 || physmem > npages) 1319 physmem = npages; 1320 1321 /* 1322 * root_is_ramdisk is set via /etc/system when the ramdisk miniroot 1323 * is mounted as root. This memory is held down by OBP and unlike 1324 * the stub boot_archive is never released. 1325 * 1326 * In order to get things sized correctly on lower memory 1327 * machines (where the memory used by the ramdisk represents 1328 * a significant portion of memory), physmem is adjusted. 1329 * 1330 * This is done by subtracting the ramdisk_size which is set 1331 * to the size of the ramdisk (in Kb) in /etc/system at the 1332 * time the miniroot archive is constructed. 1333 */ 1334 if (root_is_ramdisk == B_TRUE) { 1335 ramdisk_npages = (ramdisk_size * 1024) / PAGESIZE; 1336 physmem -= ramdisk_npages; 1337 } 1338 1339 if (kpm_enable && (ndata_alloc_kpm(&ndata, kpm_npages) != 0)) 1340 cmn_err(CE_PANIC, "no more nucleus memory after kpm alloc"); 1341 1342 /* 1343 * Allocate space for the interrupt vector table. 1344 */ 1345 memspace = prom_alloc((caddr_t)intr_vec_table, IVSIZE, MMU_PAGESIZE); 1346 if (memspace != (caddr_t)intr_vec_table) 1347 prom_panic("interrupt vector table allocation failure"); 1348 1349 /* 1350 * Between now and when we finish copying in the memory lists, 1351 * allocations happen so the space gets fragmented and the 1352 * lists longer. Leave enough space for lists twice as 1353 * long as we have now; then roundup to a pagesize. 1354 */ 1355 memlist_sz = sizeof (struct memlist) * (prom_phys_installed_len() + 1356 prom_phys_avail_len() + prom_virt_avail_len()); 1357 memlist_sz *= 2; 1358 memlist_sz = roundup(memlist_sz, PAGESIZE); 1359 memspace = ndata_alloc(&ndata, memlist_sz, ecache_alignsize); 1360 if (memspace == NULL) 1361 cmn_err(CE_PANIC, "no more nucleus memory after memlist alloc"); 1362 1363 memlist = (struct memlist *)memspace; 1364 memlist_end = (char *)memspace + memlist_sz; 1365 PRM_DEBUG(memlist); 1366 PRM_DEBUG(memlist_end); 1367 1368 PRM_DEBUG(sysbase); 1369 PRM_DEBUG(syslimit); 1370 kernelheap_init((void *)sysbase, (void *)syslimit, 1371 (caddr_t)sysbase + PAGESIZE, NULL, NULL); 1372 1373 /* 1374 * Take the most current snapshot we can by calling mem-update. 1375 */ 1376 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1377 &boot_physavail, &boot_physavail_len, 1378 &boot_virtavail, &boot_virtavail_len); 1379 1380 /* 1381 * Remove the space used by prom_alloc from the kernel heap 1382 * plus the area actually used by the OBP (if any) 1383 * ignoring virtual addresses in virt_avail, above syslimit. 1384 */ 1385 virt_avail = memlist; 1386 copy_memlist(boot_virtavail, boot_virtavail_len, &memlist); 1387 1388 for (cur = virt_avail; cur->ml_next; cur = cur->ml_next) { 1389 uint64_t range_base, range_size; 1390 1391 if ((range_base = cur->ml_address + cur->ml_size) < 1392 (uint64_t)sysbase) 1393 continue; 1394 if (range_base >= (uint64_t)syslimit) 1395 break; 1396 /* 1397 * Limit the range to end at syslimit. 1398 */ 1399 range_size = MIN(cur->ml_next->ml_address, 1400 (uint64_t)syslimit) - range_base; 1401 (void) vmem_xalloc(heap_arena, (size_t)range_size, PAGESIZE, 1402 0, 0, (void *)range_base, (void *)(range_base + range_size), 1403 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 1404 } 1405 1406 phys_avail = memlist; 1407 copy_memlist(boot_physavail, boot_physavail_len, &memlist); 1408 1409 /* 1410 * Add any extra memory at the end of the ndata region if there's at 1411 * least a page to add. There might be a few more pages available in 1412 * the middle of the ndata region, but for now they are ignored. 1413 */ 1414 nalloc_base = ndata_extra_base(&ndata, MMU_PAGESIZE, nalloc_end); 1415 if (nalloc_base == NULL) 1416 nalloc_base = nalloc_end; 1417 ndata_remain_sz = nalloc_end - nalloc_base; 1418 1419 /* 1420 * Copy physinstalled list into kernel space. 1421 */ 1422 phys_install = memlist; 1423 copy_memlist(boot_physinstalled, boot_physinstalled_len, &memlist); 1424 1425 /* 1426 * Create list of physical addrs we don't need pp's for: 1427 * kernel text 4M page 1428 * kernel data 4M page - ndata_remain_sz 1429 * kmem64 pages 1430 * 1431 * NB if adding any pages here, make sure no kpm page 1432 * overlaps can occur (see ASSERTs in kphysm_memsegs) 1433 */ 1434 nopp_list = memlist; 1435 memlist_new(va_to_pa(s_text), MMU_PAGESIZE4M, &memlist); 1436 memlist_add(va_to_pa(s_data), MMU_PAGESIZE4M - ndata_remain_sz, 1437 &memlist, &nopp_list); 1438 1439 /* Don't add to nopp_list if kmem64 was allocated in smchunks */ 1440 if (!kmem64_smchunks) 1441 memlist_add(kmem64_pabase, kmem64_sz, &memlist, &nopp_list); 1442 1443 if ((caddr_t)memlist > (memspace + memlist_sz)) 1444 prom_panic("memlist overflow"); 1445 1446 /* 1447 * Size the pcf array based on the number of cpus in the box at 1448 * boot time. 1449 */ 1450 pcf_init(); 1451 1452 /* 1453 * Initialize the page structures from the memory lists. 1454 */ 1455 kphysm_init(); 1456 1457 availrmem_initial = availrmem = freemem; 1458 PRM_DEBUG(availrmem); 1459 1460 /* 1461 * Some of the locks depend on page_hashsz being set! 1462 * kmem_init() depends on this; so, keep it here. 1463 */ 1464 page_lock_init(); 1465 1466 /* 1467 * Initialize kernel memory allocator. 1468 */ 1469 kmem_init(); 1470 1471 /* 1472 * Factor in colorequiv to check additional 'equivalent' bins 1473 */ 1474 if (&page_set_colorequiv_arr_cpu != NULL) 1475 page_set_colorequiv_arr_cpu(); 1476 else 1477 page_set_colorequiv_arr(); 1478 1479 /* 1480 * Initialize bp_mapin(). 1481 */ 1482 bp_init(shm_alignment, HAT_STRICTORDER); 1483 1484 /* 1485 * Reserve space for MPO mblock structs from the 32-bit heap. 1486 */ 1487 1488 if (mpo_heap32_bufsz > (size_t)0) { 1489 (void) vmem_xalloc(heap32_arena, mpo_heap32_bufsz, 1490 PAGESIZE, 0, 0, mpo_heap32_buf, 1491 mpo_heap32_buf + mpo_heap32_bufsz, 1492 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 1493 } 1494 mem_config_init(); 1495 } 1496 1497 static void 1498 startup_modules(void) 1499 { 1500 int nhblk1, nhblk8; 1501 size_t nhblksz; 1502 pgcnt_t pages_per_hblk; 1503 size_t hme8blk_sz, hme1blk_sz; 1504 1505 /* 1506 * The system file /etc/system was read already under startup_memlist. 1507 */ 1508 if (&set_platform_defaults) 1509 set_platform_defaults(); 1510 1511 /* 1512 * Calculate default settings of system parameters based upon 1513 * maxusers, yet allow to be overridden via the /etc/system file. 1514 */ 1515 param_calc(0); 1516 1517 mod_setup(); 1518 1519 /* 1520 * If we are running firmware that isn't 64-bit ready 1521 * then complain and halt. 1522 */ 1523 do_prom_version_check(); 1524 1525 /* 1526 * Initialize system parameters 1527 */ 1528 param_init(); 1529 1530 /* 1531 * maxmem is the amount of physical memory we're playing with. 1532 */ 1533 maxmem = physmem; 1534 1535 /* Set segkp limits. */ 1536 ncbase = kdi_segdebugbase; 1537 ncend = kdi_segdebugbase; 1538 1539 /* 1540 * Initialize the hat layer. 1541 */ 1542 hat_init(); 1543 1544 /* 1545 * Initialize segment management stuff. 1546 */ 1547 seg_init(); 1548 1549 /* 1550 * Create the va>tte handler, so the prom can understand 1551 * kernel translations. The handler is installed later, just 1552 * as we are about to take over the trap table from the prom. 1553 */ 1554 create_va_to_tte(); 1555 1556 /* 1557 * Load the forthdebugger (optional) 1558 */ 1559 forthdebug_init(); 1560 1561 /* 1562 * Create OBP node for console input callbacks 1563 * if it is needed. 1564 */ 1565 startup_create_io_node(); 1566 1567 if (modloadonly("fs", "specfs") == -1) 1568 halt("Can't load specfs"); 1569 1570 if (modloadonly("fs", "devfs") == -1) 1571 halt("Can't load devfs"); 1572 1573 if (modloadonly("fs", "procfs") == -1) 1574 halt("Can't load procfs"); 1575 1576 if (modloadonly("misc", "swapgeneric") == -1) 1577 halt("Can't load swapgeneric"); 1578 1579 (void) modloadonly("sys", "lbl_edition"); 1580 1581 dispinit(); 1582 1583 /* 1584 * Infer meanings to the members of the idprom buffer. 1585 */ 1586 parse_idprom(); 1587 1588 /* Read cluster configuration data. */ 1589 clconf_init(); 1590 1591 setup_ddi(); 1592 1593 /* 1594 * Lets take this opportunity to load the root device. 1595 */ 1596 if (loadrootmodules() != 0) 1597 debug_enter("Can't load the root filesystem"); 1598 1599 /* 1600 * Load tod driver module for the tod part found on this system. 1601 * Recompute the cpu frequency/delays based on tod as tod part 1602 * tends to keep time more accurately. 1603 */ 1604 if (&load_tod_module) 1605 load_tod_module(); 1606 1607 /* 1608 * Allow platforms to load modules which might 1609 * be needed after bootops are gone. 1610 */ 1611 if (&load_platform_modules) 1612 load_platform_modules(); 1613 1614 setcpudelay(); 1615 1616 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1617 &boot_physavail, &boot_physavail_len, 1618 &boot_virtavail, &boot_virtavail_len); 1619 1620 /* 1621 * Calculation and allocation of hmeblks needed to remap 1622 * the memory allocated by PROM till now. 1623 * Overestimate the number of hblk1 elements by assuming 1624 * worst case of TTE64K mappings. 1625 * sfmmu_hblk_alloc will panic if this calculation is wrong. 1626 */ 1627 bop_alloc_pages = btopr(kmem64_end - kmem64_base); 1628 pages_per_hblk = btop(HMEBLK_SPAN(TTE64K)); 1629 bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk); 1630 nhblk1 = bop_alloc_pages / pages_per_hblk + hblk1_min; 1631 1632 bop_alloc_pages = size_virtalloc(boot_virtavail, boot_virtavail_len); 1633 1634 /* sfmmu_init_nucleus_hblks expects properly aligned data structures */ 1635 hme8blk_sz = roundup(HME8BLK_SZ, sizeof (int64_t)); 1636 hme1blk_sz = roundup(HME1BLK_SZ, sizeof (int64_t)); 1637 1638 bop_alloc_pages += btopr(nhblk1 * hme1blk_sz); 1639 1640 pages_per_hblk = btop(HMEBLK_SPAN(TTE8K)); 1641 nhblk8 = 0; 1642 while (bop_alloc_pages > 1) { 1643 bop_alloc_pages = roundup(bop_alloc_pages, pages_per_hblk); 1644 nhblk8 += bop_alloc_pages /= pages_per_hblk; 1645 bop_alloc_pages *= hme8blk_sz; 1646 bop_alloc_pages = btopr(bop_alloc_pages); 1647 } 1648 nhblk8 += 2; 1649 1650 /* 1651 * Since hblk8's can hold up to 64k of mappings aligned on a 64k 1652 * boundary, the number of hblk8's needed to map the entries in the 1653 * boot_virtavail list needs to be adjusted to take this into 1654 * consideration. Thus, we need to add additional hblk8's since it 1655 * is possible that an hblk8 will not have all 8 slots used due to 1656 * alignment constraints. Since there were boot_virtavail_len entries 1657 * in that list, we need to add that many hblk8's to the number 1658 * already calculated to make sure we don't underestimate. 1659 */ 1660 nhblk8 += boot_virtavail_len; 1661 nhblksz = nhblk8 * hme8blk_sz + nhblk1 * hme1blk_sz; 1662 1663 /* Allocate in pagesize chunks */ 1664 nhblksz = roundup(nhblksz, MMU_PAGESIZE); 1665 hblk_base = kmem_zalloc(nhblksz, KM_SLEEP); 1666 sfmmu_init_nucleus_hblks(hblk_base, nhblksz, nhblk8, nhblk1); 1667 } 1668 1669 static void 1670 startup_bop_gone(void) 1671 { 1672 1673 /* 1674 * Destroy the MD initialized at startup 1675 * The startup initializes the MD framework 1676 * using prom and BOP alloc free it now. 1677 */ 1678 mach_descrip_startup_fini(); 1679 1680 /* 1681 * We're done with prom allocations. 1682 */ 1683 bop_fini(); 1684 1685 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1686 &boot_physavail, &boot_physavail_len, 1687 &boot_virtavail, &boot_virtavail_len); 1688 1689 /* 1690 * setup physically contiguous area twice as large as the ecache. 1691 * this is used while doing displacement flush of ecaches 1692 */ 1693 if (&ecache_flush_address) { 1694 ecache_flushaddr = ecache_flush_address(); 1695 if (ecache_flushaddr == (uint64_t)-1) { 1696 cmn_err(CE_PANIC, 1697 "startup: no memory to set ecache_flushaddr"); 1698 } 1699 } 1700 1701 /* 1702 * Virtual available next. 1703 */ 1704 ASSERT(virt_avail != NULL); 1705 memlist_free_list(virt_avail); 1706 virt_avail = memlist; 1707 copy_memlist(boot_virtavail, boot_virtavail_len, &memlist); 1708 1709 } 1710 1711 1712 /* 1713 * startup_fixup_physavail - called from mach_sfmmu.c after the final 1714 * allocations have been performed. We can't call it in startup_bop_gone 1715 * since later operations can cause obp to allocate more memory. 1716 */ 1717 void 1718 startup_fixup_physavail(void) 1719 { 1720 struct memlist *cur; 1721 size_t kmem64_overmap_size = kmem64_aligned_end - kmem64_end; 1722 1723 PRM_DEBUG(kmem64_overmap_size); 1724 1725 /* 1726 * take the most current snapshot we can by calling mem-update 1727 */ 1728 copy_boot_memlists(&boot_physinstalled, &boot_physinstalled_len, 1729 &boot_physavail, &boot_physavail_len, 1730 &boot_virtavail, &boot_virtavail_len); 1731 1732 /* 1733 * Copy phys_avail list, again. 1734 * Both the kernel/boot and the prom have been allocating 1735 * from the original list we copied earlier. 1736 */ 1737 cur = memlist; 1738 copy_memlist(boot_physavail, boot_physavail_len, &memlist); 1739 1740 /* 1741 * Add any unused kmem64 memory from overmapped page 1742 * (Note: va_to_pa does not work for kmem64_end) 1743 */ 1744 if (kmem64_overmap_size) { 1745 memlist_add(kmem64_pabase + (kmem64_end - kmem64_base), 1746 kmem64_overmap_size, &memlist, &cur); 1747 } 1748 1749 /* 1750 * Add any extra memory after e_data we added to the phys_avail list 1751 * back to the old list. 1752 */ 1753 if (ndata_remain_sz >= MMU_PAGESIZE) 1754 memlist_add(va_to_pa(nalloc_base), 1755 (uint64_t)ndata_remain_sz, &memlist, &cur); 1756 1757 /* 1758 * There isn't any bounds checking on the memlist area 1759 * so ensure it hasn't overgrown. 1760 */ 1761 if ((caddr_t)memlist > (caddr_t)memlist_end) 1762 cmn_err(CE_PANIC, "startup: memlist size exceeded"); 1763 1764 /* 1765 * The kernel removes the pages that were allocated for it from 1766 * the freelist, but we now have to find any -extra- pages that 1767 * the prom has allocated for it's own book-keeping, and remove 1768 * them from the freelist too. sigh. 1769 */ 1770 sync_memlists(phys_avail, cur); 1771 1772 ASSERT(phys_avail != NULL); 1773 1774 old_phys_avail = phys_avail; 1775 phys_avail = cur; 1776 } 1777 1778 void 1779 update_kcage_ranges(uint64_t addr, uint64_t len) 1780 { 1781 pfn_t base = btop(addr); 1782 pgcnt_t num = btop(len); 1783 int rv; 1784 1785 rv = kcage_range_add(base, num, kcage_startup_dir); 1786 1787 if (rv == ENOMEM) { 1788 cmn_err(CE_WARN, "%ld megabytes not available to kernel cage", 1789 (len == 0 ? 0 : BYTES_TO_MB(len))); 1790 } else if (rv != 0) { 1791 /* catch this in debug kernels */ 1792 ASSERT(0); 1793 1794 cmn_err(CE_WARN, "unexpected kcage_range_add" 1795 " return value %d", rv); 1796 } 1797 } 1798 1799 static void 1800 startup_vm(void) 1801 { 1802 size_t i; 1803 struct segmap_crargs a; 1804 struct segkpm_crargs b; 1805 1806 uint64_t avmem; 1807 caddr_t va; 1808 pgcnt_t max_phys_segkp; 1809 int mnode; 1810 1811 extern int use_brk_lpg, use_stk_lpg; 1812 1813 /* 1814 * get prom's mappings, create hments for them and switch 1815 * to the kernel context. 1816 */ 1817 hat_kern_setup(); 1818 1819 /* 1820 * Take over trap table 1821 */ 1822 setup_trap_table(); 1823 1824 /* 1825 * Install the va>tte handler, so that the prom can handle 1826 * misses and understand the kernel table layout in case 1827 * we need call into the prom. 1828 */ 1829 install_va_to_tte(); 1830 1831 /* 1832 * Set a flag to indicate that the tba has been taken over. 1833 */ 1834 tba_taken_over = 1; 1835 1836 /* initialize MMU primary context register */ 1837 mmu_init_kcontext(); 1838 1839 /* 1840 * The boot cpu can now take interrupts, x-calls, x-traps 1841 */ 1842 CPUSET_ADD(cpu_ready_set, CPU->cpu_id); 1843 CPU->cpu_flags |= (CPU_READY | CPU_ENABLE | CPU_EXISTS); 1844 1845 /* 1846 * Set a flag to tell write_scb_int() that it can access V_TBR_WR_ADDR. 1847 */ 1848 tbr_wr_addr_inited = 1; 1849 1850 /* 1851 * Initialize VM system, and map kernel address space. 1852 */ 1853 kvm_init(); 1854 1855 ASSERT(old_phys_avail != NULL && phys_avail != NULL); 1856 if (kernel_cage_enable) { 1857 diff_memlists(phys_avail, old_phys_avail, update_kcage_ranges); 1858 } 1859 memlist_free_list(old_phys_avail); 1860 1861 /* 1862 * If the following is true, someone has patched 1863 * phsymem to be less than the number of pages that 1864 * the system actually has. Remove pages until system 1865 * memory is limited to the requested amount. Since we 1866 * have allocated page structures for all pages, we 1867 * correct the amount of memory we want to remove 1868 * by the size of the memory used to hold page structures 1869 * for the non-used pages. 1870 */ 1871 if (physmem + ramdisk_npages < npages) { 1872 pgcnt_t diff, off; 1873 struct page *pp; 1874 struct seg kseg; 1875 1876 cmn_err(CE_WARN, "limiting physmem to %ld pages", physmem); 1877 1878 off = 0; 1879 diff = npages - (physmem + ramdisk_npages); 1880 diff -= mmu_btopr(diff * sizeof (struct page)); 1881 kseg.s_as = &kas; 1882 while (diff--) { 1883 pp = page_create_va(&unused_pages_vp, (offset_t)off, 1884 MMU_PAGESIZE, PG_WAIT | PG_EXCL, 1885 &kseg, (caddr_t)off); 1886 if (pp == NULL) 1887 cmn_err(CE_PANIC, "limited physmem too much!"); 1888 page_io_unlock(pp); 1889 page_downgrade(pp); 1890 availrmem--; 1891 off += MMU_PAGESIZE; 1892 } 1893 } 1894 1895 /* 1896 * When printing memory, show the total as physmem less 1897 * that stolen by a debugger. 1898 */ 1899 cmn_err(CE_CONT, "?mem = %ldK (0x%lx000)\n", 1900 (ulong_t)(physinstalled) << (PAGESHIFT - 10), 1901 (ulong_t)(physinstalled) << (PAGESHIFT - 12)); 1902 1903 avmem = (uint64_t)freemem << PAGESHIFT; 1904 cmn_err(CE_CONT, "?avail mem = %lld\n", (unsigned long long)avmem); 1905 1906 /* 1907 * For small memory systems disable automatic large pages. 1908 */ 1909 if (physmem < privm_lpg_min_physmem) { 1910 use_brk_lpg = 0; 1911 use_stk_lpg = 0; 1912 } 1913 1914 /* 1915 * Perform platform specific freelist processing 1916 */ 1917 if (&plat_freelist_process) { 1918 for (mnode = 0; mnode < max_mem_nodes; mnode++) 1919 if (mem_node_config[mnode].exists) 1920 plat_freelist_process(mnode); 1921 } 1922 1923 /* 1924 * Initialize the segkp segment type. We position it 1925 * after the configured tables and buffers (whose end 1926 * is given by econtig) and before V_WKBASE_ADDR. 1927 * Also in this area is segkmap (size SEGMAPSIZE). 1928 */ 1929 1930 /* XXX - cache alignment? */ 1931 va = (caddr_t)SEGKPBASE; 1932 ASSERT(((uintptr_t)va & PAGEOFFSET) == 0); 1933 1934 max_phys_segkp = (physmem * 2); 1935 1936 if (segkpsize < btop(SEGKPMINSIZE) || segkpsize > btop(SEGKPMAXSIZE)) { 1937 segkpsize = btop(SEGKPDEFSIZE); 1938 cmn_err(CE_WARN, "Illegal value for segkpsize. " 1939 "segkpsize has been reset to %ld pages", segkpsize); 1940 } 1941 1942 i = ptob(MIN(segkpsize, max_phys_segkp)); 1943 1944 rw_enter(&kas.a_lock, RW_WRITER); 1945 if (seg_attach(&kas, va, i, segkp) < 0) 1946 cmn_err(CE_PANIC, "startup: cannot attach segkp"); 1947 if (segkp_create(segkp) != 0) 1948 cmn_err(CE_PANIC, "startup: segkp_create failed"); 1949 rw_exit(&kas.a_lock); 1950 1951 /* 1952 * kpm segment 1953 */ 1954 segmap_kpm = kpm_enable && 1955 segmap_kpm && PAGESIZE == MAXBSIZE; 1956 1957 if (kpm_enable) { 1958 rw_enter(&kas.a_lock, RW_WRITER); 1959 1960 /* 1961 * The segkpm virtual range range is larger than the 1962 * actual physical memory size and also covers gaps in 1963 * the physical address range for the following reasons: 1964 * . keep conversion between segkpm and physical addresses 1965 * simple, cheap and unambiguous. 1966 * . avoid extension/shrink of the the segkpm in case of DR. 1967 * . avoid complexity for handling of virtual addressed 1968 * caches, segkpm and the regular mapping scheme must be 1969 * kept in sync wrt. the virtual color of mapped pages. 1970 * Any accesses to virtual segkpm ranges not backed by 1971 * physical memory will fall through the memseg pfn hash 1972 * and will be handled in segkpm_fault. 1973 * Additional kpm_size spaces needed for vac alias prevention. 1974 */ 1975 if (seg_attach(&kas, kpm_vbase, kpm_size * vac_colors, 1976 segkpm) < 0) 1977 cmn_err(CE_PANIC, "cannot attach segkpm"); 1978 1979 b.prot = PROT_READ | PROT_WRITE; 1980 b.nvcolors = shm_alignment >> MMU_PAGESHIFT; 1981 1982 if (segkpm_create(segkpm, (caddr_t)&b) != 0) 1983 panic("segkpm_create segkpm"); 1984 1985 rw_exit(&kas.a_lock); 1986 1987 mach_kpm_init(); 1988 } 1989 1990 va = kpm_vbase + (kpm_size * vac_colors); 1991 1992 if (!segzio_fromheap) { 1993 size_t size; 1994 size_t physmem_b = mmu_ptob(physmem); 1995 1996 /* size is in bytes, segziosize is in pages */ 1997 if (segziosize == 0) { 1998 size = physmem_b; 1999 } else { 2000 size = mmu_ptob(segziosize); 2001 } 2002 2003 if (size < SEGZIOMINSIZE) { 2004 size = SEGZIOMINSIZE; 2005 } else if (size > SEGZIOMAXSIZE) { 2006 size = SEGZIOMAXSIZE; 2007 /* 2008 * On 64-bit x86, we only have 2TB of KVA. This exists 2009 * for parity with x86. 2010 * 2011 * SEGZIOMAXSIZE is capped at 512gb so that segzio 2012 * doesn't consume all of KVA. However, if we have a 2013 * system that has more thant 512gb of physical memory, 2014 * we can actually consume about half of the difference 2015 * between 512gb and the rest of the available physical 2016 * memory. 2017 */ 2018 if (physmem_b > SEGZIOMAXSIZE) { 2019 size += (physmem_b - SEGZIOMAXSIZE) / 2; 2020 } 2021 } 2022 segziosize = mmu_btop(roundup(size, MMU_PAGESIZE)); 2023 /* put the base of the ZIO segment after the kpm segment */ 2024 segzio_base = va; 2025 va += mmu_ptob(segziosize); 2026 PRM_DEBUG(segziosize); 2027 PRM_DEBUG(segzio_base); 2028 2029 /* 2030 * On some platforms, kvm_init is called after the kpm 2031 * sizes have been determined. On SPARC, kvm_init is called 2032 * before, so we have to attach the kzioseg after kvm is 2033 * initialized, otherwise we'll try to allocate from the boot 2034 * area since the kernel heap hasn't yet been configured. 2035 */ 2036 rw_enter(&kas.a_lock, RW_WRITER); 2037 2038 (void) seg_attach(&kas, segzio_base, mmu_ptob(segziosize), 2039 &kzioseg); 2040 (void) segkmem_zio_create(&kzioseg); 2041 2042 /* create zio area covering new segment */ 2043 segkmem_zio_init(segzio_base, mmu_ptob(segziosize)); 2044 2045 rw_exit(&kas.a_lock); 2046 } 2047 2048 if (ppvm_enable) { 2049 caddr_t ppvm_max; 2050 2051 /* 2052 * ppvm refers to the static VA space used to map 2053 * the page_t's for dynamically added memory. 2054 * 2055 * ppvm_base should not cross a potential VA hole. 2056 * 2057 * ppvm_size should be large enough to map the 2058 * page_t's needed to manage all of KPM range. 2059 */ 2060 ppvm_size = 2061 roundup(mmu_btop(kpm_size * vac_colors) * sizeof (page_t), 2062 MMU_PAGESIZE); 2063 ppvm_max = (caddr_t)(0ull - ppvm_size); 2064 ppvm_base = (page_t *)va; 2065 2066 if ((caddr_t)ppvm_base <= hole_end) { 2067 cmn_err(CE_WARN, 2068 "Memory DR disabled: invalid DR map base: 0x%p\n", 2069 (void *)ppvm_base); 2070 ppvm_enable = 0; 2071 } else if ((caddr_t)ppvm_base > ppvm_max) { 2072 uint64_t diff = (caddr_t)ppvm_base - ppvm_max; 2073 2074 cmn_err(CE_WARN, 2075 "Memory DR disabled: insufficient DR map size:" 2076 " 0x%lx (needed 0x%lx)\n", 2077 ppvm_size - diff, ppvm_size); 2078 ppvm_enable = 0; 2079 } 2080 PRM_DEBUG(ppvm_size); 2081 PRM_DEBUG(ppvm_base); 2082 } 2083 2084 /* 2085 * Now create generic mapping segment. This mapping 2086 * goes SEGMAPSIZE beyond SEGMAPBASE. But if the total 2087 * virtual address is greater than the amount of free 2088 * memory that is available, then we trim back the 2089 * segment size to that amount 2090 */ 2091 va = (caddr_t)SEGMAPBASE; 2092 2093 /* 2094 * 1201049: segkmap base address must be MAXBSIZE aligned 2095 */ 2096 ASSERT(((uintptr_t)va & MAXBOFFSET) == 0); 2097 2098 /* 2099 * Set size of segmap to percentage of freemem at boot, 2100 * but stay within the allowable range 2101 * Note we take percentage before converting from pages 2102 * to bytes to avoid an overflow on 32-bit kernels. 2103 */ 2104 i = mmu_ptob((freemem * segmap_percent) / 100); 2105 2106 if (i < MINMAPSIZE) 2107 i = MINMAPSIZE; 2108 2109 if (i > MIN(SEGMAPSIZE, mmu_ptob(freemem))) 2110 i = MIN(SEGMAPSIZE, mmu_ptob(freemem)); 2111 2112 i &= MAXBMASK; /* 1201049: segkmap size must be MAXBSIZE aligned */ 2113 2114 rw_enter(&kas.a_lock, RW_WRITER); 2115 if (seg_attach(&kas, va, i, segkmap) < 0) 2116 cmn_err(CE_PANIC, "cannot attach segkmap"); 2117 2118 a.prot = PROT_READ | PROT_WRITE; 2119 a.shmsize = shm_alignment; 2120 a.nfreelist = 0; /* use segmap driver defaults */ 2121 2122 if (segmap_create(segkmap, (caddr_t)&a) != 0) 2123 panic("segmap_create segkmap"); 2124 rw_exit(&kas.a_lock); 2125 2126 segdev_init(); 2127 } 2128 2129 static void 2130 startup_end(void) 2131 { 2132 if ((caddr_t)memlist > (caddr_t)memlist_end) 2133 panic("memlist overflow 2"); 2134 memlist_free_block((caddr_t)memlist, 2135 ((caddr_t)memlist_end - (caddr_t)memlist)); 2136 memlist = NULL; 2137 2138 /* enable page_relocation since OBP is now done */ 2139 page_relocate_ready = 1; 2140 2141 /* 2142 * Perform tasks that get done after most of the VM 2143 * initialization has been done but before the clock 2144 * and other devices get started. 2145 */ 2146 kern_setup1(); 2147 2148 /* 2149 * Perform CPC initialization for this CPU. 2150 */ 2151 kcpc_hw_init(); 2152 2153 /* 2154 * Intialize the VM arenas for allocating physically 2155 * contiguus memory chunk for interrupt queues snd 2156 * allocate/register boot cpu's queues, if any and 2157 * allocate dump buffer for sun4v systems to store 2158 * extra crash information during crash dump 2159 */ 2160 contig_mem_init(); 2161 mach_descrip_init(); 2162 2163 if (cpu_intrq_setup(CPU)) { 2164 cmn_err(CE_PANIC, "cpu%d: setup failed", CPU->cpu_id); 2165 } 2166 cpu_intrq_register(CPU); 2167 mach_htraptrace_setup(CPU->cpu_id); 2168 mach_htraptrace_configure(CPU->cpu_id); 2169 mach_dump_buffer_init(); 2170 2171 /* 2172 * Initialize interrupt related stuff 2173 */ 2174 cpu_intr_alloc(CPU, NINTR_THREADS); 2175 2176 (void) splzs(); /* allow hi clock ints but not zs */ 2177 2178 /* 2179 * Initialize errors. 2180 */ 2181 error_init(); 2182 2183 /* 2184 * Note that we may have already used kernel bcopy before this 2185 * point - but if you really care about this, adb the use_hw_* 2186 * variables to 0 before rebooting. 2187 */ 2188 mach_hw_copy_limit(); 2189 2190 /* 2191 * Install the "real" preemption guards before DDI services 2192 * are available. 2193 */ 2194 (void) prom_set_preprom(kern_preprom); 2195 (void) prom_set_postprom(kern_postprom); 2196 CPU->cpu_m.mutex_ready = 1; 2197 2198 /* 2199 * Initialize segnf (kernel support for non-faulting loads). 2200 */ 2201 segnf_init(); 2202 2203 /* 2204 * Configure the root devinfo node. 2205 */ 2206 configure(); /* set up devices */ 2207 mach_cpu_halt_idle(); 2208 } 2209 2210 2211 void 2212 post_startup(void) 2213 { 2214 #ifdef PTL1_PANIC_DEBUG 2215 extern void init_ptl1_thread(void); 2216 #endif /* PTL1_PANIC_DEBUG */ 2217 extern void abort_sequence_init(void); 2218 2219 /* 2220 * Set the system wide, processor-specific flags to be passed 2221 * to userland via the aux vector for performance hints and 2222 * instruction set extensions. 2223 */ 2224 bind_hwcap(); 2225 2226 /* 2227 * Startup memory scrubber (if any) 2228 */ 2229 mach_memscrub(); 2230 2231 /* 2232 * Allocate soft interrupt to handle abort sequence. 2233 */ 2234 abort_sequence_init(); 2235 2236 /* 2237 * Configure the rest of the system. 2238 * Perform forceloading tasks for /etc/system. 2239 */ 2240 (void) mod_sysctl(SYS_FORCELOAD, NULL); 2241 /* 2242 * ON4.0: Force /proc module in until clock interrupt handle fixed 2243 * ON4.0: This must be fixed or restated in /etc/systems. 2244 */ 2245 (void) modload("fs", "procfs"); 2246 2247 /* load machine class specific drivers */ 2248 load_mach_drivers(); 2249 2250 /* load platform specific drivers */ 2251 if (&load_platform_drivers) 2252 load_platform_drivers(); 2253 2254 /* load vis simulation module, if we are running w/fpu off */ 2255 if (!fpu_exists) { 2256 if (modload("misc", "vis") == -1) 2257 halt("Can't load vis"); 2258 } 2259 2260 mach_fpras(); 2261 2262 maxmem = freemem; 2263 2264 pg_init(); 2265 2266 #ifdef PTL1_PANIC_DEBUG 2267 init_ptl1_thread(); 2268 #endif /* PTL1_PANIC_DEBUG */ 2269 } 2270 2271 #ifdef PTL1_PANIC_DEBUG 2272 int ptl1_panic_test = 0; 2273 int ptl1_panic_xc_one_test = 0; 2274 int ptl1_panic_xc_all_test = 0; 2275 int ptl1_panic_xt_one_test = 0; 2276 int ptl1_panic_xt_all_test = 0; 2277 kthread_id_t ptl1_thread_p = NULL; 2278 kcondvar_t ptl1_cv; 2279 kmutex_t ptl1_mutex; 2280 int ptl1_recurse_count_threshold = 0x40; 2281 int ptl1_recurse_trap_threshold = 0x3d; 2282 extern void ptl1_recurse(int, int); 2283 extern void ptl1_panic_xt(int, int); 2284 2285 /* 2286 * Called once per second by timeout() to wake up 2287 * the ptl1_panic thread to see if it should cause 2288 * a trap to the ptl1_panic() code. 2289 */ 2290 /* ARGSUSED */ 2291 static void 2292 ptl1_wakeup(void *arg) 2293 { 2294 mutex_enter(&ptl1_mutex); 2295 cv_signal(&ptl1_cv); 2296 mutex_exit(&ptl1_mutex); 2297 } 2298 2299 /* 2300 * ptl1_panic cross call function: 2301 * Needed because xc_one() and xc_some() can pass 2302 * 64 bit args but ptl1_recurse() expects ints. 2303 */ 2304 static void 2305 ptl1_panic_xc(void) 2306 { 2307 ptl1_recurse(ptl1_recurse_count_threshold, 2308 ptl1_recurse_trap_threshold); 2309 } 2310 2311 /* 2312 * The ptl1 thread waits for a global flag to be set 2313 * and uses the recurse thresholds to set the stack depth 2314 * to cause a ptl1_panic() directly via a call to ptl1_recurse 2315 * or indirectly via the cross call and cross trap functions. 2316 * 2317 * This is useful testing stack overflows and normal 2318 * ptl1_panic() states with a know stack frame. 2319 * 2320 * ptl1_recurse() is an asm function in ptl1_panic.s that 2321 * sets the {In, Local, Out, and Global} registers to a 2322 * know state on the stack and just prior to causing a 2323 * test ptl1_panic trap. 2324 */ 2325 static void 2326 ptl1_thread(void) 2327 { 2328 mutex_enter(&ptl1_mutex); 2329 while (ptl1_thread_p) { 2330 cpuset_t other_cpus; 2331 int cpu_id; 2332 int my_cpu_id; 2333 int target_cpu_id; 2334 int target_found; 2335 2336 if (ptl1_panic_test) { 2337 ptl1_recurse(ptl1_recurse_count_threshold, 2338 ptl1_recurse_trap_threshold); 2339 } 2340 2341 /* 2342 * Find potential targets for x-call and x-trap, 2343 * if any exist while preempt is disabled we 2344 * start a ptl1_panic if requested via a 2345 * globals. 2346 */ 2347 kpreempt_disable(); 2348 my_cpu_id = CPU->cpu_id; 2349 other_cpus = cpu_ready_set; 2350 CPUSET_DEL(other_cpus, CPU->cpu_id); 2351 target_found = 0; 2352 if (!CPUSET_ISNULL(other_cpus)) { 2353 /* 2354 * Pick the first one 2355 */ 2356 for (cpu_id = 0; cpu_id < NCPU; cpu_id++) { 2357 if (cpu_id == my_cpu_id) 2358 continue; 2359 2360 if (CPU_XCALL_READY(cpu_id)) { 2361 target_cpu_id = cpu_id; 2362 target_found = 1; 2363 break; 2364 } 2365 } 2366 ASSERT(target_found); 2367 2368 if (ptl1_panic_xc_one_test) { 2369 xc_one(target_cpu_id, 2370 (xcfunc_t *)ptl1_panic_xc, 0, 0); 2371 } 2372 if (ptl1_panic_xc_all_test) { 2373 xc_some(other_cpus, 2374 (xcfunc_t *)ptl1_panic_xc, 0, 0); 2375 } 2376 if (ptl1_panic_xt_one_test) { 2377 xt_one(target_cpu_id, 2378 (xcfunc_t *)ptl1_panic_xt, 0, 0); 2379 } 2380 if (ptl1_panic_xt_all_test) { 2381 xt_some(other_cpus, 2382 (xcfunc_t *)ptl1_panic_xt, 0, 0); 2383 } 2384 } 2385 kpreempt_enable(); 2386 (void) timeout(ptl1_wakeup, NULL, hz); 2387 (void) cv_wait(&ptl1_cv, &ptl1_mutex); 2388 } 2389 mutex_exit(&ptl1_mutex); 2390 } 2391 2392 /* 2393 * Called during early startup to create the ptl1_thread 2394 */ 2395 void 2396 init_ptl1_thread(void) 2397 { 2398 ptl1_thread_p = thread_create(NULL, 0, ptl1_thread, NULL, 0, 2399 &p0, TS_RUN, 0); 2400 } 2401 #endif /* PTL1_PANIC_DEBUG */ 2402 2403 2404 static void 2405 memlist_new(uint64_t start, uint64_t len, struct memlist **memlistp) 2406 { 2407 struct memlist *new; 2408 2409 new = *memlistp; 2410 new->ml_address = start; 2411 new->ml_size = len; 2412 *memlistp = new + 1; 2413 } 2414 2415 /* 2416 * Add to a memory list. 2417 * start = start of new memory segment 2418 * len = length of new memory segment in bytes 2419 * memlistp = pointer to array of available memory segment structures 2420 * curmemlistp = memory list to which to add segment. 2421 */ 2422 static void 2423 memlist_add(uint64_t start, uint64_t len, struct memlist **memlistp, 2424 struct memlist **curmemlistp) 2425 { 2426 struct memlist *new = *memlistp; 2427 2428 memlist_new(start, len, memlistp); 2429 memlist_insert(new, curmemlistp); 2430 } 2431 2432 static int 2433 ndata_alloc_memseg(struct memlist *ndata, size_t avail) 2434 { 2435 int nseg; 2436 size_t memseg_sz; 2437 struct memseg *msp; 2438 2439 /* 2440 * The memseg list is for the chunks of physical memory that 2441 * will be managed by the vm system. The number calculated is 2442 * a guess as boot may fragment it more when memory allocations 2443 * are made before kphysm_init(). 2444 */ 2445 memseg_sz = (avail + 10) * sizeof (struct memseg); 2446 memseg_sz = roundup(memseg_sz, PAGESIZE); 2447 nseg = memseg_sz / sizeof (struct memseg); 2448 msp = ndata_alloc(ndata, memseg_sz, ecache_alignsize); 2449 if (msp == NULL) 2450 return (1); 2451 PRM_DEBUG(memseg_free); 2452 2453 while (nseg--) { 2454 msp->next = memseg_free; 2455 memseg_free = msp; 2456 msp++; 2457 } 2458 return (0); 2459 } 2460 2461 /* 2462 * In the case of architectures that support dynamic addition of 2463 * memory at run-time there are two cases where memsegs need to 2464 * be initialized and added to the memseg list. 2465 * 1) memsegs that are constructed at startup. 2466 * 2) memsegs that are constructed at run-time on 2467 * hot-plug capable architectures. 2468 * This code was originally part of the function kphysm_init(). 2469 */ 2470 2471 static void 2472 memseg_list_add(struct memseg *memsegp) 2473 { 2474 struct memseg **prev_memsegp; 2475 pgcnt_t num; 2476 2477 /* insert in memseg list, decreasing number of pages order */ 2478 2479 num = MSEG_NPAGES(memsegp); 2480 2481 for (prev_memsegp = &memsegs; *prev_memsegp; 2482 prev_memsegp = &((*prev_memsegp)->next)) { 2483 if (num > MSEG_NPAGES(*prev_memsegp)) 2484 break; 2485 } 2486 2487 memsegp->next = *prev_memsegp; 2488 *prev_memsegp = memsegp; 2489 2490 if (kpm_enable) { 2491 memsegp->nextpa = (memsegp->next) ? 2492 va_to_pa(memsegp->next) : MSEG_NULLPTR_PA; 2493 2494 if (prev_memsegp != &memsegs) { 2495 struct memseg *msp; 2496 msp = (struct memseg *)((caddr_t)prev_memsegp - 2497 offsetof(struct memseg, next)); 2498 msp->nextpa = va_to_pa(memsegp); 2499 } else { 2500 memsegspa = va_to_pa(memsegs); 2501 } 2502 } 2503 } 2504 2505 /* 2506 * PSM add_physmem_cb(). US-II and newer processors have some 2507 * flavor of the prefetch capability implemented. We exploit 2508 * this capability for optimum performance. 2509 */ 2510 #define PREFETCH_BYTES 64 2511 2512 void 2513 add_physmem_cb(page_t *pp, pfn_t pnum) 2514 { 2515 extern void prefetch_page_w(void *); 2516 2517 pp->p_pagenum = pnum; 2518 2519 /* 2520 * Prefetch one more page_t into E$. To prevent future 2521 * mishaps with the sizeof(page_t) changing on us, we 2522 * catch this on debug kernels if we can't bring in the 2523 * entire hpage with 2 PREFETCH_BYTES reads. See 2524 * also, sun4u/cpu/cpu_module.c 2525 */ 2526 /*LINTED*/ 2527 ASSERT(sizeof (page_t) <= 2*PREFETCH_BYTES); 2528 prefetch_page_w((char *)pp); 2529 } 2530 2531 /* 2532 * Find memseg with given pfn 2533 */ 2534 static struct memseg * 2535 memseg_find(pfn_t base, pfn_t *next) 2536 { 2537 struct memseg *seg; 2538 2539 if (next != NULL) 2540 *next = LONG_MAX; 2541 for (seg = memsegs; seg != NULL; seg = seg->next) { 2542 if (base >= seg->pages_base && base < seg->pages_end) 2543 return (seg); 2544 if (next != NULL && seg->pages_base > base && 2545 seg->pages_base < *next) 2546 *next = seg->pages_base; 2547 } 2548 return (NULL); 2549 } 2550 2551 /* 2552 * Put page allocated by OBP on prom_ppages 2553 */ 2554 static void 2555 kphysm_erase(uint64_t addr, uint64_t len) 2556 { 2557 struct page *pp; 2558 struct memseg *seg; 2559 pfn_t base = btop(addr), next; 2560 pgcnt_t num = btop(len); 2561 2562 while (num != 0) { 2563 pgcnt_t off, left; 2564 2565 seg = memseg_find(base, &next); 2566 if (seg == NULL) { 2567 if (next == LONG_MAX) 2568 break; 2569 left = MIN(next - base, num); 2570 base += left, num -= left; 2571 continue; 2572 } 2573 off = base - seg->pages_base; 2574 pp = seg->pages + off; 2575 left = num - MIN(num, (seg->pages_end - seg->pages_base) - off); 2576 while (num != left) { 2577 /* 2578 * init it, lock it, and hashin on prom_pages vp. 2579 * 2580 * Mark it as NONRELOC to let DR know the page 2581 * is locked long term, otherwise DR hangs when 2582 * trying to remove those pages. 2583 * 2584 * XXX vnode offsets on the prom_ppages vnode 2585 * are page numbers (gack) for >32 bit 2586 * physical memory machines. 2587 */ 2588 PP_SETNORELOC(pp); 2589 add_physmem_cb(pp, base); 2590 if (page_trylock(pp, SE_EXCL) == 0) 2591 cmn_err(CE_PANIC, "prom page locked"); 2592 (void) page_hashin(pp, &promvp, 2593 (offset_t)base, NULL); 2594 (void) page_pp_lock(pp, 0, 1); 2595 pp++, base++, num--; 2596 } 2597 } 2598 } 2599 2600 static page_t *ppnext; 2601 static pgcnt_t ppleft; 2602 2603 static void *kpm_ppnext; 2604 static pgcnt_t kpm_ppleft; 2605 2606 /* 2607 * Create a memseg 2608 */ 2609 static void 2610 kphysm_memseg(uint64_t addr, uint64_t len) 2611 { 2612 pfn_t base = btop(addr); 2613 pgcnt_t num = btop(len); 2614 struct memseg *seg; 2615 2616 seg = memseg_free; 2617 memseg_free = seg->next; 2618 ASSERT(seg != NULL); 2619 2620 seg->pages = ppnext; 2621 seg->epages = ppnext + num; 2622 seg->pages_base = base; 2623 seg->pages_end = base + num; 2624 ppnext += num; 2625 ppleft -= num; 2626 2627 if (kpm_enable) { 2628 pgcnt_t kpnum = ptokpmpr(num); 2629 2630 if (kpnum > kpm_ppleft) 2631 panic("kphysm_memseg: kpm_pp overflow"); 2632 seg->pagespa = va_to_pa(seg->pages); 2633 seg->epagespa = va_to_pa(seg->epages); 2634 seg->kpm_pbase = kpmptop(ptokpmp(base)); 2635 seg->kpm_nkpmpgs = kpnum; 2636 /* 2637 * In the kpm_smallpage case, the kpm array 2638 * is 1-1 wrt the page array 2639 */ 2640 if (kpm_smallpages) { 2641 kpm_spage_t *kpm_pp = kpm_ppnext; 2642 2643 kpm_ppnext = kpm_pp + kpnum; 2644 seg->kpm_spages = kpm_pp; 2645 seg->kpm_pagespa = va_to_pa(seg->kpm_spages); 2646 } else { 2647 kpm_page_t *kpm_pp = kpm_ppnext; 2648 2649 kpm_ppnext = kpm_pp + kpnum; 2650 seg->kpm_pages = kpm_pp; 2651 seg->kpm_pagespa = va_to_pa(seg->kpm_pages); 2652 /* ASSERT no kpm overlaps */ 2653 ASSERT( 2654 memseg_find(base - pmodkpmp(base), NULL) == NULL); 2655 ASSERT(memseg_find( 2656 roundup(base + num, kpmpnpgs) - 1, NULL) == NULL); 2657 } 2658 kpm_ppleft -= kpnum; 2659 } 2660 2661 memseg_list_add(seg); 2662 } 2663 2664 /* 2665 * Add range to free list 2666 */ 2667 void 2668 kphysm_add(uint64_t addr, uint64_t len, int reclaim) 2669 { 2670 struct page *pp; 2671 struct memseg *seg; 2672 pfn_t base = btop(addr); 2673 pgcnt_t num = btop(len); 2674 2675 seg = memseg_find(base, NULL); 2676 ASSERT(seg != NULL); 2677 pp = seg->pages + (base - seg->pages_base); 2678 2679 if (reclaim) { 2680 struct page *rpp = pp; 2681 struct page *lpp = pp + num; 2682 2683 /* 2684 * page should be locked on prom_ppages 2685 * unhash and unlock it 2686 */ 2687 while (rpp < lpp) { 2688 ASSERT(PAGE_EXCL(rpp) && rpp->p_vnode == &promvp); 2689 ASSERT(PP_ISNORELOC(rpp)); 2690 PP_CLRNORELOC(rpp); 2691 page_pp_unlock(rpp, 0, 1); 2692 page_hashout(rpp, NULL); 2693 page_unlock(rpp); 2694 rpp++; 2695 } 2696 } 2697 2698 /* 2699 * add_physmem() initializes the PSM part of the page 2700 * struct by calling the PSM back with add_physmem_cb(). 2701 * In addition it coalesces pages into larger pages as 2702 * it initializes them. 2703 */ 2704 add_physmem(pp, num, base); 2705 } 2706 2707 /* 2708 * kphysm_init() tackles the problem of initializing physical memory. 2709 */ 2710 static void 2711 kphysm_init(void) 2712 { 2713 struct memlist *pmem; 2714 2715 ASSERT(page_hash != NULL && page_hashsz != 0); 2716 2717 ppnext = pp_base; 2718 ppleft = npages; 2719 kpm_ppnext = kpm_pp_base; 2720 kpm_ppleft = kpm_npages; 2721 2722 /* 2723 * installed pages not on nopp_memlist go in memseg list 2724 */ 2725 diff_memlists(phys_install, nopp_list, kphysm_memseg); 2726 2727 /* 2728 * Free the avail list 2729 */ 2730 for (pmem = phys_avail; pmem != NULL; pmem = pmem->ml_next) 2731 kphysm_add(pmem->ml_address, pmem->ml_size, 0); 2732 2733 /* 2734 * Erase pages that aren't available 2735 */ 2736 diff_memlists(phys_install, phys_avail, kphysm_erase); 2737 2738 build_pfn_hash(); 2739 } 2740 2741 /* 2742 * Kernel VM initialization. 2743 * Assumptions about kernel address space ordering: 2744 * (1) gap (user space) 2745 * (2) kernel text 2746 * (3) kernel data/bss 2747 * (4) gap 2748 * (5) kernel data structures 2749 * (6) gap 2750 * (7) debugger (optional) 2751 * (8) monitor 2752 * (9) gap (possibly null) 2753 * (10) dvma 2754 * (11) devices 2755 */ 2756 static void 2757 kvm_init(void) 2758 { 2759 /* 2760 * Put the kernel segments in kernel address space. 2761 */ 2762 rw_enter(&kas.a_lock, RW_WRITER); 2763 as_avlinit(&kas); 2764 2765 (void) seg_attach(&kas, (caddr_t)KERNELBASE, 2766 (size_t)(e_moddata - KERNELBASE), &ktextseg); 2767 (void) segkmem_create(&ktextseg); 2768 2769 (void) seg_attach(&kas, (caddr_t)(KERNELBASE + MMU_PAGESIZE4M), 2770 (size_t)(MMU_PAGESIZE4M), &ktexthole); 2771 (void) segkmem_create(&ktexthole); 2772 2773 (void) seg_attach(&kas, (caddr_t)valloc_base, 2774 (size_t)(econtig32 - valloc_base), &kvalloc); 2775 (void) segkmem_create(&kvalloc); 2776 2777 if (kmem64_base) { 2778 (void) seg_attach(&kas, (caddr_t)kmem64_base, 2779 (size_t)(kmem64_end - kmem64_base), &kmem64); 2780 (void) segkmem_create(&kmem64); 2781 } 2782 2783 /* 2784 * We're about to map out /boot. This is the beginning of the 2785 * system resource management transition. We can no longer 2786 * call into /boot for I/O or memory allocations. 2787 */ 2788 (void) seg_attach(&kas, kernelheap, ekernelheap - kernelheap, &kvseg); 2789 (void) segkmem_create(&kvseg); 2790 hblk_alloc_dynamic = 1; 2791 2792 /* 2793 * we need to preallocate pages for DR operations before enabling large 2794 * page kernel heap because of memseg_remap_init() hat_unload() hack. 2795 */ 2796 memseg_remap_init(); 2797 2798 /* at this point we are ready to use large page heap */ 2799 segkmem_heap_lp_init(); 2800 2801 (void) seg_attach(&kas, (caddr_t)SYSBASE32, SYSLIMIT32 - SYSBASE32, 2802 &kvseg32); 2803 (void) segkmem_create(&kvseg32); 2804 2805 /* 2806 * Create a segment for the debugger. 2807 */ 2808 (void) seg_attach(&kas, kdi_segdebugbase, kdi_segdebugsize, &kdebugseg); 2809 (void) segkmem_create(&kdebugseg); 2810 2811 rw_exit(&kas.a_lock); 2812 } 2813 2814 char obp_tte_str[] = 2815 "h# %x constant MMU_PAGESHIFT " 2816 "h# %x constant TTE8K " 2817 "h# %x constant SFHME_SIZE " 2818 "h# %x constant SFHME_TTE " 2819 "h# %x constant HMEBLK_TAG " 2820 "h# %x constant HMEBLK_NEXT " 2821 "h# %x constant HMEBLK_MISC " 2822 "h# %x constant HMEBLK_HME1 " 2823 "h# %x constant NHMENTS " 2824 "h# %x constant HBLK_SZMASK " 2825 "h# %x constant HBLK_RANGE_SHIFT " 2826 "h# %x constant HMEBP_HBLK " 2827 "h# %x constant HMEBLK_ENDPA " 2828 "h# %x constant HMEBUCKET_SIZE " 2829 "h# %x constant HTAG_SFMMUPSZ " 2830 "h# %x constant HTAG_BSPAGE_SHIFT " 2831 "h# %x constant HTAG_REHASH_SHIFT " 2832 "h# %x constant SFMMU_INVALID_SHMERID " 2833 "h# %x constant mmu_hashcnt " 2834 "h# %p constant uhme_hash " 2835 "h# %p constant khme_hash " 2836 "h# %x constant UHMEHASH_SZ " 2837 "h# %x constant KHMEHASH_SZ " 2838 "h# %p constant KCONTEXT " 2839 "h# %p constant KHATID " 2840 "h# %x constant ASI_MEM " 2841 2842 ": PHYS-X@ ( phys -- data ) " 2843 " ASI_MEM spacex@ " 2844 "; " 2845 2846 ": PHYS-W@ ( phys -- data ) " 2847 " ASI_MEM spacew@ " 2848 "; " 2849 2850 ": PHYS-L@ ( phys -- data ) " 2851 " ASI_MEM spaceL@ " 2852 "; " 2853 2854 ": TTE_PAGE_SHIFT ( ttesz -- hmeshift ) " 2855 " 3 * MMU_PAGESHIFT + " 2856 "; " 2857 2858 ": TTE_IS_VALID ( ttep -- flag ) " 2859 " PHYS-X@ 0< " 2860 "; " 2861 2862 ": HME_HASH_SHIFT ( ttesz -- hmeshift ) " 2863 " dup TTE8K = if " 2864 " drop HBLK_RANGE_SHIFT " 2865 " else " 2866 " TTE_PAGE_SHIFT " 2867 " then " 2868 "; " 2869 2870 ": HME_HASH_BSPAGE ( addr hmeshift -- bspage ) " 2871 " tuck >> swap MMU_PAGESHIFT - << " 2872 "; " 2873 2874 ": HME_HASH_FUNCTION ( sfmmup addr hmeshift -- hmebp ) " 2875 " >> over xor swap ( hash sfmmup ) " 2876 " KHATID <> if ( hash ) " 2877 " UHMEHASH_SZ and ( bucket ) " 2878 " HMEBUCKET_SIZE * uhme_hash + ( hmebp ) " 2879 " else ( hash ) " 2880 " KHMEHASH_SZ and ( bucket ) " 2881 " HMEBUCKET_SIZE * khme_hash + ( hmebp ) " 2882 " then ( hmebp ) " 2883 "; " 2884 2885 ": HME_HASH_TABLE_SEARCH " 2886 " ( sfmmup hmebp hblktag -- sfmmup null | sfmmup hmeblkp ) " 2887 " >r hmebp_hblk + phys-x@ begin ( sfmmup hmeblkp ) ( r: hblktag ) " 2888 " dup HMEBLK_ENDPA <> if ( sfmmup hmeblkp ) ( r: hblktag ) " 2889 " dup hmeblk_tag + phys-x@ r@ = if ( sfmmup hmeblkp ) " 2890 " dup hmeblk_tag + 8 + phys-x@ 2 pick = if " 2891 " true ( sfmmup hmeblkp true ) ( r: hblktag ) " 2892 " else " 2893 " hmeblk_next + phys-x@ false " 2894 " ( sfmmup hmeblkp false ) ( r: hblktag ) " 2895 " then " 2896 " else " 2897 " hmeblk_next + phys-x@ false " 2898 " ( sfmmup hmeblkp false ) ( r: hblktag ) " 2899 " then " 2900 " else " 2901 " drop 0 true " 2902 " then " 2903 " until r> drop " 2904 "; " 2905 2906 ": HME_HASH_TAG ( sfmmup rehash addr -- hblktag ) " 2907 " over HME_HASH_SHIFT HME_HASH_BSPAGE ( sfmmup rehash bspage ) " 2908 " HTAG_BSPAGE_SHIFT << ( sfmmup rehash htag-bspage )" 2909 " swap HTAG_REHASH_SHIFT << or ( sfmmup htag-bspage-rehash )" 2910 " SFMMU_INVALID_SHMERID or nip ( hblktag ) " 2911 "; " 2912 2913 ": HBLK_TO_TTEP ( hmeblkp addr -- ttep ) " 2914 " over HMEBLK_MISC + PHYS-L@ HBLK_SZMASK and ( hmeblkp addr ttesz ) " 2915 " TTE8K = if ( hmeblkp addr ) " 2916 " MMU_PAGESHIFT >> NHMENTS 1- and ( hmeblkp hme-index ) " 2917 " else ( hmeblkp addr ) " 2918 " drop 0 ( hmeblkp 0 ) " 2919 " then ( hmeblkp hme-index ) " 2920 " SFHME_SIZE * + HMEBLK_HME1 + ( hmep ) " 2921 " SFHME_TTE + ( ttep ) " 2922 "; " 2923 2924 ": unix-tte ( addr cnum -- false | tte-data true ) " 2925 " KCONTEXT = if ( addr ) " 2926 " KHATID ( addr khatid ) " 2927 " else ( addr ) " 2928 " drop false exit ( false ) " 2929 " then " 2930 " ( addr khatid ) " 2931 " mmu_hashcnt 1+ 1 do ( addr sfmmup ) " 2932 " 2dup swap i HME_HASH_SHIFT " 2933 "( addr sfmmup sfmmup addr hmeshift ) " 2934 " HME_HASH_FUNCTION ( addr sfmmup hmebp ) " 2935 " over i 4 pick " 2936 "( addr sfmmup hmebp sfmmup rehash addr ) " 2937 " HME_HASH_TAG ( addr sfmmup hmebp hblktag ) " 2938 " HME_HASH_TABLE_SEARCH " 2939 "( addr sfmmup { null | hmeblkp } ) " 2940 " ?dup if ( addr sfmmup hmeblkp ) " 2941 " nip swap HBLK_TO_TTEP ( ttep ) " 2942 " dup TTE_IS_VALID if ( valid-ttep ) " 2943 " PHYS-X@ true ( tte-data true ) " 2944 " else ( invalid-tte ) " 2945 " drop false ( false ) " 2946 " then ( false | tte-data true ) " 2947 " unloop exit ( false | tte-data true ) " 2948 " then ( addr sfmmup ) " 2949 " loop ( addr sfmmup ) " 2950 " 2drop false ( false ) " 2951 "; " 2952 ; 2953 2954 void 2955 create_va_to_tte(void) 2956 { 2957 char *bp; 2958 extern int khmehash_num, uhmehash_num; 2959 extern struct hmehash_bucket *khme_hash, *uhme_hash; 2960 2961 #define OFFSET(type, field) ((uintptr_t)(&((type *)0)->field)) 2962 2963 bp = (char *)kobj_zalloc(MMU_PAGESIZE, KM_SLEEP); 2964 2965 /* 2966 * Teach obp how to parse our sw ttes. 2967 */ 2968 (void) sprintf(bp, obp_tte_str, 2969 MMU_PAGESHIFT, 2970 TTE8K, 2971 sizeof (struct sf_hment), 2972 OFFSET(struct sf_hment, hme_tte), 2973 OFFSET(struct hme_blk, hblk_tag), 2974 OFFSET(struct hme_blk, hblk_nextpa), 2975 OFFSET(struct hme_blk, hblk_misc), 2976 OFFSET(struct hme_blk, hblk_hme), 2977 NHMENTS, 2978 HBLK_SZMASK, 2979 HBLK_RANGE_SHIFT, 2980 OFFSET(struct hmehash_bucket, hmeh_nextpa), 2981 HMEBLK_ENDPA, 2982 sizeof (struct hmehash_bucket), 2983 HTAG_SFMMUPSZ, 2984 HTAG_BSPAGE_SHIFT, 2985 HTAG_REHASH_SHIFT, 2986 SFMMU_INVALID_SHMERID, 2987 mmu_hashcnt, 2988 (caddr_t)va_to_pa((caddr_t)uhme_hash), 2989 (caddr_t)va_to_pa((caddr_t)khme_hash), 2990 UHMEHASH_SZ, 2991 KHMEHASH_SZ, 2992 KCONTEXT, 2993 KHATID, 2994 ASI_MEM); 2995 prom_interpret(bp, 0, 0, 0, 0, 0); 2996 2997 kobj_free(bp, MMU_PAGESIZE); 2998 } 2999 3000 void 3001 install_va_to_tte(void) 3002 { 3003 /* 3004 * advise prom that it can use unix-tte 3005 */ 3006 prom_interpret("' unix-tte is va>tte-data", 0, 0, 0, 0, 0); 3007 } 3008 3009 /* 3010 * Here we add "device-type=console" for /os-io node, for currently 3011 * our kernel console output only supports displaying text and 3012 * performing cursor-positioning operations (through kernel framebuffer 3013 * driver) and it doesn't support other functionalities required for a 3014 * standard "display" device as specified in 1275 spec. The main missing 3015 * interface defined by the 1275 spec is "draw-logo". 3016 * also see the comments above prom_stdout_is_framebuffer(). 3017 */ 3018 static char *create_node = 3019 "\" /\" find-device " 3020 "new-device " 3021 "\" os-io\" device-name " 3022 "\" "OBP_DISPLAY_CONSOLE"\" device-type " 3023 ": cb-r/w ( adr,len method$ -- #read/#written ) " 3024 " 2>r swap 2 2r> ['] $callback catch if " 3025 " 2drop 3drop 0 " 3026 " then " 3027 "; " 3028 ": read ( adr,len -- #read ) " 3029 " \" read\" ['] cb-r/w catch if 2drop 2drop -2 exit then " 3030 " ( retN ... ret1 N ) " 3031 " ?dup if " 3032 " swap >r 1- 0 ?do drop loop r> " 3033 " else " 3034 " -2 " 3035 " then " 3036 "; " 3037 ": write ( adr,len -- #written ) " 3038 " \" write\" ['] cb-r/w catch if 2drop 2drop 0 exit then " 3039 " ( retN ... ret1 N ) " 3040 " ?dup if " 3041 " swap >r 1- 0 ?do drop loop r> " 3042 " else " 3043 " 0 " 3044 " then " 3045 "; " 3046 ": poll-tty ( -- ) ; " 3047 ": install-abort ( -- ) ['] poll-tty d# 10 alarm ; " 3048 ": remove-abort ( -- ) ['] poll-tty 0 alarm ; " 3049 ": cb-give/take ( $method -- ) " 3050 " 0 -rot ['] $callback catch ?dup if " 3051 " >r 2drop 2drop r> throw " 3052 " else " 3053 " 0 ?do drop loop " 3054 " then " 3055 "; " 3056 ": give ( -- ) \" exit-input\" cb-give/take ; " 3057 ": take ( -- ) \" enter-input\" cb-give/take ; " 3058 ": open ( -- ok? ) true ; " 3059 ": close ( -- ) ; " 3060 "finish-device " 3061 "device-end "; 3062 3063 /* 3064 * Create the OBP input/output node (FCode serial driver). 3065 * It is needed for both USB console keyboard and for 3066 * the kernel terminal emulator. It is too early to check for a 3067 * kernel console compatible framebuffer now, so we create this 3068 * so that we're ready if we need to enable kernel terminal emulation. 3069 * 3070 * When the USB software takes over the input device at the time 3071 * consconfig runs, OBP's stdin is redirected to this node. 3072 * Whenever the FORTH user interface is used after this switch, 3073 * the node will call back into the kernel for console input. 3074 * If a serial device such as ttya or a UART with a Type 5 keyboard 3075 * attached is used, OBP takes over the serial device when the system 3076 * goes to the debugger after the system is booted. This sharing 3077 * of the relatively simple serial device is difficult but possible. 3078 * Sharing the USB host controller is impossible due its complexity. 3079 * 3080 * Similarly to USB keyboard input redirection, after consconfig_dacf 3081 * configures a kernel console framebuffer as the standard output 3082 * device, OBP's stdout is switched to to vector through the 3083 * /os-io node into the kernel terminal emulator. 3084 */ 3085 static void 3086 startup_create_io_node(void) 3087 { 3088 prom_interpret(create_node, 0, 0, 0, 0, 0); 3089 } 3090 3091 3092 static void 3093 do_prom_version_check(void) 3094 { 3095 int i; 3096 pnode_t node; 3097 char buf[64]; 3098 static char drev[] = "Down-rev firmware detected%s\n" 3099 "\tPlease upgrade to the following minimum version:\n" 3100 "\t\t%s\n"; 3101 3102 i = prom_version_check(buf, sizeof (buf), &node); 3103 3104 if (i == PROM_VER64_OK) 3105 return; 3106 3107 if (i == PROM_VER64_UPGRADE) { 3108 cmn_err(CE_WARN, drev, "", buf); 3109 3110 #ifdef DEBUG 3111 prom_enter_mon(); /* Type 'go' to continue */ 3112 cmn_err(CE_WARN, "Booting with down-rev firmware\n"); 3113 return; 3114 #else 3115 halt(0); 3116 #endif 3117 } 3118 3119 /* 3120 * The other possibility is that this is a server running 3121 * good firmware, but down-rev firmware was detected on at 3122 * least one other cpu board. We just complain if we see 3123 * that. 3124 */ 3125 cmn_err(CE_WARN, drev, " on one or more CPU boards", buf); 3126 } 3127 3128 3129 /* 3130 * Must be defined in platform dependent code. 3131 */ 3132 extern caddr_t modtext; 3133 extern size_t modtext_sz; 3134 extern caddr_t moddata; 3135 3136 #define HEAPTEXT_ARENA(addr) \ 3137 ((uintptr_t)(addr) < KERNELBASE + 2 * MMU_PAGESIZE4M ? 0 : \ 3138 (((uintptr_t)(addr) - HEAPTEXT_BASE) / \ 3139 (HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) + 1)) 3140 3141 #define HEAPTEXT_OVERSIZED(addr) \ 3142 ((uintptr_t)(addr) >= HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE) 3143 3144 #define HEAPTEXT_IN_NUCLEUSDATA(addr) \ 3145 (((uintptr_t)(addr) >= KERNELBASE + 2 * MMU_PAGESIZE4M) && \ 3146 ((uintptr_t)(addr) < KERNELBASE + 3 * MMU_PAGESIZE4M)) 3147 3148 vmem_t *texthole_source[HEAPTEXT_NARENAS]; 3149 vmem_t *texthole_arena[HEAPTEXT_NARENAS]; 3150 kmutex_t texthole_lock; 3151 3152 char kern_bootargs[OBP_MAXPATHLEN]; 3153 char kern_bootfile[OBP_MAXPATHLEN]; 3154 3155 void 3156 kobj_vmem_init(vmem_t **text_arena, vmem_t **data_arena) 3157 { 3158 uintptr_t addr, limit; 3159 3160 addr = HEAPTEXT_BASE; 3161 limit = addr + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE; 3162 3163 /* 3164 * Before we initialize the text_arena, we want to punch holes in the 3165 * underlying heaptext_arena. This guarantees that for any text 3166 * address we can find a text hole less than HEAPTEXT_MAPPED away. 3167 */ 3168 for (; addr + HEAPTEXT_UNMAPPED <= limit; 3169 addr += HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED) { 3170 (void) vmem_xalloc(heaptext_arena, HEAPTEXT_UNMAPPED, PAGESIZE, 3171 0, 0, (void *)addr, (void *)(addr + HEAPTEXT_UNMAPPED), 3172 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 3173 } 3174 3175 /* 3176 * Allocate one page at the oversize to break up the text region 3177 * from the oversized region. 3178 */ 3179 (void) vmem_xalloc(heaptext_arena, PAGESIZE, PAGESIZE, 0, 0, 3180 (void *)limit, (void *)(limit + PAGESIZE), 3181 VM_NOSLEEP | VM_BESTFIT | VM_PANIC); 3182 3183 *text_arena = vmem_create("module_text", modtext_sz ? modtext : NULL, 3184 modtext_sz, sizeof (uintptr_t), segkmem_alloc, segkmem_free, 3185 heaptext_arena, 0, VM_SLEEP); 3186 *data_arena = vmem_create("module_data", moddata, MODDATA, 1, 3187 segkmem_alloc, segkmem_free, heap32_arena, 0, VM_SLEEP); 3188 } 3189 3190 caddr_t 3191 kobj_text_alloc(vmem_t *arena, size_t size) 3192 { 3193 caddr_t rval, better; 3194 3195 /* 3196 * First, try a sleeping allocation. 3197 */ 3198 rval = vmem_alloc(arena, size, VM_SLEEP | VM_BESTFIT); 3199 3200 if (size >= HEAPTEXT_MAPPED || !HEAPTEXT_OVERSIZED(rval)) 3201 return (rval); 3202 3203 /* 3204 * We didn't get the area that we wanted. We're going to try to do an 3205 * allocation with explicit constraints. 3206 */ 3207 better = vmem_xalloc(arena, size, sizeof (uintptr_t), 0, 0, NULL, 3208 (void *)(HEAPTEXT_BASE + HEAPTEXT_SIZE - HEAPTEXT_OVERSIZE), 3209 VM_NOSLEEP | VM_BESTFIT); 3210 3211 if (better != NULL) { 3212 /* 3213 * That worked. Free our first attempt and return. 3214 */ 3215 vmem_free(arena, rval, size); 3216 return (better); 3217 } 3218 3219 /* 3220 * That didn't work; we'll have to return our first attempt. 3221 */ 3222 return (rval); 3223 } 3224 3225 caddr_t 3226 kobj_texthole_alloc(caddr_t addr, size_t size) 3227 { 3228 int arena = HEAPTEXT_ARENA(addr); 3229 char c[30]; 3230 uintptr_t base; 3231 3232 if (HEAPTEXT_OVERSIZED(addr) || HEAPTEXT_IN_NUCLEUSDATA(addr)) { 3233 /* 3234 * If this is an oversized allocation or it is allocated in 3235 * the nucleus data page, there is no text hole available for 3236 * it; return NULL. 3237 */ 3238 return (NULL); 3239 } 3240 3241 mutex_enter(&texthole_lock); 3242 3243 if (texthole_arena[arena] == NULL) { 3244 ASSERT(texthole_source[arena] == NULL); 3245 3246 if (arena == 0) { 3247 texthole_source[0] = vmem_create("module_text_holesrc", 3248 (void *)(KERNELBASE + MMU_PAGESIZE4M), 3249 MMU_PAGESIZE4M, PAGESIZE, NULL, NULL, NULL, 3250 0, VM_SLEEP); 3251 } else { 3252 base = HEAPTEXT_BASE + 3253 (arena - 1) * (HEAPTEXT_MAPPED + HEAPTEXT_UNMAPPED); 3254 3255 (void) snprintf(c, sizeof (c), 3256 "heaptext_holesrc_%d", arena); 3257 3258 texthole_source[arena] = vmem_create(c, (void *)base, 3259 HEAPTEXT_UNMAPPED, PAGESIZE, NULL, NULL, NULL, 3260 0, VM_SLEEP); 3261 } 3262 3263 (void) snprintf(c, sizeof (c), "heaptext_hole_%d", arena); 3264 3265 texthole_arena[arena] = vmem_create(c, NULL, 0, 3266 sizeof (uint32_t), segkmem_alloc_permanent, segkmem_free, 3267 texthole_source[arena], 0, VM_SLEEP); 3268 } 3269 3270 mutex_exit(&texthole_lock); 3271 3272 ASSERT(texthole_arena[arena] != NULL); 3273 ASSERT(arena >= 0 && arena < HEAPTEXT_NARENAS); 3274 return (vmem_alloc(texthole_arena[arena], size, 3275 VM_BESTFIT | VM_NOSLEEP)); 3276 } 3277 3278 void 3279 kobj_texthole_free(caddr_t addr, size_t size) 3280 { 3281 int arena = HEAPTEXT_ARENA(addr); 3282 3283 ASSERT(arena >= 0 && arena < HEAPTEXT_NARENAS); 3284 ASSERT(texthole_arena[arena] != NULL); 3285 vmem_free(texthole_arena[arena], addr, size); 3286 } 3287 3288 void 3289 release_bootstrap(void) 3290 { 3291 if (&cif_init) 3292 cif_init(); 3293 } 3294