1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved. 23 */ 24 /* 25 * Copyright (c) 2010, Intel Corporation. 26 * All rights reserved. 27 * Copyright 2018 Joyent, Inc. 28 */ 29 30 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */ 31 /* All Rights Reserved */ 32 33 /* 34 * Portions of this source code were derived from Berkeley 4.3 BSD 35 * under license from the Regents of the University of California. 36 */ 37 38 /* 39 * UNIX machine dependent virtual memory support. 40 */ 41 42 #include <sys/types.h> 43 #include <sys/param.h> 44 #include <sys/systm.h> 45 #include <sys/user.h> 46 #include <sys/proc.h> 47 #include <sys/kmem.h> 48 #include <sys/vmem.h> 49 #include <sys/buf.h> 50 #include <sys/cpuvar.h> 51 #include <sys/lgrp.h> 52 #include <sys/disp.h> 53 #include <sys/vm.h> 54 #include <sys/mman.h> 55 #include <sys/vnode.h> 56 #include <sys/cred.h> 57 #include <sys/exec.h> 58 #include <sys/exechdr.h> 59 #include <sys/debug.h> 60 #include <sys/vmsystm.h> 61 #include <sys/swap.h> 62 #include <sys/dumphdr.h> 63 #include <sys/random.h> 64 65 #include <vm/hat.h> 66 #include <vm/as.h> 67 #include <vm/seg.h> 68 #include <vm/seg_kp.h> 69 #include <vm/seg_vn.h> 70 #include <vm/page.h> 71 #include <vm/seg_kmem.h> 72 #include <vm/seg_kpm.h> 73 #include <vm/vm_dep.h> 74 75 #include <sys/cpu.h> 76 #include <sys/vm_machparam.h> 77 #include <sys/memlist.h> 78 #include <sys/bootconf.h> /* XXX the memlist stuff belongs in memlist_plat.h */ 79 #include <vm/hat_i86.h> 80 #include <sys/x86_archext.h> 81 #include <sys/elf_386.h> 82 #include <sys/cmn_err.h> 83 #include <sys/archsystm.h> 84 #include <sys/machsystm.h> 85 #include <sys/secflags.h> 86 87 #include <sys/vtrace.h> 88 #include <sys/ddidmareq.h> 89 #include <sys/promif.h> 90 #include <sys/memnode.h> 91 #include <sys/stack.h> 92 #include <util/qsort.h> 93 #include <sys/taskq.h> 94 95 #ifdef __xpv 96 97 #include <sys/hypervisor.h> 98 #include <sys/xen_mmu.h> 99 #include <sys/balloon_impl.h> 100 101 /* 102 * domain 0 pages usable for DMA are kept pre-allocated and kept in 103 * distinct lists, ordered by increasing mfn. 104 */ 105 static kmutex_t io_pool_lock; 106 static kmutex_t contig_list_lock; 107 static page_t *io_pool_4g; /* pool for 32 bit dma limited devices */ 108 static page_t *io_pool_16m; /* pool for 24 bit dma limited legacy devices */ 109 static long io_pool_cnt; 110 static long io_pool_cnt_max = 0; 111 #define DEFAULT_IO_POOL_MIN 128 112 static long io_pool_cnt_min = DEFAULT_IO_POOL_MIN; 113 static long io_pool_cnt_lowater = 0; 114 static long io_pool_shrink_attempts; /* how many times did we try to shrink */ 115 static long io_pool_shrinks; /* how many times did we really shrink */ 116 static long io_pool_grows; /* how many times did we grow */ 117 static mfn_t start_mfn = 1; 118 static caddr_t io_pool_kva; /* use to alloc pages when needed */ 119 120 static int create_contig_pfnlist(uint_t); 121 122 /* 123 * percentage of phys mem to hold in the i/o pool 124 */ 125 #define DEFAULT_IO_POOL_PCT 2 126 static long io_pool_physmem_pct = DEFAULT_IO_POOL_PCT; 127 static void page_io_pool_sub(page_t **, page_t *, page_t *); 128 int ioalloc_dbg = 0; 129 130 #endif /* __xpv */ 131 132 uint_t vac_colors = 1; 133 134 int largepagesupport = 0; 135 extern uint_t page_create_new; 136 extern uint_t page_create_exists; 137 extern uint_t page_create_putbacks; 138 /* 139 * Allow users to disable the kernel's use of SSE. 140 */ 141 extern int use_sse_pagecopy, use_sse_pagezero; 142 143 /* 144 * combined memory ranges from mnode and memranges[] to manage single 145 * mnode/mtype dimension in the page lists. 146 */ 147 typedef struct { 148 pfn_t mnr_pfnlo; 149 pfn_t mnr_pfnhi; 150 int mnr_mnode; 151 int mnr_memrange; /* index into memranges[] */ 152 int mnr_next; /* next lower PA mnoderange */ 153 int mnr_exists; 154 /* maintain page list stats */ 155 pgcnt_t mnr_mt_clpgcnt; /* cache list cnt */ 156 pgcnt_t mnr_mt_flpgcnt[MMU_PAGE_SIZES]; /* free list cnt per szc */ 157 pgcnt_t mnr_mt_totcnt; /* sum of cache and free lists */ 158 #ifdef DEBUG 159 struct mnr_mts { /* mnode/mtype szc stats */ 160 pgcnt_t mnr_mts_pgcnt; 161 int mnr_mts_colors; 162 pgcnt_t *mnr_mtsc_pgcnt; 163 } *mnr_mts; 164 #endif 165 } mnoderange_t; 166 167 #define MEMRANGEHI(mtype) \ 168 ((mtype > 0) ? memranges[mtype - 1] - 1: physmax) 169 #define MEMRANGELO(mtype) (memranges[mtype]) 170 171 #define MTYPE_FREEMEM(mt) (mnoderanges[mt].mnr_mt_totcnt) 172 173 /* 174 * As the PC architecture evolved memory up was clumped into several 175 * ranges for various historical I/O devices to do DMA. 176 * < 16Meg - ISA bus 177 * < 2Gig - ??? 178 * < 4Gig - PCI bus or drivers that don't understand PAE mode 179 * 180 * These are listed in reverse order, so that we can skip over unused 181 * ranges on machines with small memories. 182 * 183 * For now under the Hypervisor, we'll only ever have one memrange. 184 */ 185 #define PFN_4GIG 0x100000 186 #define PFN_16MEG 0x1000 187 /* Indices into the memory range (arch_memranges) array. */ 188 #define MRI_4G 0 189 #define MRI_2G 1 190 #define MRI_16M 2 191 #define MRI_0 3 192 static pfn_t arch_memranges[NUM_MEM_RANGES] = { 193 PFN_4GIG, /* pfn range for 4G and above */ 194 0x80000, /* pfn range for 2G-4G */ 195 PFN_16MEG, /* pfn range for 16M-2G */ 196 0x00000, /* pfn range for 0-16M */ 197 }; 198 pfn_t *memranges = &arch_memranges[0]; 199 int nranges = NUM_MEM_RANGES; 200 201 /* 202 * This combines mem_node_config and memranges into one data 203 * structure to be used for page list management. 204 */ 205 mnoderange_t *mnoderanges; 206 int mnoderangecnt; 207 int mtype4g; 208 int mtype16m; 209 int mtypetop; /* index of highest pfn'ed mnoderange */ 210 211 /* 212 * 4g memory management variables for systems with more than 4g of memory: 213 * 214 * physical memory below 4g is required for 32bit dma devices and, currently, 215 * for kmem memory. On systems with more than 4g of memory, the pool of memory 216 * below 4g can be depleted without any paging activity given that there is 217 * likely to be sufficient memory above 4g. 218 * 219 * physmax4g is set true if the largest pfn is over 4g. The rest of the 220 * 4g memory management code is enabled only when physmax4g is true. 221 * 222 * maxmem4g is the count of the maximum number of pages on the page lists 223 * with physical addresses below 4g. It can be a lot less then 4g given that 224 * BIOS may reserve large chunks of space below 4g for hot plug pci devices, 225 * agp aperture etc. 226 * 227 * freemem4g maintains the count of the number of available pages on the 228 * page lists with physical addresses below 4g. 229 * 230 * DESFREE4G specifies the desired amount of below 4g memory. It defaults to 231 * 6% (desfree4gshift = 4) of maxmem4g. 232 * 233 * RESTRICT4G_ALLOC returns true if freemem4g falls below DESFREE4G 234 * and the amount of physical memory above 4g is greater than freemem4g. 235 * In this case, page_get_* routines will restrict below 4g allocations 236 * for requests that don't specifically require it. 237 */ 238 239 #define DESFREE4G (maxmem4g >> desfree4gshift) 240 241 #define RESTRICT4G_ALLOC \ 242 (physmax4g && (freemem4g < DESFREE4G) && ((freemem4g << 1) < freemem)) 243 244 static pgcnt_t maxmem4g; 245 static pgcnt_t freemem4g; 246 static int physmax4g; 247 static int desfree4gshift = 4; /* maxmem4g shift to derive DESFREE4G */ 248 249 /* 250 * 16m memory management: 251 * 252 * reserve some amount of physical memory below 16m for legacy devices. 253 * 254 * RESTRICT16M_ALLOC returns true if an there are sufficient free pages above 255 * 16m or if the 16m pool drops below DESFREE16M. 256 * 257 * In this case, general page allocations via page_get_{free,cache}list 258 * routines will be restricted from allocating from the 16m pool. Allocations 259 * that require specific pfn ranges (page_get_anylist) and PG_PANIC allocations 260 * are not restricted. 261 */ 262 263 #define FREEMEM16M MTYPE_FREEMEM(mtype16m) 264 #define DESFREE16M desfree16m 265 #define RESTRICT16M_ALLOC(freemem, pgcnt, flags) \ 266 ((freemem != 0) && ((flags & PG_PANIC) == 0) && \ 267 ((freemem >= (FREEMEM16M)) || \ 268 (FREEMEM16M < (DESFREE16M + pgcnt)))) 269 270 static pgcnt_t desfree16m = 0x380; 271 272 /* 273 * This can be patched via /etc/system to allow old non-PAE aware device 274 * drivers to use kmem_alloc'd memory on 32 bit systems with > 4Gig RAM. 275 */ 276 int restricted_kmemalloc = 0; 277 278 #ifdef VM_STATS 279 struct { 280 ulong_t pga_alloc; 281 ulong_t pga_notfullrange; 282 ulong_t pga_nulldmaattr; 283 ulong_t pga_allocok; 284 ulong_t pga_allocfailed; 285 ulong_t pgma_alloc; 286 ulong_t pgma_allocok; 287 ulong_t pgma_allocfailed; 288 ulong_t pgma_allocempty; 289 } pga_vmstats; 290 #endif 291 292 uint_t mmu_page_sizes; 293 294 /* How many page sizes the users can see */ 295 uint_t mmu_exported_page_sizes; 296 297 /* page sizes that legacy applications can see */ 298 uint_t mmu_legacy_page_sizes; 299 300 /* 301 * Number of pages in 1 GB. Don't enable automatic large pages if we have 302 * fewer than this many pages. 303 */ 304 pgcnt_t shm_lpg_min_physmem = 1 << (30 - MMU_PAGESHIFT); 305 pgcnt_t privm_lpg_min_physmem = 1 << (30 - MMU_PAGESHIFT); 306 307 /* 308 * Maximum and default segment size tunables for user private 309 * and shared anon memory, and user text and initialized data. 310 * These can be patched via /etc/system to allow large pages 311 * to be used for mapping application private and shared anon memory. 312 */ 313 size_t mcntl0_lpsize = MMU_PAGESIZE; 314 size_t max_uheap_lpsize = MMU_PAGESIZE; 315 size_t default_uheap_lpsize = MMU_PAGESIZE; 316 size_t max_ustack_lpsize = MMU_PAGESIZE; 317 size_t default_ustack_lpsize = MMU_PAGESIZE; 318 size_t max_privmap_lpsize = MMU_PAGESIZE; 319 size_t max_uidata_lpsize = MMU_PAGESIZE; 320 size_t max_utext_lpsize = MMU_PAGESIZE; 321 size_t max_shm_lpsize = MMU_PAGESIZE; 322 323 324 /* 325 * initialized by page_coloring_init(). 326 */ 327 uint_t page_colors; 328 uint_t page_colors_mask; 329 uint_t page_coloring_shift; 330 int cpu_page_colors; 331 static uint_t l2_colors; 332 333 /* 334 * Page freelists and cachelists are dynamically allocated once mnoderangecnt 335 * and page_colors are calculated from the l2 cache n-way set size. Within a 336 * mnode range, the page freelist and cachelist are hashed into bins based on 337 * color. This makes it easier to search for a page within a specific memory 338 * range. 339 */ 340 #define PAGE_COLORS_MIN 16 341 342 page_t ****page_freelists; 343 page_t ***page_cachelists; 344 345 346 /* 347 * Used by page layer to know about page sizes 348 */ 349 hw_pagesize_t hw_page_array[MAX_NUM_LEVEL + 1]; 350 351 kmutex_t *fpc_mutex[NPC_MUTEX]; 352 kmutex_t *cpc_mutex[NPC_MUTEX]; 353 354 /* Lock to protect mnoderanges array for memory DR operations. */ 355 static kmutex_t mnoderange_lock; 356 357 /* 358 * Only let one thread at a time try to coalesce large pages, to 359 * prevent them from working against each other. 360 */ 361 static kmutex_t contig_lock; 362 #define CONTIG_LOCK() mutex_enter(&contig_lock); 363 #define CONTIG_UNLOCK() mutex_exit(&contig_lock); 364 365 #define PFN_16M (mmu_btop((uint64_t)0x1000000)) 366 367 caddr_t 368 i86devmap(pfn_t pf, pgcnt_t pgcnt, uint_t prot) 369 { 370 caddr_t addr; 371 caddr_t addr1; 372 page_t *pp; 373 374 addr1 = addr = vmem_alloc(heap_arena, mmu_ptob(pgcnt), VM_SLEEP); 375 376 for (; pgcnt != 0; addr += MMU_PAGESIZE, ++pf, --pgcnt) { 377 pp = page_numtopp_nolock(pf); 378 if (pp == NULL) { 379 hat_devload(kas.a_hat, addr, MMU_PAGESIZE, pf, 380 prot | HAT_NOSYNC, HAT_LOAD_LOCK); 381 } else { 382 hat_memload(kas.a_hat, addr, pp, 383 prot | HAT_NOSYNC, HAT_LOAD_LOCK); 384 } 385 } 386 387 return (addr1); 388 } 389 390 /* 391 * This routine is like page_numtopp, but accepts only free pages, which 392 * it allocates (unfrees) and returns with the exclusive lock held. 393 * It is used by machdep.c/dma_init() to find contiguous free pages. 394 */ 395 page_t * 396 page_numtopp_alloc(pfn_t pfnum) 397 { 398 page_t *pp; 399 400 retry: 401 pp = page_numtopp_nolock(pfnum); 402 if (pp == NULL) { 403 return (NULL); 404 } 405 406 if (!page_trylock(pp, SE_EXCL)) { 407 return (NULL); 408 } 409 410 if (page_pptonum(pp) != pfnum) { 411 page_unlock(pp); 412 goto retry; 413 } 414 415 if (!PP_ISFREE(pp)) { 416 page_unlock(pp); 417 return (NULL); 418 } 419 if (pp->p_szc) { 420 page_demote_free_pages(pp); 421 page_unlock(pp); 422 goto retry; 423 } 424 425 /* If associated with a vnode, destroy mappings */ 426 427 if (pp->p_vnode) { 428 429 page_destroy_free(pp); 430 431 if (!page_lock(pp, SE_EXCL, (kmutex_t *)NULL, P_NO_RECLAIM)) { 432 return (NULL); 433 } 434 435 if (page_pptonum(pp) != pfnum) { 436 page_unlock(pp); 437 goto retry; 438 } 439 } 440 441 if (!PP_ISFREE(pp)) { 442 page_unlock(pp); 443 return (NULL); 444 } 445 446 if (!page_reclaim(pp, (kmutex_t *)NULL)) 447 return (NULL); 448 449 return (pp); 450 } 451 452 /* 453 * Return the optimum page size for a given mapping 454 */ 455 /*ARGSUSED*/ 456 size_t 457 map_pgsz(int maptype, struct proc *p, caddr_t addr, size_t len, int memcntl) 458 { 459 level_t l = 0; 460 size_t pgsz = MMU_PAGESIZE; 461 size_t max_lpsize; 462 uint_t mszc; 463 464 ASSERT(maptype != MAPPGSZ_VA); 465 466 if (maptype != MAPPGSZ_ISM && physmem < privm_lpg_min_physmem) { 467 return (MMU_PAGESIZE); 468 } 469 470 switch (maptype) { 471 case MAPPGSZ_HEAP: 472 case MAPPGSZ_STK: 473 max_lpsize = memcntl ? mcntl0_lpsize : (maptype == 474 MAPPGSZ_HEAP ? max_uheap_lpsize : max_ustack_lpsize); 475 if (max_lpsize == MMU_PAGESIZE) { 476 return (MMU_PAGESIZE); 477 } 478 if (len == 0) { 479 len = (maptype == MAPPGSZ_HEAP) ? p->p_brkbase + 480 p->p_brksize - p->p_bssbase : p->p_stksize; 481 } 482 len = (maptype == MAPPGSZ_HEAP) ? MAX(len, 483 default_uheap_lpsize) : MAX(len, default_ustack_lpsize); 484 485 /* 486 * use the pages size that best fits len 487 */ 488 for (l = mmu.umax_page_level; l > 0; --l) { 489 if (LEVEL_SIZE(l) > max_lpsize || len < LEVEL_SIZE(l)) { 490 continue; 491 } else { 492 pgsz = LEVEL_SIZE(l); 493 } 494 break; 495 } 496 497 mszc = (maptype == MAPPGSZ_HEAP ? p->p_brkpageszc : 498 p->p_stkpageszc); 499 if (addr == 0 && (pgsz < hw_page_array[mszc].hp_size)) { 500 pgsz = hw_page_array[mszc].hp_size; 501 } 502 return (pgsz); 503 504 case MAPPGSZ_ISM: 505 for (l = mmu.umax_page_level; l > 0; --l) { 506 if (len >= LEVEL_SIZE(l)) 507 return (LEVEL_SIZE(l)); 508 } 509 return (LEVEL_SIZE(0)); 510 } 511 return (pgsz); 512 } 513 514 static uint_t 515 map_szcvec(caddr_t addr, size_t size, uintptr_t off, size_t max_lpsize, 516 size_t min_physmem) 517 { 518 caddr_t eaddr = addr + size; 519 uint_t szcvec = 0; 520 caddr_t raddr; 521 caddr_t readdr; 522 size_t pgsz; 523 int i; 524 525 if (physmem < min_physmem || max_lpsize <= MMU_PAGESIZE) { 526 return (0); 527 } 528 529 for (i = mmu_exported_page_sizes - 1; i > 0; i--) { 530 pgsz = page_get_pagesize(i); 531 if (pgsz > max_lpsize) { 532 continue; 533 } 534 raddr = (caddr_t)P2ROUNDUP((uintptr_t)addr, pgsz); 535 readdr = (caddr_t)P2ALIGN((uintptr_t)eaddr, pgsz); 536 if (raddr < addr || raddr >= readdr) { 537 continue; 538 } 539 if (P2PHASE((uintptr_t)addr ^ off, pgsz)) { 540 continue; 541 } 542 /* 543 * Set szcvec to the remaining page sizes. 544 */ 545 szcvec = ((1 << (i + 1)) - 1) & ~1; 546 break; 547 } 548 return (szcvec); 549 } 550 551 /* 552 * Return a bit vector of large page size codes that 553 * can be used to map [addr, addr + len) region. 554 */ 555 /*ARGSUSED*/ 556 uint_t 557 map_pgszcvec(caddr_t addr, size_t size, uintptr_t off, int flags, int type, 558 int memcntl) 559 { 560 size_t max_lpsize = mcntl0_lpsize; 561 562 if (mmu.max_page_level == 0) 563 return (0); 564 565 if (flags & MAP_TEXT) { 566 if (!memcntl) 567 max_lpsize = max_utext_lpsize; 568 return (map_szcvec(addr, size, off, max_lpsize, 569 shm_lpg_min_physmem)); 570 571 } else if (flags & MAP_INITDATA) { 572 if (!memcntl) 573 max_lpsize = max_uidata_lpsize; 574 return (map_szcvec(addr, size, off, max_lpsize, 575 privm_lpg_min_physmem)); 576 577 } else if (type == MAPPGSZC_SHM) { 578 if (!memcntl) 579 max_lpsize = max_shm_lpsize; 580 return (map_szcvec(addr, size, off, max_lpsize, 581 shm_lpg_min_physmem)); 582 583 } else if (type == MAPPGSZC_HEAP) { 584 if (!memcntl) 585 max_lpsize = max_uheap_lpsize; 586 return (map_szcvec(addr, size, off, max_lpsize, 587 privm_lpg_min_physmem)); 588 589 } else if (type == MAPPGSZC_STACK) { 590 if (!memcntl) 591 max_lpsize = max_ustack_lpsize; 592 return (map_szcvec(addr, size, off, max_lpsize, 593 privm_lpg_min_physmem)); 594 595 } else { 596 if (!memcntl) 597 max_lpsize = max_privmap_lpsize; 598 return (map_szcvec(addr, size, off, max_lpsize, 599 privm_lpg_min_physmem)); 600 } 601 } 602 603 /* 604 * Handle a pagefault. 605 */ 606 faultcode_t 607 pagefault( 608 caddr_t addr, 609 enum fault_type type, 610 enum seg_rw rw, 611 int iskernel) 612 { 613 struct as *as; 614 struct hat *hat; 615 struct proc *p; 616 kthread_t *t; 617 faultcode_t res; 618 caddr_t base; 619 size_t len; 620 int err; 621 int mapped_red; 622 uintptr_t ea; 623 624 ASSERT_STACK_ALIGNED(); 625 626 if (INVALID_VADDR(addr)) 627 return (FC_NOMAP); 628 629 mapped_red = segkp_map_red(); 630 631 if (iskernel) { 632 as = &kas; 633 hat = as->a_hat; 634 } else { 635 t = curthread; 636 p = ttoproc(t); 637 as = p->p_as; 638 hat = as->a_hat; 639 } 640 641 /* 642 * Dispatch pagefault. 643 */ 644 res = as_fault(hat, as, addr, 1, type, rw); 645 646 /* 647 * If this isn't a potential unmapped hole in the user's 648 * UNIX data or stack segments, just return status info. 649 */ 650 if (res != FC_NOMAP || iskernel) 651 goto out; 652 653 /* 654 * Check to see if we happened to faulted on a currently unmapped 655 * part of the UNIX data or stack segments. If so, create a zfod 656 * mapping there and then try calling the fault routine again. 657 */ 658 base = p->p_brkbase; 659 len = p->p_brksize; 660 661 if (addr < base || addr >= base + len) { /* data seg? */ 662 base = (caddr_t)p->p_usrstack - p->p_stksize; 663 len = p->p_stksize; 664 if (addr < base || addr >= p->p_usrstack) { /* stack seg? */ 665 /* not in either UNIX data or stack segments */ 666 res = FC_NOMAP; 667 goto out; 668 } 669 } 670 671 /* 672 * the rest of this function implements a 3.X 4.X 5.X compatibility 673 * This code is probably not needed anymore 674 */ 675 if (p->p_model == DATAMODEL_ILP32) { 676 677 /* expand the gap to the page boundaries on each side */ 678 ea = P2ROUNDUP((uintptr_t)base + len, MMU_PAGESIZE); 679 base = (caddr_t)P2ALIGN((uintptr_t)base, MMU_PAGESIZE); 680 len = ea - (uintptr_t)base; 681 682 as_rangelock(as); 683 if (as_gap(as, MMU_PAGESIZE, &base, &len, AH_CONTAIN, addr) == 684 0) { 685 err = as_map(as, base, len, segvn_create, zfod_argsp); 686 as_rangeunlock(as); 687 if (err) { 688 res = FC_MAKE_ERR(err); 689 goto out; 690 } 691 } else { 692 /* 693 * This page is already mapped by another thread after 694 * we returned from as_fault() above. We just fall 695 * through as_fault() below. 696 */ 697 as_rangeunlock(as); 698 } 699 700 res = as_fault(hat, as, addr, 1, F_INVAL, rw); 701 } 702 703 out: 704 if (mapped_red) 705 segkp_unmap_red(); 706 707 return (res); 708 } 709 710 void 711 map_addr(caddr_t *addrp, size_t len, offset_t off, int vacalign, uint_t flags) 712 { 713 struct proc *p = curproc; 714 caddr_t userlimit = (flags & _MAP_LOW32) ? 715 (caddr_t)_userlimit32 : p->p_as->a_userlimit; 716 717 map_addr_proc(addrp, len, off, vacalign, userlimit, curproc, flags); 718 } 719 720 /*ARGSUSED*/ 721 int 722 map_addr_vacalign_check(caddr_t addr, u_offset_t off) 723 { 724 return (0); 725 } 726 727 /* 728 * The maximum amount a randomized mapping will be slewed. We should perhaps 729 * arrange things so these tunables can be separate for mmap, mmapobj, and 730 * ld.so 731 */ 732 size_t aslr_max_map_skew = 256 * 1024 * 1024; /* 256MB */ 733 734 /* 735 * map_addr_proc() is the routine called when the system is to 736 * choose an address for the user. We will pick an address 737 * range which is the highest available below userlimit. 738 * 739 * Every mapping will have a redzone of a single page on either side of 740 * the request. This is done to leave one page unmapped between segments. 741 * This is not required, but it's useful for the user because if their 742 * program strays across a segment boundary, it will catch a fault 743 * immediately making debugging a little easier. Currently the redzone 744 * is mandatory. 745 * 746 * addrp is a value/result parameter. 747 * On input it is a hint from the user to be used in a completely 748 * machine dependent fashion. We decide to completely ignore this hint. 749 * If MAP_ALIGN was specified, addrp contains the minimal alignment, which 750 * must be some "power of two" multiple of pagesize. 751 * 752 * On output it is NULL if no address can be found in the current 753 * processes address space or else an address that is currently 754 * not mapped for len bytes with a page of red zone on either side. 755 * 756 * vacalign is not needed on x86 (it's for viturally addressed caches) 757 */ 758 /*ARGSUSED*/ 759 void 760 map_addr_proc( 761 caddr_t *addrp, 762 size_t len, 763 offset_t off, 764 int vacalign, 765 caddr_t userlimit, 766 struct proc *p, 767 uint_t flags) 768 { 769 struct as *as = p->p_as; 770 caddr_t addr; 771 caddr_t base; 772 size_t slen; 773 size_t align_amount; 774 775 ASSERT32(userlimit == as->a_userlimit); 776 777 base = p->p_brkbase; 778 #if defined(__amd64) 779 /* 780 * XX64 Yes, this needs more work. 781 */ 782 if (p->p_model == DATAMODEL_NATIVE) { 783 if (userlimit < as->a_userlimit) { 784 /* 785 * This happens when a program wants to map 786 * something in a range that's accessible to a 787 * program in a smaller address space. For example, 788 * a 64-bit program calling mmap32(2) to guarantee 789 * that the returned address is below 4Gbytes. 790 */ 791 ASSERT((uintptr_t)userlimit < ADDRESS_C(0xffffffff)); 792 793 if (userlimit > base) 794 slen = userlimit - base; 795 else { 796 *addrp = NULL; 797 return; 798 } 799 } else { 800 /* 801 * XX64 This layout is probably wrong .. but in 802 * the event we make the amd64 address space look 803 * like sparcv9 i.e. with the stack -above- the 804 * heap, this bit of code might even be correct. 805 */ 806 slen = p->p_usrstack - base - 807 ((p->p_stk_ctl + PAGEOFFSET) & PAGEMASK); 808 } 809 } else 810 #endif 811 slen = userlimit - base; 812 813 /* Make len be a multiple of PAGESIZE */ 814 len = (len + PAGEOFFSET) & PAGEMASK; 815 816 /* 817 * figure out what the alignment should be 818 * 819 * XX64 -- is there an ELF_AMD64_MAXPGSZ or is it the same???? 820 */ 821 if (len <= ELF_386_MAXPGSZ) { 822 /* 823 * Align virtual addresses to ensure that ELF shared libraries 824 * are mapped with the appropriate alignment constraints by 825 * the run-time linker. 826 */ 827 align_amount = ELF_386_MAXPGSZ; 828 } else { 829 /* 830 * For 32-bit processes, only those which have specified 831 * MAP_ALIGN and an addr will be aligned on a larger page size. 832 * Not doing so can potentially waste up to 1G of process 833 * address space. 834 */ 835 int lvl = (p->p_model == DATAMODEL_ILP32) ? 1 : 836 mmu.umax_page_level; 837 838 while (lvl && len < LEVEL_SIZE(lvl)) 839 --lvl; 840 841 align_amount = LEVEL_SIZE(lvl); 842 } 843 if ((flags & MAP_ALIGN) && ((uintptr_t)*addrp > align_amount)) 844 align_amount = (uintptr_t)*addrp; 845 846 ASSERT(ISP2(align_amount)); 847 ASSERT(align_amount == 0 || align_amount >= PAGESIZE); 848 849 off = off & (align_amount - 1); 850 851 /* 852 * Look for a large enough hole starting below userlimit. 853 * After finding it, use the upper part. 854 */ 855 if (as_gap_aligned(as, len, &base, &slen, AH_HI, NULL, align_amount, 856 PAGESIZE, off) == 0) { 857 caddr_t as_addr; 858 859 /* 860 * addr is the highest possible address to use since we have 861 * a PAGESIZE redzone at the beginning and end. 862 */ 863 addr = base + slen - (PAGESIZE + len); 864 as_addr = addr; 865 /* 866 * Round address DOWN to the alignment amount and 867 * add the offset in. 868 * If addr is greater than as_addr, len would not be large 869 * enough to include the redzone, so we must adjust down 870 * by the alignment amount. 871 */ 872 addr = (caddr_t)((uintptr_t)addr & (~(align_amount - 1))); 873 addr += (uintptr_t)off; 874 if (addr > as_addr) { 875 addr -= align_amount; 876 } 877 878 /* 879 * If randomization is requested, slew the allocation 880 * backwards, within the same gap, by a random amount. 881 */ 882 if (flags & _MAP_RANDOMIZE) { 883 uint32_t slew; 884 885 (void) random_get_pseudo_bytes((uint8_t *)&slew, 886 sizeof (slew)); 887 888 slew = slew % MIN(aslr_max_map_skew, (addr - base)); 889 addr -= P2ALIGN(slew, align_amount); 890 } 891 892 ASSERT(addr > base); 893 ASSERT(addr + len < base + slen); 894 ASSERT(((uintptr_t)addr & (align_amount - 1)) == 895 ((uintptr_t)(off))); 896 *addrp = addr; 897 } else { 898 *addrp = NULL; /* no more virtual space */ 899 } 900 } 901 902 int valid_va_range_aligned_wraparound; 903 904 /* 905 * Determine whether [*basep, *basep + *lenp) contains a mappable range of 906 * addresses at least "minlen" long, where the base of the range is at "off" 907 * phase from an "align" boundary and there is space for a "redzone"-sized 908 * redzone on either side of the range. On success, 1 is returned and *basep 909 * and *lenp are adjusted to describe the acceptable range (including 910 * the redzone). On failure, 0 is returned. 911 */ 912 /*ARGSUSED3*/ 913 int 914 valid_va_range_aligned(caddr_t *basep, size_t *lenp, size_t minlen, int dir, 915 size_t align, size_t redzone, size_t off) 916 { 917 uintptr_t hi, lo; 918 size_t tot_len; 919 920 ASSERT(align == 0 ? off == 0 : off < align); 921 ASSERT(ISP2(align)); 922 ASSERT(align == 0 || align >= PAGESIZE); 923 924 lo = (uintptr_t)*basep; 925 hi = lo + *lenp; 926 tot_len = minlen + 2 * redzone; /* need at least this much space */ 927 928 /* 929 * If hi rolled over the top, try cutting back. 930 */ 931 if (hi < lo) { 932 *lenp = 0UL - lo - 1UL; 933 /* See if this really happens. If so, then we figure out why */ 934 valid_va_range_aligned_wraparound++; 935 hi = lo + *lenp; 936 } 937 if (*lenp < tot_len) { 938 return (0); 939 } 940 941 #if defined(__amd64) 942 /* 943 * Deal with a possible hole in the address range between 944 * hole_start and hole_end that should never be mapped. 945 */ 946 if (lo < hole_start) { 947 if (hi > hole_start) { 948 if (hi < hole_end) { 949 hi = hole_start; 950 } else { 951 /* lo < hole_start && hi >= hole_end */ 952 if (dir == AH_LO) { 953 /* 954 * prefer lowest range 955 */ 956 if (hole_start - lo >= tot_len) 957 hi = hole_start; 958 else if (hi - hole_end >= tot_len) 959 lo = hole_end; 960 else 961 return (0); 962 } else { 963 /* 964 * prefer highest range 965 */ 966 if (hi - hole_end >= tot_len) 967 lo = hole_end; 968 else if (hole_start - lo >= tot_len) 969 hi = hole_start; 970 else 971 return (0); 972 } 973 } 974 } 975 } else { 976 /* lo >= hole_start */ 977 if (hi < hole_end) 978 return (0); 979 if (lo < hole_end) 980 lo = hole_end; 981 } 982 #endif 983 984 if (hi - lo < tot_len) 985 return (0); 986 987 if (align > 1) { 988 uintptr_t tlo = lo + redzone; 989 uintptr_t thi = hi - redzone; 990 tlo = (uintptr_t)P2PHASEUP(tlo, align, off); 991 if (tlo < lo + redzone) { 992 return (0); 993 } 994 if (thi < tlo || thi - tlo < minlen) { 995 return (0); 996 } 997 } 998 999 *basep = (caddr_t)lo; 1000 *lenp = hi - lo; 1001 return (1); 1002 } 1003 1004 /* 1005 * Determine whether [*basep, *basep + *lenp) contains a mappable range of 1006 * addresses at least "minlen" long. On success, 1 is returned and *basep 1007 * and *lenp are adjusted to describe the acceptable range. On failure, 0 1008 * is returned. 1009 */ 1010 int 1011 valid_va_range(caddr_t *basep, size_t *lenp, size_t minlen, int dir) 1012 { 1013 return (valid_va_range_aligned(basep, lenp, minlen, dir, 0, 0, 0)); 1014 } 1015 1016 /* 1017 * Default to forbidding the first 64k of address space. This protects most 1018 * reasonably sized structures from dereferences through NULL: 1019 * ((foo_t *)0)->bar 1020 */ 1021 uintptr_t forbidden_null_mapping_sz = 0x10000; 1022 1023 /* 1024 * Determine whether [addr, addr+len] are valid user addresses. 1025 */ 1026 /*ARGSUSED*/ 1027 int 1028 valid_usr_range(caddr_t addr, size_t len, uint_t prot, struct as *as, 1029 caddr_t userlimit) 1030 { 1031 caddr_t eaddr = addr + len; 1032 1033 if (eaddr <= addr || addr >= userlimit || eaddr > userlimit) 1034 return (RANGE_BADADDR); 1035 1036 if ((addr <= (caddr_t)forbidden_null_mapping_sz) && 1037 as->a_proc != NULL && 1038 secflag_enabled(as->a_proc, PROC_SEC_FORBIDNULLMAP)) 1039 return (RANGE_BADADDR); 1040 1041 #if defined(__amd64) 1042 /* 1043 * Check for the VA hole 1044 */ 1045 if (eaddr > (caddr_t)hole_start && addr < (caddr_t)hole_end) 1046 return (RANGE_BADADDR); 1047 #endif 1048 1049 return (RANGE_OKAY); 1050 } 1051 1052 /* 1053 * Return 1 if the page frame is onboard memory, else 0. 1054 */ 1055 int 1056 pf_is_memory(pfn_t pf) 1057 { 1058 if (pfn_is_foreign(pf)) 1059 return (0); 1060 return (address_in_memlist(phys_install, pfn_to_pa(pf), 1)); 1061 } 1062 1063 /* 1064 * return the memrange containing pfn 1065 */ 1066 int 1067 memrange_num(pfn_t pfn) 1068 { 1069 int n; 1070 1071 for (n = 0; n < nranges - 1; ++n) { 1072 if (pfn >= memranges[n]) 1073 break; 1074 } 1075 return (n); 1076 } 1077 1078 /* 1079 * return the mnoderange containing pfn 1080 */ 1081 /*ARGSUSED*/ 1082 int 1083 pfn_2_mtype(pfn_t pfn) 1084 { 1085 #if defined(__xpv) 1086 return (0); 1087 #else 1088 int n; 1089 1090 /* Always start from highest pfn and work our way down */ 1091 for (n = mtypetop; n != -1; n = mnoderanges[n].mnr_next) { 1092 if (pfn >= mnoderanges[n].mnr_pfnlo) { 1093 break; 1094 } 1095 } 1096 return (n); 1097 #endif 1098 } 1099 1100 #if !defined(__xpv) 1101 /* 1102 * is_contigpage_free: 1103 * returns a page list of contiguous pages. It minimally has to return 1104 * minctg pages. Caller determines minctg based on the scatter-gather 1105 * list length. 1106 * 1107 * pfnp is set to the next page frame to search on return. 1108 */ 1109 static page_t * 1110 is_contigpage_free( 1111 pfn_t *pfnp, 1112 pgcnt_t *pgcnt, 1113 pgcnt_t minctg, 1114 uint64_t pfnseg, 1115 int iolock) 1116 { 1117 int i = 0; 1118 pfn_t pfn = *pfnp; 1119 page_t *pp; 1120 page_t *plist = NULL; 1121 1122 /* 1123 * fail if pfn + minctg crosses a segment boundary. 1124 * Adjust for next starting pfn to begin at segment boundary. 1125 */ 1126 1127 if (((*pfnp + minctg - 1) & pfnseg) < (*pfnp & pfnseg)) { 1128 *pfnp = roundup(*pfnp, pfnseg + 1); 1129 return (NULL); 1130 } 1131 1132 do { 1133 retry: 1134 pp = page_numtopp_nolock(pfn + i); 1135 if ((pp == NULL) || IS_DUMP_PAGE(pp) || 1136 (page_trylock(pp, SE_EXCL) == 0)) { 1137 (*pfnp)++; 1138 break; 1139 } 1140 if (page_pptonum(pp) != pfn + i) { 1141 page_unlock(pp); 1142 goto retry; 1143 } 1144 1145 if (!(PP_ISFREE(pp))) { 1146 page_unlock(pp); 1147 (*pfnp)++; 1148 break; 1149 } 1150 1151 if (!PP_ISAGED(pp)) { 1152 page_list_sub(pp, PG_CACHE_LIST); 1153 page_hashout(pp, (kmutex_t *)NULL); 1154 } else { 1155 page_list_sub(pp, PG_FREE_LIST); 1156 } 1157 1158 if (iolock) 1159 page_io_lock(pp); 1160 page_list_concat(&plist, &pp); 1161 1162 /* 1163 * exit loop when pgcnt satisfied or segment boundary reached. 1164 */ 1165 1166 } while ((++i < *pgcnt) && ((pfn + i) & pfnseg)); 1167 1168 *pfnp += i; /* set to next pfn to search */ 1169 1170 if (i >= minctg) { 1171 *pgcnt -= i; 1172 return (plist); 1173 } 1174 1175 /* 1176 * failure: minctg not satisfied. 1177 * 1178 * if next request crosses segment boundary, set next pfn 1179 * to search from the segment boundary. 1180 */ 1181 if (((*pfnp + minctg - 1) & pfnseg) < (*pfnp & pfnseg)) 1182 *pfnp = roundup(*pfnp, pfnseg + 1); 1183 1184 /* clean up any pages already allocated */ 1185 1186 while (plist) { 1187 pp = plist; 1188 page_sub(&plist, pp); 1189 page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL); 1190 if (iolock) 1191 page_io_unlock(pp); 1192 page_unlock(pp); 1193 } 1194 1195 return (NULL); 1196 } 1197 #endif /* !__xpv */ 1198 1199 /* 1200 * verify that pages being returned from allocator have correct DMA attribute 1201 */ 1202 #ifndef DEBUG 1203 #define check_dma(a, b, c) (void)(0) 1204 #else 1205 static void 1206 check_dma(ddi_dma_attr_t *dma_attr, page_t *pp, int cnt) 1207 { 1208 if (dma_attr == NULL) 1209 return; 1210 1211 while (cnt-- > 0) { 1212 if (pa_to_ma(pfn_to_pa(pp->p_pagenum)) < 1213 dma_attr->dma_attr_addr_lo) 1214 panic("PFN (pp=%p) below dma_attr_addr_lo", (void *)pp); 1215 if (pa_to_ma(pfn_to_pa(pp->p_pagenum)) >= 1216 dma_attr->dma_attr_addr_hi) 1217 panic("PFN (pp=%p) above dma_attr_addr_hi", (void *)pp); 1218 pp = pp->p_next; 1219 } 1220 } 1221 #endif 1222 1223 #if !defined(__xpv) 1224 static page_t * 1225 page_get_contigpage(pgcnt_t *pgcnt, ddi_dma_attr_t *mattr, int iolock) 1226 { 1227 pfn_t pfn; 1228 int sgllen; 1229 uint64_t pfnseg; 1230 pgcnt_t minctg; 1231 page_t *pplist = NULL, *plist; 1232 uint64_t lo, hi; 1233 pgcnt_t pfnalign = 0; 1234 static pfn_t startpfn; 1235 static pgcnt_t lastctgcnt; 1236 uintptr_t align; 1237 1238 CONTIG_LOCK(); 1239 1240 if (mattr) { 1241 lo = mmu_btop((mattr->dma_attr_addr_lo + MMU_PAGEOFFSET)); 1242 hi = mmu_btop(mattr->dma_attr_addr_hi); 1243 if (hi >= physmax) 1244 hi = physmax - 1; 1245 sgllen = mattr->dma_attr_sgllen; 1246 pfnseg = mmu_btop(mattr->dma_attr_seg); 1247 1248 align = maxbit(mattr->dma_attr_align, mattr->dma_attr_minxfer); 1249 if (align > MMU_PAGESIZE) 1250 pfnalign = mmu_btop(align); 1251 1252 /* 1253 * in order to satisfy the request, must minimally 1254 * acquire minctg contiguous pages 1255 */ 1256 minctg = howmany(*pgcnt, sgllen); 1257 1258 ASSERT(hi >= lo); 1259 1260 /* 1261 * start from where last searched if the minctg >= lastctgcnt 1262 */ 1263 if (minctg < lastctgcnt || startpfn < lo || startpfn > hi) 1264 startpfn = lo; 1265 } else { 1266 hi = physmax - 1; 1267 lo = 0; 1268 sgllen = 1; 1269 pfnseg = mmu.highest_pfn; 1270 minctg = *pgcnt; 1271 1272 if (minctg < lastctgcnt) 1273 startpfn = lo; 1274 } 1275 lastctgcnt = minctg; 1276 1277 ASSERT(pfnseg + 1 >= (uint64_t)minctg); 1278 1279 /* conserve 16m memory - start search above 16m when possible */ 1280 if (hi > PFN_16M && startpfn < PFN_16M) 1281 startpfn = PFN_16M; 1282 1283 pfn = startpfn; 1284 if (pfnalign) 1285 pfn = P2ROUNDUP(pfn, pfnalign); 1286 1287 while (pfn + minctg - 1 <= hi) { 1288 1289 plist = is_contigpage_free(&pfn, pgcnt, minctg, pfnseg, iolock); 1290 if (plist) { 1291 page_list_concat(&pplist, &plist); 1292 sgllen--; 1293 /* 1294 * return when contig pages no longer needed 1295 */ 1296 if (!*pgcnt || ((*pgcnt <= sgllen) && !pfnalign)) { 1297 startpfn = pfn; 1298 CONTIG_UNLOCK(); 1299 check_dma(mattr, pplist, *pgcnt); 1300 return (pplist); 1301 } 1302 minctg = howmany(*pgcnt, sgllen); 1303 } 1304 if (pfnalign) 1305 pfn = P2ROUNDUP(pfn, pfnalign); 1306 } 1307 1308 /* cannot find contig pages in specified range */ 1309 if (startpfn == lo) { 1310 CONTIG_UNLOCK(); 1311 return (NULL); 1312 } 1313 1314 /* did not start with lo previously */ 1315 pfn = lo; 1316 if (pfnalign) 1317 pfn = P2ROUNDUP(pfn, pfnalign); 1318 1319 /* allow search to go above startpfn */ 1320 while (pfn < startpfn) { 1321 1322 plist = is_contigpage_free(&pfn, pgcnt, minctg, pfnseg, iolock); 1323 if (plist != NULL) { 1324 1325 page_list_concat(&pplist, &plist); 1326 sgllen--; 1327 1328 /* 1329 * return when contig pages no longer needed 1330 */ 1331 if (!*pgcnt || ((*pgcnt <= sgllen) && !pfnalign)) { 1332 startpfn = pfn; 1333 CONTIG_UNLOCK(); 1334 check_dma(mattr, pplist, *pgcnt); 1335 return (pplist); 1336 } 1337 minctg = howmany(*pgcnt, sgllen); 1338 } 1339 if (pfnalign) 1340 pfn = P2ROUNDUP(pfn, pfnalign); 1341 } 1342 CONTIG_UNLOCK(); 1343 return (NULL); 1344 } 1345 #endif /* !__xpv */ 1346 1347 /* 1348 * mnode_range_cnt() calculates the number of memory ranges for mnode and 1349 * memranges[]. Used to determine the size of page lists and mnoderanges. 1350 */ 1351 int 1352 mnode_range_cnt(int mnode) 1353 { 1354 #if defined(__xpv) 1355 ASSERT(mnode == 0); 1356 return (1); 1357 #else /* __xpv */ 1358 int mri; 1359 int mnrcnt = 0; 1360 1361 if (mem_node_config[mnode].exists != 0) { 1362 mri = nranges - 1; 1363 1364 /* find the memranges index below contained in mnode range */ 1365 1366 while (MEMRANGEHI(mri) < mem_node_config[mnode].physbase) 1367 mri--; 1368 1369 /* 1370 * increment mnode range counter when memranges or mnode 1371 * boundary is reached. 1372 */ 1373 while (mri >= 0 && 1374 mem_node_config[mnode].physmax >= MEMRANGELO(mri)) { 1375 mnrcnt++; 1376 if (mem_node_config[mnode].physmax > MEMRANGEHI(mri)) 1377 mri--; 1378 else 1379 break; 1380 } 1381 } 1382 ASSERT(mnrcnt <= MAX_MNODE_MRANGES); 1383 return (mnrcnt); 1384 #endif /* __xpv */ 1385 } 1386 1387 /* 1388 * mnode_range_setup() initializes mnoderanges. 1389 */ 1390 void 1391 mnode_range_setup(mnoderange_t *mnoderanges) 1392 { 1393 mnoderange_t *mp = mnoderanges; 1394 int mnode, mri; 1395 int mindex = 0; /* current index into mnoderanges array */ 1396 int i, j; 1397 pfn_t hipfn; 1398 int last, hi; 1399 1400 for (mnode = 0; mnode < max_mem_nodes; mnode++) { 1401 if (mem_node_config[mnode].exists == 0) 1402 continue; 1403 1404 mri = nranges - 1; 1405 1406 while (MEMRANGEHI(mri) < mem_node_config[mnode].physbase) 1407 mri--; 1408 1409 while (mri >= 0 && mem_node_config[mnode].physmax >= 1410 MEMRANGELO(mri)) { 1411 mnoderanges->mnr_pfnlo = MAX(MEMRANGELO(mri), 1412 mem_node_config[mnode].physbase); 1413 mnoderanges->mnr_pfnhi = MIN(MEMRANGEHI(mri), 1414 mem_node_config[mnode].physmax); 1415 mnoderanges->mnr_mnode = mnode; 1416 mnoderanges->mnr_memrange = mri; 1417 mnoderanges->mnr_exists = 1; 1418 mnoderanges++; 1419 mindex++; 1420 if (mem_node_config[mnode].physmax > MEMRANGEHI(mri)) 1421 mri--; 1422 else 1423 break; 1424 } 1425 } 1426 1427 /* 1428 * For now do a simple sort of the mnoderanges array to fill in 1429 * the mnr_next fields. Since mindex is expected to be relatively 1430 * small, using a simple O(N^2) algorithm. 1431 */ 1432 for (i = 0; i < mindex; i++) { 1433 if (mp[i].mnr_pfnlo == 0) /* find lowest */ 1434 break; 1435 } 1436 ASSERT(i < mindex); 1437 last = i; 1438 mtype16m = last; 1439 mp[last].mnr_next = -1; 1440 for (i = 0; i < mindex - 1; i++) { 1441 hipfn = (pfn_t)(-1); 1442 hi = -1; 1443 /* find next highest mnode range */ 1444 for (j = 0; j < mindex; j++) { 1445 if (mp[j].mnr_pfnlo > mp[last].mnr_pfnlo && 1446 mp[j].mnr_pfnlo < hipfn) { 1447 hipfn = mp[j].mnr_pfnlo; 1448 hi = j; 1449 } 1450 } 1451 mp[hi].mnr_next = last; 1452 last = hi; 1453 } 1454 mtypetop = last; 1455 } 1456 1457 #ifndef __xpv 1458 /* 1459 * Update mnoderanges for memory hot-add DR operations. 1460 */ 1461 static void 1462 mnode_range_add(int mnode) 1463 { 1464 int *prev; 1465 int n, mri; 1466 pfn_t start, end; 1467 extern void membar_sync(void); 1468 1469 ASSERT(0 <= mnode && mnode < max_mem_nodes); 1470 ASSERT(mem_node_config[mnode].exists); 1471 start = mem_node_config[mnode].physbase; 1472 end = mem_node_config[mnode].physmax; 1473 ASSERT(start <= end); 1474 mutex_enter(&mnoderange_lock); 1475 1476 #ifdef DEBUG 1477 /* Check whether it interleaves with other memory nodes. */ 1478 for (n = mtypetop; n != -1; n = mnoderanges[n].mnr_next) { 1479 ASSERT(mnoderanges[n].mnr_exists); 1480 if (mnoderanges[n].mnr_mnode == mnode) 1481 continue; 1482 ASSERT(start > mnoderanges[n].mnr_pfnhi || 1483 end < mnoderanges[n].mnr_pfnlo); 1484 } 1485 #endif /* DEBUG */ 1486 1487 mri = nranges - 1; 1488 while (MEMRANGEHI(mri) < mem_node_config[mnode].physbase) 1489 mri--; 1490 while (mri >= 0 && mem_node_config[mnode].physmax >= MEMRANGELO(mri)) { 1491 /* Check whether mtype already exists. */ 1492 for (n = mtypetop; n != -1; n = mnoderanges[n].mnr_next) { 1493 if (mnoderanges[n].mnr_mnode == mnode && 1494 mnoderanges[n].mnr_memrange == mri) { 1495 mnoderanges[n].mnr_pfnlo = MAX(MEMRANGELO(mri), 1496 start); 1497 mnoderanges[n].mnr_pfnhi = MIN(MEMRANGEHI(mri), 1498 end); 1499 break; 1500 } 1501 } 1502 1503 /* Add a new entry if it doesn't exist yet. */ 1504 if (n == -1) { 1505 /* Try to find an unused entry in mnoderanges array. */ 1506 for (n = 0; n < mnoderangecnt; n++) { 1507 if (mnoderanges[n].mnr_exists == 0) 1508 break; 1509 } 1510 ASSERT(n < mnoderangecnt); 1511 mnoderanges[n].mnr_pfnlo = MAX(MEMRANGELO(mri), start); 1512 mnoderanges[n].mnr_pfnhi = MIN(MEMRANGEHI(mri), end); 1513 mnoderanges[n].mnr_mnode = mnode; 1514 mnoderanges[n].mnr_memrange = mri; 1515 mnoderanges[n].mnr_exists = 1; 1516 /* Page 0 should always be present. */ 1517 for (prev = &mtypetop; 1518 mnoderanges[*prev].mnr_pfnlo > start; 1519 prev = &mnoderanges[*prev].mnr_next) { 1520 ASSERT(mnoderanges[*prev].mnr_next >= 0); 1521 ASSERT(mnoderanges[*prev].mnr_pfnlo > end); 1522 } 1523 mnoderanges[n].mnr_next = *prev; 1524 membar_sync(); 1525 *prev = n; 1526 } 1527 1528 if (mem_node_config[mnode].physmax > MEMRANGEHI(mri)) 1529 mri--; 1530 else 1531 break; 1532 } 1533 1534 mutex_exit(&mnoderange_lock); 1535 } 1536 1537 /* 1538 * Update mnoderanges for memory hot-removal DR operations. 1539 */ 1540 static void 1541 mnode_range_del(int mnode) 1542 { 1543 _NOTE(ARGUNUSED(mnode)); 1544 ASSERT(0 <= mnode && mnode < max_mem_nodes); 1545 /* TODO: support deletion operation. */ 1546 ASSERT(0); 1547 } 1548 1549 void 1550 plat_slice_add(pfn_t start, pfn_t end) 1551 { 1552 mem_node_add_slice(start, end); 1553 if (plat_dr_enabled()) { 1554 mnode_range_add(PFN_2_MEM_NODE(start)); 1555 } 1556 } 1557 1558 void 1559 plat_slice_del(pfn_t start, pfn_t end) 1560 { 1561 ASSERT(PFN_2_MEM_NODE(start) == PFN_2_MEM_NODE(end)); 1562 ASSERT(plat_dr_enabled()); 1563 mnode_range_del(PFN_2_MEM_NODE(start)); 1564 mem_node_del_slice(start, end); 1565 } 1566 #endif /* __xpv */ 1567 1568 /*ARGSUSED*/ 1569 int 1570 mtype_init(vnode_t *vp, caddr_t vaddr, uint_t *flags, size_t pgsz) 1571 { 1572 int mtype = mtypetop; 1573 1574 #if !defined(__xpv) 1575 #if defined(__i386) 1576 /* 1577 * set the mtype range 1578 * - kmem requests need to be below 4g if restricted_kmemalloc is set. 1579 * - for non kmem requests, set range to above 4g if memory below 4g 1580 * runs low. 1581 */ 1582 if (restricted_kmemalloc && VN_ISKAS(vp) && 1583 (caddr_t)(vaddr) >= kernelheap && 1584 (caddr_t)(vaddr) < ekernelheap) { 1585 ASSERT(physmax4g); 1586 mtype = mtype4g; 1587 if (RESTRICT16M_ALLOC(freemem4g - btop(pgsz), 1588 btop(pgsz), *flags)) { 1589 *flags |= PGI_MT_RANGE16M; 1590 } else { 1591 VM_STAT_ADD(vmm_vmstats.unrestrict16mcnt); 1592 VM_STAT_COND_ADD((*flags & PG_PANIC), 1593 vmm_vmstats.pgpanicalloc); 1594 *flags |= PGI_MT_RANGE0; 1595 } 1596 return (mtype); 1597 } 1598 #endif /* __i386 */ 1599 1600 if (RESTRICT4G_ALLOC) { 1601 VM_STAT_ADD(vmm_vmstats.restrict4gcnt); 1602 /* here only for > 4g systems */ 1603 *flags |= PGI_MT_RANGE4G; 1604 } else if (RESTRICT16M_ALLOC(freemem, btop(pgsz), *flags)) { 1605 *flags |= PGI_MT_RANGE16M; 1606 } else { 1607 VM_STAT_ADD(vmm_vmstats.unrestrict16mcnt); 1608 VM_STAT_COND_ADD((*flags & PG_PANIC), vmm_vmstats.pgpanicalloc); 1609 *flags |= PGI_MT_RANGE0; 1610 } 1611 #endif /* !__xpv */ 1612 return (mtype); 1613 } 1614 1615 1616 /* mtype init for page_get_replacement_page */ 1617 /*ARGSUSED*/ 1618 int 1619 mtype_pgr_init(int *flags, page_t *pp, int mnode, pgcnt_t pgcnt) 1620 { 1621 int mtype = mtypetop; 1622 #if !defined(__xpv) 1623 if (RESTRICT16M_ALLOC(freemem, pgcnt, *flags)) { 1624 *flags |= PGI_MT_RANGE16M; 1625 } else { 1626 VM_STAT_ADD(vmm_vmstats.unrestrict16mcnt); 1627 *flags |= PGI_MT_RANGE0; 1628 } 1629 #endif 1630 return (mtype); 1631 } 1632 1633 /* 1634 * Determine if the mnode range specified in mtype contains memory belonging 1635 * to memory node mnode. If flags & PGI_MT_RANGE is set then mtype contains 1636 * the range from high pfn to 0, 16m or 4g. 1637 * 1638 * Return first mnode range type index found otherwise return -1 if none found. 1639 */ 1640 int 1641 mtype_func(int mnode, int mtype, uint_t flags) 1642 { 1643 if (flags & PGI_MT_RANGE) { 1644 int mnr_lim = MRI_0; 1645 1646 if (flags & PGI_MT_NEXT) { 1647 mtype = mnoderanges[mtype].mnr_next; 1648 } 1649 if (flags & PGI_MT_RANGE4G) 1650 mnr_lim = MRI_4G; /* exclude 0-4g range */ 1651 else if (flags & PGI_MT_RANGE16M) 1652 mnr_lim = MRI_16M; /* exclude 0-16m range */ 1653 while (mtype != -1 && 1654 mnoderanges[mtype].mnr_memrange <= mnr_lim) { 1655 if (mnoderanges[mtype].mnr_mnode == mnode) 1656 return (mtype); 1657 mtype = mnoderanges[mtype].mnr_next; 1658 } 1659 } else if (mnoderanges[mtype].mnr_mnode == mnode) { 1660 return (mtype); 1661 } 1662 return (-1); 1663 } 1664 1665 /* 1666 * Update the page list max counts with the pfn range specified by the 1667 * input parameters. 1668 */ 1669 void 1670 mtype_modify_max(pfn_t startpfn, long cnt) 1671 { 1672 int mtype; 1673 pgcnt_t inc; 1674 spgcnt_t scnt = (spgcnt_t)(cnt); 1675 pgcnt_t acnt = ABS(scnt); 1676 pfn_t endpfn = startpfn + acnt; 1677 pfn_t pfn, lo; 1678 1679 if (!physmax4g) 1680 return; 1681 1682 mtype = mtypetop; 1683 for (pfn = endpfn; pfn > startpfn; ) { 1684 ASSERT(mtype != -1); 1685 lo = mnoderanges[mtype].mnr_pfnlo; 1686 if (pfn > lo) { 1687 if (startpfn >= lo) { 1688 inc = pfn - startpfn; 1689 } else { 1690 inc = pfn - lo; 1691 } 1692 if (mnoderanges[mtype].mnr_memrange != MRI_4G) { 1693 if (scnt > 0) 1694 maxmem4g += inc; 1695 else 1696 maxmem4g -= inc; 1697 } 1698 pfn -= inc; 1699 } 1700 mtype = mnoderanges[mtype].mnr_next; 1701 } 1702 } 1703 1704 int 1705 mtype_2_mrange(int mtype) 1706 { 1707 return (mnoderanges[mtype].mnr_memrange); 1708 } 1709 1710 void 1711 mnodetype_2_pfn(int mnode, int mtype, pfn_t *pfnlo, pfn_t *pfnhi) 1712 { 1713 _NOTE(ARGUNUSED(mnode)); 1714 ASSERT(mnoderanges[mtype].mnr_mnode == mnode); 1715 *pfnlo = mnoderanges[mtype].mnr_pfnlo; 1716 *pfnhi = mnoderanges[mtype].mnr_pfnhi; 1717 } 1718 1719 size_t 1720 plcnt_sz(size_t ctrs_sz) 1721 { 1722 #ifdef DEBUG 1723 int szc, colors; 1724 1725 ctrs_sz += mnoderangecnt * sizeof (struct mnr_mts) * mmu_page_sizes; 1726 for (szc = 0; szc < mmu_page_sizes; szc++) { 1727 colors = page_get_pagecolors(szc); 1728 ctrs_sz += mnoderangecnt * sizeof (pgcnt_t) * colors; 1729 } 1730 #endif 1731 return (ctrs_sz); 1732 } 1733 1734 caddr_t 1735 plcnt_init(caddr_t addr) 1736 { 1737 #ifdef DEBUG 1738 int mt, szc, colors; 1739 1740 for (mt = 0; mt < mnoderangecnt; mt++) { 1741 mnoderanges[mt].mnr_mts = (struct mnr_mts *)addr; 1742 addr += (sizeof (struct mnr_mts) * mmu_page_sizes); 1743 for (szc = 0; szc < mmu_page_sizes; szc++) { 1744 colors = page_get_pagecolors(szc); 1745 mnoderanges[mt].mnr_mts[szc].mnr_mts_colors = colors; 1746 mnoderanges[mt].mnr_mts[szc].mnr_mtsc_pgcnt = 1747 (pgcnt_t *)addr; 1748 addr += (sizeof (pgcnt_t) * colors); 1749 } 1750 } 1751 #endif 1752 return (addr); 1753 } 1754 1755 void 1756 plcnt_inc_dec(page_t *pp, int mtype, int szc, long cnt, int flags) 1757 { 1758 _NOTE(ARGUNUSED(pp)); 1759 #ifdef DEBUG 1760 int bin = PP_2_BIN(pp); 1761 1762 atomic_add_long(&mnoderanges[mtype].mnr_mts[szc].mnr_mts_pgcnt, cnt); 1763 atomic_add_long(&mnoderanges[mtype].mnr_mts[szc].mnr_mtsc_pgcnt[bin], 1764 cnt); 1765 #endif 1766 ASSERT(mtype == PP_2_MTYPE(pp)); 1767 if (physmax4g && mnoderanges[mtype].mnr_memrange != MRI_4G) 1768 atomic_add_long(&freemem4g, cnt); 1769 if (flags & PG_CACHE_LIST) 1770 atomic_add_long(&mnoderanges[mtype].mnr_mt_clpgcnt, cnt); 1771 else 1772 atomic_add_long(&mnoderanges[mtype].mnr_mt_flpgcnt[szc], cnt); 1773 atomic_add_long(&mnoderanges[mtype].mnr_mt_totcnt, cnt); 1774 } 1775 1776 /* 1777 * Returns the free page count for mnode 1778 */ 1779 int 1780 mnode_pgcnt(int mnode) 1781 { 1782 int mtype = mtypetop; 1783 int flags = PGI_MT_RANGE0; 1784 pgcnt_t pgcnt = 0; 1785 1786 mtype = mtype_func(mnode, mtype, flags); 1787 1788 while (mtype != -1) { 1789 pgcnt += MTYPE_FREEMEM(mtype); 1790 mtype = mtype_func(mnode, mtype, flags | PGI_MT_NEXT); 1791 } 1792 return (pgcnt); 1793 } 1794 1795 /* 1796 * Initialize page coloring variables based on the l2 cache parameters. 1797 * Calculate and return memory needed for page coloring data structures. 1798 */ 1799 size_t 1800 page_coloring_init(uint_t l2_sz, int l2_linesz, int l2_assoc) 1801 { 1802 _NOTE(ARGUNUSED(l2_linesz)); 1803 size_t colorsz = 0; 1804 int i; 1805 int colors; 1806 1807 #if defined(__xpv) 1808 /* 1809 * Hypervisor domains currently don't have any concept of NUMA. 1810 * Hence we'll act like there is only 1 memrange. 1811 */ 1812 i = memrange_num(1); 1813 #else /* !__xpv */ 1814 /* 1815 * Reduce the memory ranges lists if we don't have large amounts 1816 * of memory. This avoids searching known empty free lists. 1817 * To support memory DR operations, we need to keep memory ranges 1818 * for possible memory hot-add operations. 1819 */ 1820 if (plat_dr_physmax > physmax) 1821 i = memrange_num(plat_dr_physmax); 1822 else 1823 i = memrange_num(physmax); 1824 #if defined(__i386) 1825 if (i > MRI_4G) 1826 restricted_kmemalloc = 0; 1827 #endif 1828 /* physmax greater than 4g */ 1829 if (i == MRI_4G) 1830 physmax4g = 1; 1831 #endif /* !__xpv */ 1832 memranges += i; 1833 nranges -= i; 1834 1835 ASSERT(mmu_page_sizes <= MMU_PAGE_SIZES); 1836 1837 ASSERT(ISP2(l2_linesz)); 1838 ASSERT(l2_sz > MMU_PAGESIZE); 1839 1840 /* l2_assoc is 0 for fully associative l2 cache */ 1841 if (l2_assoc) 1842 l2_colors = MAX(1, l2_sz / (l2_assoc * MMU_PAGESIZE)); 1843 else 1844 l2_colors = 1; 1845 1846 ASSERT(ISP2(l2_colors)); 1847 1848 /* for scalability, configure at least PAGE_COLORS_MIN color bins */ 1849 page_colors = MAX(l2_colors, PAGE_COLORS_MIN); 1850 1851 /* 1852 * cpu_page_colors is non-zero when a page color may be spread across 1853 * multiple bins. 1854 */ 1855 if (l2_colors < page_colors) 1856 cpu_page_colors = l2_colors; 1857 1858 ASSERT(ISP2(page_colors)); 1859 1860 page_colors_mask = page_colors - 1; 1861 1862 ASSERT(ISP2(CPUSETSIZE())); 1863 page_coloring_shift = lowbit(CPUSETSIZE()); 1864 1865 /* initialize number of colors per page size */ 1866 for (i = 0; i <= mmu.max_page_level; i++) { 1867 hw_page_array[i].hp_size = LEVEL_SIZE(i); 1868 hw_page_array[i].hp_shift = LEVEL_SHIFT(i); 1869 hw_page_array[i].hp_pgcnt = LEVEL_SIZE(i) >> LEVEL_SHIFT(0); 1870 hw_page_array[i].hp_colors = (page_colors_mask >> 1871 (hw_page_array[i].hp_shift - hw_page_array[0].hp_shift)) 1872 + 1; 1873 colorequivszc[i] = 0; 1874 } 1875 1876 /* 1877 * The value of cpu_page_colors determines if additional color bins 1878 * need to be checked for a particular color in the page_get routines. 1879 */ 1880 if (cpu_page_colors != 0) { 1881 1882 int a = lowbit(page_colors) - lowbit(cpu_page_colors); 1883 ASSERT(a > 0); 1884 ASSERT(a < 16); 1885 1886 for (i = 0; i <= mmu.max_page_level; i++) { 1887 if ((colors = hw_page_array[i].hp_colors) <= 1) { 1888 colorequivszc[i] = 0; 1889 continue; 1890 } 1891 while ((colors >> a) == 0) 1892 a--; 1893 ASSERT(a >= 0); 1894 1895 /* higher 4 bits encodes color equiv mask */ 1896 colorequivszc[i] = (a << 4); 1897 } 1898 } 1899 1900 /* factor in colorequiv to check additional 'equivalent' bins. */ 1901 if (colorequiv > 1) { 1902 1903 int a = lowbit(colorequiv) - 1; 1904 if (a > 15) 1905 a = 15; 1906 1907 for (i = 0; i <= mmu.max_page_level; i++) { 1908 if ((colors = hw_page_array[i].hp_colors) <= 1) { 1909 continue; 1910 } 1911 while ((colors >> a) == 0) 1912 a--; 1913 if ((a << 4) > colorequivszc[i]) { 1914 colorequivszc[i] = (a << 4); 1915 } 1916 } 1917 } 1918 1919 /* size for mnoderanges */ 1920 for (mnoderangecnt = 0, i = 0; i < max_mem_nodes; i++) 1921 mnoderangecnt += mnode_range_cnt(i); 1922 if (plat_dr_support_memory()) { 1923 /* 1924 * Reserve enough space for memory DR operations. 1925 * Two extra mnoderanges for possbile fragmentations, 1926 * one for the 2G boundary and the other for the 4G boundary. 1927 * We don't expect a memory board crossing the 16M boundary 1928 * for memory hot-add operations on x86 platforms. 1929 */ 1930 mnoderangecnt += 2 + max_mem_nodes - lgrp_plat_node_cnt; 1931 } 1932 colorsz = mnoderangecnt * sizeof (mnoderange_t); 1933 1934 /* size for fpc_mutex and cpc_mutex */ 1935 colorsz += (2 * max_mem_nodes * sizeof (kmutex_t) * NPC_MUTEX); 1936 1937 /* size of page_freelists */ 1938 colorsz += mnoderangecnt * sizeof (page_t ***); 1939 colorsz += mnoderangecnt * mmu_page_sizes * sizeof (page_t **); 1940 1941 for (i = 0; i < mmu_page_sizes; i++) { 1942 colors = page_get_pagecolors(i); 1943 colorsz += mnoderangecnt * colors * sizeof (page_t *); 1944 } 1945 1946 /* size of page_cachelists */ 1947 colorsz += mnoderangecnt * sizeof (page_t **); 1948 colorsz += mnoderangecnt * page_colors * sizeof (page_t *); 1949 1950 return (colorsz); 1951 } 1952 1953 /* 1954 * Called once at startup to configure page_coloring data structures and 1955 * does the 1st page_free()/page_freelist_add(). 1956 */ 1957 void 1958 page_coloring_setup(caddr_t pcmemaddr) 1959 { 1960 int i; 1961 int j; 1962 int k; 1963 caddr_t addr; 1964 int colors; 1965 1966 /* 1967 * do page coloring setup 1968 */ 1969 addr = pcmemaddr; 1970 1971 mnoderanges = (mnoderange_t *)addr; 1972 addr += (mnoderangecnt * sizeof (mnoderange_t)); 1973 1974 mnode_range_setup(mnoderanges); 1975 1976 if (physmax4g) 1977 mtype4g = pfn_2_mtype(0xfffff); 1978 1979 for (k = 0; k < NPC_MUTEX; k++) { 1980 fpc_mutex[k] = (kmutex_t *)addr; 1981 addr += (max_mem_nodes * sizeof (kmutex_t)); 1982 } 1983 for (k = 0; k < NPC_MUTEX; k++) { 1984 cpc_mutex[k] = (kmutex_t *)addr; 1985 addr += (max_mem_nodes * sizeof (kmutex_t)); 1986 } 1987 page_freelists = (page_t ****)addr; 1988 addr += (mnoderangecnt * sizeof (page_t ***)); 1989 1990 page_cachelists = (page_t ***)addr; 1991 addr += (mnoderangecnt * sizeof (page_t **)); 1992 1993 for (i = 0; i < mnoderangecnt; i++) { 1994 page_freelists[i] = (page_t ***)addr; 1995 addr += (mmu_page_sizes * sizeof (page_t **)); 1996 1997 for (j = 0; j < mmu_page_sizes; j++) { 1998 colors = page_get_pagecolors(j); 1999 page_freelists[i][j] = (page_t **)addr; 2000 addr += (colors * sizeof (page_t *)); 2001 } 2002 page_cachelists[i] = (page_t **)addr; 2003 addr += (page_colors * sizeof (page_t *)); 2004 } 2005 } 2006 2007 #if defined(__xpv) 2008 /* 2009 * Give back 10% of the io_pool pages to the free list. 2010 * Don't shrink the pool below some absolute minimum. 2011 */ 2012 static void 2013 page_io_pool_shrink() 2014 { 2015 int retcnt; 2016 page_t *pp, *pp_first, *pp_last, **curpool; 2017 mfn_t mfn; 2018 int bothpools = 0; 2019 2020 mutex_enter(&io_pool_lock); 2021 io_pool_shrink_attempts++; /* should be a kstat? */ 2022 retcnt = io_pool_cnt / 10; 2023 if (io_pool_cnt - retcnt < io_pool_cnt_min) 2024 retcnt = io_pool_cnt - io_pool_cnt_min; 2025 if (retcnt <= 0) 2026 goto done; 2027 io_pool_shrinks++; /* should be a kstat? */ 2028 curpool = &io_pool_4g; 2029 domore: 2030 /* 2031 * Loop through taking pages from the end of the list 2032 * (highest mfns) till amount to return reached. 2033 */ 2034 for (pp = *curpool; pp && retcnt > 0; ) { 2035 pp_first = pp_last = pp->p_prev; 2036 if (pp_first == *curpool) 2037 break; 2038 retcnt--; 2039 io_pool_cnt--; 2040 page_io_pool_sub(curpool, pp_first, pp_last); 2041 if ((mfn = pfn_to_mfn(pp->p_pagenum)) < start_mfn) 2042 start_mfn = mfn; 2043 page_free(pp_first, 1); 2044 pp = *curpool; 2045 } 2046 if (retcnt != 0 && !bothpools) { 2047 /* 2048 * If not enough found in less constrained pool try the 2049 * more constrained one. 2050 */ 2051 curpool = &io_pool_16m; 2052 bothpools = 1; 2053 goto domore; 2054 } 2055 done: 2056 mutex_exit(&io_pool_lock); 2057 } 2058 2059 #endif /* __xpv */ 2060 2061 uint_t 2062 page_create_update_flags_x86(uint_t flags) 2063 { 2064 #if defined(__xpv) 2065 /* 2066 * Check this is an urgent allocation and free pages are depleted. 2067 */ 2068 if (!(flags & PG_WAIT) && freemem < desfree) 2069 page_io_pool_shrink(); 2070 #else /* !__xpv */ 2071 /* 2072 * page_create_get_something may call this because 4g memory may be 2073 * depleted. Set flags to allow for relocation of base page below 2074 * 4g if necessary. 2075 */ 2076 if (physmax4g) 2077 flags |= (PGI_PGCPSZC0 | PGI_PGCPHIPRI); 2078 #endif /* __xpv */ 2079 return (flags); 2080 } 2081 2082 /*ARGSUSED*/ 2083 int 2084 bp_color(struct buf *bp) 2085 { 2086 return (0); 2087 } 2088 2089 #if defined(__xpv) 2090 2091 /* 2092 * Take pages out of an io_pool 2093 */ 2094 static void 2095 page_io_pool_sub(page_t **poolp, page_t *pp_first, page_t *pp_last) 2096 { 2097 if (*poolp == pp_first) { 2098 *poolp = pp_last->p_next; 2099 if (*poolp == pp_first) 2100 *poolp = NULL; 2101 } 2102 pp_first->p_prev->p_next = pp_last->p_next; 2103 pp_last->p_next->p_prev = pp_first->p_prev; 2104 pp_first->p_prev = pp_last; 2105 pp_last->p_next = pp_first; 2106 } 2107 2108 /* 2109 * Put a page on the io_pool list. The list is ordered by increasing MFN. 2110 */ 2111 static void 2112 page_io_pool_add(page_t **poolp, page_t *pp) 2113 { 2114 page_t *look; 2115 mfn_t mfn = mfn_list[pp->p_pagenum]; 2116 2117 if (*poolp == NULL) { 2118 *poolp = pp; 2119 pp->p_next = pp; 2120 pp->p_prev = pp; 2121 return; 2122 } 2123 2124 /* 2125 * Since we try to take pages from the high end of the pool 2126 * chances are good that the pages to be put on the list will 2127 * go at or near the end of the list. so start at the end and 2128 * work backwards. 2129 */ 2130 look = (*poolp)->p_prev; 2131 while (mfn < mfn_list[look->p_pagenum]) { 2132 look = look->p_prev; 2133 if (look == (*poolp)->p_prev) 2134 break; /* backed all the way to front of list */ 2135 } 2136 2137 /* insert after look */ 2138 pp->p_prev = look; 2139 pp->p_next = look->p_next; 2140 pp->p_next->p_prev = pp; 2141 look->p_next = pp; 2142 if (mfn < mfn_list[(*poolp)->p_pagenum]) { 2143 /* 2144 * we inserted a new first list element 2145 * adjust pool pointer to newly inserted element 2146 */ 2147 *poolp = pp; 2148 } 2149 } 2150 2151 /* 2152 * Add a page to the io_pool. Setting the force flag will force the page 2153 * into the io_pool no matter what. 2154 */ 2155 static void 2156 add_page_to_pool(page_t *pp, int force) 2157 { 2158 page_t *highest; 2159 page_t *freep = NULL; 2160 2161 mutex_enter(&io_pool_lock); 2162 /* 2163 * Always keep the scarce low memory pages 2164 */ 2165 if (mfn_list[pp->p_pagenum] < PFN_16MEG) { 2166 ++io_pool_cnt; 2167 page_io_pool_add(&io_pool_16m, pp); 2168 goto done; 2169 } 2170 if (io_pool_cnt < io_pool_cnt_max || force || io_pool_4g == NULL) { 2171 ++io_pool_cnt; 2172 page_io_pool_add(&io_pool_4g, pp); 2173 } else { 2174 highest = io_pool_4g->p_prev; 2175 if (mfn_list[pp->p_pagenum] < mfn_list[highest->p_pagenum]) { 2176 page_io_pool_sub(&io_pool_4g, highest, highest); 2177 page_io_pool_add(&io_pool_4g, pp); 2178 freep = highest; 2179 } else { 2180 freep = pp; 2181 } 2182 } 2183 done: 2184 mutex_exit(&io_pool_lock); 2185 if (freep) 2186 page_free(freep, 1); 2187 } 2188 2189 2190 int contig_pfn_cnt; /* no of pfns in the contig pfn list */ 2191 int contig_pfn_max; /* capacity of the contig pfn list */ 2192 int next_alloc_pfn; /* next position in list to start a contig search */ 2193 int contig_pfnlist_updates; /* pfn list update count */ 2194 int contig_pfnlist_builds; /* how many times have we (re)built list */ 2195 int contig_pfnlist_buildfailed; /* how many times has list build failed */ 2196 int create_contig_pending; /* nonzero means taskq creating contig list */ 2197 pfn_t *contig_pfn_list = NULL; /* list of contig pfns in ascending mfn order */ 2198 2199 /* 2200 * Function to use in sorting a list of pfns by their underlying mfns. 2201 */ 2202 static int 2203 mfn_compare(const void *pfnp1, const void *pfnp2) 2204 { 2205 mfn_t mfn1 = mfn_list[*(pfn_t *)pfnp1]; 2206 mfn_t mfn2 = mfn_list[*(pfn_t *)pfnp2]; 2207 2208 if (mfn1 > mfn2) 2209 return (1); 2210 if (mfn1 < mfn2) 2211 return (-1); 2212 return (0); 2213 } 2214 2215 /* 2216 * Compact the contig_pfn_list by tossing all the non-contiguous 2217 * elements from the list. 2218 */ 2219 static void 2220 compact_contig_pfn_list(void) 2221 { 2222 pfn_t pfn, lapfn, prev_lapfn; 2223 mfn_t mfn; 2224 int i, newcnt = 0; 2225 2226 prev_lapfn = 0; 2227 for (i = 0; i < contig_pfn_cnt - 1; i++) { 2228 pfn = contig_pfn_list[i]; 2229 lapfn = contig_pfn_list[i + 1]; 2230 mfn = mfn_list[pfn]; 2231 /* 2232 * See if next pfn is for a contig mfn 2233 */ 2234 if (mfn_list[lapfn] != mfn + 1) 2235 continue; 2236 /* 2237 * pfn and lookahead are both put in list 2238 * unless pfn is the previous lookahead. 2239 */ 2240 if (pfn != prev_lapfn) 2241 contig_pfn_list[newcnt++] = pfn; 2242 contig_pfn_list[newcnt++] = lapfn; 2243 prev_lapfn = lapfn; 2244 } 2245 for (i = newcnt; i < contig_pfn_cnt; i++) 2246 contig_pfn_list[i] = 0; 2247 contig_pfn_cnt = newcnt; 2248 } 2249 2250 /*ARGSUSED*/ 2251 static void 2252 call_create_contiglist(void *arg) 2253 { 2254 (void) create_contig_pfnlist(PG_WAIT); 2255 } 2256 2257 /* 2258 * Create list of freelist pfns that have underlying 2259 * contiguous mfns. The list is kept in ascending mfn order. 2260 * returns 1 if list created else 0. 2261 */ 2262 static int 2263 create_contig_pfnlist(uint_t flags) 2264 { 2265 pfn_t pfn; 2266 page_t *pp; 2267 int ret = 1; 2268 2269 mutex_enter(&contig_list_lock); 2270 if (contig_pfn_list != NULL) 2271 goto out; 2272 contig_pfn_max = freemem + (freemem / 10); 2273 contig_pfn_list = kmem_zalloc(contig_pfn_max * sizeof (pfn_t), 2274 (flags & PG_WAIT) ? KM_SLEEP : KM_NOSLEEP); 2275 if (contig_pfn_list == NULL) { 2276 /* 2277 * If we could not create the contig list (because 2278 * we could not sleep for memory). Dispatch a taskq that can 2279 * sleep to get the memory. 2280 */ 2281 if (!create_contig_pending) { 2282 if (taskq_dispatch(system_taskq, call_create_contiglist, 2283 NULL, TQ_NOSLEEP) != NULL) 2284 create_contig_pending = 1; 2285 } 2286 contig_pfnlist_buildfailed++; /* count list build failures */ 2287 ret = 0; 2288 goto out; 2289 } 2290 create_contig_pending = 0; 2291 ASSERT(contig_pfn_cnt == 0); 2292 for (pfn = 0; pfn < mfn_count; pfn++) { 2293 pp = page_numtopp_nolock(pfn); 2294 if (pp == NULL || !PP_ISFREE(pp)) 2295 continue; 2296 contig_pfn_list[contig_pfn_cnt] = pfn; 2297 if (++contig_pfn_cnt == contig_pfn_max) 2298 break; 2299 } 2300 /* 2301 * Sanity check the new list. 2302 */ 2303 if (contig_pfn_cnt < 2) { /* no contig pfns */ 2304 contig_pfn_cnt = 0; 2305 contig_pfnlist_buildfailed++; 2306 kmem_free(contig_pfn_list, contig_pfn_max * sizeof (pfn_t)); 2307 contig_pfn_list = NULL; 2308 contig_pfn_max = 0; 2309 ret = 0; 2310 goto out; 2311 } 2312 qsort(contig_pfn_list, contig_pfn_cnt, sizeof (pfn_t), mfn_compare); 2313 compact_contig_pfn_list(); 2314 /* 2315 * Make sure next search of the newly created contiguous pfn 2316 * list starts at the beginning of the list. 2317 */ 2318 next_alloc_pfn = 0; 2319 contig_pfnlist_builds++; /* count list builds */ 2320 out: 2321 mutex_exit(&contig_list_lock); 2322 return (ret); 2323 } 2324 2325 2326 /* 2327 * Toss the current contig pfnlist. Someone is about to do a massive 2328 * update to pfn<->mfn mappings. So we have them destroy the list and lock 2329 * it till they are done with their update. 2330 */ 2331 void 2332 clear_and_lock_contig_pfnlist() 2333 { 2334 pfn_t *listp = NULL; 2335 size_t listsize; 2336 2337 mutex_enter(&contig_list_lock); 2338 if (contig_pfn_list != NULL) { 2339 listp = contig_pfn_list; 2340 listsize = contig_pfn_max * sizeof (pfn_t); 2341 contig_pfn_list = NULL; 2342 contig_pfn_max = contig_pfn_cnt = 0; 2343 } 2344 if (listp != NULL) 2345 kmem_free(listp, listsize); 2346 } 2347 2348 /* 2349 * Unlock the contig_pfn_list. The next attempted use of it will cause 2350 * it to be re-created. 2351 */ 2352 void 2353 unlock_contig_pfnlist() 2354 { 2355 mutex_exit(&contig_list_lock); 2356 } 2357 2358 /* 2359 * Update the contiguous pfn list in response to a pfn <-> mfn reassignment 2360 */ 2361 void 2362 update_contig_pfnlist(pfn_t pfn, mfn_t oldmfn, mfn_t newmfn) 2363 { 2364 int probe_hi, probe_lo, probe_pos, insert_after, insert_point; 2365 pfn_t probe_pfn; 2366 mfn_t probe_mfn; 2367 int drop_lock = 0; 2368 2369 if (mutex_owner(&contig_list_lock) != curthread) { 2370 drop_lock = 1; 2371 mutex_enter(&contig_list_lock); 2372 } 2373 if (contig_pfn_list == NULL) 2374 goto done; 2375 contig_pfnlist_updates++; 2376 /* 2377 * Find the pfn in the current list. Use a binary chop to locate it. 2378 */ 2379 probe_hi = contig_pfn_cnt - 1; 2380 probe_lo = 0; 2381 probe_pos = (probe_hi + probe_lo) / 2; 2382 while ((probe_pfn = contig_pfn_list[probe_pos]) != pfn) { 2383 if (probe_pos == probe_lo) { /* pfn not in list */ 2384 probe_pos = -1; 2385 break; 2386 } 2387 if (pfn_to_mfn(probe_pfn) <= oldmfn) 2388 probe_lo = probe_pos; 2389 else 2390 probe_hi = probe_pos; 2391 probe_pos = (probe_hi + probe_lo) / 2; 2392 } 2393 if (probe_pos >= 0) { 2394 /* 2395 * Remove pfn from list and ensure next alloc 2396 * position stays in bounds. 2397 */ 2398 if (--contig_pfn_cnt <= next_alloc_pfn) 2399 next_alloc_pfn = 0; 2400 if (contig_pfn_cnt < 2) { /* no contig pfns */ 2401 contig_pfn_cnt = 0; 2402 kmem_free(contig_pfn_list, 2403 contig_pfn_max * sizeof (pfn_t)); 2404 contig_pfn_list = NULL; 2405 contig_pfn_max = 0; 2406 goto done; 2407 } 2408 ovbcopy(&contig_pfn_list[probe_pos + 1], 2409 &contig_pfn_list[probe_pos], 2410 (contig_pfn_cnt - probe_pos) * sizeof (pfn_t)); 2411 } 2412 if (newmfn == MFN_INVALID) 2413 goto done; 2414 /* 2415 * Check if new mfn has adjacent mfns in the list 2416 */ 2417 probe_hi = contig_pfn_cnt - 1; 2418 probe_lo = 0; 2419 insert_after = -2; 2420 do { 2421 probe_pos = (probe_hi + probe_lo) / 2; 2422 probe_mfn = pfn_to_mfn(contig_pfn_list[probe_pos]); 2423 if (newmfn == probe_mfn + 1) 2424 insert_after = probe_pos; 2425 else if (newmfn == probe_mfn - 1) 2426 insert_after = probe_pos - 1; 2427 if (probe_pos == probe_lo) 2428 break; 2429 if (probe_mfn <= newmfn) 2430 probe_lo = probe_pos; 2431 else 2432 probe_hi = probe_pos; 2433 } while (insert_after == -2); 2434 /* 2435 * If there is space in the list and there are adjacent mfns 2436 * insert the pfn in to its proper place in the list. 2437 */ 2438 if (insert_after != -2 && contig_pfn_cnt + 1 <= contig_pfn_max) { 2439 insert_point = insert_after + 1; 2440 ovbcopy(&contig_pfn_list[insert_point], 2441 &contig_pfn_list[insert_point + 1], 2442 (contig_pfn_cnt - insert_point) * sizeof (pfn_t)); 2443 contig_pfn_list[insert_point] = pfn; 2444 contig_pfn_cnt++; 2445 } 2446 done: 2447 if (drop_lock) 2448 mutex_exit(&contig_list_lock); 2449 } 2450 2451 /* 2452 * Called to (re-)populate the io_pool from the free page lists. 2453 */ 2454 long 2455 populate_io_pool(void) 2456 { 2457 pfn_t pfn; 2458 mfn_t mfn, max_mfn; 2459 page_t *pp; 2460 2461 /* 2462 * Figure out the bounds of the pool on first invocation. 2463 * We use a percentage of memory for the io pool size. 2464 * we allow that to shrink, but not to less than a fixed minimum 2465 */ 2466 if (io_pool_cnt_max == 0) { 2467 io_pool_cnt_max = physmem / (100 / io_pool_physmem_pct); 2468 io_pool_cnt_lowater = io_pool_cnt_max; 2469 /* 2470 * This is the first time in populate_io_pool, grab a va to use 2471 * when we need to allocate pages. 2472 */ 2473 io_pool_kva = vmem_alloc(heap_arena, PAGESIZE, VM_SLEEP); 2474 } 2475 /* 2476 * If we are out of pages in the pool, then grow the size of the pool 2477 */ 2478 if (io_pool_cnt == 0) { 2479 /* 2480 * Grow the max size of the io pool by 5%, but never more than 2481 * 25% of physical memory. 2482 */ 2483 if (io_pool_cnt_max < physmem / 4) 2484 io_pool_cnt_max += io_pool_cnt_max / 20; 2485 } 2486 io_pool_grows++; /* should be a kstat? */ 2487 2488 /* 2489 * Get highest mfn on this platform, but limit to the 32 bit DMA max. 2490 */ 2491 (void) mfn_to_pfn(start_mfn); 2492 max_mfn = MIN(cached_max_mfn, PFN_4GIG); 2493 for (mfn = start_mfn; mfn < max_mfn; start_mfn = ++mfn) { 2494 pfn = mfn_to_pfn(mfn); 2495 if (pfn & PFN_IS_FOREIGN_MFN) 2496 continue; 2497 /* 2498 * try to allocate it from free pages 2499 */ 2500 pp = page_numtopp_alloc(pfn); 2501 if (pp == NULL) 2502 continue; 2503 PP_CLRFREE(pp); 2504 add_page_to_pool(pp, 1); 2505 if (io_pool_cnt >= io_pool_cnt_max) 2506 break; 2507 } 2508 2509 return (io_pool_cnt); 2510 } 2511 2512 /* 2513 * Destroy a page that was being used for DMA I/O. It may or 2514 * may not actually go back to the io_pool. 2515 */ 2516 void 2517 page_destroy_io(page_t *pp) 2518 { 2519 mfn_t mfn = mfn_list[pp->p_pagenum]; 2520 2521 /* 2522 * When the page was alloc'd a reservation was made, release it now 2523 */ 2524 page_unresv(1); 2525 /* 2526 * Unload translations, if any, then hash out the 2527 * page to erase its identity. 2528 */ 2529 (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD); 2530 page_hashout(pp, NULL); 2531 2532 /* 2533 * If the page came from the free lists, just put it back to them. 2534 * DomU pages always go on the free lists as well. 2535 */ 2536 if (!DOMAIN_IS_INITDOMAIN(xen_info) || mfn >= PFN_4GIG) { 2537 page_free(pp, 1); 2538 return; 2539 } 2540 2541 add_page_to_pool(pp, 0); 2542 } 2543 2544 2545 long contig_searches; /* count of times contig pages requested */ 2546 long contig_search_restarts; /* count of contig ranges tried */ 2547 long contig_search_failed; /* count of contig alloc failures */ 2548 2549 /* 2550 * Free partial page list 2551 */ 2552 static void 2553 free_partial_list(page_t **pplist) 2554 { 2555 page_t *pp; 2556 2557 while (*pplist != NULL) { 2558 pp = *pplist; 2559 page_io_pool_sub(pplist, pp, pp); 2560 page_free(pp, 1); 2561 } 2562 } 2563 2564 /* 2565 * Look thru the contiguous pfns that are not part of the io_pool for 2566 * contiguous free pages. Return a list of the found pages or NULL. 2567 */ 2568 page_t * 2569 find_contig_free(uint_t npages, uint_t flags, uint64_t pfnseg, 2570 pgcnt_t pfnalign) 2571 { 2572 page_t *pp, *plist = NULL; 2573 mfn_t mfn, prev_mfn, start_mfn; 2574 pfn_t pfn; 2575 int pages_needed, pages_requested; 2576 int search_start; 2577 2578 /* 2579 * create the contig pfn list if not already done 2580 */ 2581 retry: 2582 mutex_enter(&contig_list_lock); 2583 if (contig_pfn_list == NULL) { 2584 mutex_exit(&contig_list_lock); 2585 if (!create_contig_pfnlist(flags)) { 2586 return (NULL); 2587 } 2588 goto retry; 2589 } 2590 contig_searches++; 2591 /* 2592 * Search contiguous pfn list for physically contiguous pages not in 2593 * the io_pool. Start the search where the last search left off. 2594 */ 2595 pages_requested = pages_needed = npages; 2596 search_start = next_alloc_pfn; 2597 start_mfn = prev_mfn = 0; 2598 while (pages_needed) { 2599 pfn = contig_pfn_list[next_alloc_pfn]; 2600 mfn = pfn_to_mfn(pfn); 2601 /* 2602 * Check if mfn is first one or contig to previous one and 2603 * if page corresponding to mfn is free and that mfn 2604 * range is not crossing a segment boundary. 2605 */ 2606 if ((prev_mfn == 0 || mfn == prev_mfn + 1) && 2607 (pp = page_numtopp_alloc(pfn)) != NULL && 2608 !((mfn & pfnseg) < (start_mfn & pfnseg))) { 2609 PP_CLRFREE(pp); 2610 page_io_pool_add(&plist, pp); 2611 pages_needed--; 2612 if (prev_mfn == 0) { 2613 if (pfnalign && 2614 mfn != P2ROUNDUP(mfn, pfnalign)) { 2615 /* 2616 * not properly aligned 2617 */ 2618 contig_search_restarts++; 2619 free_partial_list(&plist); 2620 pages_needed = pages_requested; 2621 start_mfn = prev_mfn = 0; 2622 goto skip; 2623 } 2624 start_mfn = mfn; 2625 } 2626 prev_mfn = mfn; 2627 } else { 2628 contig_search_restarts++; 2629 free_partial_list(&plist); 2630 pages_needed = pages_requested; 2631 start_mfn = prev_mfn = 0; 2632 } 2633 skip: 2634 if (++next_alloc_pfn == contig_pfn_cnt) 2635 next_alloc_pfn = 0; 2636 if (next_alloc_pfn == search_start) 2637 break; /* all pfns searched */ 2638 } 2639 mutex_exit(&contig_list_lock); 2640 if (pages_needed) { 2641 contig_search_failed++; 2642 /* 2643 * Failed to find enough contig pages. 2644 * free partial page list 2645 */ 2646 free_partial_list(&plist); 2647 } 2648 return (plist); 2649 } 2650 2651 /* 2652 * Search the reserved io pool pages for a page range with the 2653 * desired characteristics. 2654 */ 2655 page_t * 2656 page_io_pool_alloc(ddi_dma_attr_t *mattr, int contig, pgcnt_t minctg) 2657 { 2658 page_t *pp_first, *pp_last; 2659 page_t *pp, **poolp; 2660 pgcnt_t nwanted, pfnalign; 2661 uint64_t pfnseg; 2662 mfn_t mfn, tmfn, hi_mfn, lo_mfn; 2663 int align, attempt = 0; 2664 2665 if (minctg == 1) 2666 contig = 0; 2667 lo_mfn = mmu_btop(mattr->dma_attr_addr_lo); 2668 hi_mfn = mmu_btop(mattr->dma_attr_addr_hi); 2669 pfnseg = mmu_btop(mattr->dma_attr_seg); 2670 align = maxbit(mattr->dma_attr_align, mattr->dma_attr_minxfer); 2671 if (align > MMU_PAGESIZE) 2672 pfnalign = mmu_btop(align); 2673 else 2674 pfnalign = 0; 2675 2676 try_again: 2677 /* 2678 * See if we want pages for a legacy device 2679 */ 2680 if (hi_mfn < PFN_16MEG) 2681 poolp = &io_pool_16m; 2682 else 2683 poolp = &io_pool_4g; 2684 try_smaller: 2685 /* 2686 * Take pages from I/O pool. We'll use pages from the highest 2687 * MFN range possible. 2688 */ 2689 pp_first = pp_last = NULL; 2690 mutex_enter(&io_pool_lock); 2691 nwanted = minctg; 2692 for (pp = *poolp; pp && nwanted > 0; ) { 2693 pp = pp->p_prev; 2694 2695 /* 2696 * skip pages above allowable range 2697 */ 2698 mfn = mfn_list[pp->p_pagenum]; 2699 if (hi_mfn < mfn) 2700 goto skip; 2701 2702 /* 2703 * stop at pages below allowable range 2704 */ 2705 if (lo_mfn > mfn) 2706 break; 2707 restart: 2708 if (pp_last == NULL) { 2709 /* 2710 * Check alignment 2711 */ 2712 tmfn = mfn - (minctg - 1); 2713 if (pfnalign && tmfn != P2ROUNDUP(tmfn, pfnalign)) 2714 goto skip; /* not properly aligned */ 2715 /* 2716 * Check segment 2717 */ 2718 if ((mfn & pfnseg) < (tmfn & pfnseg)) 2719 goto skip; /* crosses seg boundary */ 2720 /* 2721 * Start building page list 2722 */ 2723 pp_first = pp_last = pp; 2724 nwanted--; 2725 } else { 2726 /* 2727 * check physical contiguity if required 2728 */ 2729 if (contig && 2730 mfn_list[pp_first->p_pagenum] != mfn + 1) { 2731 /* 2732 * not a contiguous page, restart list. 2733 */ 2734 pp_last = NULL; 2735 nwanted = minctg; 2736 goto restart; 2737 } else { /* add page to list */ 2738 pp_first = pp; 2739 nwanted--; 2740 } 2741 } 2742 skip: 2743 if (pp == *poolp) 2744 break; 2745 } 2746 2747 /* 2748 * If we didn't find memory. Try the more constrained pool, then 2749 * sweep free pages into the DMA pool and try again. 2750 */ 2751 if (nwanted != 0) { 2752 mutex_exit(&io_pool_lock); 2753 /* 2754 * If we were looking in the less constrained pool and 2755 * didn't find pages, try the more constrained pool. 2756 */ 2757 if (poolp == &io_pool_4g) { 2758 poolp = &io_pool_16m; 2759 goto try_smaller; 2760 } 2761 kmem_reap(); 2762 if (++attempt < 4) { 2763 /* 2764 * Grab some more io_pool pages 2765 */ 2766 (void) populate_io_pool(); 2767 goto try_again; /* go around and retry */ 2768 } 2769 return (NULL); 2770 } 2771 /* 2772 * Found the pages, now snip them from the list 2773 */ 2774 page_io_pool_sub(poolp, pp_first, pp_last); 2775 io_pool_cnt -= minctg; 2776 /* 2777 * reset low water mark 2778 */ 2779 if (io_pool_cnt < io_pool_cnt_lowater) 2780 io_pool_cnt_lowater = io_pool_cnt; 2781 mutex_exit(&io_pool_lock); 2782 return (pp_first); 2783 } 2784 2785 page_t * 2786 page_swap_with_hypervisor(struct vnode *vp, u_offset_t off, caddr_t vaddr, 2787 ddi_dma_attr_t *mattr, uint_t flags, pgcnt_t minctg) 2788 { 2789 uint_t kflags; 2790 int order, extra, extpages, i, contig, nbits, extents; 2791 page_t *pp, *expp, *pp_first, **pplist = NULL; 2792 mfn_t *mfnlist = NULL; 2793 2794 contig = flags & PG_PHYSCONTIG; 2795 if (minctg == 1) 2796 contig = 0; 2797 flags &= ~PG_PHYSCONTIG; 2798 kflags = flags & PG_WAIT ? KM_SLEEP : KM_NOSLEEP; 2799 /* 2800 * Hypervisor will allocate extents, if we want contig 2801 * pages extent must be >= minctg 2802 */ 2803 if (contig) { 2804 order = highbit(minctg) - 1; 2805 if (minctg & ((1 << order) - 1)) 2806 order++; 2807 extpages = 1 << order; 2808 } else { 2809 order = 0; 2810 extpages = minctg; 2811 } 2812 if (extpages > minctg) { 2813 extra = extpages - minctg; 2814 if (!page_resv(extra, kflags)) 2815 return (NULL); 2816 } 2817 pp_first = NULL; 2818 pplist = kmem_alloc(extpages * sizeof (page_t *), kflags); 2819 if (pplist == NULL) 2820 goto balloon_fail; 2821 mfnlist = kmem_alloc(extpages * sizeof (mfn_t), kflags); 2822 if (mfnlist == NULL) 2823 goto balloon_fail; 2824 pp = page_create_va(vp, off, minctg * PAGESIZE, flags, &kvseg, vaddr); 2825 if (pp == NULL) 2826 goto balloon_fail; 2827 pp_first = pp; 2828 if (extpages > minctg) { 2829 /* 2830 * fill out the rest of extent pages to swap 2831 * with the hypervisor 2832 */ 2833 for (i = 0; i < extra; i++) { 2834 expp = page_create_va(vp, 2835 (u_offset_t)(uintptr_t)io_pool_kva, 2836 PAGESIZE, flags, &kvseg, io_pool_kva); 2837 if (expp == NULL) 2838 goto balloon_fail; 2839 (void) hat_pageunload(expp, HAT_FORCE_PGUNLOAD); 2840 page_io_unlock(expp); 2841 page_hashout(expp, NULL); 2842 page_io_lock(expp); 2843 /* 2844 * add page to end of list 2845 */ 2846 expp->p_prev = pp_first->p_prev; 2847 expp->p_next = pp_first; 2848 expp->p_prev->p_next = expp; 2849 pp_first->p_prev = expp; 2850 } 2851 2852 } 2853 for (i = 0; i < extpages; i++) { 2854 pplist[i] = pp; 2855 pp = pp->p_next; 2856 } 2857 nbits = highbit(mattr->dma_attr_addr_hi); 2858 extents = contig ? 1 : minctg; 2859 if (balloon_replace_pages(extents, pplist, nbits, order, 2860 mfnlist) != extents) { 2861 if (ioalloc_dbg) 2862 cmn_err(CE_NOTE, "request to hypervisor" 2863 " for %d pages, maxaddr %" PRIx64 " failed", 2864 extpages, mattr->dma_attr_addr_hi); 2865 goto balloon_fail; 2866 } 2867 2868 kmem_free(pplist, extpages * sizeof (page_t *)); 2869 kmem_free(mfnlist, extpages * sizeof (mfn_t)); 2870 /* 2871 * Return any excess pages to free list 2872 */ 2873 if (extpages > minctg) { 2874 for (i = 0; i < extra; i++) { 2875 pp = pp_first->p_prev; 2876 page_sub(&pp_first, pp); 2877 page_io_unlock(pp); 2878 page_unresv(1); 2879 page_free(pp, 1); 2880 } 2881 } 2882 return (pp_first); 2883 balloon_fail: 2884 /* 2885 * Return pages to free list and return failure 2886 */ 2887 while (pp_first != NULL) { 2888 pp = pp_first; 2889 page_sub(&pp_first, pp); 2890 page_io_unlock(pp); 2891 if (pp->p_vnode != NULL) 2892 page_hashout(pp, NULL); 2893 page_free(pp, 1); 2894 } 2895 if (pplist) 2896 kmem_free(pplist, extpages * sizeof (page_t *)); 2897 if (mfnlist) 2898 kmem_free(mfnlist, extpages * sizeof (mfn_t)); 2899 page_unresv(extpages - minctg); 2900 return (NULL); 2901 } 2902 2903 static void 2904 return_partial_alloc(page_t *plist) 2905 { 2906 page_t *pp; 2907 2908 while (plist != NULL) { 2909 pp = plist; 2910 page_sub(&plist, pp); 2911 page_io_unlock(pp); 2912 page_destroy_io(pp); 2913 } 2914 } 2915 2916 static page_t * 2917 page_get_contigpages( 2918 struct vnode *vp, 2919 u_offset_t off, 2920 int *npagesp, 2921 uint_t flags, 2922 caddr_t vaddr, 2923 ddi_dma_attr_t *mattr) 2924 { 2925 mfn_t max_mfn = HYPERVISOR_memory_op(XENMEM_maximum_ram_page, NULL); 2926 page_t *plist; /* list to return */ 2927 page_t *pp, *mcpl; 2928 int contig, anyaddr, npages, getone = 0; 2929 mfn_t lo_mfn; 2930 mfn_t hi_mfn; 2931 pgcnt_t pfnalign = 0; 2932 int align, sgllen; 2933 uint64_t pfnseg; 2934 pgcnt_t minctg; 2935 2936 npages = *npagesp; 2937 ASSERT(mattr != NULL); 2938 lo_mfn = mmu_btop(mattr->dma_attr_addr_lo); 2939 hi_mfn = mmu_btop(mattr->dma_attr_addr_hi); 2940 sgllen = mattr->dma_attr_sgllen; 2941 pfnseg = mmu_btop(mattr->dma_attr_seg); 2942 align = maxbit(mattr->dma_attr_align, mattr->dma_attr_minxfer); 2943 if (align > MMU_PAGESIZE) 2944 pfnalign = mmu_btop(align); 2945 2946 contig = flags & PG_PHYSCONTIG; 2947 if (npages == -1) { 2948 npages = 1; 2949 pfnalign = 0; 2950 } 2951 /* 2952 * Clear the contig flag if only one page is needed. 2953 */ 2954 if (npages == 1) { 2955 getone = 1; 2956 contig = 0; 2957 } 2958 2959 /* 2960 * Check if any page in the system is fine. 2961 */ 2962 anyaddr = lo_mfn == 0 && hi_mfn >= max_mfn; 2963 if (!contig && anyaddr && !pfnalign) { 2964 flags &= ~PG_PHYSCONTIG; 2965 plist = page_create_va(vp, off, npages * MMU_PAGESIZE, 2966 flags, &kvseg, vaddr); 2967 if (plist != NULL) { 2968 *npagesp = 0; 2969 return (plist); 2970 } 2971 } 2972 plist = NULL; 2973 minctg = howmany(npages, sgllen); 2974 while (npages > sgllen || getone) { 2975 if (minctg > npages) 2976 minctg = npages; 2977 mcpl = NULL; 2978 /* 2979 * We could want contig pages with no address range limits. 2980 */ 2981 if (anyaddr && contig) { 2982 /* 2983 * Look for free contig pages to satisfy the request. 2984 */ 2985 mcpl = find_contig_free(minctg, flags, pfnseg, 2986 pfnalign); 2987 } 2988 /* 2989 * Try the reserved io pools next 2990 */ 2991 if (mcpl == NULL) 2992 mcpl = page_io_pool_alloc(mattr, contig, minctg); 2993 if (mcpl != NULL) { 2994 pp = mcpl; 2995 do { 2996 if (!page_hashin(pp, vp, off, NULL)) { 2997 panic("page_get_contigpages:" 2998 " hashin failed" 2999 " pp %p, vp %p, off %llx", 3000 (void *)pp, (void *)vp, off); 3001 } 3002 off += MMU_PAGESIZE; 3003 PP_CLRFREE(pp); 3004 PP_CLRAGED(pp); 3005 page_set_props(pp, P_REF); 3006 page_io_lock(pp); 3007 pp = pp->p_next; 3008 } while (pp != mcpl); 3009 } else { 3010 /* 3011 * Hypervisor exchange doesn't handle segment or 3012 * alignment constraints 3013 */ 3014 if (mattr->dma_attr_seg < mattr->dma_attr_addr_hi || 3015 pfnalign) 3016 goto fail; 3017 /* 3018 * Try exchanging pages with the hypervisor 3019 */ 3020 mcpl = page_swap_with_hypervisor(vp, off, vaddr, mattr, 3021 flags, minctg); 3022 if (mcpl == NULL) 3023 goto fail; 3024 off += minctg * MMU_PAGESIZE; 3025 } 3026 check_dma(mattr, mcpl, minctg); 3027 /* 3028 * Here with a minctg run of contiguous pages, add them to the 3029 * list we will return for this request. 3030 */ 3031 page_list_concat(&plist, &mcpl); 3032 npages -= minctg; 3033 *npagesp = npages; 3034 sgllen--; 3035 if (getone) 3036 break; 3037 } 3038 return (plist); 3039 fail: 3040 return_partial_alloc(plist); 3041 return (NULL); 3042 } 3043 3044 /* 3045 * Allocator for domain 0 I/O pages. We match the required 3046 * DMA attributes and contiguity constraints. 3047 */ 3048 /*ARGSUSED*/ 3049 page_t * 3050 page_create_io( 3051 struct vnode *vp, 3052 u_offset_t off, 3053 uint_t bytes, 3054 uint_t flags, 3055 struct as *as, 3056 caddr_t vaddr, 3057 ddi_dma_attr_t *mattr) 3058 { 3059 page_t *plist = NULL, *pp; 3060 int npages = 0, contig, anyaddr, pages_req; 3061 mfn_t lo_mfn; 3062 mfn_t hi_mfn; 3063 pgcnt_t pfnalign = 0; 3064 int align; 3065 int is_domu = 0; 3066 int dummy, bytes_got; 3067 mfn_t max_mfn = HYPERVISOR_memory_op(XENMEM_maximum_ram_page, NULL); 3068 3069 ASSERT(mattr != NULL); 3070 lo_mfn = mmu_btop(mattr->dma_attr_addr_lo); 3071 hi_mfn = mmu_btop(mattr->dma_attr_addr_hi); 3072 align = maxbit(mattr->dma_attr_align, mattr->dma_attr_minxfer); 3073 if (align > MMU_PAGESIZE) 3074 pfnalign = mmu_btop(align); 3075 3076 /* 3077 * Clear the contig flag if only one page is needed or the scatter 3078 * gather list length is >= npages. 3079 */ 3080 pages_req = npages = mmu_btopr(bytes); 3081 contig = (flags & PG_PHYSCONTIG); 3082 bytes = P2ROUNDUP(bytes, MMU_PAGESIZE); 3083 if (bytes == MMU_PAGESIZE || mattr->dma_attr_sgllen >= npages) 3084 contig = 0; 3085 3086 /* 3087 * Check if any old page in the system is fine. 3088 * DomU should always go down this path. 3089 */ 3090 is_domu = !DOMAIN_IS_INITDOMAIN(xen_info); 3091 anyaddr = lo_mfn == 0 && hi_mfn >= max_mfn && !pfnalign; 3092 if ((!contig && anyaddr) || is_domu) { 3093 flags &= ~PG_PHYSCONTIG; 3094 plist = page_create_va(vp, off, bytes, flags, &kvseg, vaddr); 3095 if (plist != NULL) 3096 return (plist); 3097 else if (is_domu) 3098 return (NULL); /* no memory available */ 3099 } 3100 /* 3101 * DomU should never reach here 3102 */ 3103 if (contig) { 3104 plist = page_get_contigpages(vp, off, &npages, flags, vaddr, 3105 mattr); 3106 if (plist == NULL) 3107 goto fail; 3108 bytes_got = (pages_req - npages) << MMU_PAGESHIFT; 3109 vaddr += bytes_got; 3110 off += bytes_got; 3111 /* 3112 * We now have all the contiguous pages we need, but 3113 * we may still need additional non-contiguous pages. 3114 */ 3115 } 3116 /* 3117 * now loop collecting the requested number of pages, these do 3118 * not have to be contiguous pages but we will use the contig 3119 * page alloc code to get the pages since it will honor any 3120 * other constraints the pages may have. 3121 */ 3122 while (npages--) { 3123 dummy = -1; 3124 pp = page_get_contigpages(vp, off, &dummy, flags, vaddr, mattr); 3125 if (pp == NULL) 3126 goto fail; 3127 page_add(&plist, pp); 3128 vaddr += MMU_PAGESIZE; 3129 off += MMU_PAGESIZE; 3130 } 3131 return (plist); 3132 fail: 3133 /* 3134 * Failed to get enough pages, return ones we did get 3135 */ 3136 return_partial_alloc(plist); 3137 return (NULL); 3138 } 3139 3140 /* 3141 * Lock and return the page with the highest mfn that we can find. last_mfn 3142 * holds the last one found, so the next search can start from there. We 3143 * also keep a counter so that we don't loop forever if the machine has no 3144 * free pages. 3145 * 3146 * This is called from the balloon thread to find pages to give away. new_high 3147 * is used when new mfn's have been added to the system - we will reset our 3148 * search if the new mfn's are higher than our current search position. 3149 */ 3150 page_t * 3151 page_get_high_mfn(mfn_t new_high) 3152 { 3153 static mfn_t last_mfn = 0; 3154 pfn_t pfn; 3155 page_t *pp; 3156 ulong_t loop_count = 0; 3157 3158 if (new_high > last_mfn) 3159 last_mfn = new_high; 3160 3161 for (; loop_count < mfn_count; loop_count++, last_mfn--) { 3162 if (last_mfn == 0) { 3163 last_mfn = cached_max_mfn; 3164 } 3165 3166 pfn = mfn_to_pfn(last_mfn); 3167 if (pfn & PFN_IS_FOREIGN_MFN) 3168 continue; 3169 3170 /* See if the page is free. If so, lock it. */ 3171 pp = page_numtopp_alloc(pfn); 3172 if (pp == NULL) 3173 continue; 3174 PP_CLRFREE(pp); 3175 3176 ASSERT(PAGE_EXCL(pp)); 3177 ASSERT(pp->p_vnode == NULL); 3178 ASSERT(!hat_page_is_mapped(pp)); 3179 last_mfn--; 3180 return (pp); 3181 } 3182 return (NULL); 3183 } 3184 3185 #else /* !__xpv */ 3186 3187 /* 3188 * get a page from any list with the given mnode 3189 */ 3190 static page_t * 3191 page_get_mnode_anylist(ulong_t origbin, uchar_t szc, uint_t flags, 3192 int mnode, int mtype, ddi_dma_attr_t *dma_attr) 3193 { 3194 kmutex_t *pcm; 3195 int i; 3196 page_t *pp; 3197 page_t *first_pp; 3198 uint64_t pgaddr; 3199 ulong_t bin; 3200 int mtypestart; 3201 int plw_initialized; 3202 page_list_walker_t plw; 3203 3204 VM_STAT_ADD(pga_vmstats.pgma_alloc); 3205 3206 ASSERT((flags & PG_MATCH_COLOR) == 0); 3207 ASSERT(szc == 0); 3208 ASSERT(dma_attr != NULL); 3209 3210 MTYPE_START(mnode, mtype, flags); 3211 if (mtype < 0) { 3212 VM_STAT_ADD(pga_vmstats.pgma_allocempty); 3213 return (NULL); 3214 } 3215 3216 mtypestart = mtype; 3217 3218 bin = origbin; 3219 3220 /* 3221 * check up to page_colors + 1 bins - origbin may be checked twice 3222 * because of BIN_STEP skip 3223 */ 3224 do { 3225 plw_initialized = 0; 3226 3227 for (plw.plw_count = 0; 3228 plw.plw_count < page_colors; plw.plw_count++) { 3229 3230 if (PAGE_FREELISTS(mnode, szc, bin, mtype) == NULL) 3231 goto nextfreebin; 3232 3233 pcm = PC_BIN_MUTEX(mnode, bin, PG_FREE_LIST); 3234 mutex_enter(pcm); 3235 pp = PAGE_FREELISTS(mnode, szc, bin, mtype); 3236 first_pp = pp; 3237 while (pp != NULL) { 3238 if (IS_DUMP_PAGE(pp) || page_trylock(pp, 3239 SE_EXCL) == 0) { 3240 pp = pp->p_next; 3241 if (pp == first_pp) { 3242 pp = NULL; 3243 } 3244 continue; 3245 } 3246 3247 ASSERT(PP_ISFREE(pp)); 3248 ASSERT(PP_ISAGED(pp)); 3249 ASSERT(pp->p_vnode == NULL); 3250 ASSERT(pp->p_hash == NULL); 3251 ASSERT(pp->p_offset == (u_offset_t)-1); 3252 ASSERT(pp->p_szc == szc); 3253 ASSERT(PFN_2_MEM_NODE(pp->p_pagenum) == mnode); 3254 /* check if page within DMA attributes */ 3255 pgaddr = pa_to_ma(pfn_to_pa(pp->p_pagenum)); 3256 if ((pgaddr >= dma_attr->dma_attr_addr_lo) && 3257 (pgaddr + MMU_PAGESIZE - 1 <= 3258 dma_attr->dma_attr_addr_hi)) { 3259 break; 3260 } 3261 3262 /* continue looking */ 3263 page_unlock(pp); 3264 pp = pp->p_next; 3265 if (pp == first_pp) 3266 pp = NULL; 3267 3268 } 3269 if (pp != NULL) { 3270 ASSERT(mtype == PP_2_MTYPE(pp)); 3271 ASSERT(pp->p_szc == 0); 3272 3273 /* found a page with specified DMA attributes */ 3274 page_sub(&PAGE_FREELISTS(mnode, szc, bin, 3275 mtype), pp); 3276 page_ctr_sub(mnode, mtype, pp, PG_FREE_LIST); 3277 3278 if ((PP_ISFREE(pp) == 0) || 3279 (PP_ISAGED(pp) == 0)) { 3280 cmn_err(CE_PANIC, "page %p is not free", 3281 (void *)pp); 3282 } 3283 3284 mutex_exit(pcm); 3285 check_dma(dma_attr, pp, 1); 3286 VM_STAT_ADD(pga_vmstats.pgma_allocok); 3287 return (pp); 3288 } 3289 mutex_exit(pcm); 3290 nextfreebin: 3291 if (plw_initialized == 0) { 3292 page_list_walk_init(szc, 0, bin, 1, 0, &plw); 3293 ASSERT(plw.plw_ceq_dif == page_colors); 3294 plw_initialized = 1; 3295 } 3296 3297 if (plw.plw_do_split) { 3298 pp = page_freelist_split(szc, bin, mnode, 3299 mtype, 3300 mmu_btop(dma_attr->dma_attr_addr_lo), 3301 mmu_btop(dma_attr->dma_attr_addr_hi + 1), 3302 &plw); 3303 if (pp != NULL) { 3304 check_dma(dma_attr, pp, 1); 3305 return (pp); 3306 } 3307 } 3308 3309 bin = page_list_walk_next_bin(szc, bin, &plw); 3310 } 3311 3312 MTYPE_NEXT(mnode, mtype, flags); 3313 } while (mtype >= 0); 3314 3315 /* failed to find a page in the freelist; try it in the cachelist */ 3316 3317 /* reset mtype start for cachelist search */ 3318 mtype = mtypestart; 3319 ASSERT(mtype >= 0); 3320 3321 /* start with the bin of matching color */ 3322 bin = origbin; 3323 3324 do { 3325 for (i = 0; i <= page_colors; i++) { 3326 if (PAGE_CACHELISTS(mnode, bin, mtype) == NULL) 3327 goto nextcachebin; 3328 pcm = PC_BIN_MUTEX(mnode, bin, PG_CACHE_LIST); 3329 mutex_enter(pcm); 3330 pp = PAGE_CACHELISTS(mnode, bin, mtype); 3331 first_pp = pp; 3332 while (pp != NULL) { 3333 if (IS_DUMP_PAGE(pp) || page_trylock(pp, 3334 SE_EXCL) == 0) { 3335 pp = pp->p_next; 3336 if (pp == first_pp) 3337 pp = NULL; 3338 continue; 3339 } 3340 ASSERT(pp->p_vnode); 3341 ASSERT(PP_ISAGED(pp) == 0); 3342 ASSERT(pp->p_szc == 0); 3343 ASSERT(PFN_2_MEM_NODE(pp->p_pagenum) == mnode); 3344 3345 /* check if page within DMA attributes */ 3346 3347 pgaddr = pa_to_ma(pfn_to_pa(pp->p_pagenum)); 3348 if ((pgaddr >= dma_attr->dma_attr_addr_lo) && 3349 (pgaddr + MMU_PAGESIZE - 1 <= 3350 dma_attr->dma_attr_addr_hi)) { 3351 break; 3352 } 3353 3354 /* continue looking */ 3355 page_unlock(pp); 3356 pp = pp->p_next; 3357 if (pp == first_pp) 3358 pp = NULL; 3359 } 3360 3361 if (pp != NULL) { 3362 ASSERT(mtype == PP_2_MTYPE(pp)); 3363 ASSERT(pp->p_szc == 0); 3364 3365 /* found a page with specified DMA attributes */ 3366 page_sub(&PAGE_CACHELISTS(mnode, bin, 3367 mtype), pp); 3368 page_ctr_sub(mnode, mtype, pp, PG_CACHE_LIST); 3369 3370 mutex_exit(pcm); 3371 ASSERT(pp->p_vnode); 3372 ASSERT(PP_ISAGED(pp) == 0); 3373 check_dma(dma_attr, pp, 1); 3374 VM_STAT_ADD(pga_vmstats.pgma_allocok); 3375 return (pp); 3376 } 3377 mutex_exit(pcm); 3378 nextcachebin: 3379 bin += (i == 0) ? BIN_STEP : 1; 3380 bin &= page_colors_mask; 3381 } 3382 MTYPE_NEXT(mnode, mtype, flags); 3383 } while (mtype >= 0); 3384 3385 VM_STAT_ADD(pga_vmstats.pgma_allocfailed); 3386 return (NULL); 3387 } 3388 3389 /* 3390 * This function is similar to page_get_freelist()/page_get_cachelist() 3391 * but it searches both the lists to find a page with the specified 3392 * color (or no color) and DMA attributes. The search is done in the 3393 * freelist first and then in the cache list within the highest memory 3394 * range (based on DMA attributes) before searching in the lower 3395 * memory ranges. 3396 * 3397 * Note: This function is called only by page_create_io(). 3398 */ 3399 /*ARGSUSED*/ 3400 static page_t * 3401 page_get_anylist(struct vnode *vp, u_offset_t off, struct as *as, caddr_t vaddr, 3402 size_t size, uint_t flags, ddi_dma_attr_t *dma_attr, lgrp_t *lgrp) 3403 { 3404 uint_t bin; 3405 int mtype; 3406 page_t *pp; 3407 int n; 3408 int m; 3409 int szc; 3410 int fullrange; 3411 int mnode; 3412 int local_failed_stat = 0; 3413 lgrp_mnode_cookie_t lgrp_cookie; 3414 3415 VM_STAT_ADD(pga_vmstats.pga_alloc); 3416 3417 /* only base pagesize currently supported */ 3418 if (size != MMU_PAGESIZE) 3419 return (NULL); 3420 3421 /* 3422 * If we're passed a specific lgroup, we use it. Otherwise, 3423 * assume first-touch placement is desired. 3424 */ 3425 if (!LGRP_EXISTS(lgrp)) 3426 lgrp = lgrp_home_lgrp(); 3427 3428 /* LINTED */ 3429 AS_2_BIN(as, seg, vp, vaddr, bin, 0); 3430 3431 /* 3432 * Only hold one freelist or cachelist lock at a time, that way we 3433 * can start anywhere and not have to worry about lock 3434 * ordering. 3435 */ 3436 if (dma_attr == NULL) { 3437 n = mtype16m; 3438 m = mtypetop; 3439 fullrange = 1; 3440 VM_STAT_ADD(pga_vmstats.pga_nulldmaattr); 3441 } else { 3442 pfn_t pfnlo = mmu_btop(dma_attr->dma_attr_addr_lo); 3443 pfn_t pfnhi = mmu_btop(dma_attr->dma_attr_addr_hi); 3444 3445 /* 3446 * We can guarantee alignment only for page boundary. 3447 */ 3448 if (dma_attr->dma_attr_align > MMU_PAGESIZE) 3449 return (NULL); 3450 3451 /* Sanity check the dma_attr */ 3452 if (pfnlo > pfnhi) 3453 return (NULL); 3454 3455 n = pfn_2_mtype(pfnlo); 3456 m = pfn_2_mtype(pfnhi); 3457 3458 fullrange = ((pfnlo == mnoderanges[n].mnr_pfnlo) && 3459 (pfnhi >= mnoderanges[m].mnr_pfnhi)); 3460 } 3461 VM_STAT_COND_ADD(fullrange == 0, pga_vmstats.pga_notfullrange); 3462 3463 szc = 0; 3464 3465 /* cylcing thru mtype handled by RANGE0 if n == mtype16m */ 3466 if (n == mtype16m) { 3467 flags |= PGI_MT_RANGE0; 3468 n = m; 3469 } 3470 3471 /* 3472 * Try local memory node first, but try remote if we can't 3473 * get a page of the right color. 3474 */ 3475 LGRP_MNODE_COOKIE_INIT(lgrp_cookie, lgrp, LGRP_SRCH_HIER); 3476 while ((mnode = lgrp_memnode_choose(&lgrp_cookie)) >= 0) { 3477 /* 3478 * allocate pages from high pfn to low. 3479 */ 3480 mtype = m; 3481 do { 3482 if (fullrange != 0) { 3483 pp = page_get_mnode_freelist(mnode, 3484 bin, mtype, szc, flags); 3485 if (pp == NULL) { 3486 pp = page_get_mnode_cachelist( 3487 bin, flags, mnode, mtype); 3488 } 3489 } else { 3490 pp = page_get_mnode_anylist(bin, szc, 3491 flags, mnode, mtype, dma_attr); 3492 } 3493 if (pp != NULL) { 3494 VM_STAT_ADD(pga_vmstats.pga_allocok); 3495 check_dma(dma_attr, pp, 1); 3496 return (pp); 3497 } 3498 } while (mtype != n && 3499 (mtype = mnoderanges[mtype].mnr_next) != -1); 3500 if (!local_failed_stat) { 3501 lgrp_stat_add(lgrp->lgrp_id, LGRP_NUM_ALLOC_FAIL, 1); 3502 local_failed_stat = 1; 3503 } 3504 } 3505 VM_STAT_ADD(pga_vmstats.pga_allocfailed); 3506 3507 return (NULL); 3508 } 3509 3510 /* 3511 * page_create_io() 3512 * 3513 * This function is a copy of page_create_va() with an additional 3514 * argument 'mattr' that specifies DMA memory requirements to 3515 * the page list functions. This function is used by the segkmem 3516 * allocator so it is only to create new pages (i.e PG_EXCL is 3517 * set). 3518 * 3519 * Note: This interface is currently used by x86 PSM only and is 3520 * not fully specified so the commitment level is only for 3521 * private interface specific to x86. This interface uses PSM 3522 * specific page_get_anylist() interface. 3523 */ 3524 3525 #define PAGE_HASH_SEARCH(index, pp, vp, off) { \ 3526 for ((pp) = page_hash[(index)]; (pp); (pp) = (pp)->p_hash) { \ 3527 if ((pp)->p_vnode == (vp) && (pp)->p_offset == (off)) \ 3528 break; \ 3529 } \ 3530 } 3531 3532 3533 page_t * 3534 page_create_io( 3535 struct vnode *vp, 3536 u_offset_t off, 3537 uint_t bytes, 3538 uint_t flags, 3539 struct as *as, 3540 caddr_t vaddr, 3541 ddi_dma_attr_t *mattr) /* DMA memory attributes if any */ 3542 { 3543 page_t *plist = NULL; 3544 uint_t plist_len = 0; 3545 pgcnt_t npages; 3546 page_t *npp = NULL; 3547 uint_t pages_req; 3548 page_t *pp; 3549 kmutex_t *phm = NULL; 3550 uint_t index; 3551 3552 TRACE_4(TR_FAC_VM, TR_PAGE_CREATE_START, 3553 "page_create_start:vp %p off %llx bytes %u flags %x", 3554 vp, off, bytes, flags); 3555 3556 ASSERT((flags & ~(PG_EXCL | PG_WAIT | PG_PHYSCONTIG)) == 0); 3557 3558 pages_req = npages = mmu_btopr(bytes); 3559 3560 /* 3561 * Do the freemem and pcf accounting. 3562 */ 3563 if (!page_create_wait(npages, flags)) { 3564 return (NULL); 3565 } 3566 3567 TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_SUCCESS, 3568 "page_create_success:vp %p off %llx", vp, off); 3569 3570 /* 3571 * If satisfying this request has left us with too little 3572 * memory, start the wheels turning to get some back. The 3573 * first clause of the test prevents waking up the pageout 3574 * daemon in situations where it would decide that there's 3575 * nothing to do. 3576 */ 3577 if (nscan < desscan && freemem < minfree) { 3578 TRACE_1(TR_FAC_VM, TR_PAGEOUT_CV_SIGNAL, 3579 "pageout_cv_signal:freemem %ld", freemem); 3580 cv_signal(&proc_pageout->p_cv); 3581 } 3582 3583 if (flags & PG_PHYSCONTIG) { 3584 3585 plist = page_get_contigpage(&npages, mattr, 1); 3586 if (plist == NULL) { 3587 page_create_putback(npages); 3588 return (NULL); 3589 } 3590 3591 pp = plist; 3592 3593 do { 3594 if (!page_hashin(pp, vp, off, NULL)) { 3595 panic("pg_creat_io: hashin failed %p %p %llx", 3596 (void *)pp, (void *)vp, off); 3597 } 3598 VM_STAT_ADD(page_create_new); 3599 off += MMU_PAGESIZE; 3600 PP_CLRFREE(pp); 3601 PP_CLRAGED(pp); 3602 page_set_props(pp, P_REF); 3603 pp = pp->p_next; 3604 } while (pp != plist); 3605 3606 if (!npages) { 3607 check_dma(mattr, plist, pages_req); 3608 return (plist); 3609 } else { 3610 vaddr += (pages_req - npages) << MMU_PAGESHIFT; 3611 } 3612 3613 /* 3614 * fall-thru: 3615 * 3616 * page_get_contigpage returns when npages <= sgllen. 3617 * Grab the rest of the non-contig pages below from anylist. 3618 */ 3619 } 3620 3621 /* 3622 * Loop around collecting the requested number of pages. 3623 * Most of the time, we have to `create' a new page. With 3624 * this in mind, pull the page off the free list before 3625 * getting the hash lock. This will minimize the hash 3626 * lock hold time, nesting, and the like. If it turns 3627 * out we don't need the page, we put it back at the end. 3628 */ 3629 while (npages--) { 3630 phm = NULL; 3631 3632 index = PAGE_HASH_FUNC(vp, off); 3633 top: 3634 ASSERT(phm == NULL); 3635 ASSERT(index == PAGE_HASH_FUNC(vp, off)); 3636 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 3637 3638 if (npp == NULL) { 3639 /* 3640 * Try to get the page of any color either from 3641 * the freelist or from the cache list. 3642 */ 3643 npp = page_get_anylist(vp, off, as, vaddr, MMU_PAGESIZE, 3644 flags & ~PG_MATCH_COLOR, mattr, NULL); 3645 if (npp == NULL) { 3646 if (mattr == NULL) { 3647 /* 3648 * Not looking for a special page; 3649 * panic! 3650 */ 3651 panic("no page found %d", (int)npages); 3652 } 3653 /* 3654 * No page found! This can happen 3655 * if we are looking for a page 3656 * within a specific memory range 3657 * for DMA purposes. If PG_WAIT is 3658 * specified then we wait for a 3659 * while and then try again. The 3660 * wait could be forever if we 3661 * don't get the page(s) we need. 3662 * 3663 * Note: XXX We really need a mechanism 3664 * to wait for pages in the desired 3665 * range. For now, we wait for any 3666 * pages and see if we can use it. 3667 */ 3668 3669 if ((mattr != NULL) && (flags & PG_WAIT)) { 3670 delay(10); 3671 goto top; 3672 } 3673 goto fail; /* undo accounting stuff */ 3674 } 3675 3676 if (PP_ISAGED(npp) == 0) { 3677 /* 3678 * Since this page came from the 3679 * cachelist, we must destroy the 3680 * old vnode association. 3681 */ 3682 page_hashout(npp, (kmutex_t *)NULL); 3683 } 3684 } 3685 3686 /* 3687 * We own this page! 3688 */ 3689 ASSERT(PAGE_EXCL(npp)); 3690 ASSERT(npp->p_vnode == NULL); 3691 ASSERT(!hat_page_is_mapped(npp)); 3692 PP_CLRFREE(npp); 3693 PP_CLRAGED(npp); 3694 3695 /* 3696 * Here we have a page in our hot little mits and are 3697 * just waiting to stuff it on the appropriate lists. 3698 * Get the mutex and check to see if it really does 3699 * not exist. 3700 */ 3701 phm = PAGE_HASH_MUTEX(index); 3702 mutex_enter(phm); 3703 PAGE_HASH_SEARCH(index, pp, vp, off); 3704 if (pp == NULL) { 3705 VM_STAT_ADD(page_create_new); 3706 pp = npp; 3707 npp = NULL; 3708 if (!page_hashin(pp, vp, off, phm)) { 3709 /* 3710 * Since we hold the page hash mutex and 3711 * just searched for this page, page_hashin 3712 * had better not fail. If it does, that 3713 * means somethread did not follow the 3714 * page hash mutex rules. Panic now and 3715 * get it over with. As usual, go down 3716 * holding all the locks. 3717 */ 3718 ASSERT(MUTEX_HELD(phm)); 3719 panic("page_create: hashin fail %p %p %llx %p", 3720 (void *)pp, (void *)vp, off, (void *)phm); 3721 3722 } 3723 ASSERT(MUTEX_HELD(phm)); 3724 mutex_exit(phm); 3725 phm = NULL; 3726 3727 /* 3728 * Hat layer locking need not be done to set 3729 * the following bits since the page is not hashed 3730 * and was on the free list (i.e., had no mappings). 3731 * 3732 * Set the reference bit to protect 3733 * against immediate pageout 3734 * 3735 * XXXmh modify freelist code to set reference 3736 * bit so we don't have to do it here. 3737 */ 3738 page_set_props(pp, P_REF); 3739 } else { 3740 ASSERT(MUTEX_HELD(phm)); 3741 mutex_exit(phm); 3742 phm = NULL; 3743 /* 3744 * NOTE: This should not happen for pages associated 3745 * with kernel vnode 'kvp'. 3746 */ 3747 /* XX64 - to debug why this happens! */ 3748 ASSERT(!VN_ISKAS(vp)); 3749 if (VN_ISKAS(vp)) 3750 cmn_err(CE_NOTE, 3751 "page_create: page not expected " 3752 "in hash list for kernel vnode - pp 0x%p", 3753 (void *)pp); 3754 VM_STAT_ADD(page_create_exists); 3755 goto fail; 3756 } 3757 3758 /* 3759 * Got a page! It is locked. Acquire the i/o 3760 * lock since we are going to use the p_next and 3761 * p_prev fields to link the requested pages together. 3762 */ 3763 page_io_lock(pp); 3764 page_add(&plist, pp); 3765 plist = plist->p_next; 3766 off += MMU_PAGESIZE; 3767 vaddr += MMU_PAGESIZE; 3768 } 3769 3770 check_dma(mattr, plist, pages_req); 3771 return (plist); 3772 3773 fail: 3774 if (npp != NULL) { 3775 /* 3776 * Did not need this page after all. 3777 * Put it back on the free list. 3778 */ 3779 VM_STAT_ADD(page_create_putbacks); 3780 PP_SETFREE(npp); 3781 PP_SETAGED(npp); 3782 npp->p_offset = (u_offset_t)-1; 3783 page_list_add(npp, PG_FREE_LIST | PG_LIST_TAIL); 3784 page_unlock(npp); 3785 } 3786 3787 /* 3788 * Give up the pages we already got. 3789 */ 3790 while (plist != NULL) { 3791 pp = plist; 3792 page_sub(&plist, pp); 3793 page_io_unlock(pp); 3794 plist_len++; 3795 /*LINTED: constant in conditional ctx*/ 3796 VN_DISPOSE(pp, B_INVAL, 0, kcred); 3797 } 3798 3799 /* 3800 * VN_DISPOSE does freemem accounting for the pages in plist 3801 * by calling page_free. So, we need to undo the pcf accounting 3802 * for only the remaining pages. 3803 */ 3804 VM_STAT_ADD(page_create_putbacks); 3805 page_create_putback(pages_req - plist_len); 3806 3807 return (NULL); 3808 } 3809 #endif /* !__xpv */ 3810 3811 3812 /* 3813 * Copy the data from the physical page represented by "frompp" to 3814 * that represented by "topp". ppcopy uses CPU->cpu_caddr1 and 3815 * CPU->cpu_caddr2. It assumes that no one uses either map at interrupt 3816 * level and no one sleeps with an active mapping there. 3817 * 3818 * Note that the ref/mod bits in the page_t's are not affected by 3819 * this operation, hence it is up to the caller to update them appropriately. 3820 */ 3821 int 3822 ppcopy(page_t *frompp, page_t *topp) 3823 { 3824 caddr_t pp_addr1; 3825 caddr_t pp_addr2; 3826 hat_mempte_t pte1; 3827 hat_mempte_t pte2; 3828 kmutex_t *ppaddr_mutex; 3829 label_t ljb; 3830 int ret = 1; 3831 3832 ASSERT_STACK_ALIGNED(); 3833 ASSERT(PAGE_LOCKED(frompp)); 3834 ASSERT(PAGE_LOCKED(topp)); 3835 3836 if (kpm_enable) { 3837 pp_addr1 = hat_kpm_page2va(frompp, 0); 3838 pp_addr2 = hat_kpm_page2va(topp, 0); 3839 kpreempt_disable(); 3840 } else { 3841 /* 3842 * disable pre-emption so that CPU can't change 3843 */ 3844 kpreempt_disable(); 3845 3846 pp_addr1 = CPU->cpu_caddr1; 3847 pp_addr2 = CPU->cpu_caddr2; 3848 pte1 = CPU->cpu_caddr1pte; 3849 pte2 = CPU->cpu_caddr2pte; 3850 3851 ppaddr_mutex = &CPU->cpu_ppaddr_mutex; 3852 mutex_enter(ppaddr_mutex); 3853 3854 hat_mempte_remap(page_pptonum(frompp), pp_addr1, pte1, 3855 PROT_READ | HAT_STORECACHING_OK, HAT_LOAD_NOCONSIST); 3856 hat_mempte_remap(page_pptonum(topp), pp_addr2, pte2, 3857 PROT_READ | PROT_WRITE | HAT_STORECACHING_OK, 3858 HAT_LOAD_NOCONSIST); 3859 } 3860 3861 if (on_fault(&ljb)) { 3862 ret = 0; 3863 goto faulted; 3864 } 3865 if (use_sse_pagecopy) 3866 #ifdef __xpv 3867 page_copy_no_xmm(pp_addr2, pp_addr1); 3868 #else 3869 hwblkpagecopy(pp_addr1, pp_addr2); 3870 #endif 3871 else 3872 bcopy(pp_addr1, pp_addr2, PAGESIZE); 3873 3874 no_fault(); 3875 faulted: 3876 if (!kpm_enable) { 3877 #ifdef __xpv 3878 /* 3879 * We can't leave unused mappings laying about under the 3880 * hypervisor, so blow them away. 3881 */ 3882 if (HYPERVISOR_update_va_mapping((uintptr_t)pp_addr1, 0, 3883 UVMF_INVLPG | UVMF_LOCAL) < 0) 3884 panic("HYPERVISOR_update_va_mapping() failed"); 3885 if (HYPERVISOR_update_va_mapping((uintptr_t)pp_addr2, 0, 3886 UVMF_INVLPG | UVMF_LOCAL) < 0) 3887 panic("HYPERVISOR_update_va_mapping() failed"); 3888 #endif 3889 mutex_exit(ppaddr_mutex); 3890 } 3891 kpreempt_enable(); 3892 return (ret); 3893 } 3894 3895 void 3896 pagezero(page_t *pp, uint_t off, uint_t len) 3897 { 3898 ASSERT(PAGE_LOCKED(pp)); 3899 pfnzero(page_pptonum(pp), off, len); 3900 } 3901 3902 /* 3903 * Zero the physical page from off to off + len given by pfn 3904 * without changing the reference and modified bits of page. 3905 * 3906 * We use this using CPU private page address #2, see ppcopy() for more info. 3907 * pfnzero() must not be called at interrupt level. 3908 */ 3909 void 3910 pfnzero(pfn_t pfn, uint_t off, uint_t len) 3911 { 3912 caddr_t pp_addr2; 3913 hat_mempte_t pte2; 3914 kmutex_t *ppaddr_mutex = NULL; 3915 3916 ASSERT_STACK_ALIGNED(); 3917 ASSERT(len <= MMU_PAGESIZE); 3918 ASSERT(off <= MMU_PAGESIZE); 3919 ASSERT(off + len <= MMU_PAGESIZE); 3920 3921 if (kpm_enable && !pfn_is_foreign(pfn)) { 3922 pp_addr2 = hat_kpm_pfn2va(pfn); 3923 kpreempt_disable(); 3924 } else { 3925 kpreempt_disable(); 3926 3927 pp_addr2 = CPU->cpu_caddr2; 3928 pte2 = CPU->cpu_caddr2pte; 3929 3930 ppaddr_mutex = &CPU->cpu_ppaddr_mutex; 3931 mutex_enter(ppaddr_mutex); 3932 3933 hat_mempte_remap(pfn, pp_addr2, pte2, 3934 PROT_READ | PROT_WRITE | HAT_STORECACHING_OK, 3935 HAT_LOAD_NOCONSIST); 3936 } 3937 3938 if (use_sse_pagezero) { 3939 #ifdef __xpv 3940 uint_t rem; 3941 3942 /* 3943 * zero a byte at a time until properly aligned for 3944 * block_zero_no_xmm(). 3945 */ 3946 while (!P2NPHASE(off, ((uint_t)BLOCKZEROALIGN)) && len-- > 0) 3947 pp_addr2[off++] = 0; 3948 3949 /* 3950 * Now use faster block_zero_no_xmm() for any range 3951 * that is properly aligned and sized. 3952 */ 3953 rem = P2PHASE(len, ((uint_t)BLOCKZEROALIGN)); 3954 len -= rem; 3955 if (len != 0) { 3956 block_zero_no_xmm(pp_addr2 + off, len); 3957 off += len; 3958 } 3959 3960 /* 3961 * zero remainder with byte stores. 3962 */ 3963 while (rem-- > 0) 3964 pp_addr2[off++] = 0; 3965 #else 3966 hwblkclr(pp_addr2 + off, len); 3967 #endif 3968 } else { 3969 bzero(pp_addr2 + off, len); 3970 } 3971 3972 if (!kpm_enable || pfn_is_foreign(pfn)) { 3973 #ifdef __xpv 3974 /* 3975 * On the hypervisor this page might get used for a page 3976 * table before any intervening change to this mapping, 3977 * so blow it away. 3978 */ 3979 if (HYPERVISOR_update_va_mapping((uintptr_t)pp_addr2, 0, 3980 UVMF_INVLPG) < 0) 3981 panic("HYPERVISOR_update_va_mapping() failed"); 3982 #endif 3983 mutex_exit(ppaddr_mutex); 3984 } 3985 3986 kpreempt_enable(); 3987 } 3988 3989 /* 3990 * Platform-dependent page scrub call. 3991 */ 3992 void 3993 pagescrub(page_t *pp, uint_t off, uint_t len) 3994 { 3995 /* 3996 * For now, we rely on the fact that pagezero() will 3997 * always clear UEs. 3998 */ 3999 pagezero(pp, off, len); 4000 } 4001 4002 /* 4003 * set up two private addresses for use on a given CPU for use in ppcopy() 4004 */ 4005 void 4006 setup_vaddr_for_ppcopy(struct cpu *cpup) 4007 { 4008 void *addr; 4009 hat_mempte_t pte_pa; 4010 4011 addr = vmem_alloc(heap_arena, mmu_ptob(1), VM_SLEEP); 4012 pte_pa = hat_mempte_setup(addr); 4013 cpup->cpu_caddr1 = addr; 4014 cpup->cpu_caddr1pte = pte_pa; 4015 4016 addr = vmem_alloc(heap_arena, mmu_ptob(1), VM_SLEEP); 4017 pte_pa = hat_mempte_setup(addr); 4018 cpup->cpu_caddr2 = addr; 4019 cpup->cpu_caddr2pte = pte_pa; 4020 4021 mutex_init(&cpup->cpu_ppaddr_mutex, NULL, MUTEX_DEFAULT, NULL); 4022 } 4023 4024 /* 4025 * Undo setup_vaddr_for_ppcopy 4026 */ 4027 void 4028 teardown_vaddr_for_ppcopy(struct cpu *cpup) 4029 { 4030 mutex_destroy(&cpup->cpu_ppaddr_mutex); 4031 4032 hat_mempte_release(cpup->cpu_caddr2, cpup->cpu_caddr2pte); 4033 cpup->cpu_caddr2pte = 0; 4034 vmem_free(heap_arena, cpup->cpu_caddr2, mmu_ptob(1)); 4035 cpup->cpu_caddr2 = 0; 4036 4037 hat_mempte_release(cpup->cpu_caddr1, cpup->cpu_caddr1pte); 4038 cpup->cpu_caddr1pte = 0; 4039 vmem_free(heap_arena, cpup->cpu_caddr1, mmu_ptob(1)); 4040 cpup->cpu_caddr1 = 0; 4041 } 4042 4043 /* 4044 * Function for flushing D-cache when performing module relocations 4045 * to an alternate mapping. Unnecessary on Intel / AMD platforms. 4046 */ 4047 void 4048 dcache_flushall() 4049 {} 4050 4051 /* 4052 * Allocate a memory page. The argument 'seed' can be any pseudo-random 4053 * number to vary where the pages come from. This is quite a hacked up 4054 * method -- it works for now, but really needs to be fixed up a bit. 4055 * 4056 * We currently use page_create_va() on the kvp with fake offsets, 4057 * segments and virt address. This is pretty bogus, but was copied from the 4058 * old hat_i86.c code. A better approach would be to specify either mnode 4059 * random or mnode local and takes a page from whatever color has the MOST 4060 * available - this would have a minimal impact on page coloring. 4061 */ 4062 page_t * 4063 page_get_physical(uintptr_t seed) 4064 { 4065 page_t *pp; 4066 u_offset_t offset; 4067 static struct seg tmpseg; 4068 static uintptr_t ctr = 0; 4069 4070 /* 4071 * This code is gross, we really need a simpler page allocator. 4072 * 4073 * We need to assign an offset for the page to call page_create_va() 4074 * To avoid conflicts with other pages, we get creative with the offset. 4075 * For 32 bits, we need an offset > 4Gig 4076 * For 64 bits, need an offset somewhere in the VA hole. 4077 */ 4078 offset = seed; 4079 if (offset > kernelbase) 4080 offset -= kernelbase; 4081 offset <<= MMU_PAGESHIFT; 4082 #if defined(__amd64) 4083 offset += mmu.hole_start; /* something in VA hole */ 4084 #else 4085 offset += 1ULL << 40; /* something > 4 Gig */ 4086 #endif 4087 4088 if (page_resv(1, KM_NOSLEEP) == 0) 4089 return (NULL); 4090 4091 #ifdef DEBUG 4092 pp = page_exists(&kvp, offset); 4093 if (pp != NULL) 4094 panic("page already exists %p", (void *)pp); 4095 #endif 4096 4097 pp = page_create_va(&kvp, offset, MMU_PAGESIZE, PG_EXCL, 4098 &tmpseg, (caddr_t)(ctr += MMU_PAGESIZE)); /* changing VA usage */ 4099 if (pp != NULL) { 4100 page_io_unlock(pp); 4101 page_downgrade(pp); 4102 } 4103 return (pp); 4104 } 4105