1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 /* Copyright (c) 1983, 1984, 1985, 1986, 1987, 1988, 1989 AT&T */ 27 /* All Rights Reserved */ 28 29 /* 30 * University Copyright- Copyright (c) 1982, 1986, 1988 31 * The Regents of the University of California 32 * All Rights Reserved 33 * 34 * University Acknowledgment- Portions of this document are derived from 35 * software developed by the University of California, Berkeley, and its 36 * contributors. 37 */ 38 39 /* 40 * VM - physical page management. 41 */ 42 43 #include <sys/types.h> 44 #include <sys/t_lock.h> 45 #include <sys/param.h> 46 #include <sys/systm.h> 47 #include <sys/errno.h> 48 #include <sys/time.h> 49 #include <sys/vnode.h> 50 #include <sys/vm.h> 51 #include <sys/vtrace.h> 52 #include <sys/swap.h> 53 #include <sys/cmn_err.h> 54 #include <sys/tuneable.h> 55 #include <sys/sysmacros.h> 56 #include <sys/cpuvar.h> 57 #include <sys/callb.h> 58 #include <sys/debug.h> 59 #include <sys/tnf_probe.h> 60 #include <sys/condvar_impl.h> 61 #include <sys/mem_config.h> 62 #include <sys/mem_cage.h> 63 #include <sys/kmem.h> 64 #include <sys/atomic.h> 65 #include <sys/strlog.h> 66 #include <sys/mman.h> 67 #include <sys/ontrap.h> 68 #include <sys/lgrp.h> 69 #include <sys/vfs.h> 70 71 #include <vm/hat.h> 72 #include <vm/anon.h> 73 #include <vm/page.h> 74 #include <vm/seg.h> 75 #include <vm/pvn.h> 76 #include <vm/seg_kmem.h> 77 #include <vm/vm_dep.h> 78 #include <sys/vm_usage.h> 79 #include <fs/fs_subr.h> 80 #include <sys/ddi.h> 81 #include <sys/modctl.h> 82 83 static int nopageage = 0; 84 85 static pgcnt_t max_page_get; /* max page_get request size in pages */ 86 pgcnt_t total_pages = 0; /* total number of pages (used by /proc) */ 87 88 /* 89 * freemem_lock protects all freemem variables: 90 * availrmem. Also this lock protects the globals which track the 91 * availrmem changes for accurate kernel footprint calculation. 92 * See below for an explanation of these 93 * globals. 94 */ 95 kmutex_t freemem_lock; 96 pgcnt_t availrmem; 97 pgcnt_t availrmem_initial; 98 99 /* 100 * These globals track availrmem changes to get a more accurate 101 * estimate of tke kernel size. Historically pp_kernel is used for 102 * kernel size and is based on availrmem. But availrmem is adjusted for 103 * locked pages in the system not just for kernel locked pages. 104 * These new counters will track the pages locked through segvn and 105 * by explicit user locking. 106 * 107 * pages_locked : How many pages are locked because of user specified 108 * locking through mlock or plock. 109 * 110 * pages_useclaim,pages_claimed : These two variables track the 111 * claim adjustments because of the protection changes on a segvn segment. 112 * 113 * All these globals are protected by the same lock which protects availrmem. 114 */ 115 pgcnt_t pages_locked = 0; 116 pgcnt_t pages_useclaim = 0; 117 pgcnt_t pages_claimed = 0; 118 119 120 /* 121 * new_freemem_lock protects freemem, freemem_wait & freemem_cv. 122 */ 123 static kmutex_t new_freemem_lock; 124 static uint_t freemem_wait; /* someone waiting for freemem */ 125 static kcondvar_t freemem_cv; 126 127 /* 128 * The logical page free list is maintained as two lists, the 'free' 129 * and the 'cache' lists. 130 * The free list contains those pages that should be reused first. 131 * 132 * The implementation of the lists is machine dependent. 133 * page_get_freelist(), page_get_cachelist(), 134 * page_list_sub(), and page_list_add() 135 * form the interface to the machine dependent implementation. 136 * 137 * Pages with p_free set are on the cache list. 138 * Pages with p_free and p_age set are on the free list, 139 * 140 * A page may be locked while on either list. 141 */ 142 143 /* 144 * free list accounting stuff. 145 * 146 * 147 * Spread out the value for the number of pages on the 148 * page free and page cache lists. If there is just one 149 * value, then it must be under just one lock. 150 * The lock contention and cache traffic are a real bother. 151 * 152 * When we acquire and then drop a single pcf lock 153 * we can start in the middle of the array of pcf structures. 154 * If we acquire more than one pcf lock at a time, we need to 155 * start at the front to avoid deadlocking. 156 * 157 * pcf_count holds the number of pages in each pool. 158 * 159 * pcf_block is set when page_create_get_something() has asked the 160 * PSM page freelist and page cachelist routines without specifying 161 * a color and nothing came back. This is used to block anything 162 * else from moving pages from one list to the other while the 163 * lists are searched again. If a page is freeed while pcf_block is 164 * set, then pcf_reserve is incremented. pcgs_unblock() takes care 165 * of clearning pcf_block, doing the wakeups, etc. 166 */ 167 168 #define MAX_PCF_FANOUT NCPU 169 static uint_t pcf_fanout = 1; /* Will get changed at boot time */ 170 static uint_t pcf_fanout_mask = 0; 171 172 struct pcf { 173 kmutex_t pcf_lock; /* protects the structure */ 174 uint_t pcf_count; /* page count */ 175 uint_t pcf_wait; /* number of waiters */ 176 uint_t pcf_block; /* pcgs flag to page_free() */ 177 uint_t pcf_reserve; /* pages freed after pcf_block set */ 178 uint_t pcf_fill[10]; /* to line up on the caches */ 179 }; 180 181 /* 182 * PCF_INDEX hash needs to be dynamic (every so often the hash changes where 183 * it will hash the cpu to). This is done to prevent a drain condition 184 * from happening. This drain condition will occur when pcf_count decrement 185 * occurs on cpu A and the increment of pcf_count always occurs on cpu B. An 186 * example of this shows up with device interrupts. The dma buffer is allocated 187 * by the cpu requesting the IO thus the pcf_count is decremented based on that. 188 * When the memory is returned by the interrupt thread, the pcf_count will be 189 * incremented based on the cpu servicing the interrupt. 190 */ 191 static struct pcf pcf[MAX_PCF_FANOUT]; 192 #define PCF_INDEX() ((int)(((long)CPU->cpu_seqid) + \ 193 (randtick() >> 24)) & (pcf_fanout_mask)) 194 195 static int pcf_decrement_bucket(pgcnt_t); 196 static int pcf_decrement_multiple(pgcnt_t *, pgcnt_t, int); 197 198 kmutex_t pcgs_lock; /* serializes page_create_get_ */ 199 kmutex_t pcgs_cagelock; /* serializes NOSLEEP cage allocs */ 200 kmutex_t pcgs_wait_lock; /* used for delay in pcgs */ 201 static kcondvar_t pcgs_cv; /* cv for delay in pcgs */ 202 203 #ifdef VM_STATS 204 205 /* 206 * No locks, but so what, they are only statistics. 207 */ 208 209 static struct page_tcnt { 210 int pc_free_cache; /* free's into cache list */ 211 int pc_free_dontneed; /* free's with dontneed */ 212 int pc_free_pageout; /* free's from pageout */ 213 int pc_free_free; /* free's into free list */ 214 int pc_free_pages; /* free's into large page free list */ 215 int pc_destroy_pages; /* large page destroy's */ 216 int pc_get_cache; /* get's from cache list */ 217 int pc_get_free; /* get's from free list */ 218 int pc_reclaim; /* reclaim's */ 219 int pc_abortfree; /* abort's of free pages */ 220 int pc_find_hit; /* find's that find page */ 221 int pc_find_miss; /* find's that don't find page */ 222 int pc_destroy_free; /* # of free pages destroyed */ 223 #define PC_HASH_CNT (4*PAGE_HASHAVELEN) 224 int pc_find_hashlen[PC_HASH_CNT+1]; 225 int pc_addclaim_pages; 226 int pc_subclaim_pages; 227 int pc_free_replacement_page[2]; 228 int pc_try_demote_pages[6]; 229 int pc_demote_pages[2]; 230 } pagecnt; 231 232 uint_t hashin_count; 233 uint_t hashin_not_held; 234 uint_t hashin_already; 235 236 uint_t hashout_count; 237 uint_t hashout_not_held; 238 239 uint_t page_create_count; 240 uint_t page_create_not_enough; 241 uint_t page_create_not_enough_again; 242 uint_t page_create_zero; 243 uint_t page_create_hashout; 244 uint_t page_create_page_lock_failed; 245 uint_t page_create_trylock_failed; 246 uint_t page_create_found_one; 247 uint_t page_create_hashin_failed; 248 uint_t page_create_dropped_phm; 249 250 uint_t page_create_new; 251 uint_t page_create_exists; 252 uint_t page_create_putbacks; 253 uint_t page_create_overshoot; 254 255 uint_t page_reclaim_zero; 256 uint_t page_reclaim_zero_locked; 257 258 uint_t page_rename_exists; 259 uint_t page_rename_count; 260 261 uint_t page_lookup_cnt[20]; 262 uint_t page_lookup_nowait_cnt[10]; 263 uint_t page_find_cnt; 264 uint_t page_exists_cnt; 265 uint_t page_exists_forreal_cnt; 266 uint_t page_lookup_dev_cnt; 267 uint_t get_cachelist_cnt; 268 uint_t page_create_cnt[10]; 269 uint_t alloc_pages[9]; 270 uint_t page_exphcontg[19]; 271 uint_t page_create_large_cnt[10]; 272 273 /* 274 * Collects statistics. 275 */ 276 #define PAGE_HASH_SEARCH(index, pp, vp, off) { \ 277 uint_t mylen = 0; \ 278 \ 279 for ((pp) = page_hash[(index)]; (pp); (pp) = (pp)->p_hash, mylen++) { \ 280 if ((pp)->p_vnode == (vp) && (pp)->p_offset == (off)) \ 281 break; \ 282 } \ 283 if ((pp) != NULL) \ 284 pagecnt.pc_find_hit++; \ 285 else \ 286 pagecnt.pc_find_miss++; \ 287 if (mylen > PC_HASH_CNT) \ 288 mylen = PC_HASH_CNT; \ 289 pagecnt.pc_find_hashlen[mylen]++; \ 290 } 291 292 #else /* VM_STATS */ 293 294 /* 295 * Don't collect statistics 296 */ 297 #define PAGE_HASH_SEARCH(index, pp, vp, off) { \ 298 for ((pp) = page_hash[(index)]; (pp); (pp) = (pp)->p_hash) { \ 299 if ((pp)->p_vnode == (vp) && (pp)->p_offset == (off)) \ 300 break; \ 301 } \ 302 } 303 304 #endif /* VM_STATS */ 305 306 307 308 #ifdef DEBUG 309 #define MEMSEG_SEARCH_STATS 310 #endif 311 312 #ifdef MEMSEG_SEARCH_STATS 313 struct memseg_stats { 314 uint_t nsearch; 315 uint_t nlastwon; 316 uint_t nhashwon; 317 uint_t nnotfound; 318 } memseg_stats; 319 320 #define MEMSEG_STAT_INCR(v) \ 321 atomic_add_32(&memseg_stats.v, 1) 322 #else 323 #define MEMSEG_STAT_INCR(x) 324 #endif 325 326 struct memseg *memsegs; /* list of memory segments */ 327 328 /* 329 * /etc/system tunable to control large page allocation hueristic. 330 * 331 * Setting to LPAP_LOCAL will heavily prefer the local lgroup over remote lgroup 332 * for large page allocation requests. If a large page is not readily 333 * avaliable on the local freelists we will go through additional effort 334 * to create a large page, potentially moving smaller pages around to coalesce 335 * larger pages in the local lgroup. 336 * Default value of LPAP_DEFAULT will go to remote freelists if large pages 337 * are not readily available in the local lgroup. 338 */ 339 enum lpap { 340 LPAP_DEFAULT, /* default large page allocation policy */ 341 LPAP_LOCAL /* local large page allocation policy */ 342 }; 343 344 enum lpap lpg_alloc_prefer = LPAP_DEFAULT; 345 346 static void page_init_mem_config(void); 347 static int page_do_hashin(page_t *, vnode_t *, u_offset_t); 348 static void page_do_hashout(page_t *); 349 static void page_capture_init(); 350 int page_capture_take_action(page_t *, uint_t, void *); 351 352 static void page_demote_vp_pages(page_t *); 353 354 355 void 356 pcf_init(void) 357 358 { 359 if (boot_ncpus != -1) { 360 pcf_fanout = boot_ncpus; 361 } else { 362 pcf_fanout = max_ncpus; 363 } 364 #ifdef sun4v 365 /* 366 * Force at least 4 buckets if possible for sun4v. 367 */ 368 pcf_fanout = MAX(pcf_fanout, 4); 369 #endif /* sun4v */ 370 371 /* 372 * Round up to the nearest power of 2. 373 */ 374 pcf_fanout = MIN(pcf_fanout, MAX_PCF_FANOUT); 375 if (!ISP2(pcf_fanout)) { 376 pcf_fanout = 1 << highbit(pcf_fanout); 377 378 if (pcf_fanout > MAX_PCF_FANOUT) { 379 pcf_fanout = 1 << (highbit(MAX_PCF_FANOUT) - 1); 380 } 381 } 382 pcf_fanout_mask = pcf_fanout - 1; 383 } 384 385 /* 386 * vm subsystem related initialization 387 */ 388 void 389 vm_init(void) 390 { 391 boolean_t callb_vm_cpr(void *, int); 392 393 (void) callb_add(callb_vm_cpr, 0, CB_CL_CPR_VM, "vm"); 394 page_init_mem_config(); 395 page_retire_init(); 396 vm_usage_init(); 397 page_capture_init(); 398 } 399 400 /* 401 * This function is called at startup and when memory is added or deleted. 402 */ 403 void 404 init_pages_pp_maximum() 405 { 406 static pgcnt_t p_min; 407 static pgcnt_t pages_pp_maximum_startup; 408 static pgcnt_t avrmem_delta; 409 static int init_done; 410 static int user_set; /* true if set in /etc/system */ 411 412 if (init_done == 0) { 413 414 /* If the user specified a value, save it */ 415 if (pages_pp_maximum != 0) { 416 user_set = 1; 417 pages_pp_maximum_startup = pages_pp_maximum; 418 } 419 420 /* 421 * Setting of pages_pp_maximum is based first time 422 * on the value of availrmem just after the start-up 423 * allocations. To preserve this relationship at run 424 * time, use a delta from availrmem_initial. 425 */ 426 ASSERT(availrmem_initial >= availrmem); 427 avrmem_delta = availrmem_initial - availrmem; 428 429 /* The allowable floor of pages_pp_maximum */ 430 p_min = tune.t_minarmem + 100; 431 432 /* Make sure we don't come through here again. */ 433 init_done = 1; 434 } 435 /* 436 * Determine pages_pp_maximum, the number of currently available 437 * pages (availrmem) that can't be `locked'. If not set by 438 * the user, we set it to 4% of the currently available memory 439 * plus 4MB. 440 * But we also insist that it be greater than tune.t_minarmem; 441 * otherwise a process could lock down a lot of memory, get swapped 442 * out, and never have enough to get swapped back in. 443 */ 444 if (user_set) 445 pages_pp_maximum = pages_pp_maximum_startup; 446 else 447 pages_pp_maximum = ((availrmem_initial - avrmem_delta) / 25) 448 + btop(4 * 1024 * 1024); 449 450 if (pages_pp_maximum <= p_min) { 451 pages_pp_maximum = p_min; 452 } 453 } 454 455 void 456 set_max_page_get(pgcnt_t target_total_pages) 457 { 458 max_page_get = target_total_pages / 2; 459 } 460 461 static pgcnt_t pending_delete; 462 463 /*ARGSUSED*/ 464 static void 465 page_mem_config_post_add( 466 void *arg, 467 pgcnt_t delta_pages) 468 { 469 set_max_page_get(total_pages - pending_delete); 470 init_pages_pp_maximum(); 471 } 472 473 /*ARGSUSED*/ 474 static int 475 page_mem_config_pre_del( 476 void *arg, 477 pgcnt_t delta_pages) 478 { 479 pgcnt_t nv; 480 481 nv = atomic_add_long_nv(&pending_delete, (spgcnt_t)delta_pages); 482 set_max_page_get(total_pages - nv); 483 return (0); 484 } 485 486 /*ARGSUSED*/ 487 static void 488 page_mem_config_post_del( 489 void *arg, 490 pgcnt_t delta_pages, 491 int cancelled) 492 { 493 pgcnt_t nv; 494 495 nv = atomic_add_long_nv(&pending_delete, -(spgcnt_t)delta_pages); 496 set_max_page_get(total_pages - nv); 497 if (!cancelled) 498 init_pages_pp_maximum(); 499 } 500 501 static kphysm_setup_vector_t page_mem_config_vec = { 502 KPHYSM_SETUP_VECTOR_VERSION, 503 page_mem_config_post_add, 504 page_mem_config_pre_del, 505 page_mem_config_post_del, 506 }; 507 508 static void 509 page_init_mem_config(void) 510 { 511 int ret; 512 513 ret = kphysm_setup_func_register(&page_mem_config_vec, (void *)NULL); 514 ASSERT(ret == 0); 515 } 516 517 /* 518 * Evenly spread out the PCF counters for large free pages 519 */ 520 static void 521 page_free_large_ctr(pgcnt_t npages) 522 { 523 static struct pcf *p = pcf; 524 pgcnt_t lump; 525 526 freemem += npages; 527 528 lump = roundup(npages, pcf_fanout) / pcf_fanout; 529 530 while (npages > 0) { 531 532 ASSERT(!p->pcf_block); 533 534 if (lump < npages) { 535 p->pcf_count += (uint_t)lump; 536 npages -= lump; 537 } else { 538 p->pcf_count += (uint_t)npages; 539 npages = 0; 540 } 541 542 ASSERT(!p->pcf_wait); 543 544 if (++p > &pcf[pcf_fanout - 1]) 545 p = pcf; 546 } 547 548 ASSERT(npages == 0); 549 } 550 551 /* 552 * Add a physical chunk of memory to the system free lists during startup. 553 * Platform specific startup() allocates the memory for the page structs. 554 * 555 * num - number of page structures 556 * base - page number (pfn) to be associated with the first page. 557 * 558 * Since we are doing this during startup (ie. single threaded), we will 559 * use shortcut routines to avoid any locking overhead while putting all 560 * these pages on the freelists. 561 * 562 * NOTE: Any changes performed to page_free(), must also be performed to 563 * add_physmem() since this is how we initialize all page_t's at 564 * boot time. 565 */ 566 void 567 add_physmem( 568 page_t *pp, 569 pgcnt_t num, 570 pfn_t pnum) 571 { 572 page_t *root = NULL; 573 uint_t szc = page_num_pagesizes() - 1; 574 pgcnt_t large = page_get_pagecnt(szc); 575 pgcnt_t cnt = 0; 576 577 TRACE_2(TR_FAC_VM, TR_PAGE_INIT, 578 "add_physmem:pp %p num %lu", pp, num); 579 580 /* 581 * Arbitrarily limit the max page_get request 582 * to 1/2 of the page structs we have. 583 */ 584 total_pages += num; 585 set_max_page_get(total_pages); 586 587 PLCNT_MODIFY_MAX(pnum, (long)num); 588 589 /* 590 * The physical space for the pages array 591 * representing ram pages has already been 592 * allocated. Here we initialize each lock 593 * in the page structure, and put each on 594 * the free list 595 */ 596 for (; num; pp++, pnum++, num--) { 597 598 /* 599 * this needs to fill in the page number 600 * and do any other arch specific initialization 601 */ 602 add_physmem_cb(pp, pnum); 603 604 pp->p_lckcnt = 0; 605 pp->p_cowcnt = 0; 606 pp->p_slckcnt = 0; 607 608 /* 609 * Initialize the page lock as unlocked, since nobody 610 * can see or access this page yet. 611 */ 612 pp->p_selock = 0; 613 614 /* 615 * Initialize IO lock 616 */ 617 page_iolock_init(pp); 618 619 /* 620 * initialize other fields in the page_t 621 */ 622 PP_SETFREE(pp); 623 page_clr_all_props(pp, 0); 624 PP_SETAGED(pp); 625 pp->p_offset = (u_offset_t)-1; 626 pp->p_next = pp; 627 pp->p_prev = pp; 628 629 /* 630 * Simple case: System doesn't support large pages. 631 */ 632 if (szc == 0) { 633 pp->p_szc = 0; 634 page_free_at_startup(pp); 635 continue; 636 } 637 638 /* 639 * Handle unaligned pages, we collect them up onto 640 * the root page until we have a full large page. 641 */ 642 if (!IS_P2ALIGNED(pnum, large)) { 643 644 /* 645 * If not in a large page, 646 * just free as small page. 647 */ 648 if (root == NULL) { 649 pp->p_szc = 0; 650 page_free_at_startup(pp); 651 continue; 652 } 653 654 /* 655 * Link a constituent page into the large page. 656 */ 657 pp->p_szc = szc; 658 page_list_concat(&root, &pp); 659 660 /* 661 * When large page is fully formed, free it. 662 */ 663 if (++cnt == large) { 664 page_free_large_ctr(cnt); 665 page_list_add_pages(root, PG_LIST_ISINIT); 666 root = NULL; 667 cnt = 0; 668 } 669 continue; 670 } 671 672 /* 673 * At this point we have a page number which 674 * is aligned. We assert that we aren't already 675 * in a different large page. 676 */ 677 ASSERT(IS_P2ALIGNED(pnum, large)); 678 ASSERT(root == NULL && cnt == 0); 679 680 /* 681 * If insufficient number of pages left to form 682 * a large page, just free the small page. 683 */ 684 if (num < large) { 685 pp->p_szc = 0; 686 page_free_at_startup(pp); 687 continue; 688 } 689 690 /* 691 * Otherwise start a new large page. 692 */ 693 pp->p_szc = szc; 694 cnt++; 695 root = pp; 696 } 697 ASSERT(root == NULL && cnt == 0); 698 } 699 700 /* 701 * Find a page representing the specified [vp, offset]. 702 * If we find the page but it is intransit coming in, 703 * it will have an "exclusive" lock and we wait for 704 * the i/o to complete. A page found on the free list 705 * is always reclaimed and then locked. On success, the page 706 * is locked, its data is valid and it isn't on the free 707 * list, while a NULL is returned if the page doesn't exist. 708 */ 709 page_t * 710 page_lookup(vnode_t *vp, u_offset_t off, se_t se) 711 { 712 return (page_lookup_create(vp, off, se, NULL, NULL, 0)); 713 } 714 715 /* 716 * Find a page representing the specified [vp, offset]. 717 * We either return the one we found or, if passed in, 718 * create one with identity of [vp, offset] of the 719 * pre-allocated page. If we find existing page but it is 720 * intransit coming in, it will have an "exclusive" lock 721 * and we wait for the i/o to complete. A page found on 722 * the free list is always reclaimed and then locked. 723 * On success, the page is locked, its data is valid and 724 * it isn't on the free list, while a NULL is returned 725 * if the page doesn't exist and newpp is NULL; 726 */ 727 page_t * 728 page_lookup_create( 729 vnode_t *vp, 730 u_offset_t off, 731 se_t se, 732 page_t *newpp, 733 spgcnt_t *nrelocp, 734 int flags) 735 { 736 page_t *pp; 737 kmutex_t *phm; 738 ulong_t index; 739 uint_t hash_locked; 740 uint_t es; 741 742 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 743 VM_STAT_ADD(page_lookup_cnt[0]); 744 ASSERT(newpp ? PAGE_EXCL(newpp) : 1); 745 746 /* 747 * Acquire the appropriate page hash lock since 748 * we have to search the hash list. Pages that 749 * hash to this list can't change identity while 750 * this lock is held. 751 */ 752 hash_locked = 0; 753 index = PAGE_HASH_FUNC(vp, off); 754 phm = NULL; 755 top: 756 PAGE_HASH_SEARCH(index, pp, vp, off); 757 if (pp != NULL) { 758 VM_STAT_ADD(page_lookup_cnt[1]); 759 es = (newpp != NULL) ? 1 : 0; 760 es |= flags; 761 if (!hash_locked) { 762 VM_STAT_ADD(page_lookup_cnt[2]); 763 if (!page_try_reclaim_lock(pp, se, es)) { 764 /* 765 * On a miss, acquire the phm. Then 766 * next time, page_lock() will be called, 767 * causing a wait if the page is busy. 768 * just looping with page_trylock() would 769 * get pretty boring. 770 */ 771 VM_STAT_ADD(page_lookup_cnt[3]); 772 phm = PAGE_HASH_MUTEX(index); 773 mutex_enter(phm); 774 hash_locked = 1; 775 goto top; 776 } 777 } else { 778 VM_STAT_ADD(page_lookup_cnt[4]); 779 if (!page_lock_es(pp, se, phm, P_RECLAIM, es)) { 780 VM_STAT_ADD(page_lookup_cnt[5]); 781 goto top; 782 } 783 } 784 785 /* 786 * Since `pp' is locked it can not change identity now. 787 * Reconfirm we locked the correct page. 788 * 789 * Both the p_vnode and p_offset *must* be cast volatile 790 * to force a reload of their values: The PAGE_HASH_SEARCH 791 * macro will have stuffed p_vnode and p_offset into 792 * registers before calling page_trylock(); another thread, 793 * actually holding the hash lock, could have changed the 794 * page's identity in memory, but our registers would not 795 * be changed, fooling the reconfirmation. If the hash 796 * lock was held during the search, the casting would 797 * not be needed. 798 */ 799 VM_STAT_ADD(page_lookup_cnt[6]); 800 if (((volatile struct vnode *)(pp->p_vnode) != vp) || 801 ((volatile u_offset_t)(pp->p_offset) != off)) { 802 VM_STAT_ADD(page_lookup_cnt[7]); 803 if (hash_locked) { 804 panic("page_lookup_create: lost page %p", 805 (void *)pp); 806 /*NOTREACHED*/ 807 } 808 page_unlock(pp); 809 phm = PAGE_HASH_MUTEX(index); 810 mutex_enter(phm); 811 hash_locked = 1; 812 goto top; 813 } 814 815 /* 816 * If page_trylock() was called, then pp may still be on 817 * the cachelist (can't be on the free list, it would not 818 * have been found in the search). If it is on the 819 * cachelist it must be pulled now. To pull the page from 820 * the cachelist, it must be exclusively locked. 821 * 822 * The other big difference between page_trylock() and 823 * page_lock(), is that page_lock() will pull the 824 * page from whatever free list (the cache list in this 825 * case) the page is on. If page_trylock() was used 826 * above, then we have to do the reclaim ourselves. 827 */ 828 if ((!hash_locked) && (PP_ISFREE(pp))) { 829 ASSERT(PP_ISAGED(pp) == 0); 830 VM_STAT_ADD(page_lookup_cnt[8]); 831 832 /* 833 * page_relcaim will insure that we 834 * have this page exclusively 835 */ 836 837 if (!page_reclaim(pp, NULL)) { 838 /* 839 * Page_reclaim dropped whatever lock 840 * we held. 841 */ 842 VM_STAT_ADD(page_lookup_cnt[9]); 843 phm = PAGE_HASH_MUTEX(index); 844 mutex_enter(phm); 845 hash_locked = 1; 846 goto top; 847 } else if (se == SE_SHARED && newpp == NULL) { 848 VM_STAT_ADD(page_lookup_cnt[10]); 849 page_downgrade(pp); 850 } 851 } 852 853 if (hash_locked) { 854 mutex_exit(phm); 855 } 856 857 if (newpp != NULL && pp->p_szc < newpp->p_szc && 858 PAGE_EXCL(pp) && nrelocp != NULL) { 859 ASSERT(nrelocp != NULL); 860 (void) page_relocate(&pp, &newpp, 1, 1, nrelocp, 861 NULL); 862 if (*nrelocp > 0) { 863 VM_STAT_COND_ADD(*nrelocp == 1, 864 page_lookup_cnt[11]); 865 VM_STAT_COND_ADD(*nrelocp > 1, 866 page_lookup_cnt[12]); 867 pp = newpp; 868 se = SE_EXCL; 869 } else { 870 if (se == SE_SHARED) { 871 page_downgrade(pp); 872 } 873 VM_STAT_ADD(page_lookup_cnt[13]); 874 } 875 } else if (newpp != NULL && nrelocp != NULL) { 876 if (PAGE_EXCL(pp) && se == SE_SHARED) { 877 page_downgrade(pp); 878 } 879 VM_STAT_COND_ADD(pp->p_szc < newpp->p_szc, 880 page_lookup_cnt[14]); 881 VM_STAT_COND_ADD(pp->p_szc == newpp->p_szc, 882 page_lookup_cnt[15]); 883 VM_STAT_COND_ADD(pp->p_szc > newpp->p_szc, 884 page_lookup_cnt[16]); 885 } else if (newpp != NULL && PAGE_EXCL(pp)) { 886 se = SE_EXCL; 887 } 888 } else if (!hash_locked) { 889 VM_STAT_ADD(page_lookup_cnt[17]); 890 phm = PAGE_HASH_MUTEX(index); 891 mutex_enter(phm); 892 hash_locked = 1; 893 goto top; 894 } else if (newpp != NULL) { 895 /* 896 * If we have a preallocated page then 897 * insert it now and basically behave like 898 * page_create. 899 */ 900 VM_STAT_ADD(page_lookup_cnt[18]); 901 /* 902 * Since we hold the page hash mutex and 903 * just searched for this page, page_hashin 904 * had better not fail. If it does, that 905 * means some thread did not follow the 906 * page hash mutex rules. Panic now and 907 * get it over with. As usual, go down 908 * holding all the locks. 909 */ 910 ASSERT(MUTEX_HELD(phm)); 911 if (!page_hashin(newpp, vp, off, phm)) { 912 ASSERT(MUTEX_HELD(phm)); 913 panic("page_lookup_create: hashin failed %p %p %llx %p", 914 (void *)newpp, (void *)vp, off, (void *)phm); 915 /*NOTREACHED*/ 916 } 917 ASSERT(MUTEX_HELD(phm)); 918 mutex_exit(phm); 919 phm = NULL; 920 page_set_props(newpp, P_REF); 921 page_io_lock(newpp); 922 pp = newpp; 923 se = SE_EXCL; 924 } else { 925 VM_STAT_ADD(page_lookup_cnt[19]); 926 mutex_exit(phm); 927 } 928 929 ASSERT(pp ? PAGE_LOCKED_SE(pp, se) : 1); 930 931 ASSERT(pp ? ((PP_ISFREE(pp) == 0) && (PP_ISAGED(pp) == 0)) : 1); 932 933 return (pp); 934 } 935 936 /* 937 * Search the hash list for the page representing the 938 * specified [vp, offset] and return it locked. Skip 939 * free pages and pages that cannot be locked as requested. 940 * Used while attempting to kluster pages. 941 */ 942 page_t * 943 page_lookup_nowait(vnode_t *vp, u_offset_t off, se_t se) 944 { 945 page_t *pp; 946 kmutex_t *phm; 947 ulong_t index; 948 uint_t locked; 949 950 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 951 VM_STAT_ADD(page_lookup_nowait_cnt[0]); 952 953 index = PAGE_HASH_FUNC(vp, off); 954 PAGE_HASH_SEARCH(index, pp, vp, off); 955 locked = 0; 956 if (pp == NULL) { 957 top: 958 VM_STAT_ADD(page_lookup_nowait_cnt[1]); 959 locked = 1; 960 phm = PAGE_HASH_MUTEX(index); 961 mutex_enter(phm); 962 PAGE_HASH_SEARCH(index, pp, vp, off); 963 } 964 965 if (pp == NULL || PP_ISFREE(pp)) { 966 VM_STAT_ADD(page_lookup_nowait_cnt[2]); 967 pp = NULL; 968 } else { 969 if (!page_trylock(pp, se)) { 970 VM_STAT_ADD(page_lookup_nowait_cnt[3]); 971 pp = NULL; 972 } else { 973 VM_STAT_ADD(page_lookup_nowait_cnt[4]); 974 /* 975 * See the comment in page_lookup() 976 */ 977 if (((volatile struct vnode *)(pp->p_vnode) != vp) || 978 ((u_offset_t)(pp->p_offset) != off)) { 979 VM_STAT_ADD(page_lookup_nowait_cnt[5]); 980 if (locked) { 981 panic("page_lookup_nowait %p", 982 (void *)pp); 983 /*NOTREACHED*/ 984 } 985 page_unlock(pp); 986 goto top; 987 } 988 if (PP_ISFREE(pp)) { 989 VM_STAT_ADD(page_lookup_nowait_cnt[6]); 990 page_unlock(pp); 991 pp = NULL; 992 } 993 } 994 } 995 if (locked) { 996 VM_STAT_ADD(page_lookup_nowait_cnt[7]); 997 mutex_exit(phm); 998 } 999 1000 ASSERT(pp ? PAGE_LOCKED_SE(pp, se) : 1); 1001 1002 return (pp); 1003 } 1004 1005 /* 1006 * Search the hash list for a page with the specified [vp, off] 1007 * that is known to exist and is already locked. This routine 1008 * is typically used by segment SOFTUNLOCK routines. 1009 */ 1010 page_t * 1011 page_find(vnode_t *vp, u_offset_t off) 1012 { 1013 page_t *pp; 1014 kmutex_t *phm; 1015 ulong_t index; 1016 1017 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 1018 VM_STAT_ADD(page_find_cnt); 1019 1020 index = PAGE_HASH_FUNC(vp, off); 1021 phm = PAGE_HASH_MUTEX(index); 1022 1023 mutex_enter(phm); 1024 PAGE_HASH_SEARCH(index, pp, vp, off); 1025 mutex_exit(phm); 1026 1027 ASSERT(pp == NULL || PAGE_LOCKED(pp) || panicstr); 1028 return (pp); 1029 } 1030 1031 /* 1032 * Determine whether a page with the specified [vp, off] 1033 * currently exists in the system. Obviously this should 1034 * only be considered as a hint since nothing prevents the 1035 * page from disappearing or appearing immediately after 1036 * the return from this routine. Subsequently, we don't 1037 * even bother to lock the list. 1038 */ 1039 page_t * 1040 page_exists(vnode_t *vp, u_offset_t off) 1041 { 1042 page_t *pp; 1043 ulong_t index; 1044 1045 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 1046 VM_STAT_ADD(page_exists_cnt); 1047 1048 index = PAGE_HASH_FUNC(vp, off); 1049 PAGE_HASH_SEARCH(index, pp, vp, off); 1050 1051 return (pp); 1052 } 1053 1054 /* 1055 * Determine if physically contiguous pages exist for [vp, off] - [vp, off + 1056 * page_size(szc)) range. if they exist and ppa is not NULL fill ppa array 1057 * with these pages locked SHARED. If necessary reclaim pages from 1058 * freelist. Return 1 if contiguous pages exist and 0 otherwise. 1059 * 1060 * If we fail to lock pages still return 1 if pages exist and contiguous. 1061 * But in this case return value is just a hint. ppa array won't be filled. 1062 * Caller should initialize ppa[0] as NULL to distinguish return value. 1063 * 1064 * Returns 0 if pages don't exist or not physically contiguous. 1065 * 1066 * This routine doesn't work for anonymous(swapfs) pages. 1067 */ 1068 int 1069 page_exists_physcontig(vnode_t *vp, u_offset_t off, uint_t szc, page_t *ppa[]) 1070 { 1071 pgcnt_t pages; 1072 pfn_t pfn; 1073 page_t *rootpp; 1074 pgcnt_t i; 1075 pgcnt_t j; 1076 u_offset_t save_off = off; 1077 ulong_t index; 1078 kmutex_t *phm; 1079 page_t *pp; 1080 uint_t pszc; 1081 int loopcnt = 0; 1082 1083 ASSERT(szc != 0); 1084 ASSERT(vp != NULL); 1085 ASSERT(!IS_SWAPFSVP(vp)); 1086 ASSERT(!VN_ISKAS(vp)); 1087 1088 again: 1089 if (++loopcnt > 3) { 1090 VM_STAT_ADD(page_exphcontg[0]); 1091 return (0); 1092 } 1093 1094 index = PAGE_HASH_FUNC(vp, off); 1095 phm = PAGE_HASH_MUTEX(index); 1096 1097 mutex_enter(phm); 1098 PAGE_HASH_SEARCH(index, pp, vp, off); 1099 mutex_exit(phm); 1100 1101 VM_STAT_ADD(page_exphcontg[1]); 1102 1103 if (pp == NULL) { 1104 VM_STAT_ADD(page_exphcontg[2]); 1105 return (0); 1106 } 1107 1108 pages = page_get_pagecnt(szc); 1109 rootpp = pp; 1110 pfn = rootpp->p_pagenum; 1111 1112 if ((pszc = pp->p_szc) >= szc && ppa != NULL) { 1113 VM_STAT_ADD(page_exphcontg[3]); 1114 if (!page_trylock(pp, SE_SHARED)) { 1115 VM_STAT_ADD(page_exphcontg[4]); 1116 return (1); 1117 } 1118 if (pp->p_szc != pszc || pp->p_vnode != vp || 1119 pp->p_offset != off) { 1120 VM_STAT_ADD(page_exphcontg[5]); 1121 page_unlock(pp); 1122 off = save_off; 1123 goto again; 1124 } 1125 /* 1126 * szc was non zero and vnode and offset matched after we 1127 * locked the page it means it can't become free on us. 1128 */ 1129 ASSERT(!PP_ISFREE(pp)); 1130 if (!IS_P2ALIGNED(pfn, pages)) { 1131 page_unlock(pp); 1132 return (0); 1133 } 1134 ppa[0] = pp; 1135 pp++; 1136 off += PAGESIZE; 1137 pfn++; 1138 for (i = 1; i < pages; i++, pp++, off += PAGESIZE, pfn++) { 1139 if (!page_trylock(pp, SE_SHARED)) { 1140 VM_STAT_ADD(page_exphcontg[6]); 1141 pp--; 1142 while (i-- > 0) { 1143 page_unlock(pp); 1144 pp--; 1145 } 1146 ppa[0] = NULL; 1147 return (1); 1148 } 1149 if (pp->p_szc != pszc) { 1150 VM_STAT_ADD(page_exphcontg[7]); 1151 page_unlock(pp); 1152 pp--; 1153 while (i-- > 0) { 1154 page_unlock(pp); 1155 pp--; 1156 } 1157 ppa[0] = NULL; 1158 off = save_off; 1159 goto again; 1160 } 1161 /* 1162 * szc the same as for previous already locked pages 1163 * with right identity. Since this page had correct 1164 * szc after we locked it can't get freed or destroyed 1165 * and therefore must have the expected identity. 1166 */ 1167 ASSERT(!PP_ISFREE(pp)); 1168 if (pp->p_vnode != vp || 1169 pp->p_offset != off) { 1170 panic("page_exists_physcontig: " 1171 "large page identity doesn't match"); 1172 } 1173 ppa[i] = pp; 1174 ASSERT(pp->p_pagenum == pfn); 1175 } 1176 VM_STAT_ADD(page_exphcontg[8]); 1177 ppa[pages] = NULL; 1178 return (1); 1179 } else if (pszc >= szc) { 1180 VM_STAT_ADD(page_exphcontg[9]); 1181 if (!IS_P2ALIGNED(pfn, pages)) { 1182 return (0); 1183 } 1184 return (1); 1185 } 1186 1187 if (!IS_P2ALIGNED(pfn, pages)) { 1188 VM_STAT_ADD(page_exphcontg[10]); 1189 return (0); 1190 } 1191 1192 if (page_numtomemseg_nolock(pfn) != 1193 page_numtomemseg_nolock(pfn + pages - 1)) { 1194 VM_STAT_ADD(page_exphcontg[11]); 1195 return (0); 1196 } 1197 1198 /* 1199 * We loop up 4 times across pages to promote page size. 1200 * We're extra cautious to promote page size atomically with respect 1201 * to everybody else. But we can probably optimize into 1 loop if 1202 * this becomes an issue. 1203 */ 1204 1205 for (i = 0; i < pages; i++, pp++, off += PAGESIZE, pfn++) { 1206 ASSERT(pp->p_pagenum == pfn); 1207 if (!page_trylock(pp, SE_EXCL)) { 1208 VM_STAT_ADD(page_exphcontg[12]); 1209 break; 1210 } 1211 if (pp->p_vnode != vp || 1212 pp->p_offset != off) { 1213 VM_STAT_ADD(page_exphcontg[13]); 1214 page_unlock(pp); 1215 break; 1216 } 1217 if (pp->p_szc >= szc) { 1218 ASSERT(i == 0); 1219 page_unlock(pp); 1220 off = save_off; 1221 goto again; 1222 } 1223 } 1224 1225 if (i != pages) { 1226 VM_STAT_ADD(page_exphcontg[14]); 1227 --pp; 1228 while (i-- > 0) { 1229 page_unlock(pp); 1230 --pp; 1231 } 1232 return (0); 1233 } 1234 1235 pp = rootpp; 1236 for (i = 0; i < pages; i++, pp++) { 1237 if (PP_ISFREE(pp)) { 1238 VM_STAT_ADD(page_exphcontg[15]); 1239 ASSERT(!PP_ISAGED(pp)); 1240 ASSERT(pp->p_szc == 0); 1241 if (!page_reclaim(pp, NULL)) { 1242 break; 1243 } 1244 } else { 1245 ASSERT(pp->p_szc < szc); 1246 VM_STAT_ADD(page_exphcontg[16]); 1247 (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD); 1248 } 1249 } 1250 if (i < pages) { 1251 VM_STAT_ADD(page_exphcontg[17]); 1252 /* 1253 * page_reclaim failed because we were out of memory. 1254 * drop the rest of the locks and return because this page 1255 * must be already reallocated anyway. 1256 */ 1257 pp = rootpp; 1258 for (j = 0; j < pages; j++, pp++) { 1259 if (j != i) { 1260 page_unlock(pp); 1261 } 1262 } 1263 return (0); 1264 } 1265 1266 off = save_off; 1267 pp = rootpp; 1268 for (i = 0; i < pages; i++, pp++, off += PAGESIZE) { 1269 ASSERT(PAGE_EXCL(pp)); 1270 ASSERT(!PP_ISFREE(pp)); 1271 ASSERT(!hat_page_is_mapped(pp)); 1272 ASSERT(pp->p_vnode == vp); 1273 ASSERT(pp->p_offset == off); 1274 pp->p_szc = szc; 1275 } 1276 pp = rootpp; 1277 for (i = 0; i < pages; i++, pp++) { 1278 if (ppa == NULL) { 1279 page_unlock(pp); 1280 } else { 1281 ppa[i] = pp; 1282 page_downgrade(ppa[i]); 1283 } 1284 } 1285 if (ppa != NULL) { 1286 ppa[pages] = NULL; 1287 } 1288 VM_STAT_ADD(page_exphcontg[18]); 1289 ASSERT(vp->v_pages != NULL); 1290 return (1); 1291 } 1292 1293 /* 1294 * Determine whether a page with the specified [vp, off] 1295 * currently exists in the system and if so return its 1296 * size code. Obviously this should only be considered as 1297 * a hint since nothing prevents the page from disappearing 1298 * or appearing immediately after the return from this routine. 1299 */ 1300 int 1301 page_exists_forreal(vnode_t *vp, u_offset_t off, uint_t *szc) 1302 { 1303 page_t *pp; 1304 kmutex_t *phm; 1305 ulong_t index; 1306 int rc = 0; 1307 1308 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 1309 ASSERT(szc != NULL); 1310 VM_STAT_ADD(page_exists_forreal_cnt); 1311 1312 index = PAGE_HASH_FUNC(vp, off); 1313 phm = PAGE_HASH_MUTEX(index); 1314 1315 mutex_enter(phm); 1316 PAGE_HASH_SEARCH(index, pp, vp, off); 1317 if (pp != NULL) { 1318 *szc = pp->p_szc; 1319 rc = 1; 1320 } 1321 mutex_exit(phm); 1322 return (rc); 1323 } 1324 1325 /* wakeup threads waiting for pages in page_create_get_something() */ 1326 void 1327 wakeup_pcgs(void) 1328 { 1329 if (!CV_HAS_WAITERS(&pcgs_cv)) 1330 return; 1331 cv_broadcast(&pcgs_cv); 1332 } 1333 1334 /* 1335 * 'freemem' is used all over the kernel as an indication of how many 1336 * pages are free (either on the cache list or on the free page list) 1337 * in the system. In very few places is a really accurate 'freemem' 1338 * needed. To avoid contention of the lock protecting a the 1339 * single freemem, it was spread out into NCPU buckets. Set_freemem 1340 * sets freemem to the total of all NCPU buckets. It is called from 1341 * clock() on each TICK. 1342 */ 1343 void 1344 set_freemem() 1345 { 1346 struct pcf *p; 1347 ulong_t t; 1348 uint_t i; 1349 1350 t = 0; 1351 p = pcf; 1352 for (i = 0; i < pcf_fanout; i++) { 1353 t += p->pcf_count; 1354 p++; 1355 } 1356 freemem = t; 1357 1358 /* 1359 * Don't worry about grabbing mutex. It's not that 1360 * critical if we miss a tick or two. This is 1361 * where we wakeup possible delayers in 1362 * page_create_get_something(). 1363 */ 1364 wakeup_pcgs(); 1365 } 1366 1367 ulong_t 1368 get_freemem() 1369 { 1370 struct pcf *p; 1371 ulong_t t; 1372 uint_t i; 1373 1374 t = 0; 1375 p = pcf; 1376 for (i = 0; i < pcf_fanout; i++) { 1377 t += p->pcf_count; 1378 p++; 1379 } 1380 /* 1381 * We just calculated it, might as well set it. 1382 */ 1383 freemem = t; 1384 return (t); 1385 } 1386 1387 /* 1388 * Acquire all of the page cache & free (pcf) locks. 1389 */ 1390 void 1391 pcf_acquire_all() 1392 { 1393 struct pcf *p; 1394 uint_t i; 1395 1396 p = pcf; 1397 for (i = 0; i < pcf_fanout; i++) { 1398 mutex_enter(&p->pcf_lock); 1399 p++; 1400 } 1401 } 1402 1403 /* 1404 * Release all the pcf_locks. 1405 */ 1406 void 1407 pcf_release_all() 1408 { 1409 struct pcf *p; 1410 uint_t i; 1411 1412 p = pcf; 1413 for (i = 0; i < pcf_fanout; i++) { 1414 mutex_exit(&p->pcf_lock); 1415 p++; 1416 } 1417 } 1418 1419 /* 1420 * Inform the VM system that we need some pages freed up. 1421 * Calls must be symmetric, e.g.: 1422 * 1423 * page_needfree(100); 1424 * wait a bit; 1425 * page_needfree(-100); 1426 */ 1427 void 1428 page_needfree(spgcnt_t npages) 1429 { 1430 mutex_enter(&new_freemem_lock); 1431 needfree += npages; 1432 mutex_exit(&new_freemem_lock); 1433 } 1434 1435 /* 1436 * Throttle for page_create(): try to prevent freemem from dropping 1437 * below throttlefree. We can't provide a 100% guarantee because 1438 * KM_NOSLEEP allocations, page_reclaim(), and various other things 1439 * nibble away at the freelist. However, we can block all PG_WAIT 1440 * allocations until memory becomes available. The motivation is 1441 * that several things can fall apart when there's no free memory: 1442 * 1443 * (1) If pageout() needs memory to push a page, the system deadlocks. 1444 * 1445 * (2) By (broken) specification, timeout(9F) can neither fail nor 1446 * block, so it has no choice but to panic the system if it 1447 * cannot allocate a callout structure. 1448 * 1449 * (3) Like timeout(), ddi_set_callback() cannot fail and cannot block; 1450 * it panics if it cannot allocate a callback structure. 1451 * 1452 * (4) Untold numbers of third-party drivers have not yet been hardened 1453 * against KM_NOSLEEP and/or allocb() failures; they simply assume 1454 * success and panic the system with a data fault on failure. 1455 * (The long-term solution to this particular problem is to ship 1456 * hostile fault-injecting DEBUG kernels with the DDK.) 1457 * 1458 * It is theoretically impossible to guarantee success of non-blocking 1459 * allocations, but in practice, this throttle is very hard to break. 1460 */ 1461 static int 1462 page_create_throttle(pgcnt_t npages, int flags) 1463 { 1464 ulong_t fm; 1465 uint_t i; 1466 pgcnt_t tf; /* effective value of throttlefree */ 1467 1468 /* 1469 * Never deny pages when: 1470 * - it's a thread that cannot block [NOMEMWAIT()] 1471 * - the allocation cannot block and must not fail 1472 * - the allocation cannot block and is pageout dispensated 1473 */ 1474 if (NOMEMWAIT() || 1475 ((flags & (PG_WAIT | PG_PANIC)) == PG_PANIC) || 1476 ((flags & (PG_WAIT | PG_PUSHPAGE)) == PG_PUSHPAGE)) 1477 return (1); 1478 1479 /* 1480 * If the allocation can't block, we look favorably upon it 1481 * unless we're below pageout_reserve. In that case we fail 1482 * the allocation because we want to make sure there are a few 1483 * pages available for pageout. 1484 */ 1485 if ((flags & PG_WAIT) == 0) 1486 return (freemem >= npages + pageout_reserve); 1487 1488 /* Calculate the effective throttlefree value */ 1489 tf = throttlefree - 1490 ((flags & PG_PUSHPAGE) ? pageout_reserve : 0); 1491 1492 cv_signal(&proc_pageout->p_cv); 1493 1494 for (;;) { 1495 fm = 0; 1496 pcf_acquire_all(); 1497 mutex_enter(&new_freemem_lock); 1498 for (i = 0; i < pcf_fanout; i++) { 1499 fm += pcf[i].pcf_count; 1500 pcf[i].pcf_wait++; 1501 mutex_exit(&pcf[i].pcf_lock); 1502 } 1503 freemem = fm; 1504 if (freemem >= npages + tf) { 1505 mutex_exit(&new_freemem_lock); 1506 break; 1507 } 1508 needfree += npages; 1509 freemem_wait++; 1510 cv_wait(&freemem_cv, &new_freemem_lock); 1511 freemem_wait--; 1512 needfree -= npages; 1513 mutex_exit(&new_freemem_lock); 1514 } 1515 return (1); 1516 } 1517 1518 /* 1519 * page_create_wait() is called to either coalesce pages from the 1520 * different pcf buckets or to wait because there simply are not 1521 * enough pages to satisfy the caller's request. 1522 * 1523 * Sadly, this is called from platform/vm/vm_machdep.c 1524 */ 1525 int 1526 page_create_wait(pgcnt_t npages, uint_t flags) 1527 { 1528 pgcnt_t total; 1529 uint_t i; 1530 struct pcf *p; 1531 1532 /* 1533 * Wait until there are enough free pages to satisfy our 1534 * entire request. 1535 * We set needfree += npages before prodding pageout, to make sure 1536 * it does real work when npages > lotsfree > freemem. 1537 */ 1538 VM_STAT_ADD(page_create_not_enough); 1539 1540 ASSERT(!kcage_on ? !(flags & PG_NORELOC) : 1); 1541 checkagain: 1542 if ((flags & PG_NORELOC) && 1543 kcage_freemem < kcage_throttlefree + npages) 1544 (void) kcage_create_throttle(npages, flags); 1545 1546 if (freemem < npages + throttlefree) 1547 if (!page_create_throttle(npages, flags)) 1548 return (0); 1549 1550 if (pcf_decrement_bucket(npages) || 1551 pcf_decrement_multiple(&total, npages, 0)) 1552 return (1); 1553 1554 /* 1555 * All of the pcf locks are held, there are not enough pages 1556 * to satisfy the request (npages < total). 1557 * Be sure to acquire the new_freemem_lock before dropping 1558 * the pcf locks. This prevents dropping wakeups in page_free(). 1559 * The order is always pcf_lock then new_freemem_lock. 1560 * 1561 * Since we hold all the pcf locks, it is a good time to set freemem. 1562 * 1563 * If the caller does not want to wait, return now. 1564 * Else turn the pageout daemon loose to find something 1565 * and wait till it does. 1566 * 1567 */ 1568 freemem = total; 1569 1570 if ((flags & PG_WAIT) == 0) { 1571 pcf_release_all(); 1572 1573 TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_NOMEM, 1574 "page_create_nomem:npages %ld freemem %ld", npages, freemem); 1575 return (0); 1576 } 1577 1578 ASSERT(proc_pageout != NULL); 1579 cv_signal(&proc_pageout->p_cv); 1580 1581 TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_SLEEP_START, 1582 "page_create_sleep_start: freemem %ld needfree %ld", 1583 freemem, needfree); 1584 1585 /* 1586 * We are going to wait. 1587 * We currently hold all of the pcf_locks, 1588 * get the new_freemem_lock (it protects freemem_wait), 1589 * before dropping the pcf_locks. 1590 */ 1591 mutex_enter(&new_freemem_lock); 1592 1593 p = pcf; 1594 for (i = 0; i < pcf_fanout; i++) { 1595 p->pcf_wait++; 1596 mutex_exit(&p->pcf_lock); 1597 p++; 1598 } 1599 1600 needfree += npages; 1601 freemem_wait++; 1602 1603 cv_wait(&freemem_cv, &new_freemem_lock); 1604 1605 freemem_wait--; 1606 needfree -= npages; 1607 1608 mutex_exit(&new_freemem_lock); 1609 1610 TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_SLEEP_END, 1611 "page_create_sleep_end: freemem %ld needfree %ld", 1612 freemem, needfree); 1613 1614 VM_STAT_ADD(page_create_not_enough_again); 1615 goto checkagain; 1616 } 1617 /* 1618 * A routine to do the opposite of page_create_wait(). 1619 */ 1620 void 1621 page_create_putback(spgcnt_t npages) 1622 { 1623 struct pcf *p; 1624 pgcnt_t lump; 1625 uint_t *which; 1626 1627 /* 1628 * When a contiguous lump is broken up, we have to 1629 * deal with lots of pages (min 64) so lets spread 1630 * the wealth around. 1631 */ 1632 lump = roundup(npages, pcf_fanout) / pcf_fanout; 1633 freemem += npages; 1634 1635 for (p = pcf; (npages > 0) && (p < &pcf[pcf_fanout]); p++) { 1636 which = &p->pcf_count; 1637 1638 mutex_enter(&p->pcf_lock); 1639 1640 if (p->pcf_block) { 1641 which = &p->pcf_reserve; 1642 } 1643 1644 if (lump < npages) { 1645 *which += (uint_t)lump; 1646 npages -= lump; 1647 } else { 1648 *which += (uint_t)npages; 1649 npages = 0; 1650 } 1651 1652 if (p->pcf_wait) { 1653 mutex_enter(&new_freemem_lock); 1654 /* 1655 * Check to see if some other thread 1656 * is actually waiting. Another bucket 1657 * may have woken it up by now. If there 1658 * are no waiters, then set our pcf_wait 1659 * count to zero to avoid coming in here 1660 * next time. 1661 */ 1662 if (freemem_wait) { 1663 if (npages > 1) { 1664 cv_broadcast(&freemem_cv); 1665 } else { 1666 cv_signal(&freemem_cv); 1667 } 1668 p->pcf_wait--; 1669 } else { 1670 p->pcf_wait = 0; 1671 } 1672 mutex_exit(&new_freemem_lock); 1673 } 1674 mutex_exit(&p->pcf_lock); 1675 } 1676 ASSERT(npages == 0); 1677 } 1678 1679 /* 1680 * A helper routine for page_create_get_something. 1681 * The indenting got to deep down there. 1682 * Unblock the pcf counters. Any pages freed after 1683 * pcf_block got set are moved to pcf_count and 1684 * wakeups (cv_broadcast() or cv_signal()) are done as needed. 1685 */ 1686 static void 1687 pcgs_unblock(void) 1688 { 1689 int i; 1690 struct pcf *p; 1691 1692 /* Update freemem while we're here. */ 1693 freemem = 0; 1694 p = pcf; 1695 for (i = 0; i < pcf_fanout; i++) { 1696 mutex_enter(&p->pcf_lock); 1697 ASSERT(p->pcf_count == 0); 1698 p->pcf_count = p->pcf_reserve; 1699 p->pcf_block = 0; 1700 freemem += p->pcf_count; 1701 if (p->pcf_wait) { 1702 mutex_enter(&new_freemem_lock); 1703 if (freemem_wait) { 1704 if (p->pcf_reserve > 1) { 1705 cv_broadcast(&freemem_cv); 1706 p->pcf_wait = 0; 1707 } else { 1708 cv_signal(&freemem_cv); 1709 p->pcf_wait--; 1710 } 1711 } else { 1712 p->pcf_wait = 0; 1713 } 1714 mutex_exit(&new_freemem_lock); 1715 } 1716 p->pcf_reserve = 0; 1717 mutex_exit(&p->pcf_lock); 1718 p++; 1719 } 1720 } 1721 1722 /* 1723 * Called from page_create_va() when both the cache and free lists 1724 * have been checked once. 1725 * 1726 * Either returns a page or panics since the accounting was done 1727 * way before we got here. 1728 * 1729 * We don't come here often, so leave the accounting on permanently. 1730 */ 1731 1732 #define MAX_PCGS 100 1733 1734 #ifdef DEBUG 1735 #define PCGS_TRIES 100 1736 #else /* DEBUG */ 1737 #define PCGS_TRIES 10 1738 #endif /* DEBUG */ 1739 1740 #ifdef VM_STATS 1741 uint_t pcgs_counts[PCGS_TRIES]; 1742 uint_t pcgs_too_many; 1743 uint_t pcgs_entered; 1744 uint_t pcgs_entered_noreloc; 1745 uint_t pcgs_locked; 1746 uint_t pcgs_cagelocked; 1747 #endif /* VM_STATS */ 1748 1749 static page_t * 1750 page_create_get_something(vnode_t *vp, u_offset_t off, struct seg *seg, 1751 caddr_t vaddr, uint_t flags) 1752 { 1753 uint_t count; 1754 page_t *pp; 1755 uint_t locked, i; 1756 struct pcf *p; 1757 lgrp_t *lgrp; 1758 int cagelocked = 0; 1759 1760 VM_STAT_ADD(pcgs_entered); 1761 1762 /* 1763 * Tap any reserve freelists: if we fail now, we'll die 1764 * since the page(s) we're looking for have already been 1765 * accounted for. 1766 */ 1767 flags |= PG_PANIC; 1768 1769 if ((flags & PG_NORELOC) != 0) { 1770 VM_STAT_ADD(pcgs_entered_noreloc); 1771 /* 1772 * Requests for free pages from critical threads 1773 * such as pageout still won't throttle here, but 1774 * we must try again, to give the cageout thread 1775 * another chance to catch up. Since we already 1776 * accounted for the pages, we had better get them 1777 * this time. 1778 * 1779 * N.B. All non-critical threads acquire the pcgs_cagelock 1780 * to serialize access to the freelists. This implements a 1781 * turnstile-type synchornization to avoid starvation of 1782 * critical requests for PG_NORELOC memory by non-critical 1783 * threads: all non-critical threads must acquire a 'ticket' 1784 * before passing through, which entails making sure 1785 * kcage_freemem won't fall below minfree prior to grabbing 1786 * pages from the freelists. 1787 */ 1788 if (kcage_create_throttle(1, flags) == KCT_NONCRIT) { 1789 mutex_enter(&pcgs_cagelock); 1790 cagelocked = 1; 1791 VM_STAT_ADD(pcgs_cagelocked); 1792 } 1793 } 1794 1795 /* 1796 * Time to get serious. 1797 * We failed to get a `correctly colored' page from both the 1798 * free and cache lists. 1799 * We escalate in stage. 1800 * 1801 * First try both lists without worring about color. 1802 * 1803 * Then, grab all page accounting locks (ie. pcf[]) and 1804 * steal any pages that they have and set the pcf_block flag to 1805 * stop deletions from the lists. This will help because 1806 * a page can get added to the free list while we are looking 1807 * at the cache list, then another page could be added to the cache 1808 * list allowing the page on the free list to be removed as we 1809 * move from looking at the cache list to the free list. This 1810 * could happen over and over. We would never find the page 1811 * we have accounted for. 1812 * 1813 * Noreloc pages are a subset of the global (relocatable) page pool. 1814 * They are not tracked separately in the pcf bins, so it is 1815 * impossible to know when doing pcf accounting if the available 1816 * page(s) are noreloc pages or not. When looking for a noreloc page 1817 * it is quite easy to end up here even if the global (relocatable) 1818 * page pool has plenty of free pages but the noreloc pool is empty. 1819 * 1820 * When the noreloc pool is empty (or low), additional noreloc pages 1821 * are created by converting pages from the global page pool. This 1822 * process will stall during pcf accounting if the pcf bins are 1823 * already locked. Such is the case when a noreloc allocation is 1824 * looping here in page_create_get_something waiting for more noreloc 1825 * pages to appear. 1826 * 1827 * Short of adding a new field to the pcf bins to accurately track 1828 * the number of free noreloc pages, we instead do not grab the 1829 * pcgs_lock, do not set the pcf blocks and do not timeout when 1830 * allocating a noreloc page. This allows noreloc allocations to 1831 * loop without blocking global page pool allocations. 1832 * 1833 * NOTE: the behaviour of page_create_get_something has not changed 1834 * for the case of global page pool allocations. 1835 */ 1836 1837 flags &= ~PG_MATCH_COLOR; 1838 locked = 0; 1839 #if defined(__i386) || defined(__amd64) 1840 flags = page_create_update_flags_x86(flags); 1841 #endif 1842 1843 lgrp = lgrp_mem_choose(seg, vaddr, PAGESIZE); 1844 1845 for (count = 0; kcage_on || count < MAX_PCGS; count++) { 1846 pp = page_get_freelist(vp, off, seg, vaddr, PAGESIZE, 1847 flags, lgrp); 1848 if (pp == NULL) { 1849 pp = page_get_cachelist(vp, off, seg, vaddr, 1850 flags, lgrp); 1851 } 1852 if (pp == NULL) { 1853 /* 1854 * Serialize. Don't fight with other pcgs(). 1855 */ 1856 if (!locked && (!kcage_on || !(flags & PG_NORELOC))) { 1857 mutex_enter(&pcgs_lock); 1858 VM_STAT_ADD(pcgs_locked); 1859 locked = 1; 1860 p = pcf; 1861 for (i = 0; i < pcf_fanout; i++) { 1862 mutex_enter(&p->pcf_lock); 1863 ASSERT(p->pcf_block == 0); 1864 p->pcf_block = 1; 1865 p->pcf_reserve = p->pcf_count; 1866 p->pcf_count = 0; 1867 mutex_exit(&p->pcf_lock); 1868 p++; 1869 } 1870 freemem = 0; 1871 } 1872 1873 if (count) { 1874 /* 1875 * Since page_free() puts pages on 1876 * a list then accounts for it, we 1877 * just have to wait for page_free() 1878 * to unlock any page it was working 1879 * with. The page_lock()-page_reclaim() 1880 * path falls in the same boat. 1881 * 1882 * We don't need to check on the 1883 * PG_WAIT flag, we have already 1884 * accounted for the page we are 1885 * looking for in page_create_va(). 1886 * 1887 * We just wait a moment to let any 1888 * locked pages on the lists free up, 1889 * then continue around and try again. 1890 * 1891 * Will be awakened by set_freemem(). 1892 */ 1893 mutex_enter(&pcgs_wait_lock); 1894 cv_wait(&pcgs_cv, &pcgs_wait_lock); 1895 mutex_exit(&pcgs_wait_lock); 1896 } 1897 } else { 1898 #ifdef VM_STATS 1899 if (count >= PCGS_TRIES) { 1900 VM_STAT_ADD(pcgs_too_many); 1901 } else { 1902 VM_STAT_ADD(pcgs_counts[count]); 1903 } 1904 #endif 1905 if (locked) { 1906 pcgs_unblock(); 1907 mutex_exit(&pcgs_lock); 1908 } 1909 if (cagelocked) 1910 mutex_exit(&pcgs_cagelock); 1911 return (pp); 1912 } 1913 } 1914 /* 1915 * we go down holding the pcf locks. 1916 */ 1917 panic("no %spage found %d", 1918 ((flags & PG_NORELOC) ? "non-reloc " : ""), count); 1919 /*NOTREACHED*/ 1920 } 1921 1922 /* 1923 * Create enough pages for "bytes" worth of data starting at 1924 * "off" in "vp". 1925 * 1926 * Where flag must be one of: 1927 * 1928 * PG_EXCL: Exclusive create (fail if any page already 1929 * exists in the page cache) which does not 1930 * wait for memory to become available. 1931 * 1932 * PG_WAIT: Non-exclusive create which can wait for 1933 * memory to become available. 1934 * 1935 * PG_PHYSCONTIG: Allocate physically contiguous pages. 1936 * (Not Supported) 1937 * 1938 * A doubly linked list of pages is returned to the caller. Each page 1939 * on the list has the "exclusive" (p_selock) lock and "iolock" (p_iolock) 1940 * lock. 1941 * 1942 * Unable to change the parameters to page_create() in a minor release, 1943 * we renamed page_create() to page_create_va(), changed all known calls 1944 * from page_create() to page_create_va(), and created this wrapper. 1945 * 1946 * Upon a major release, we should break compatibility by deleting this 1947 * wrapper, and replacing all the strings "page_create_va", with "page_create". 1948 * 1949 * NOTE: There is a copy of this interface as page_create_io() in 1950 * i86/vm/vm_machdep.c. Any bugs fixed here should be applied 1951 * there. 1952 */ 1953 page_t * 1954 page_create(vnode_t *vp, u_offset_t off, size_t bytes, uint_t flags) 1955 { 1956 caddr_t random_vaddr; 1957 struct seg kseg; 1958 1959 #ifdef DEBUG 1960 cmn_err(CE_WARN, "Using deprecated interface page_create: caller %p", 1961 (void *)caller()); 1962 #endif 1963 1964 random_vaddr = (caddr_t)(((uintptr_t)vp >> 7) ^ 1965 (uintptr_t)(off >> PAGESHIFT)); 1966 kseg.s_as = &kas; 1967 1968 return (page_create_va(vp, off, bytes, flags, &kseg, random_vaddr)); 1969 } 1970 1971 #ifdef DEBUG 1972 uint32_t pg_alloc_pgs_mtbf = 0; 1973 #endif 1974 1975 /* 1976 * Used for large page support. It will attempt to allocate 1977 * a large page(s) off the freelist. 1978 * 1979 * Returns non zero on failure. 1980 */ 1981 int 1982 page_alloc_pages(struct vnode *vp, struct seg *seg, caddr_t addr, 1983 page_t **basepp, page_t *ppa[], uint_t szc, int anypgsz, int pgflags) 1984 { 1985 pgcnt_t npgs, curnpgs, totpgs; 1986 size_t pgsz; 1987 page_t *pplist = NULL, *pp; 1988 int err = 0; 1989 lgrp_t *lgrp; 1990 1991 ASSERT(szc != 0 && szc <= (page_num_pagesizes() - 1)); 1992 ASSERT(pgflags == 0 || pgflags == PG_LOCAL); 1993 1994 /* 1995 * Check if system heavily prefers local large pages over remote 1996 * on systems with multiple lgroups. 1997 */ 1998 if (lpg_alloc_prefer == LPAP_LOCAL && nlgrps > 1) { 1999 pgflags = PG_LOCAL; 2000 } 2001 2002 VM_STAT_ADD(alloc_pages[0]); 2003 2004 #ifdef DEBUG 2005 if (pg_alloc_pgs_mtbf && !(gethrtime() % pg_alloc_pgs_mtbf)) { 2006 return (ENOMEM); 2007 } 2008 #endif 2009 2010 /* 2011 * One must be NULL but not both. 2012 * And one must be non NULL but not both. 2013 */ 2014 ASSERT(basepp != NULL || ppa != NULL); 2015 ASSERT(basepp == NULL || ppa == NULL); 2016 2017 #if defined(__i386) || defined(__amd64) 2018 while (page_chk_freelist(szc) == 0) { 2019 VM_STAT_ADD(alloc_pages[8]); 2020 if (anypgsz == 0 || --szc == 0) 2021 return (ENOMEM); 2022 } 2023 #endif 2024 2025 pgsz = page_get_pagesize(szc); 2026 totpgs = curnpgs = npgs = pgsz >> PAGESHIFT; 2027 2028 ASSERT(((uintptr_t)addr & (pgsz - 1)) == 0); 2029 2030 (void) page_create_wait(npgs, PG_WAIT); 2031 2032 while (npgs && szc) { 2033 lgrp = lgrp_mem_choose(seg, addr, pgsz); 2034 if (pgflags == PG_LOCAL) { 2035 pp = page_get_freelist(vp, 0, seg, addr, pgsz, 2036 pgflags, lgrp); 2037 if (pp == NULL) { 2038 pp = page_get_freelist(vp, 0, seg, addr, pgsz, 2039 0, lgrp); 2040 } 2041 } else { 2042 pp = page_get_freelist(vp, 0, seg, addr, pgsz, 2043 0, lgrp); 2044 } 2045 if (pp != NULL) { 2046 VM_STAT_ADD(alloc_pages[1]); 2047 page_list_concat(&pplist, &pp); 2048 ASSERT(npgs >= curnpgs); 2049 npgs -= curnpgs; 2050 } else if (anypgsz) { 2051 VM_STAT_ADD(alloc_pages[2]); 2052 szc--; 2053 pgsz = page_get_pagesize(szc); 2054 curnpgs = pgsz >> PAGESHIFT; 2055 } else { 2056 VM_STAT_ADD(alloc_pages[3]); 2057 ASSERT(npgs == totpgs); 2058 page_create_putback(npgs); 2059 return (ENOMEM); 2060 } 2061 } 2062 if (szc == 0) { 2063 VM_STAT_ADD(alloc_pages[4]); 2064 ASSERT(npgs != 0); 2065 page_create_putback(npgs); 2066 err = ENOMEM; 2067 } else if (basepp != NULL) { 2068 ASSERT(npgs == 0); 2069 ASSERT(ppa == NULL); 2070 *basepp = pplist; 2071 } 2072 2073 npgs = totpgs - npgs; 2074 pp = pplist; 2075 2076 /* 2077 * Clear the free and age bits. Also if we were passed in a ppa then 2078 * fill it in with all the constituent pages from the large page. But 2079 * if we failed to allocate all the pages just free what we got. 2080 */ 2081 while (npgs != 0) { 2082 ASSERT(PP_ISFREE(pp)); 2083 ASSERT(PP_ISAGED(pp)); 2084 if (ppa != NULL || err != 0) { 2085 if (err == 0) { 2086 VM_STAT_ADD(alloc_pages[5]); 2087 PP_CLRFREE(pp); 2088 PP_CLRAGED(pp); 2089 page_sub(&pplist, pp); 2090 *ppa++ = pp; 2091 npgs--; 2092 } else { 2093 VM_STAT_ADD(alloc_pages[6]); 2094 ASSERT(pp->p_szc != 0); 2095 curnpgs = page_get_pagecnt(pp->p_szc); 2096 page_list_break(&pp, &pplist, curnpgs); 2097 page_list_add_pages(pp, 0); 2098 page_create_putback(curnpgs); 2099 ASSERT(npgs >= curnpgs); 2100 npgs -= curnpgs; 2101 } 2102 pp = pplist; 2103 } else { 2104 VM_STAT_ADD(alloc_pages[7]); 2105 PP_CLRFREE(pp); 2106 PP_CLRAGED(pp); 2107 pp = pp->p_next; 2108 npgs--; 2109 } 2110 } 2111 return (err); 2112 } 2113 2114 /* 2115 * Get a single large page off of the freelists, and set it up for use. 2116 * Number of bytes requested must be a supported page size. 2117 * 2118 * Note that this call may fail even if there is sufficient 2119 * memory available or PG_WAIT is set, so the caller must 2120 * be willing to fallback on page_create_va(), block and retry, 2121 * or fail the requester. 2122 */ 2123 page_t * 2124 page_create_va_large(vnode_t *vp, u_offset_t off, size_t bytes, uint_t flags, 2125 struct seg *seg, caddr_t vaddr, void *arg) 2126 { 2127 pgcnt_t npages; 2128 page_t *pp; 2129 page_t *rootpp; 2130 lgrp_t *lgrp; 2131 lgrp_id_t *lgrpid = (lgrp_id_t *)arg; 2132 2133 ASSERT(vp != NULL); 2134 2135 ASSERT((flags & ~(PG_EXCL | PG_WAIT | 2136 PG_NORELOC | PG_PANIC | PG_PUSHPAGE)) == 0); 2137 /* but no others */ 2138 2139 ASSERT((flags & PG_EXCL) == PG_EXCL); 2140 2141 npages = btop(bytes); 2142 2143 if (!kcage_on || panicstr) { 2144 /* 2145 * Cage is OFF, or we are single threaded in 2146 * panic, so make everything a RELOC request. 2147 */ 2148 flags &= ~PG_NORELOC; 2149 } 2150 2151 /* 2152 * Make sure there's adequate physical memory available. 2153 * Note: PG_WAIT is ignored here. 2154 */ 2155 if (freemem <= throttlefree + npages) { 2156 VM_STAT_ADD(page_create_large_cnt[1]); 2157 return (NULL); 2158 } 2159 2160 /* 2161 * If cage is on, dampen draw from cage when available 2162 * cage space is low. 2163 */ 2164 if ((flags & (PG_NORELOC | PG_WAIT)) == (PG_NORELOC | PG_WAIT) && 2165 kcage_freemem < kcage_throttlefree + npages) { 2166 2167 /* 2168 * The cage is on, the caller wants PG_NORELOC 2169 * pages and available cage memory is very low. 2170 * Call kcage_create_throttle() to attempt to 2171 * control demand on the cage. 2172 */ 2173 if (kcage_create_throttle(npages, flags) == KCT_FAILURE) { 2174 VM_STAT_ADD(page_create_large_cnt[2]); 2175 return (NULL); 2176 } 2177 } 2178 2179 if (!pcf_decrement_bucket(npages) && 2180 !pcf_decrement_multiple(NULL, npages, 1)) { 2181 VM_STAT_ADD(page_create_large_cnt[4]); 2182 return (NULL); 2183 } 2184 2185 /* 2186 * This is where this function behaves fundamentally differently 2187 * than page_create_va(); since we're intending to map the page 2188 * with a single TTE, we have to get it as a physically contiguous 2189 * hardware pagesize chunk. If we can't, we fail. 2190 */ 2191 if (lgrpid != NULL && *lgrpid >= 0 && *lgrpid <= lgrp_alloc_max && 2192 LGRP_EXISTS(lgrp_table[*lgrpid])) 2193 lgrp = lgrp_table[*lgrpid]; 2194 else 2195 lgrp = lgrp_mem_choose(seg, vaddr, bytes); 2196 2197 if ((rootpp = page_get_freelist(&kvp, off, seg, vaddr, 2198 bytes, flags & ~PG_MATCH_COLOR, lgrp)) == NULL) { 2199 page_create_putback(npages); 2200 VM_STAT_ADD(page_create_large_cnt[5]); 2201 return (NULL); 2202 } 2203 2204 /* 2205 * if we got the page with the wrong mtype give it back this is a 2206 * workaround for CR 6249718. When CR 6249718 is fixed we never get 2207 * inside "if" and the workaround becomes just a nop 2208 */ 2209 if (kcage_on && (flags & PG_NORELOC) && !PP_ISNORELOC(rootpp)) { 2210 page_list_add_pages(rootpp, 0); 2211 page_create_putback(npages); 2212 VM_STAT_ADD(page_create_large_cnt[6]); 2213 return (NULL); 2214 } 2215 2216 /* 2217 * If satisfying this request has left us with too little 2218 * memory, start the wheels turning to get some back. The 2219 * first clause of the test prevents waking up the pageout 2220 * daemon in situations where it would decide that there's 2221 * nothing to do. 2222 */ 2223 if (nscan < desscan && freemem < minfree) { 2224 TRACE_1(TR_FAC_VM, TR_PAGEOUT_CV_SIGNAL, 2225 "pageout_cv_signal:freemem %ld", freemem); 2226 cv_signal(&proc_pageout->p_cv); 2227 } 2228 2229 pp = rootpp; 2230 while (npages--) { 2231 ASSERT(PAGE_EXCL(pp)); 2232 ASSERT(pp->p_vnode == NULL); 2233 ASSERT(!hat_page_is_mapped(pp)); 2234 PP_CLRFREE(pp); 2235 PP_CLRAGED(pp); 2236 if (!page_hashin(pp, vp, off, NULL)) 2237 panic("page_create_large: hashin failed: page %p", 2238 (void *)pp); 2239 page_io_lock(pp); 2240 off += PAGESIZE; 2241 pp = pp->p_next; 2242 } 2243 2244 VM_STAT_ADD(page_create_large_cnt[0]); 2245 return (rootpp); 2246 } 2247 2248 page_t * 2249 page_create_va(vnode_t *vp, u_offset_t off, size_t bytes, uint_t flags, 2250 struct seg *seg, caddr_t vaddr) 2251 { 2252 page_t *plist = NULL; 2253 pgcnt_t npages; 2254 pgcnt_t found_on_free = 0; 2255 pgcnt_t pages_req; 2256 page_t *npp = NULL; 2257 struct pcf *p; 2258 lgrp_t *lgrp; 2259 2260 TRACE_4(TR_FAC_VM, TR_PAGE_CREATE_START, 2261 "page_create_start:vp %p off %llx bytes %lu flags %x", 2262 vp, off, bytes, flags); 2263 2264 ASSERT(bytes != 0 && vp != NULL); 2265 2266 if ((flags & PG_EXCL) == 0 && (flags & PG_WAIT) == 0) { 2267 panic("page_create: invalid flags"); 2268 /*NOTREACHED*/ 2269 } 2270 ASSERT((flags & ~(PG_EXCL | PG_WAIT | 2271 PG_NORELOC | PG_PANIC | PG_PUSHPAGE)) == 0); 2272 /* but no others */ 2273 2274 pages_req = npages = btopr(bytes); 2275 /* 2276 * Try to see whether request is too large to *ever* be 2277 * satisfied, in order to prevent deadlock. We arbitrarily 2278 * decide to limit maximum size requests to max_page_get. 2279 */ 2280 if (npages >= max_page_get) { 2281 if ((flags & PG_WAIT) == 0) { 2282 TRACE_4(TR_FAC_VM, TR_PAGE_CREATE_TOOBIG, 2283 "page_create_toobig:vp %p off %llx npages " 2284 "%lu max_page_get %lu", 2285 vp, off, npages, max_page_get); 2286 return (NULL); 2287 } else { 2288 cmn_err(CE_WARN, 2289 "Request for too much kernel memory " 2290 "(%lu bytes), will hang forever", bytes); 2291 for (;;) 2292 delay(1000000000); 2293 } 2294 } 2295 2296 if (!kcage_on || panicstr) { 2297 /* 2298 * Cage is OFF, or we are single threaded in 2299 * panic, so make everything a RELOC request. 2300 */ 2301 flags &= ~PG_NORELOC; 2302 } 2303 2304 if (freemem <= throttlefree + npages) 2305 if (!page_create_throttle(npages, flags)) 2306 return (NULL); 2307 2308 /* 2309 * If cage is on, dampen draw from cage when available 2310 * cage space is low. 2311 */ 2312 if ((flags & PG_NORELOC) && 2313 kcage_freemem < kcage_throttlefree + npages) { 2314 2315 /* 2316 * The cage is on, the caller wants PG_NORELOC 2317 * pages and available cage memory is very low. 2318 * Call kcage_create_throttle() to attempt to 2319 * control demand on the cage. 2320 */ 2321 if (kcage_create_throttle(npages, flags) == KCT_FAILURE) 2322 return (NULL); 2323 } 2324 2325 VM_STAT_ADD(page_create_cnt[0]); 2326 2327 if (!pcf_decrement_bucket(npages)) { 2328 /* 2329 * Have to look harder. If npages is greater than 2330 * one, then we might have to coalesce the counters. 2331 * 2332 * Go wait. We come back having accounted 2333 * for the memory. 2334 */ 2335 VM_STAT_ADD(page_create_cnt[1]); 2336 if (!page_create_wait(npages, flags)) { 2337 VM_STAT_ADD(page_create_cnt[2]); 2338 return (NULL); 2339 } 2340 } 2341 2342 TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_SUCCESS, 2343 "page_create_success:vp %p off %llx", vp, off); 2344 2345 /* 2346 * If satisfying this request has left us with too little 2347 * memory, start the wheels turning to get some back. The 2348 * first clause of the test prevents waking up the pageout 2349 * daemon in situations where it would decide that there's 2350 * nothing to do. 2351 */ 2352 if (nscan < desscan && freemem < minfree) { 2353 TRACE_1(TR_FAC_VM, TR_PAGEOUT_CV_SIGNAL, 2354 "pageout_cv_signal:freemem %ld", freemem); 2355 cv_signal(&proc_pageout->p_cv); 2356 } 2357 2358 /* 2359 * Loop around collecting the requested number of pages. 2360 * Most of the time, we have to `create' a new page. With 2361 * this in mind, pull the page off the free list before 2362 * getting the hash lock. This will minimize the hash 2363 * lock hold time, nesting, and the like. If it turns 2364 * out we don't need the page, we put it back at the end. 2365 */ 2366 while (npages--) { 2367 page_t *pp; 2368 kmutex_t *phm = NULL; 2369 ulong_t index; 2370 2371 index = PAGE_HASH_FUNC(vp, off); 2372 top: 2373 ASSERT(phm == NULL); 2374 ASSERT(index == PAGE_HASH_FUNC(vp, off)); 2375 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 2376 2377 if (npp == NULL) { 2378 /* 2379 * Try to get a page from the freelist (ie, 2380 * a page with no [vp, off] tag). If that 2381 * fails, use the cachelist. 2382 * 2383 * During the first attempt at both the free 2384 * and cache lists we try for the correct color. 2385 */ 2386 /* 2387 * XXXX-how do we deal with virtual indexed 2388 * caches and and colors? 2389 */ 2390 VM_STAT_ADD(page_create_cnt[4]); 2391 /* 2392 * Get lgroup to allocate next page of shared memory 2393 * from and use it to specify where to allocate 2394 * the physical memory 2395 */ 2396 lgrp = lgrp_mem_choose(seg, vaddr, PAGESIZE); 2397 npp = page_get_freelist(vp, off, seg, vaddr, PAGESIZE, 2398 flags | PG_MATCH_COLOR, lgrp); 2399 if (npp == NULL) { 2400 npp = page_get_cachelist(vp, off, seg, 2401 vaddr, flags | PG_MATCH_COLOR, lgrp); 2402 if (npp == NULL) { 2403 npp = page_create_get_something(vp, 2404 off, seg, vaddr, 2405 flags & ~PG_MATCH_COLOR); 2406 } 2407 2408 if (PP_ISAGED(npp) == 0) { 2409 /* 2410 * Since this page came from the 2411 * cachelist, we must destroy the 2412 * old vnode association. 2413 */ 2414 page_hashout(npp, NULL); 2415 } 2416 } 2417 } 2418 2419 /* 2420 * We own this page! 2421 */ 2422 ASSERT(PAGE_EXCL(npp)); 2423 ASSERT(npp->p_vnode == NULL); 2424 ASSERT(!hat_page_is_mapped(npp)); 2425 PP_CLRFREE(npp); 2426 PP_CLRAGED(npp); 2427 2428 /* 2429 * Here we have a page in our hot little mits and are 2430 * just waiting to stuff it on the appropriate lists. 2431 * Get the mutex and check to see if it really does 2432 * not exist. 2433 */ 2434 phm = PAGE_HASH_MUTEX(index); 2435 mutex_enter(phm); 2436 PAGE_HASH_SEARCH(index, pp, vp, off); 2437 if (pp == NULL) { 2438 VM_STAT_ADD(page_create_new); 2439 pp = npp; 2440 npp = NULL; 2441 if (!page_hashin(pp, vp, off, phm)) { 2442 /* 2443 * Since we hold the page hash mutex and 2444 * just searched for this page, page_hashin 2445 * had better not fail. If it does, that 2446 * means somethread did not follow the 2447 * page hash mutex rules. Panic now and 2448 * get it over with. As usual, go down 2449 * holding all the locks. 2450 */ 2451 ASSERT(MUTEX_HELD(phm)); 2452 panic("page_create: " 2453 "hashin failed %p %p %llx %p", 2454 (void *)pp, (void *)vp, off, (void *)phm); 2455 /*NOTREACHED*/ 2456 } 2457 ASSERT(MUTEX_HELD(phm)); 2458 mutex_exit(phm); 2459 phm = NULL; 2460 2461 /* 2462 * Hat layer locking need not be done to set 2463 * the following bits since the page is not hashed 2464 * and was on the free list (i.e., had no mappings). 2465 * 2466 * Set the reference bit to protect 2467 * against immediate pageout 2468 * 2469 * XXXmh modify freelist code to set reference 2470 * bit so we don't have to do it here. 2471 */ 2472 page_set_props(pp, P_REF); 2473 found_on_free++; 2474 } else { 2475 VM_STAT_ADD(page_create_exists); 2476 if (flags & PG_EXCL) { 2477 /* 2478 * Found an existing page, and the caller 2479 * wanted all new pages. Undo all of the work 2480 * we have done. 2481 */ 2482 mutex_exit(phm); 2483 phm = NULL; 2484 while (plist != NULL) { 2485 pp = plist; 2486 page_sub(&plist, pp); 2487 page_io_unlock(pp); 2488 /* large pages should not end up here */ 2489 ASSERT(pp->p_szc == 0); 2490 /*LINTED: constant in conditional ctx*/ 2491 VN_DISPOSE(pp, B_INVAL, 0, kcred); 2492 } 2493 VM_STAT_ADD(page_create_found_one); 2494 goto fail; 2495 } 2496 ASSERT(flags & PG_WAIT); 2497 if (!page_lock(pp, SE_EXCL, phm, P_NO_RECLAIM)) { 2498 /* 2499 * Start all over again if we blocked trying 2500 * to lock the page. 2501 */ 2502 mutex_exit(phm); 2503 VM_STAT_ADD(page_create_page_lock_failed); 2504 phm = NULL; 2505 goto top; 2506 } 2507 mutex_exit(phm); 2508 phm = NULL; 2509 2510 if (PP_ISFREE(pp)) { 2511 ASSERT(PP_ISAGED(pp) == 0); 2512 VM_STAT_ADD(pagecnt.pc_get_cache); 2513 page_list_sub(pp, PG_CACHE_LIST); 2514 PP_CLRFREE(pp); 2515 found_on_free++; 2516 } 2517 } 2518 2519 /* 2520 * Got a page! It is locked. Acquire the i/o 2521 * lock since we are going to use the p_next and 2522 * p_prev fields to link the requested pages together. 2523 */ 2524 page_io_lock(pp); 2525 page_add(&plist, pp); 2526 plist = plist->p_next; 2527 off += PAGESIZE; 2528 vaddr += PAGESIZE; 2529 } 2530 2531 ASSERT((flags & PG_EXCL) ? (found_on_free == pages_req) : 1); 2532 fail: 2533 if (npp != NULL) { 2534 /* 2535 * Did not need this page after all. 2536 * Put it back on the free list. 2537 */ 2538 VM_STAT_ADD(page_create_putbacks); 2539 PP_SETFREE(npp); 2540 PP_SETAGED(npp); 2541 npp->p_offset = (u_offset_t)-1; 2542 page_list_add(npp, PG_FREE_LIST | PG_LIST_TAIL); 2543 page_unlock(npp); 2544 2545 } 2546 2547 ASSERT(pages_req >= found_on_free); 2548 2549 { 2550 uint_t overshoot = (uint_t)(pages_req - found_on_free); 2551 2552 if (overshoot) { 2553 VM_STAT_ADD(page_create_overshoot); 2554 p = &pcf[PCF_INDEX()]; 2555 mutex_enter(&p->pcf_lock); 2556 if (p->pcf_block) { 2557 p->pcf_reserve += overshoot; 2558 } else { 2559 p->pcf_count += overshoot; 2560 if (p->pcf_wait) { 2561 mutex_enter(&new_freemem_lock); 2562 if (freemem_wait) { 2563 cv_signal(&freemem_cv); 2564 p->pcf_wait--; 2565 } else { 2566 p->pcf_wait = 0; 2567 } 2568 mutex_exit(&new_freemem_lock); 2569 } 2570 } 2571 mutex_exit(&p->pcf_lock); 2572 /* freemem is approximate, so this test OK */ 2573 if (!p->pcf_block) 2574 freemem += overshoot; 2575 } 2576 } 2577 2578 return (plist); 2579 } 2580 2581 /* 2582 * One or more constituent pages of this large page has been marked 2583 * toxic. Simply demote the large page to PAGESIZE pages and let 2584 * page_free() handle it. This routine should only be called by 2585 * large page free routines (page_free_pages() and page_destroy_pages(). 2586 * All pages are locked SE_EXCL and have already been marked free. 2587 */ 2588 static void 2589 page_free_toxic_pages(page_t *rootpp) 2590 { 2591 page_t *tpp; 2592 pgcnt_t i, pgcnt = page_get_pagecnt(rootpp->p_szc); 2593 uint_t szc = rootpp->p_szc; 2594 2595 for (i = 0, tpp = rootpp; i < pgcnt; i++, tpp = tpp->p_next) { 2596 ASSERT(tpp->p_szc == szc); 2597 ASSERT((PAGE_EXCL(tpp) && 2598 !page_iolock_assert(tpp)) || panicstr); 2599 tpp->p_szc = 0; 2600 } 2601 2602 while (rootpp != NULL) { 2603 tpp = rootpp; 2604 page_sub(&rootpp, tpp); 2605 ASSERT(PP_ISFREE(tpp)); 2606 PP_CLRFREE(tpp); 2607 page_free(tpp, 1); 2608 } 2609 } 2610 2611 /* 2612 * Put page on the "free" list. 2613 * The free list is really two lists maintained by 2614 * the PSM of whatever machine we happen to be on. 2615 */ 2616 void 2617 page_free(page_t *pp, int dontneed) 2618 { 2619 struct pcf *p; 2620 uint_t pcf_index; 2621 2622 ASSERT((PAGE_EXCL(pp) && 2623 !page_iolock_assert(pp)) || panicstr); 2624 2625 if (PP_ISFREE(pp)) { 2626 panic("page_free: page %p is free", (void *)pp); 2627 } 2628 2629 if (pp->p_szc != 0) { 2630 if (pp->p_vnode == NULL || IS_SWAPFSVP(pp->p_vnode) || 2631 PP_ISKAS(pp)) { 2632 panic("page_free: anon or kernel " 2633 "or no vnode large page %p", (void *)pp); 2634 } 2635 page_demote_vp_pages(pp); 2636 ASSERT(pp->p_szc == 0); 2637 } 2638 2639 /* 2640 * The page_struct_lock need not be acquired to examine these 2641 * fields since the page has an "exclusive" lock. 2642 */ 2643 if (hat_page_is_mapped(pp) || pp->p_lckcnt != 0 || pp->p_cowcnt != 0 || 2644 pp->p_slckcnt != 0) { 2645 panic("page_free pp=%p, pfn=%lx, lckcnt=%d, cowcnt=%d " 2646 "slckcnt = %d", (void *)pp, page_pptonum(pp), pp->p_lckcnt, 2647 pp->p_cowcnt, pp->p_slckcnt); 2648 /*NOTREACHED*/ 2649 } 2650 2651 ASSERT(!hat_page_getshare(pp)); 2652 2653 PP_SETFREE(pp); 2654 ASSERT(pp->p_vnode == NULL || !IS_VMODSORT(pp->p_vnode) || 2655 !hat_ismod(pp)); 2656 page_clr_all_props(pp, 0); 2657 ASSERT(!hat_page_getshare(pp)); 2658 2659 /* 2660 * Now we add the page to the head of the free list. 2661 * But if this page is associated with a paged vnode 2662 * then we adjust the head forward so that the page is 2663 * effectively at the end of the list. 2664 */ 2665 if (pp->p_vnode == NULL) { 2666 /* 2667 * Page has no identity, put it on the free list. 2668 */ 2669 PP_SETAGED(pp); 2670 pp->p_offset = (u_offset_t)-1; 2671 page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL); 2672 VM_STAT_ADD(pagecnt.pc_free_free); 2673 TRACE_1(TR_FAC_VM, TR_PAGE_FREE_FREE, 2674 "page_free_free:pp %p", pp); 2675 } else { 2676 PP_CLRAGED(pp); 2677 2678 if (!dontneed || nopageage) { 2679 /* move it to the tail of the list */ 2680 page_list_add(pp, PG_CACHE_LIST | PG_LIST_TAIL); 2681 2682 VM_STAT_ADD(pagecnt.pc_free_cache); 2683 TRACE_1(TR_FAC_VM, TR_PAGE_FREE_CACHE_TAIL, 2684 "page_free_cache_tail:pp %p", pp); 2685 } else { 2686 page_list_add(pp, PG_CACHE_LIST | PG_LIST_HEAD); 2687 2688 VM_STAT_ADD(pagecnt.pc_free_dontneed); 2689 TRACE_1(TR_FAC_VM, TR_PAGE_FREE_CACHE_HEAD, 2690 "page_free_cache_head:pp %p", pp); 2691 } 2692 } 2693 page_unlock(pp); 2694 2695 /* 2696 * Now do the `freemem' accounting. 2697 */ 2698 pcf_index = PCF_INDEX(); 2699 p = &pcf[pcf_index]; 2700 2701 mutex_enter(&p->pcf_lock); 2702 if (p->pcf_block) { 2703 p->pcf_reserve += 1; 2704 } else { 2705 p->pcf_count += 1; 2706 if (p->pcf_wait) { 2707 mutex_enter(&new_freemem_lock); 2708 /* 2709 * Check to see if some other thread 2710 * is actually waiting. Another bucket 2711 * may have woken it up by now. If there 2712 * are no waiters, then set our pcf_wait 2713 * count to zero to avoid coming in here 2714 * next time. Also, since only one page 2715 * was put on the free list, just wake 2716 * up one waiter. 2717 */ 2718 if (freemem_wait) { 2719 cv_signal(&freemem_cv); 2720 p->pcf_wait--; 2721 } else { 2722 p->pcf_wait = 0; 2723 } 2724 mutex_exit(&new_freemem_lock); 2725 } 2726 } 2727 mutex_exit(&p->pcf_lock); 2728 2729 /* freemem is approximate, so this test OK */ 2730 if (!p->pcf_block) 2731 freemem += 1; 2732 } 2733 2734 /* 2735 * Put page on the "free" list during intial startup. 2736 * This happens during initial single threaded execution. 2737 */ 2738 void 2739 page_free_at_startup(page_t *pp) 2740 { 2741 struct pcf *p; 2742 uint_t pcf_index; 2743 2744 page_list_add(pp, PG_FREE_LIST | PG_LIST_HEAD | PG_LIST_ISINIT); 2745 VM_STAT_ADD(pagecnt.pc_free_free); 2746 2747 /* 2748 * Now do the `freemem' accounting. 2749 */ 2750 pcf_index = PCF_INDEX(); 2751 p = &pcf[pcf_index]; 2752 2753 ASSERT(p->pcf_block == 0); 2754 ASSERT(p->pcf_wait == 0); 2755 p->pcf_count += 1; 2756 2757 /* freemem is approximate, so this is OK */ 2758 freemem += 1; 2759 } 2760 2761 void 2762 page_free_pages(page_t *pp) 2763 { 2764 page_t *tpp, *rootpp = NULL; 2765 pgcnt_t pgcnt = page_get_pagecnt(pp->p_szc); 2766 pgcnt_t i; 2767 uint_t szc = pp->p_szc; 2768 2769 VM_STAT_ADD(pagecnt.pc_free_pages); 2770 TRACE_1(TR_FAC_VM, TR_PAGE_FREE_FREE, 2771 "page_free_free:pp %p", pp); 2772 2773 ASSERT(pp->p_szc != 0 && pp->p_szc < page_num_pagesizes()); 2774 if ((page_pptonum(pp) & (pgcnt - 1)) != 0) { 2775 panic("page_free_pages: not root page %p", (void *)pp); 2776 /*NOTREACHED*/ 2777 } 2778 2779 for (i = 0, tpp = pp; i < pgcnt; i++, tpp++) { 2780 ASSERT((PAGE_EXCL(tpp) && 2781 !page_iolock_assert(tpp)) || panicstr); 2782 if (PP_ISFREE(tpp)) { 2783 panic("page_free_pages: page %p is free", (void *)tpp); 2784 /*NOTREACHED*/ 2785 } 2786 if (hat_page_is_mapped(tpp) || tpp->p_lckcnt != 0 || 2787 tpp->p_cowcnt != 0 || tpp->p_slckcnt != 0) { 2788 panic("page_free_pages %p", (void *)tpp); 2789 /*NOTREACHED*/ 2790 } 2791 2792 ASSERT(!hat_page_getshare(tpp)); 2793 ASSERT(tpp->p_vnode == NULL); 2794 ASSERT(tpp->p_szc == szc); 2795 2796 PP_SETFREE(tpp); 2797 page_clr_all_props(tpp, 0); 2798 PP_SETAGED(tpp); 2799 tpp->p_offset = (u_offset_t)-1; 2800 ASSERT(tpp->p_next == tpp); 2801 ASSERT(tpp->p_prev == tpp); 2802 page_list_concat(&rootpp, &tpp); 2803 } 2804 ASSERT(rootpp == pp); 2805 2806 page_list_add_pages(rootpp, 0); 2807 page_create_putback(pgcnt); 2808 } 2809 2810 int free_pages = 1; 2811 2812 /* 2813 * This routine attempts to return pages to the cachelist via page_release(). 2814 * It does not *have* to be successful in all cases, since the pageout scanner 2815 * will catch any pages it misses. It does need to be fast and not introduce 2816 * too much overhead. 2817 * 2818 * If a page isn't found on the unlocked sweep of the page_hash bucket, we 2819 * don't lock and retry. This is ok, since the page scanner will eventually 2820 * find any page we miss in free_vp_pages(). 2821 */ 2822 void 2823 free_vp_pages(vnode_t *vp, u_offset_t off, size_t len) 2824 { 2825 page_t *pp; 2826 u_offset_t eoff; 2827 extern int swap_in_range(vnode_t *, u_offset_t, size_t); 2828 2829 eoff = off + len; 2830 2831 if (free_pages == 0) 2832 return; 2833 if (swap_in_range(vp, off, len)) 2834 return; 2835 2836 for (; off < eoff; off += PAGESIZE) { 2837 2838 /* 2839 * find the page using a fast, but inexact search. It'll be OK 2840 * if a few pages slip through the cracks here. 2841 */ 2842 pp = page_exists(vp, off); 2843 2844 /* 2845 * If we didn't find the page (it may not exist), the page 2846 * is free, looks still in use (shared), or we can't lock it, 2847 * just give up. 2848 */ 2849 if (pp == NULL || 2850 PP_ISFREE(pp) || 2851 page_share_cnt(pp) > 0 || 2852 !page_trylock(pp, SE_EXCL)) 2853 continue; 2854 2855 /* 2856 * Once we have locked pp, verify that it's still the 2857 * correct page and not already free 2858 */ 2859 ASSERT(PAGE_LOCKED_SE(pp, SE_EXCL)); 2860 if (pp->p_vnode != vp || pp->p_offset != off || PP_ISFREE(pp)) { 2861 page_unlock(pp); 2862 continue; 2863 } 2864 2865 /* 2866 * try to release the page... 2867 */ 2868 (void) page_release(pp, 1); 2869 } 2870 } 2871 2872 /* 2873 * Reclaim the given page from the free list. 2874 * If pp is part of a large pages, only the given constituent page is reclaimed 2875 * and the large page it belonged to will be demoted. This can only happen 2876 * if the page is not on the cachelist. 2877 * 2878 * Returns 1 on success or 0 on failure. 2879 * 2880 * The page is unlocked if it can't be reclaimed (when freemem == 0). 2881 * If `lock' is non-null, it will be dropped and re-acquired if 2882 * the routine must wait while freemem is 0. 2883 * 2884 * As it turns out, boot_getpages() does this. It picks a page, 2885 * based on where OBP mapped in some address, gets its pfn, searches 2886 * the memsegs, locks the page, then pulls it off the free list! 2887 */ 2888 int 2889 page_reclaim(page_t *pp, kmutex_t *lock) 2890 { 2891 struct pcf *p; 2892 struct cpu *cpup; 2893 int enough; 2894 uint_t i; 2895 2896 ASSERT(lock != NULL ? MUTEX_HELD(lock) : 1); 2897 ASSERT(PAGE_EXCL(pp) && PP_ISFREE(pp)); 2898 2899 /* 2900 * If `freemem' is 0, we cannot reclaim this page from the 2901 * freelist, so release every lock we might hold: the page, 2902 * and the `lock' before blocking. 2903 * 2904 * The only way `freemem' can become 0 while there are pages 2905 * marked free (have their p->p_free bit set) is when the 2906 * system is low on memory and doing a page_create(). In 2907 * order to guarantee that once page_create() starts acquiring 2908 * pages it will be able to get all that it needs since `freemem' 2909 * was decreased by the requested amount. So, we need to release 2910 * this page, and let page_create() have it. 2911 * 2912 * Since `freemem' being zero is not supposed to happen, just 2913 * use the usual hash stuff as a starting point. If that bucket 2914 * is empty, then assume the worst, and start at the beginning 2915 * of the pcf array. If we always start at the beginning 2916 * when acquiring more than one pcf lock, there won't be any 2917 * deadlock problems. 2918 */ 2919 2920 /* TODO: Do we need to test kcage_freemem if PG_NORELOC(pp)? */ 2921 2922 if (freemem <= throttlefree && !page_create_throttle(1l, 0)) { 2923 pcf_acquire_all(); 2924 goto page_reclaim_nomem; 2925 } 2926 2927 enough = pcf_decrement_bucket(1); 2928 2929 if (!enough) { 2930 VM_STAT_ADD(page_reclaim_zero); 2931 /* 2932 * Check again. Its possible that some other thread 2933 * could have been right behind us, and added one 2934 * to a list somewhere. Acquire each of the pcf locks 2935 * until we find a page. 2936 */ 2937 p = pcf; 2938 for (i = 0; i < pcf_fanout; i++) { 2939 mutex_enter(&p->pcf_lock); 2940 if (p->pcf_count >= 1) { 2941 p->pcf_count -= 1; 2942 /* 2943 * freemem is not protected by any lock. Thus, 2944 * we cannot have any assertion containing 2945 * freemem here. 2946 */ 2947 freemem -= 1; 2948 enough = 1; 2949 break; 2950 } 2951 p++; 2952 } 2953 2954 if (!enough) { 2955 page_reclaim_nomem: 2956 /* 2957 * We really can't have page `pp'. 2958 * Time for the no-memory dance with 2959 * page_free(). This is just like 2960 * page_create_wait(). Plus the added 2961 * attraction of releasing whatever mutex 2962 * we held when we were called with in `lock'. 2963 * Page_unlock() will wakeup any thread 2964 * waiting around for this page. 2965 */ 2966 if (lock) { 2967 VM_STAT_ADD(page_reclaim_zero_locked); 2968 mutex_exit(lock); 2969 } 2970 page_unlock(pp); 2971 2972 /* 2973 * get this before we drop all the pcf locks. 2974 */ 2975 mutex_enter(&new_freemem_lock); 2976 2977 p = pcf; 2978 for (i = 0; i < pcf_fanout; i++) { 2979 p->pcf_wait++; 2980 mutex_exit(&p->pcf_lock); 2981 p++; 2982 } 2983 2984 freemem_wait++; 2985 cv_wait(&freemem_cv, &new_freemem_lock); 2986 freemem_wait--; 2987 2988 mutex_exit(&new_freemem_lock); 2989 2990 if (lock) { 2991 mutex_enter(lock); 2992 } 2993 return (0); 2994 } 2995 2996 /* 2997 * The pcf accounting has been done, 2998 * though none of the pcf_wait flags have been set, 2999 * drop the locks and continue on. 3000 */ 3001 while (p >= pcf) { 3002 mutex_exit(&p->pcf_lock); 3003 p--; 3004 } 3005 } 3006 3007 3008 VM_STAT_ADD(pagecnt.pc_reclaim); 3009 3010 /* 3011 * page_list_sub will handle the case where pp is a large page. 3012 * It's possible that the page was promoted while on the freelist 3013 */ 3014 if (PP_ISAGED(pp)) { 3015 page_list_sub(pp, PG_FREE_LIST); 3016 TRACE_1(TR_FAC_VM, TR_PAGE_UNFREE_FREE, 3017 "page_reclaim_free:pp %p", pp); 3018 } else { 3019 page_list_sub(pp, PG_CACHE_LIST); 3020 TRACE_1(TR_FAC_VM, TR_PAGE_UNFREE_CACHE, 3021 "page_reclaim_cache:pp %p", pp); 3022 } 3023 3024 /* 3025 * clear the p_free & p_age bits since this page is no longer 3026 * on the free list. Notice that there was a brief time where 3027 * a page is marked as free, but is not on the list. 3028 * 3029 * Set the reference bit to protect against immediate pageout. 3030 */ 3031 PP_CLRFREE(pp); 3032 PP_CLRAGED(pp); 3033 page_set_props(pp, P_REF); 3034 3035 CPU_STATS_ENTER_K(); 3036 cpup = CPU; /* get cpup now that CPU cannot change */ 3037 CPU_STATS_ADDQ(cpup, vm, pgrec, 1); 3038 CPU_STATS_ADDQ(cpup, vm, pgfrec, 1); 3039 CPU_STATS_EXIT_K(); 3040 ASSERT(pp->p_szc == 0); 3041 3042 return (1); 3043 } 3044 3045 /* 3046 * Destroy identity of the page and put it back on 3047 * the page free list. Assumes that the caller has 3048 * acquired the "exclusive" lock on the page. 3049 */ 3050 void 3051 page_destroy(page_t *pp, int dontfree) 3052 { 3053 ASSERT((PAGE_EXCL(pp) && 3054 !page_iolock_assert(pp)) || panicstr); 3055 ASSERT(pp->p_slckcnt == 0 || panicstr); 3056 3057 if (pp->p_szc != 0) { 3058 if (pp->p_vnode == NULL || IS_SWAPFSVP(pp->p_vnode) || 3059 PP_ISKAS(pp)) { 3060 panic("page_destroy: anon or kernel or no vnode " 3061 "large page %p", (void *)pp); 3062 } 3063 page_demote_vp_pages(pp); 3064 ASSERT(pp->p_szc == 0); 3065 } 3066 3067 TRACE_1(TR_FAC_VM, TR_PAGE_DESTROY, "page_destroy:pp %p", pp); 3068 3069 /* 3070 * Unload translations, if any, then hash out the 3071 * page to erase its identity. 3072 */ 3073 (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD); 3074 page_hashout(pp, NULL); 3075 3076 if (!dontfree) { 3077 /* 3078 * Acquire the "freemem_lock" for availrmem. 3079 * The page_struct_lock need not be acquired for lckcnt 3080 * and cowcnt since the page has an "exclusive" lock. 3081 * We are doing a modified version of page_pp_unlock here. 3082 */ 3083 if ((pp->p_lckcnt != 0) || (pp->p_cowcnt != 0)) { 3084 mutex_enter(&freemem_lock); 3085 if (pp->p_lckcnt != 0) { 3086 availrmem++; 3087 pages_locked--; 3088 pp->p_lckcnt = 0; 3089 } 3090 if (pp->p_cowcnt != 0) { 3091 availrmem += pp->p_cowcnt; 3092 pages_locked -= pp->p_cowcnt; 3093 pp->p_cowcnt = 0; 3094 } 3095 mutex_exit(&freemem_lock); 3096 } 3097 /* 3098 * Put the page on the "free" list. 3099 */ 3100 page_free(pp, 0); 3101 } 3102 } 3103 3104 void 3105 page_destroy_pages(page_t *pp) 3106 { 3107 3108 page_t *tpp, *rootpp = NULL; 3109 pgcnt_t pgcnt = page_get_pagecnt(pp->p_szc); 3110 pgcnt_t i, pglcks = 0; 3111 uint_t szc = pp->p_szc; 3112 3113 ASSERT(pp->p_szc != 0 && pp->p_szc < page_num_pagesizes()); 3114 3115 VM_STAT_ADD(pagecnt.pc_destroy_pages); 3116 3117 TRACE_1(TR_FAC_VM, TR_PAGE_DESTROY, "page_destroy_pages:pp %p", pp); 3118 3119 if ((page_pptonum(pp) & (pgcnt - 1)) != 0) { 3120 panic("page_destroy_pages: not root page %p", (void *)pp); 3121 /*NOTREACHED*/ 3122 } 3123 3124 for (i = 0, tpp = pp; i < pgcnt; i++, tpp++) { 3125 ASSERT((PAGE_EXCL(tpp) && 3126 !page_iolock_assert(tpp)) || panicstr); 3127 ASSERT(tpp->p_slckcnt == 0 || panicstr); 3128 (void) hat_pageunload(tpp, HAT_FORCE_PGUNLOAD); 3129 page_hashout(tpp, NULL); 3130 ASSERT(tpp->p_offset == (u_offset_t)-1); 3131 if (tpp->p_lckcnt != 0) { 3132 pglcks++; 3133 tpp->p_lckcnt = 0; 3134 } else if (tpp->p_cowcnt != 0) { 3135 pglcks += tpp->p_cowcnt; 3136 tpp->p_cowcnt = 0; 3137 } 3138 ASSERT(!hat_page_getshare(tpp)); 3139 ASSERT(tpp->p_vnode == NULL); 3140 ASSERT(tpp->p_szc == szc); 3141 3142 PP_SETFREE(tpp); 3143 page_clr_all_props(tpp, 0); 3144 PP_SETAGED(tpp); 3145 ASSERT(tpp->p_next == tpp); 3146 ASSERT(tpp->p_prev == tpp); 3147 page_list_concat(&rootpp, &tpp); 3148 } 3149 3150 ASSERT(rootpp == pp); 3151 if (pglcks != 0) { 3152 mutex_enter(&freemem_lock); 3153 availrmem += pglcks; 3154 mutex_exit(&freemem_lock); 3155 } 3156 3157 page_list_add_pages(rootpp, 0); 3158 page_create_putback(pgcnt); 3159 } 3160 3161 /* 3162 * Similar to page_destroy(), but destroys pages which are 3163 * locked and known to be on the page free list. Since 3164 * the page is known to be free and locked, no one can access 3165 * it. 3166 * 3167 * Also, the number of free pages does not change. 3168 */ 3169 void 3170 page_destroy_free(page_t *pp) 3171 { 3172 ASSERT(PAGE_EXCL(pp)); 3173 ASSERT(PP_ISFREE(pp)); 3174 ASSERT(pp->p_vnode); 3175 ASSERT(hat_page_getattr(pp, P_MOD | P_REF | P_RO) == 0); 3176 ASSERT(!hat_page_is_mapped(pp)); 3177 ASSERT(PP_ISAGED(pp) == 0); 3178 ASSERT(pp->p_szc == 0); 3179 3180 VM_STAT_ADD(pagecnt.pc_destroy_free); 3181 page_list_sub(pp, PG_CACHE_LIST); 3182 3183 page_hashout(pp, NULL); 3184 ASSERT(pp->p_vnode == NULL); 3185 ASSERT(pp->p_offset == (u_offset_t)-1); 3186 ASSERT(pp->p_hash == NULL); 3187 3188 PP_SETAGED(pp); 3189 page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL); 3190 page_unlock(pp); 3191 3192 mutex_enter(&new_freemem_lock); 3193 if (freemem_wait) { 3194 cv_signal(&freemem_cv); 3195 } 3196 mutex_exit(&new_freemem_lock); 3197 } 3198 3199 /* 3200 * Rename the page "opp" to have an identity specified 3201 * by [vp, off]. If a page already exists with this name 3202 * it is locked and destroyed. Note that the page's 3203 * translations are not unloaded during the rename. 3204 * 3205 * This routine is used by the anon layer to "steal" the 3206 * original page and is not unlike destroying a page and 3207 * creating a new page using the same page frame. 3208 * 3209 * XXX -- Could deadlock if caller 1 tries to rename A to B while 3210 * caller 2 tries to rename B to A. 3211 */ 3212 void 3213 page_rename(page_t *opp, vnode_t *vp, u_offset_t off) 3214 { 3215 page_t *pp; 3216 int olckcnt = 0; 3217 int ocowcnt = 0; 3218 kmutex_t *phm; 3219 ulong_t index; 3220 3221 ASSERT(PAGE_EXCL(opp) && !page_iolock_assert(opp)); 3222 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 3223 ASSERT(PP_ISFREE(opp) == 0); 3224 3225 VM_STAT_ADD(page_rename_count); 3226 3227 TRACE_3(TR_FAC_VM, TR_PAGE_RENAME, 3228 "page rename:pp %p vp %p off %llx", opp, vp, off); 3229 3230 /* 3231 * CacheFS may call page_rename for a large NFS page 3232 * when both CacheFS and NFS mount points are used 3233 * by applications. Demote this large page before 3234 * renaming it, to ensure that there are no "partial" 3235 * large pages left lying around. 3236 */ 3237 if (opp->p_szc != 0) { 3238 vnode_t *ovp = opp->p_vnode; 3239 ASSERT(ovp != NULL); 3240 ASSERT(!IS_SWAPFSVP(ovp)); 3241 ASSERT(!VN_ISKAS(ovp)); 3242 page_demote_vp_pages(opp); 3243 ASSERT(opp->p_szc == 0); 3244 } 3245 3246 page_hashout(opp, NULL); 3247 PP_CLRAGED(opp); 3248 3249 /* 3250 * Acquire the appropriate page hash lock, since 3251 * we're going to rename the page. 3252 */ 3253 index = PAGE_HASH_FUNC(vp, off); 3254 phm = PAGE_HASH_MUTEX(index); 3255 mutex_enter(phm); 3256 top: 3257 /* 3258 * Look for an existing page with this name and destroy it if found. 3259 * By holding the page hash lock all the way to the page_hashin() 3260 * call, we are assured that no page can be created with this 3261 * identity. In the case when the phm lock is dropped to undo any 3262 * hat layer mappings, the existing page is held with an "exclusive" 3263 * lock, again preventing another page from being created with 3264 * this identity. 3265 */ 3266 PAGE_HASH_SEARCH(index, pp, vp, off); 3267 if (pp != NULL) { 3268 VM_STAT_ADD(page_rename_exists); 3269 3270 /* 3271 * As it turns out, this is one of only two places where 3272 * page_lock() needs to hold the passed in lock in the 3273 * successful case. In all of the others, the lock could 3274 * be dropped as soon as the attempt is made to lock 3275 * the page. It is tempting to add yet another arguement, 3276 * PL_KEEP or PL_DROP, to let page_lock know what to do. 3277 */ 3278 if (!page_lock(pp, SE_EXCL, phm, P_RECLAIM)) { 3279 /* 3280 * Went to sleep because the page could not 3281 * be locked. We were woken up when the page 3282 * was unlocked, or when the page was destroyed. 3283 * In either case, `phm' was dropped while we 3284 * slept. Hence we should not just roar through 3285 * this loop. 3286 */ 3287 goto top; 3288 } 3289 3290 /* 3291 * If an existing page is a large page, then demote 3292 * it to ensure that no "partial" large pages are 3293 * "created" after page_rename. An existing page 3294 * can be a CacheFS page, and can't belong to swapfs. 3295 */ 3296 if (hat_page_is_mapped(pp)) { 3297 /* 3298 * Unload translations. Since we hold the 3299 * exclusive lock on this page, the page 3300 * can not be changed while we drop phm. 3301 * This is also not a lock protocol violation, 3302 * but rather the proper way to do things. 3303 */ 3304 mutex_exit(phm); 3305 (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD); 3306 if (pp->p_szc != 0) { 3307 ASSERT(!IS_SWAPFSVP(vp)); 3308 ASSERT(!VN_ISKAS(vp)); 3309 page_demote_vp_pages(pp); 3310 ASSERT(pp->p_szc == 0); 3311 } 3312 mutex_enter(phm); 3313 } else if (pp->p_szc != 0) { 3314 ASSERT(!IS_SWAPFSVP(vp)); 3315 ASSERT(!VN_ISKAS(vp)); 3316 mutex_exit(phm); 3317 page_demote_vp_pages(pp); 3318 ASSERT(pp->p_szc == 0); 3319 mutex_enter(phm); 3320 } 3321 page_hashout(pp, phm); 3322 } 3323 /* 3324 * Hash in the page with the new identity. 3325 */ 3326 if (!page_hashin(opp, vp, off, phm)) { 3327 /* 3328 * We were holding phm while we searched for [vp, off] 3329 * and only dropped phm if we found and locked a page. 3330 * If we can't create this page now, then some thing 3331 * is really broken. 3332 */ 3333 panic("page_rename: Can't hash in page: %p", (void *)pp); 3334 /*NOTREACHED*/ 3335 } 3336 3337 ASSERT(MUTEX_HELD(phm)); 3338 mutex_exit(phm); 3339 3340 /* 3341 * Now that we have dropped phm, lets get around to finishing up 3342 * with pp. 3343 */ 3344 if (pp != NULL) { 3345 ASSERT(!hat_page_is_mapped(pp)); 3346 /* for now large pages should not end up here */ 3347 ASSERT(pp->p_szc == 0); 3348 /* 3349 * Save the locks for transfer to the new page and then 3350 * clear them so page_free doesn't think they're important. 3351 * The page_struct_lock need not be acquired for lckcnt and 3352 * cowcnt since the page has an "exclusive" lock. 3353 */ 3354 olckcnt = pp->p_lckcnt; 3355 ocowcnt = pp->p_cowcnt; 3356 pp->p_lckcnt = pp->p_cowcnt = 0; 3357 3358 /* 3359 * Put the page on the "free" list after we drop 3360 * the lock. The less work under the lock the better. 3361 */ 3362 /*LINTED: constant in conditional context*/ 3363 VN_DISPOSE(pp, B_FREE, 0, kcred); 3364 } 3365 3366 /* 3367 * Transfer the lock count from the old page (if any). 3368 * The page_struct_lock need not be acquired for lckcnt and 3369 * cowcnt since the page has an "exclusive" lock. 3370 */ 3371 opp->p_lckcnt += olckcnt; 3372 opp->p_cowcnt += ocowcnt; 3373 } 3374 3375 /* 3376 * low level routine to add page `pp' to the hash and vp chains for [vp, offset] 3377 * 3378 * Pages are normally inserted at the start of a vnode's v_pages list. 3379 * If the vnode is VMODSORT and the page is modified, it goes at the end. 3380 * This can happen when a modified page is relocated for DR. 3381 * 3382 * Returns 1 on success and 0 on failure. 3383 */ 3384 static int 3385 page_do_hashin(page_t *pp, vnode_t *vp, u_offset_t offset) 3386 { 3387 page_t **listp; 3388 page_t *tp; 3389 ulong_t index; 3390 3391 ASSERT(PAGE_EXCL(pp)); 3392 ASSERT(vp != NULL); 3393 ASSERT(MUTEX_HELD(page_vnode_mutex(vp))); 3394 3395 /* 3396 * Be sure to set these up before the page is inserted on the hash 3397 * list. As soon as the page is placed on the list some other 3398 * thread might get confused and wonder how this page could 3399 * possibly hash to this list. 3400 */ 3401 pp->p_vnode = vp; 3402 pp->p_offset = offset; 3403 3404 /* 3405 * record if this page is on a swap vnode 3406 */ 3407 if ((vp->v_flag & VISSWAP) != 0) 3408 PP_SETSWAP(pp); 3409 3410 index = PAGE_HASH_FUNC(vp, offset); 3411 ASSERT(MUTEX_HELD(PAGE_HASH_MUTEX(index))); 3412 listp = &page_hash[index]; 3413 3414 /* 3415 * If this page is already hashed in, fail this attempt to add it. 3416 */ 3417 for (tp = *listp; tp != NULL; tp = tp->p_hash) { 3418 if (tp->p_vnode == vp && tp->p_offset == offset) { 3419 pp->p_vnode = NULL; 3420 pp->p_offset = (u_offset_t)(-1); 3421 return (0); 3422 } 3423 } 3424 pp->p_hash = *listp; 3425 *listp = pp; 3426 3427 /* 3428 * Add the page to the vnode's list of pages 3429 */ 3430 if (vp->v_pages != NULL && IS_VMODSORT(vp) && hat_ismod(pp)) 3431 listp = &vp->v_pages->p_vpprev->p_vpnext; 3432 else 3433 listp = &vp->v_pages; 3434 3435 page_vpadd(listp, pp); 3436 3437 return (1); 3438 } 3439 3440 /* 3441 * Add page `pp' to both the hash and vp chains for [vp, offset]. 3442 * 3443 * Returns 1 on success and 0 on failure. 3444 * If hold is passed in, it is not dropped. 3445 */ 3446 int 3447 page_hashin(page_t *pp, vnode_t *vp, u_offset_t offset, kmutex_t *hold) 3448 { 3449 kmutex_t *phm = NULL; 3450 kmutex_t *vphm; 3451 int rc; 3452 3453 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 3454 ASSERT(pp->p_fsdata == 0 || panicstr); 3455 3456 TRACE_3(TR_FAC_VM, TR_PAGE_HASHIN, 3457 "page_hashin:pp %p vp %p offset %llx", 3458 pp, vp, offset); 3459 3460 VM_STAT_ADD(hashin_count); 3461 3462 if (hold != NULL) 3463 phm = hold; 3464 else { 3465 VM_STAT_ADD(hashin_not_held); 3466 phm = PAGE_HASH_MUTEX(PAGE_HASH_FUNC(vp, offset)); 3467 mutex_enter(phm); 3468 } 3469 3470 vphm = page_vnode_mutex(vp); 3471 mutex_enter(vphm); 3472 rc = page_do_hashin(pp, vp, offset); 3473 mutex_exit(vphm); 3474 if (hold == NULL) 3475 mutex_exit(phm); 3476 if (rc == 0) 3477 VM_STAT_ADD(hashin_already); 3478 return (rc); 3479 } 3480 3481 /* 3482 * Remove page ``pp'' from the hash and vp chains and remove vp association. 3483 * All mutexes must be held 3484 */ 3485 static void 3486 page_do_hashout(page_t *pp) 3487 { 3488 page_t **hpp; 3489 page_t *hp; 3490 vnode_t *vp = pp->p_vnode; 3491 3492 ASSERT(vp != NULL); 3493 ASSERT(MUTEX_HELD(page_vnode_mutex(vp))); 3494 3495 /* 3496 * First, take pp off of its hash chain. 3497 */ 3498 hpp = &page_hash[PAGE_HASH_FUNC(vp, pp->p_offset)]; 3499 3500 for (;;) { 3501 hp = *hpp; 3502 if (hp == pp) 3503 break; 3504 if (hp == NULL) { 3505 panic("page_do_hashout"); 3506 /*NOTREACHED*/ 3507 } 3508 hpp = &hp->p_hash; 3509 } 3510 *hpp = pp->p_hash; 3511 3512 /* 3513 * Now remove it from its associated vnode. 3514 */ 3515 if (vp->v_pages) 3516 page_vpsub(&vp->v_pages, pp); 3517 3518 pp->p_hash = NULL; 3519 page_clr_all_props(pp, 1); 3520 PP_CLRSWAP(pp); 3521 pp->p_vnode = NULL; 3522 pp->p_offset = (u_offset_t)-1; 3523 pp->p_fsdata = 0; 3524 } 3525 3526 /* 3527 * Remove page ``pp'' from the hash and vp chains and remove vp association. 3528 * 3529 * When `phm' is non-NULL it contains the address of the mutex protecting the 3530 * hash list pp is on. It is not dropped. 3531 */ 3532 void 3533 page_hashout(page_t *pp, kmutex_t *phm) 3534 { 3535 vnode_t *vp; 3536 ulong_t index; 3537 kmutex_t *nphm; 3538 kmutex_t *vphm; 3539 kmutex_t *sep; 3540 3541 ASSERT(phm != NULL ? MUTEX_HELD(phm) : 1); 3542 ASSERT(pp->p_vnode != NULL); 3543 ASSERT((PAGE_EXCL(pp) && !page_iolock_assert(pp)) || panicstr); 3544 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(pp->p_vnode))); 3545 3546 vp = pp->p_vnode; 3547 3548 TRACE_2(TR_FAC_VM, TR_PAGE_HASHOUT, 3549 "page_hashout:pp %p vp %p", pp, vp); 3550 3551 /* Kernel probe */ 3552 TNF_PROBE_2(page_unmap, "vm pagefault", /* CSTYLED */, 3553 tnf_opaque, vnode, vp, 3554 tnf_offset, offset, pp->p_offset); 3555 3556 /* 3557 * 3558 */ 3559 VM_STAT_ADD(hashout_count); 3560 index = PAGE_HASH_FUNC(vp, pp->p_offset); 3561 if (phm == NULL) { 3562 VM_STAT_ADD(hashout_not_held); 3563 nphm = PAGE_HASH_MUTEX(index); 3564 mutex_enter(nphm); 3565 } 3566 ASSERT(phm ? phm == PAGE_HASH_MUTEX(index) : 1); 3567 3568 3569 /* 3570 * grab page vnode mutex and remove it... 3571 */ 3572 vphm = page_vnode_mutex(vp); 3573 mutex_enter(vphm); 3574 3575 page_do_hashout(pp); 3576 3577 mutex_exit(vphm); 3578 if (phm == NULL) 3579 mutex_exit(nphm); 3580 3581 /* 3582 * Wake up processes waiting for this page. The page's 3583 * identity has been changed, and is probably not the 3584 * desired page any longer. 3585 */ 3586 sep = page_se_mutex(pp); 3587 mutex_enter(sep); 3588 pp->p_selock &= ~SE_EWANTED; 3589 if (CV_HAS_WAITERS(&pp->p_cv)) 3590 cv_broadcast(&pp->p_cv); 3591 mutex_exit(sep); 3592 } 3593 3594 /* 3595 * Add the page to the front of a linked list of pages 3596 * using the p_next & p_prev pointers for the list. 3597 * The caller is responsible for protecting the list pointers. 3598 */ 3599 void 3600 page_add(page_t **ppp, page_t *pp) 3601 { 3602 ASSERT(PAGE_EXCL(pp) || (PAGE_SHARED(pp) && page_iolock_assert(pp))); 3603 3604 page_add_common(ppp, pp); 3605 } 3606 3607 3608 3609 /* 3610 * Common code for page_add() and mach_page_add() 3611 */ 3612 void 3613 page_add_common(page_t **ppp, page_t *pp) 3614 { 3615 if (*ppp == NULL) { 3616 pp->p_next = pp->p_prev = pp; 3617 } else { 3618 pp->p_next = *ppp; 3619 pp->p_prev = (*ppp)->p_prev; 3620 (*ppp)->p_prev = pp; 3621 pp->p_prev->p_next = pp; 3622 } 3623 *ppp = pp; 3624 } 3625 3626 3627 /* 3628 * Remove this page from a linked list of pages 3629 * using the p_next & p_prev pointers for the list. 3630 * 3631 * The caller is responsible for protecting the list pointers. 3632 */ 3633 void 3634 page_sub(page_t **ppp, page_t *pp) 3635 { 3636 ASSERT((PP_ISFREE(pp)) ? 1 : 3637 (PAGE_EXCL(pp)) || (PAGE_SHARED(pp) && page_iolock_assert(pp))); 3638 3639 if (*ppp == NULL || pp == NULL) { 3640 panic("page_sub: bad arg(s): pp %p, *ppp %p", 3641 (void *)pp, (void *)(*ppp)); 3642 /*NOTREACHED*/ 3643 } 3644 3645 page_sub_common(ppp, pp); 3646 } 3647 3648 3649 /* 3650 * Common code for page_sub() and mach_page_sub() 3651 */ 3652 void 3653 page_sub_common(page_t **ppp, page_t *pp) 3654 { 3655 if (*ppp == pp) 3656 *ppp = pp->p_next; /* go to next page */ 3657 3658 if (*ppp == pp) 3659 *ppp = NULL; /* page list is gone */ 3660 else { 3661 pp->p_prev->p_next = pp->p_next; 3662 pp->p_next->p_prev = pp->p_prev; 3663 } 3664 pp->p_prev = pp->p_next = pp; /* make pp a list of one */ 3665 } 3666 3667 3668 /* 3669 * Break page list cppp into two lists with npages in the first list. 3670 * The tail is returned in nppp. 3671 */ 3672 void 3673 page_list_break(page_t **oppp, page_t **nppp, pgcnt_t npages) 3674 { 3675 page_t *s1pp = *oppp; 3676 page_t *s2pp; 3677 page_t *e1pp, *e2pp; 3678 long n = 0; 3679 3680 if (s1pp == NULL) { 3681 *nppp = NULL; 3682 return; 3683 } 3684 if (npages == 0) { 3685 *nppp = s1pp; 3686 *oppp = NULL; 3687 return; 3688 } 3689 for (n = 0, s2pp = *oppp; n < npages; n++) { 3690 s2pp = s2pp->p_next; 3691 } 3692 /* Fix head and tail of new lists */ 3693 e1pp = s2pp->p_prev; 3694 e2pp = s1pp->p_prev; 3695 s1pp->p_prev = e1pp; 3696 e1pp->p_next = s1pp; 3697 s2pp->p_prev = e2pp; 3698 e2pp->p_next = s2pp; 3699 3700 /* second list empty */ 3701 if (s2pp == s1pp) { 3702 *oppp = s1pp; 3703 *nppp = NULL; 3704 } else { 3705 *oppp = s1pp; 3706 *nppp = s2pp; 3707 } 3708 } 3709 3710 /* 3711 * Concatenate page list nppp onto the end of list ppp. 3712 */ 3713 void 3714 page_list_concat(page_t **ppp, page_t **nppp) 3715 { 3716 page_t *s1pp, *s2pp, *e1pp, *e2pp; 3717 3718 if (*nppp == NULL) { 3719 return; 3720 } 3721 if (*ppp == NULL) { 3722 *ppp = *nppp; 3723 return; 3724 } 3725 s1pp = *ppp; 3726 e1pp = s1pp->p_prev; 3727 s2pp = *nppp; 3728 e2pp = s2pp->p_prev; 3729 s1pp->p_prev = e2pp; 3730 e2pp->p_next = s1pp; 3731 e1pp->p_next = s2pp; 3732 s2pp->p_prev = e1pp; 3733 } 3734 3735 /* 3736 * return the next page in the page list 3737 */ 3738 page_t * 3739 page_list_next(page_t *pp) 3740 { 3741 return (pp->p_next); 3742 } 3743 3744 3745 /* 3746 * Add the page to the front of the linked list of pages 3747 * using p_vpnext/p_vpprev pointers for the list. 3748 * 3749 * The caller is responsible for protecting the lists. 3750 */ 3751 void 3752 page_vpadd(page_t **ppp, page_t *pp) 3753 { 3754 if (*ppp == NULL) { 3755 pp->p_vpnext = pp->p_vpprev = pp; 3756 } else { 3757 pp->p_vpnext = *ppp; 3758 pp->p_vpprev = (*ppp)->p_vpprev; 3759 (*ppp)->p_vpprev = pp; 3760 pp->p_vpprev->p_vpnext = pp; 3761 } 3762 *ppp = pp; 3763 } 3764 3765 /* 3766 * Remove this page from the linked list of pages 3767 * using p_vpnext/p_vpprev pointers for the list. 3768 * 3769 * The caller is responsible for protecting the lists. 3770 */ 3771 void 3772 page_vpsub(page_t **ppp, page_t *pp) 3773 { 3774 if (*ppp == NULL || pp == NULL) { 3775 panic("page_vpsub: bad arg(s): pp %p, *ppp %p", 3776 (void *)pp, (void *)(*ppp)); 3777 /*NOTREACHED*/ 3778 } 3779 3780 if (*ppp == pp) 3781 *ppp = pp->p_vpnext; /* go to next page */ 3782 3783 if (*ppp == pp) 3784 *ppp = NULL; /* page list is gone */ 3785 else { 3786 pp->p_vpprev->p_vpnext = pp->p_vpnext; 3787 pp->p_vpnext->p_vpprev = pp->p_vpprev; 3788 } 3789 pp->p_vpprev = pp->p_vpnext = pp; /* make pp a list of one */ 3790 } 3791 3792 /* 3793 * Lock a physical page into memory "long term". Used to support "lock 3794 * in memory" functions. Accepts the page to be locked, and a cow variable 3795 * to indicate whether a the lock will travel to the new page during 3796 * a potential copy-on-write. 3797 */ 3798 int 3799 page_pp_lock( 3800 page_t *pp, /* page to be locked */ 3801 int cow, /* cow lock */ 3802 int kernel) /* must succeed -- ignore checking */ 3803 { 3804 int r = 0; /* result -- assume failure */ 3805 3806 ASSERT(PAGE_LOCKED(pp)); 3807 3808 page_struct_lock(pp); 3809 /* 3810 * Acquire the "freemem_lock" for availrmem. 3811 */ 3812 if (cow) { 3813 mutex_enter(&freemem_lock); 3814 if ((availrmem > pages_pp_maximum) && 3815 (pp->p_cowcnt < (ushort_t)PAGE_LOCK_MAXIMUM)) { 3816 availrmem--; 3817 pages_locked++; 3818 mutex_exit(&freemem_lock); 3819 r = 1; 3820 if (++pp->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) { 3821 cmn_err(CE_WARN, 3822 "COW lock limit reached on pfn 0x%lx", 3823 page_pptonum(pp)); 3824 } 3825 } else 3826 mutex_exit(&freemem_lock); 3827 } else { 3828 if (pp->p_lckcnt) { 3829 if (pp->p_lckcnt < (ushort_t)PAGE_LOCK_MAXIMUM) { 3830 r = 1; 3831 if (++pp->p_lckcnt == 3832 (ushort_t)PAGE_LOCK_MAXIMUM) { 3833 cmn_err(CE_WARN, "Page lock limit " 3834 "reached on pfn 0x%lx", 3835 page_pptonum(pp)); 3836 } 3837 } 3838 } else { 3839 if (kernel) { 3840 /* availrmem accounting done by caller */ 3841 ++pp->p_lckcnt; 3842 r = 1; 3843 } else { 3844 mutex_enter(&freemem_lock); 3845 if (availrmem > pages_pp_maximum) { 3846 availrmem--; 3847 pages_locked++; 3848 ++pp->p_lckcnt; 3849 r = 1; 3850 } 3851 mutex_exit(&freemem_lock); 3852 } 3853 } 3854 } 3855 page_struct_unlock(pp); 3856 return (r); 3857 } 3858 3859 /* 3860 * Decommit a lock on a physical page frame. Account for cow locks if 3861 * appropriate. 3862 */ 3863 void 3864 page_pp_unlock( 3865 page_t *pp, /* page to be unlocked */ 3866 int cow, /* expect cow lock */ 3867 int kernel) /* this was a kernel lock */ 3868 { 3869 ASSERT(PAGE_LOCKED(pp)); 3870 3871 page_struct_lock(pp); 3872 /* 3873 * Acquire the "freemem_lock" for availrmem. 3874 * If cowcnt or lcknt is already 0 do nothing; i.e., we 3875 * could be called to unlock even if nothing is locked. This could 3876 * happen if locked file pages were truncated (removing the lock) 3877 * and the file was grown again and new pages faulted in; the new 3878 * pages are unlocked but the segment still thinks they're locked. 3879 */ 3880 if (cow) { 3881 if (pp->p_cowcnt) { 3882 mutex_enter(&freemem_lock); 3883 pp->p_cowcnt--; 3884 availrmem++; 3885 pages_locked--; 3886 mutex_exit(&freemem_lock); 3887 } 3888 } else { 3889 if (pp->p_lckcnt && --pp->p_lckcnt == 0) { 3890 if (!kernel) { 3891 mutex_enter(&freemem_lock); 3892 availrmem++; 3893 pages_locked--; 3894 mutex_exit(&freemem_lock); 3895 } 3896 } 3897 } 3898 page_struct_unlock(pp); 3899 } 3900 3901 /* 3902 * This routine reserves availrmem for npages; 3903 * flags: KM_NOSLEEP or KM_SLEEP 3904 * returns 1 on success or 0 on failure 3905 */ 3906 int 3907 page_resv(pgcnt_t npages, uint_t flags) 3908 { 3909 mutex_enter(&freemem_lock); 3910 while (availrmem < tune.t_minarmem + npages) { 3911 if (flags & KM_NOSLEEP) { 3912 mutex_exit(&freemem_lock); 3913 return (0); 3914 } 3915 mutex_exit(&freemem_lock); 3916 page_needfree(npages); 3917 kmem_reap(); 3918 delay(hz >> 2); 3919 page_needfree(-(spgcnt_t)npages); 3920 mutex_enter(&freemem_lock); 3921 } 3922 availrmem -= npages; 3923 mutex_exit(&freemem_lock); 3924 return (1); 3925 } 3926 3927 /* 3928 * This routine unreserves availrmem for npages; 3929 */ 3930 void 3931 page_unresv(pgcnt_t npages) 3932 { 3933 mutex_enter(&freemem_lock); 3934 availrmem += npages; 3935 mutex_exit(&freemem_lock); 3936 } 3937 3938 /* 3939 * See Statement at the beginning of segvn_lockop() regarding 3940 * the way we handle cowcnts and lckcnts. 3941 * 3942 * Transfer cowcnt on 'opp' to cowcnt on 'npp' if the vpage 3943 * that breaks COW has PROT_WRITE. 3944 * 3945 * Note that, we may also break COW in case we are softlocking 3946 * on read access during physio; 3947 * in this softlock case, the vpage may not have PROT_WRITE. 3948 * So, we need to transfer lckcnt on 'opp' to lckcnt on 'npp' 3949 * if the vpage doesn't have PROT_WRITE. 3950 * 3951 * This routine is never called if we are stealing a page 3952 * in anon_private. 3953 * 3954 * The caller subtracted from availrmem for read only mapping. 3955 * if lckcnt is 1 increment availrmem. 3956 */ 3957 void 3958 page_pp_useclaim( 3959 page_t *opp, /* original page frame losing lock */ 3960 page_t *npp, /* new page frame gaining lock */ 3961 uint_t write_perm) /* set if vpage has PROT_WRITE */ 3962 { 3963 int payback = 0; 3964 3965 ASSERT(PAGE_LOCKED(opp)); 3966 ASSERT(PAGE_LOCKED(npp)); 3967 3968 page_struct_lock(opp); 3969 3970 ASSERT(npp->p_cowcnt == 0); 3971 ASSERT(npp->p_lckcnt == 0); 3972 3973 /* Don't use claim if nothing is locked (see page_pp_unlock above) */ 3974 if ((write_perm && opp->p_cowcnt != 0) || 3975 (!write_perm && opp->p_lckcnt != 0)) { 3976 3977 if (write_perm) { 3978 npp->p_cowcnt++; 3979 ASSERT(opp->p_cowcnt != 0); 3980 opp->p_cowcnt--; 3981 } else { 3982 3983 ASSERT(opp->p_lckcnt != 0); 3984 3985 /* 3986 * We didn't need availrmem decremented if p_lckcnt on 3987 * original page is 1. Here, we are unlocking 3988 * read-only copy belonging to original page and 3989 * are locking a copy belonging to new page. 3990 */ 3991 if (opp->p_lckcnt == 1) 3992 payback = 1; 3993 3994 npp->p_lckcnt++; 3995 opp->p_lckcnt--; 3996 } 3997 } 3998 if (payback) { 3999 mutex_enter(&freemem_lock); 4000 availrmem++; 4001 pages_useclaim--; 4002 mutex_exit(&freemem_lock); 4003 } 4004 page_struct_unlock(opp); 4005 } 4006 4007 /* 4008 * Simple claim adjust functions -- used to support changes in 4009 * claims due to changes in access permissions. Used by segvn_setprot(). 4010 */ 4011 int 4012 page_addclaim(page_t *pp) 4013 { 4014 int r = 0; /* result */ 4015 4016 ASSERT(PAGE_LOCKED(pp)); 4017 4018 page_struct_lock(pp); 4019 ASSERT(pp->p_lckcnt != 0); 4020 4021 if (pp->p_lckcnt == 1) { 4022 if (pp->p_cowcnt < (ushort_t)PAGE_LOCK_MAXIMUM) { 4023 --pp->p_lckcnt; 4024 r = 1; 4025 if (++pp->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) { 4026 cmn_err(CE_WARN, 4027 "COW lock limit reached on pfn 0x%lx", 4028 page_pptonum(pp)); 4029 } 4030 } 4031 } else { 4032 mutex_enter(&freemem_lock); 4033 if ((availrmem > pages_pp_maximum) && 4034 (pp->p_cowcnt < (ushort_t)PAGE_LOCK_MAXIMUM)) { 4035 --availrmem; 4036 ++pages_claimed; 4037 mutex_exit(&freemem_lock); 4038 --pp->p_lckcnt; 4039 r = 1; 4040 if (++pp->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) { 4041 cmn_err(CE_WARN, 4042 "COW lock limit reached on pfn 0x%lx", 4043 page_pptonum(pp)); 4044 } 4045 } else 4046 mutex_exit(&freemem_lock); 4047 } 4048 page_struct_unlock(pp); 4049 return (r); 4050 } 4051 4052 int 4053 page_subclaim(page_t *pp) 4054 { 4055 int r = 0; 4056 4057 ASSERT(PAGE_LOCKED(pp)); 4058 4059 page_struct_lock(pp); 4060 ASSERT(pp->p_cowcnt != 0); 4061 4062 if (pp->p_lckcnt) { 4063 if (pp->p_lckcnt < (ushort_t)PAGE_LOCK_MAXIMUM) { 4064 r = 1; 4065 /* 4066 * for availrmem 4067 */ 4068 mutex_enter(&freemem_lock); 4069 availrmem++; 4070 pages_claimed--; 4071 mutex_exit(&freemem_lock); 4072 4073 pp->p_cowcnt--; 4074 4075 if (++pp->p_lckcnt == (ushort_t)PAGE_LOCK_MAXIMUM) { 4076 cmn_err(CE_WARN, 4077 "Page lock limit reached on pfn 0x%lx", 4078 page_pptonum(pp)); 4079 } 4080 } 4081 } else { 4082 r = 1; 4083 pp->p_cowcnt--; 4084 pp->p_lckcnt++; 4085 } 4086 page_struct_unlock(pp); 4087 return (r); 4088 } 4089 4090 int 4091 page_addclaim_pages(page_t **ppa) 4092 { 4093 4094 pgcnt_t lckpgs = 0, pg_idx; 4095 4096 VM_STAT_ADD(pagecnt.pc_addclaim_pages); 4097 4098 mutex_enter(&page_llock); 4099 for (pg_idx = 0; ppa[pg_idx] != NULL; pg_idx++) { 4100 4101 ASSERT(PAGE_LOCKED(ppa[pg_idx])); 4102 ASSERT(ppa[pg_idx]->p_lckcnt != 0); 4103 if (ppa[pg_idx]->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) { 4104 mutex_exit(&page_llock); 4105 return (0); 4106 } 4107 if (ppa[pg_idx]->p_lckcnt > 1) 4108 lckpgs++; 4109 } 4110 4111 if (lckpgs != 0) { 4112 mutex_enter(&freemem_lock); 4113 if (availrmem >= pages_pp_maximum + lckpgs) { 4114 availrmem -= lckpgs; 4115 pages_claimed += lckpgs; 4116 } else { 4117 mutex_exit(&freemem_lock); 4118 mutex_exit(&page_llock); 4119 return (0); 4120 } 4121 mutex_exit(&freemem_lock); 4122 } 4123 4124 for (pg_idx = 0; ppa[pg_idx] != NULL; pg_idx++) { 4125 ppa[pg_idx]->p_lckcnt--; 4126 ppa[pg_idx]->p_cowcnt++; 4127 } 4128 mutex_exit(&page_llock); 4129 return (1); 4130 } 4131 4132 int 4133 page_subclaim_pages(page_t **ppa) 4134 { 4135 pgcnt_t ulckpgs = 0, pg_idx; 4136 4137 VM_STAT_ADD(pagecnt.pc_subclaim_pages); 4138 4139 mutex_enter(&page_llock); 4140 for (pg_idx = 0; ppa[pg_idx] != NULL; pg_idx++) { 4141 4142 ASSERT(PAGE_LOCKED(ppa[pg_idx])); 4143 ASSERT(ppa[pg_idx]->p_cowcnt != 0); 4144 if (ppa[pg_idx]->p_lckcnt == (ushort_t)PAGE_LOCK_MAXIMUM) { 4145 mutex_exit(&page_llock); 4146 return (0); 4147 } 4148 if (ppa[pg_idx]->p_lckcnt != 0) 4149 ulckpgs++; 4150 } 4151 4152 if (ulckpgs != 0) { 4153 mutex_enter(&freemem_lock); 4154 availrmem += ulckpgs; 4155 pages_claimed -= ulckpgs; 4156 mutex_exit(&freemem_lock); 4157 } 4158 4159 for (pg_idx = 0; ppa[pg_idx] != NULL; pg_idx++) { 4160 ppa[pg_idx]->p_cowcnt--; 4161 ppa[pg_idx]->p_lckcnt++; 4162 4163 } 4164 mutex_exit(&page_llock); 4165 return (1); 4166 } 4167 4168 page_t * 4169 page_numtopp(pfn_t pfnum, se_t se) 4170 { 4171 page_t *pp; 4172 4173 retry: 4174 pp = page_numtopp_nolock(pfnum); 4175 if (pp == NULL) { 4176 return ((page_t *)NULL); 4177 } 4178 4179 /* 4180 * Acquire the appropriate lock on the page. 4181 */ 4182 while (!page_lock(pp, se, (kmutex_t *)NULL, P_RECLAIM)) { 4183 if (page_pptonum(pp) != pfnum) 4184 goto retry; 4185 continue; 4186 } 4187 4188 if (page_pptonum(pp) != pfnum) { 4189 page_unlock(pp); 4190 goto retry; 4191 } 4192 4193 return (pp); 4194 } 4195 4196 page_t * 4197 page_numtopp_noreclaim(pfn_t pfnum, se_t se) 4198 { 4199 page_t *pp; 4200 4201 retry: 4202 pp = page_numtopp_nolock(pfnum); 4203 if (pp == NULL) { 4204 return ((page_t *)NULL); 4205 } 4206 4207 /* 4208 * Acquire the appropriate lock on the page. 4209 */ 4210 while (!page_lock(pp, se, (kmutex_t *)NULL, P_NO_RECLAIM)) { 4211 if (page_pptonum(pp) != pfnum) 4212 goto retry; 4213 continue; 4214 } 4215 4216 if (page_pptonum(pp) != pfnum) { 4217 page_unlock(pp); 4218 goto retry; 4219 } 4220 4221 return (pp); 4222 } 4223 4224 /* 4225 * This routine is like page_numtopp, but will only return page structs 4226 * for pages which are ok for loading into hardware using the page struct. 4227 */ 4228 page_t * 4229 page_numtopp_nowait(pfn_t pfnum, se_t se) 4230 { 4231 page_t *pp; 4232 4233 retry: 4234 pp = page_numtopp_nolock(pfnum); 4235 if (pp == NULL) { 4236 return ((page_t *)NULL); 4237 } 4238 4239 /* 4240 * Try to acquire the appropriate lock on the page. 4241 */ 4242 if (PP_ISFREE(pp)) 4243 pp = NULL; 4244 else { 4245 if (!page_trylock(pp, se)) 4246 pp = NULL; 4247 else { 4248 if (page_pptonum(pp) != pfnum) { 4249 page_unlock(pp); 4250 goto retry; 4251 } 4252 if (PP_ISFREE(pp)) { 4253 page_unlock(pp); 4254 pp = NULL; 4255 } 4256 } 4257 } 4258 return (pp); 4259 } 4260 4261 /* 4262 * Returns a count of dirty pages that are in the process 4263 * of being written out. If 'cleanit' is set, try to push the page. 4264 */ 4265 pgcnt_t 4266 page_busy(int cleanit) 4267 { 4268 page_t *page0 = page_first(); 4269 page_t *pp = page0; 4270 pgcnt_t nppbusy = 0; 4271 u_offset_t off; 4272 4273 do { 4274 vnode_t *vp = pp->p_vnode; 4275 4276 /* 4277 * A page is a candidate for syncing if it is: 4278 * 4279 * (a) On neither the freelist nor the cachelist 4280 * (b) Hashed onto a vnode 4281 * (c) Not a kernel page 4282 * (d) Dirty 4283 * (e) Not part of a swapfile 4284 * (f) a page which belongs to a real vnode; eg has a non-null 4285 * v_vfsp pointer. 4286 * (g) Backed by a filesystem which doesn't have a 4287 * stubbed-out sync operation 4288 */ 4289 if (!PP_ISFREE(pp) && vp != NULL && !VN_ISKAS(vp) && 4290 hat_ismod(pp) && !IS_SWAPVP(vp) && vp->v_vfsp != NULL && 4291 vfs_can_sync(vp->v_vfsp)) { 4292 nppbusy++; 4293 vfs_syncprogress(); 4294 4295 if (!cleanit) 4296 continue; 4297 if (!page_trylock(pp, SE_EXCL)) 4298 continue; 4299 4300 if (PP_ISFREE(pp) || vp == NULL || IS_SWAPVP(vp) || 4301 pp->p_lckcnt != 0 || pp->p_cowcnt != 0 || 4302 !(hat_pagesync(pp, 4303 HAT_SYNC_DONTZERO | HAT_SYNC_STOPON_MOD) & P_MOD)) { 4304 page_unlock(pp); 4305 continue; 4306 } 4307 off = pp->p_offset; 4308 VN_HOLD(vp); 4309 page_unlock(pp); 4310 (void) VOP_PUTPAGE(vp, off, PAGESIZE, 4311 B_ASYNC | B_FREE, kcred, NULL); 4312 VN_RELE(vp); 4313 } 4314 } while ((pp = page_next(pp)) != page0); 4315 4316 return (nppbusy); 4317 } 4318 4319 void page_invalidate_pages(void); 4320 4321 /* 4322 * callback handler to vm sub-system 4323 * 4324 * callers make sure no recursive entries to this func. 4325 */ 4326 /*ARGSUSED*/ 4327 boolean_t 4328 callb_vm_cpr(void *arg, int code) 4329 { 4330 if (code == CB_CODE_CPR_CHKPT) 4331 page_invalidate_pages(); 4332 return (B_TRUE); 4333 } 4334 4335 /* 4336 * Invalidate all pages of the system. 4337 * It shouldn't be called until all user page activities are all stopped. 4338 */ 4339 void 4340 page_invalidate_pages() 4341 { 4342 page_t *pp; 4343 page_t *page0; 4344 pgcnt_t nbusypages; 4345 int retry = 0; 4346 const int MAXRETRIES = 4; 4347 #if defined(__sparc) 4348 extern struct vnode prom_ppages; 4349 #endif /* __sparc */ 4350 4351 top: 4352 /* 4353 * Flush dirty pages and destroy the clean ones. 4354 */ 4355 nbusypages = 0; 4356 4357 pp = page0 = page_first(); 4358 do { 4359 struct vnode *vp; 4360 u_offset_t offset; 4361 int mod; 4362 4363 /* 4364 * skip the page if it has no vnode or the page associated 4365 * with the kernel vnode or prom allocated kernel mem. 4366 */ 4367 #if defined(__sparc) 4368 if ((vp = pp->p_vnode) == NULL || VN_ISKAS(vp) || 4369 vp == &prom_ppages) 4370 #else /* x86 doesn't have prom or prom_ppage */ 4371 if ((vp = pp->p_vnode) == NULL || VN_ISKAS(vp)) 4372 #endif /* __sparc */ 4373 continue; 4374 4375 /* 4376 * skip the page which is already free invalidated. 4377 */ 4378 if (PP_ISFREE(pp) && PP_ISAGED(pp)) 4379 continue; 4380 4381 /* 4382 * skip pages that are already locked or can't be "exclusively" 4383 * locked or are already free. After we lock the page, check 4384 * the free and age bits again to be sure it's not destroyed 4385 * yet. 4386 * To achieve max. parallelization, we use page_trylock instead 4387 * of page_lock so that we don't get block on individual pages 4388 * while we have thousands of other pages to process. 4389 */ 4390 if (!page_trylock(pp, SE_EXCL)) { 4391 nbusypages++; 4392 continue; 4393 } else if (PP_ISFREE(pp)) { 4394 if (!PP_ISAGED(pp)) { 4395 page_destroy_free(pp); 4396 } else { 4397 page_unlock(pp); 4398 } 4399 continue; 4400 } 4401 /* 4402 * Is this page involved in some I/O? shared? 4403 * 4404 * The page_struct_lock need not be acquired to 4405 * examine these fields since the page has an 4406 * "exclusive" lock. 4407 */ 4408 if (pp->p_lckcnt != 0 || pp->p_cowcnt != 0) { 4409 page_unlock(pp); 4410 continue; 4411 } 4412 4413 if (vp->v_type == VCHR) { 4414 panic("vp->v_type == VCHR"); 4415 /*NOTREACHED*/ 4416 } 4417 4418 if (!page_try_demote_pages(pp)) { 4419 page_unlock(pp); 4420 continue; 4421 } 4422 4423 /* 4424 * Check the modified bit. Leave the bits alone in hardware 4425 * (they will be modified if we do the putpage). 4426 */ 4427 mod = (hat_pagesync(pp, HAT_SYNC_DONTZERO | HAT_SYNC_STOPON_MOD) 4428 & P_MOD); 4429 if (mod) { 4430 offset = pp->p_offset; 4431 /* 4432 * Hold the vnode before releasing the page lock 4433 * to prevent it from being freed and re-used by 4434 * some other thread. 4435 */ 4436 VN_HOLD(vp); 4437 page_unlock(pp); 4438 /* 4439 * No error return is checked here. Callers such as 4440 * cpr deals with the dirty pages at the dump time 4441 * if this putpage fails. 4442 */ 4443 (void) VOP_PUTPAGE(vp, offset, PAGESIZE, B_INVAL, 4444 kcred, NULL); 4445 VN_RELE(vp); 4446 } else { 4447 /*LINTED: constant in conditional context*/ 4448 VN_DISPOSE(pp, B_INVAL, 0, kcred); 4449 } 4450 } while ((pp = page_next(pp)) != page0); 4451 if (nbusypages && retry++ < MAXRETRIES) { 4452 delay(1); 4453 goto top; 4454 } 4455 } 4456 4457 /* 4458 * Replace the page "old" with the page "new" on the page hash and vnode lists 4459 * 4460 * the replacement must be done in place, ie the equivalent sequence: 4461 * 4462 * vp = old->p_vnode; 4463 * off = old->p_offset; 4464 * page_do_hashout(old) 4465 * page_do_hashin(new, vp, off) 4466 * 4467 * doesn't work, since 4468 * 1) if old is the only page on the vnode, the v_pages list has a window 4469 * where it looks empty. This will break file system assumptions. 4470 * and 4471 * 2) pvn_vplist_dirty() can't deal with pages moving on the v_pages list. 4472 */ 4473 static void 4474 page_do_relocate_hash(page_t *new, page_t *old) 4475 { 4476 page_t **hash_list; 4477 vnode_t *vp = old->p_vnode; 4478 kmutex_t *sep; 4479 4480 ASSERT(PAGE_EXCL(old)); 4481 ASSERT(PAGE_EXCL(new)); 4482 ASSERT(vp != NULL); 4483 ASSERT(MUTEX_HELD(page_vnode_mutex(vp))); 4484 ASSERT(MUTEX_HELD(PAGE_HASH_MUTEX(PAGE_HASH_FUNC(vp, old->p_offset)))); 4485 4486 /* 4487 * First find old page on the page hash list 4488 */ 4489 hash_list = &page_hash[PAGE_HASH_FUNC(vp, old->p_offset)]; 4490 4491 for (;;) { 4492 if (*hash_list == old) 4493 break; 4494 if (*hash_list == NULL) { 4495 panic("page_do_hashout"); 4496 /*NOTREACHED*/ 4497 } 4498 hash_list = &(*hash_list)->p_hash; 4499 } 4500 4501 /* 4502 * update new and replace old with new on the page hash list 4503 */ 4504 new->p_vnode = old->p_vnode; 4505 new->p_offset = old->p_offset; 4506 new->p_hash = old->p_hash; 4507 *hash_list = new; 4508 4509 if ((new->p_vnode->v_flag & VISSWAP) != 0) 4510 PP_SETSWAP(new); 4511 4512 /* 4513 * replace old with new on the vnode's page list 4514 */ 4515 if (old->p_vpnext == old) { 4516 new->p_vpnext = new; 4517 new->p_vpprev = new; 4518 } else { 4519 new->p_vpnext = old->p_vpnext; 4520 new->p_vpprev = old->p_vpprev; 4521 new->p_vpnext->p_vpprev = new; 4522 new->p_vpprev->p_vpnext = new; 4523 } 4524 if (vp->v_pages == old) 4525 vp->v_pages = new; 4526 4527 /* 4528 * clear out the old page 4529 */ 4530 old->p_hash = NULL; 4531 old->p_vpnext = NULL; 4532 old->p_vpprev = NULL; 4533 old->p_vnode = NULL; 4534 PP_CLRSWAP(old); 4535 old->p_offset = (u_offset_t)-1; 4536 page_clr_all_props(old, 1); 4537 4538 /* 4539 * Wake up processes waiting for this page. The page's 4540 * identity has been changed, and is probably not the 4541 * desired page any longer. 4542 */ 4543 sep = page_se_mutex(old); 4544 mutex_enter(sep); 4545 old->p_selock &= ~SE_EWANTED; 4546 if (CV_HAS_WAITERS(&old->p_cv)) 4547 cv_broadcast(&old->p_cv); 4548 mutex_exit(sep); 4549 } 4550 4551 /* 4552 * This function moves the identity of page "pp_old" to page "pp_new". 4553 * Both pages must be locked on entry. "pp_new" is free, has no identity, 4554 * and need not be hashed out from anywhere. 4555 */ 4556 void 4557 page_relocate_hash(page_t *pp_new, page_t *pp_old) 4558 { 4559 vnode_t *vp = pp_old->p_vnode; 4560 u_offset_t off = pp_old->p_offset; 4561 kmutex_t *phm, *vphm; 4562 4563 /* 4564 * Rehash two pages 4565 */ 4566 ASSERT(PAGE_EXCL(pp_old)); 4567 ASSERT(PAGE_EXCL(pp_new)); 4568 ASSERT(vp != NULL); 4569 ASSERT(pp_new->p_vnode == NULL); 4570 4571 /* 4572 * hashout then hashin while holding the mutexes 4573 */ 4574 phm = PAGE_HASH_MUTEX(PAGE_HASH_FUNC(vp, off)); 4575 mutex_enter(phm); 4576 vphm = page_vnode_mutex(vp); 4577 mutex_enter(vphm); 4578 4579 page_do_relocate_hash(pp_new, pp_old); 4580 4581 /* The following comment preserved from page_flip(). */ 4582 pp_new->p_fsdata = pp_old->p_fsdata; 4583 pp_old->p_fsdata = 0; 4584 mutex_exit(vphm); 4585 mutex_exit(phm); 4586 4587 /* 4588 * The page_struct_lock need not be acquired for lckcnt and 4589 * cowcnt since the page has an "exclusive" lock. 4590 */ 4591 ASSERT(pp_new->p_lckcnt == 0); 4592 ASSERT(pp_new->p_cowcnt == 0); 4593 pp_new->p_lckcnt = pp_old->p_lckcnt; 4594 pp_new->p_cowcnt = pp_old->p_cowcnt; 4595 pp_old->p_lckcnt = pp_old->p_cowcnt = 0; 4596 4597 } 4598 4599 /* 4600 * Helper routine used to lock all remaining members of a 4601 * large page. The caller is responsible for passing in a locked 4602 * pp. If pp is a large page, then it succeeds in locking all the 4603 * remaining constituent pages or it returns with only the 4604 * original page locked. 4605 * 4606 * Returns 1 on success, 0 on failure. 4607 * 4608 * If success is returned this routine guarantees p_szc for all constituent 4609 * pages of a large page pp belongs to can't change. To achieve this we 4610 * recheck szc of pp after locking all constituent pages and retry if szc 4611 * changed (it could only decrease). Since hat_page_demote() needs an EXCL 4612 * lock on one of constituent pages it can't be running after all constituent 4613 * pages are locked. hat_page_demote() with a lock on a constituent page 4614 * outside of this large page (i.e. pp belonged to a larger large page) is 4615 * already done with all constituent pages of pp since the root's p_szc is 4616 * changed last. Therefore no need to synchronize with hat_page_demote() that 4617 * locked a constituent page outside of pp's current large page. 4618 */ 4619 #ifdef DEBUG 4620 uint32_t gpg_trylock_mtbf = 0; 4621 #endif 4622 4623 int 4624 group_page_trylock(page_t *pp, se_t se) 4625 { 4626 page_t *tpp; 4627 pgcnt_t npgs, i, j; 4628 uint_t pszc = pp->p_szc; 4629 4630 #ifdef DEBUG 4631 if (gpg_trylock_mtbf && !(gethrtime() % gpg_trylock_mtbf)) { 4632 return (0); 4633 } 4634 #endif 4635 4636 if (pp != PP_GROUPLEADER(pp, pszc)) { 4637 return (0); 4638 } 4639 4640 retry: 4641 ASSERT(PAGE_LOCKED_SE(pp, se)); 4642 ASSERT(!PP_ISFREE(pp)); 4643 if (pszc == 0) { 4644 return (1); 4645 } 4646 npgs = page_get_pagecnt(pszc); 4647 tpp = pp + 1; 4648 for (i = 1; i < npgs; i++, tpp++) { 4649 if (!page_trylock(tpp, se)) { 4650 tpp = pp + 1; 4651 for (j = 1; j < i; j++, tpp++) { 4652 page_unlock(tpp); 4653 } 4654 return (0); 4655 } 4656 } 4657 if (pp->p_szc != pszc) { 4658 ASSERT(pp->p_szc < pszc); 4659 ASSERT(pp->p_vnode != NULL && !PP_ISKAS(pp) && 4660 !IS_SWAPFSVP(pp->p_vnode)); 4661 tpp = pp + 1; 4662 for (i = 1; i < npgs; i++, tpp++) { 4663 page_unlock(tpp); 4664 } 4665 pszc = pp->p_szc; 4666 goto retry; 4667 } 4668 return (1); 4669 } 4670 4671 void 4672 group_page_unlock(page_t *pp) 4673 { 4674 page_t *tpp; 4675 pgcnt_t npgs, i; 4676 4677 ASSERT(PAGE_LOCKED(pp)); 4678 ASSERT(!PP_ISFREE(pp)); 4679 ASSERT(pp == PP_PAGEROOT(pp)); 4680 npgs = page_get_pagecnt(pp->p_szc); 4681 for (i = 1, tpp = pp + 1; i < npgs; i++, tpp++) { 4682 page_unlock(tpp); 4683 } 4684 } 4685 4686 /* 4687 * returns 4688 * 0 : on success and *nrelocp is number of relocated PAGESIZE pages 4689 * ERANGE : this is not a base page 4690 * EBUSY : failure to get locks on the page/pages 4691 * ENOMEM : failure to obtain replacement pages 4692 * EAGAIN : OBP has not yet completed its boot-time handoff to the kernel 4693 * EIO : An error occurred while trying to copy the page data 4694 * 4695 * Return with all constituent members of target and replacement 4696 * SE_EXCL locked. It is the callers responsibility to drop the 4697 * locks. 4698 */ 4699 int 4700 do_page_relocate( 4701 page_t **target, 4702 page_t **replacement, 4703 int grouplock, 4704 spgcnt_t *nrelocp, 4705 lgrp_t *lgrp) 4706 { 4707 page_t *first_repl; 4708 page_t *repl; 4709 page_t *targ; 4710 page_t *pl = NULL; 4711 uint_t ppattr; 4712 pfn_t pfn, repl_pfn; 4713 uint_t szc; 4714 spgcnt_t npgs, i; 4715 int repl_contig = 0; 4716 uint_t flags = 0; 4717 spgcnt_t dofree = 0; 4718 4719 *nrelocp = 0; 4720 4721 #if defined(__sparc) 4722 /* 4723 * We need to wait till OBP has completed 4724 * its boot-time handoff of its resources to the kernel 4725 * before we allow page relocation 4726 */ 4727 if (page_relocate_ready == 0) { 4728 return (EAGAIN); 4729 } 4730 #endif 4731 4732 /* 4733 * If this is not a base page, 4734 * just return with 0x0 pages relocated. 4735 */ 4736 targ = *target; 4737 ASSERT(PAGE_EXCL(targ)); 4738 ASSERT(!PP_ISFREE(targ)); 4739 szc = targ->p_szc; 4740 ASSERT(szc < mmu_page_sizes); 4741 VM_STAT_ADD(vmm_vmstats.ppr_reloc[szc]); 4742 pfn = targ->p_pagenum; 4743 if (pfn != PFN_BASE(pfn, szc)) { 4744 VM_STAT_ADD(vmm_vmstats.ppr_relocnoroot[szc]); 4745 return (ERANGE); 4746 } 4747 4748 if ((repl = *replacement) != NULL && repl->p_szc >= szc) { 4749 repl_pfn = repl->p_pagenum; 4750 if (repl_pfn != PFN_BASE(repl_pfn, szc)) { 4751 VM_STAT_ADD(vmm_vmstats.ppr_reloc_replnoroot[szc]); 4752 return (ERANGE); 4753 } 4754 repl_contig = 1; 4755 } 4756 4757 /* 4758 * We must lock all members of this large page or we cannot 4759 * relocate any part of it. 4760 */ 4761 if (grouplock != 0 && !group_page_trylock(targ, SE_EXCL)) { 4762 VM_STAT_ADD(vmm_vmstats.ppr_relocnolock[targ->p_szc]); 4763 return (EBUSY); 4764 } 4765 4766 /* 4767 * reread szc it could have been decreased before 4768 * group_page_trylock() was done. 4769 */ 4770 szc = targ->p_szc; 4771 ASSERT(szc < mmu_page_sizes); 4772 VM_STAT_ADD(vmm_vmstats.ppr_reloc[szc]); 4773 ASSERT(pfn == PFN_BASE(pfn, szc)); 4774 4775 npgs = page_get_pagecnt(targ->p_szc); 4776 4777 if (repl == NULL) { 4778 dofree = npgs; /* Size of target page in MMU pages */ 4779 if (!page_create_wait(dofree, 0)) { 4780 if (grouplock != 0) { 4781 group_page_unlock(targ); 4782 } 4783 VM_STAT_ADD(vmm_vmstats.ppr_relocnomem[szc]); 4784 return (ENOMEM); 4785 } 4786 4787 /* 4788 * seg kmem pages require that the target and replacement 4789 * page be the same pagesize. 4790 */ 4791 flags = (VN_ISKAS(targ->p_vnode)) ? PGR_SAMESZC : 0; 4792 repl = page_get_replacement_page(targ, lgrp, flags); 4793 if (repl == NULL) { 4794 if (grouplock != 0) { 4795 group_page_unlock(targ); 4796 } 4797 page_create_putback(dofree); 4798 VM_STAT_ADD(vmm_vmstats.ppr_relocnomem[szc]); 4799 return (ENOMEM); 4800 } 4801 } 4802 #ifdef DEBUG 4803 else { 4804 ASSERT(PAGE_LOCKED(repl)); 4805 } 4806 #endif /* DEBUG */ 4807 4808 #if defined(__sparc) 4809 /* 4810 * Let hat_page_relocate() complete the relocation if it's kernel page 4811 */ 4812 if (VN_ISKAS(targ->p_vnode)) { 4813 *replacement = repl; 4814 if (hat_page_relocate(target, replacement, nrelocp) != 0) { 4815 if (grouplock != 0) { 4816 group_page_unlock(targ); 4817 } 4818 if (dofree) { 4819 *replacement = NULL; 4820 page_free_replacement_page(repl); 4821 page_create_putback(dofree); 4822 } 4823 VM_STAT_ADD(vmm_vmstats.ppr_krelocfail[szc]); 4824 return (EAGAIN); 4825 } 4826 VM_STAT_ADD(vmm_vmstats.ppr_relocok[szc]); 4827 return (0); 4828 } 4829 #else 4830 #if defined(lint) 4831 dofree = dofree; 4832 #endif 4833 #endif 4834 4835 first_repl = repl; 4836 4837 for (i = 0; i < npgs; i++) { 4838 ASSERT(PAGE_EXCL(targ)); 4839 ASSERT(targ->p_slckcnt == 0); 4840 ASSERT(repl->p_slckcnt == 0); 4841 4842 (void) hat_pageunload(targ, HAT_FORCE_PGUNLOAD); 4843 4844 ASSERT(hat_page_getshare(targ) == 0); 4845 ASSERT(!PP_ISFREE(targ)); 4846 ASSERT(targ->p_pagenum == (pfn + i)); 4847 ASSERT(repl_contig == 0 || 4848 repl->p_pagenum == (repl_pfn + i)); 4849 4850 /* 4851 * Copy the page contents and attributes then 4852 * relocate the page in the page hash. 4853 */ 4854 if (ppcopy(targ, repl) == 0) { 4855 targ = *target; 4856 repl = first_repl; 4857 VM_STAT_ADD(vmm_vmstats.ppr_copyfail); 4858 if (grouplock != 0) { 4859 group_page_unlock(targ); 4860 } 4861 if (dofree) { 4862 *replacement = NULL; 4863 page_free_replacement_page(repl); 4864 page_create_putback(dofree); 4865 } 4866 return (EIO); 4867 } 4868 4869 targ++; 4870 if (repl_contig != 0) { 4871 repl++; 4872 } else { 4873 repl = repl->p_next; 4874 } 4875 } 4876 4877 repl = first_repl; 4878 targ = *target; 4879 4880 for (i = 0; i < npgs; i++) { 4881 ppattr = hat_page_getattr(targ, (P_MOD | P_REF | P_RO)); 4882 page_clr_all_props(repl, 0); 4883 page_set_props(repl, ppattr); 4884 page_relocate_hash(repl, targ); 4885 4886 ASSERT(hat_page_getshare(targ) == 0); 4887 ASSERT(hat_page_getshare(repl) == 0); 4888 /* 4889 * Now clear the props on targ, after the 4890 * page_relocate_hash(), they no longer 4891 * have any meaning. 4892 */ 4893 page_clr_all_props(targ, 0); 4894 ASSERT(targ->p_next == targ); 4895 ASSERT(targ->p_prev == targ); 4896 page_list_concat(&pl, &targ); 4897 4898 targ++; 4899 if (repl_contig != 0) { 4900 repl++; 4901 } else { 4902 repl = repl->p_next; 4903 } 4904 } 4905 /* assert that we have come full circle with repl */ 4906 ASSERT(repl_contig == 1 || first_repl == repl); 4907 4908 *target = pl; 4909 if (*replacement == NULL) { 4910 ASSERT(first_repl == repl); 4911 *replacement = repl; 4912 } 4913 VM_STAT_ADD(vmm_vmstats.ppr_relocok[szc]); 4914 *nrelocp = npgs; 4915 return (0); 4916 } 4917 /* 4918 * On success returns 0 and *nrelocp the number of PAGESIZE pages relocated. 4919 */ 4920 int 4921 page_relocate( 4922 page_t **target, 4923 page_t **replacement, 4924 int grouplock, 4925 int freetarget, 4926 spgcnt_t *nrelocp, 4927 lgrp_t *lgrp) 4928 { 4929 spgcnt_t ret; 4930 4931 /* do_page_relocate returns 0 on success or errno value */ 4932 ret = do_page_relocate(target, replacement, grouplock, nrelocp, lgrp); 4933 4934 if (ret != 0 || freetarget == 0) { 4935 return (ret); 4936 } 4937 if (*nrelocp == 1) { 4938 ASSERT(*target != NULL); 4939 page_free(*target, 1); 4940 } else { 4941 page_t *tpp = *target; 4942 uint_t szc = tpp->p_szc; 4943 pgcnt_t npgs = page_get_pagecnt(szc); 4944 ASSERT(npgs > 1); 4945 ASSERT(szc != 0); 4946 do { 4947 ASSERT(PAGE_EXCL(tpp)); 4948 ASSERT(!hat_page_is_mapped(tpp)); 4949 ASSERT(tpp->p_szc == szc); 4950 PP_SETFREE(tpp); 4951 PP_SETAGED(tpp); 4952 npgs--; 4953 } while ((tpp = tpp->p_next) != *target); 4954 ASSERT(npgs == 0); 4955 page_list_add_pages(*target, 0); 4956 npgs = page_get_pagecnt(szc); 4957 page_create_putback(npgs); 4958 } 4959 return (ret); 4960 } 4961 4962 /* 4963 * it is up to the caller to deal with pcf accounting. 4964 */ 4965 void 4966 page_free_replacement_page(page_t *pplist) 4967 { 4968 page_t *pp; 4969 4970 while (pplist != NULL) { 4971 /* 4972 * pp_targ is a linked list. 4973 */ 4974 pp = pplist; 4975 if (pp->p_szc == 0) { 4976 page_sub(&pplist, pp); 4977 page_clr_all_props(pp, 0); 4978 PP_SETFREE(pp); 4979 PP_SETAGED(pp); 4980 page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL); 4981 page_unlock(pp); 4982 VM_STAT_ADD(pagecnt.pc_free_replacement_page[0]); 4983 } else { 4984 spgcnt_t curnpgs = page_get_pagecnt(pp->p_szc); 4985 page_t *tpp; 4986 page_list_break(&pp, &pplist, curnpgs); 4987 tpp = pp; 4988 do { 4989 ASSERT(PAGE_EXCL(tpp)); 4990 ASSERT(!hat_page_is_mapped(tpp)); 4991 page_clr_all_props(tpp, 0); 4992 PP_SETFREE(tpp); 4993 PP_SETAGED(tpp); 4994 } while ((tpp = tpp->p_next) != pp); 4995 page_list_add_pages(pp, 0); 4996 VM_STAT_ADD(pagecnt.pc_free_replacement_page[1]); 4997 } 4998 } 4999 } 5000 5001 /* 5002 * Relocate target to non-relocatable replacement page. 5003 */ 5004 int 5005 page_relocate_cage(page_t **target, page_t **replacement) 5006 { 5007 page_t *tpp, *rpp; 5008 spgcnt_t pgcnt, npgs; 5009 int result; 5010 5011 tpp = *target; 5012 5013 ASSERT(PAGE_EXCL(tpp)); 5014 ASSERT(tpp->p_szc == 0); 5015 5016 pgcnt = btop(page_get_pagesize(tpp->p_szc)); 5017 5018 do { 5019 (void) page_create_wait(pgcnt, PG_WAIT | PG_NORELOC); 5020 rpp = page_get_replacement_page(tpp, NULL, PGR_NORELOC); 5021 if (rpp == NULL) { 5022 page_create_putback(pgcnt); 5023 kcage_cageout_wakeup(); 5024 } 5025 } while (rpp == NULL); 5026 5027 ASSERT(PP_ISNORELOC(rpp)); 5028 5029 result = page_relocate(&tpp, &rpp, 0, 1, &npgs, NULL); 5030 5031 if (result == 0) { 5032 *replacement = rpp; 5033 if (pgcnt != npgs) 5034 panic("page_relocate_cage: partial relocation"); 5035 } 5036 5037 return (result); 5038 } 5039 5040 /* 5041 * Release the page lock on a page, place on cachelist 5042 * tail if no longer mapped. Caller can let us know if 5043 * the page is known to be clean. 5044 */ 5045 int 5046 page_release(page_t *pp, int checkmod) 5047 { 5048 int status; 5049 5050 ASSERT(PAGE_LOCKED(pp) && !PP_ISFREE(pp) && 5051 (pp->p_vnode != NULL)); 5052 5053 if (!hat_page_is_mapped(pp) && !IS_SWAPVP(pp->p_vnode) && 5054 ((PAGE_SHARED(pp) && page_tryupgrade(pp)) || PAGE_EXCL(pp)) && 5055 pp->p_lckcnt == 0 && pp->p_cowcnt == 0 && 5056 !hat_page_is_mapped(pp)) { 5057 5058 /* 5059 * If page is modified, unlock it 5060 * 5061 * (p_nrm & P_MOD) bit has the latest stuff because: 5062 * (1) We found that this page doesn't have any mappings 5063 * _after_ holding SE_EXCL and 5064 * (2) We didn't drop SE_EXCL lock after the check in (1) 5065 */ 5066 if (checkmod && hat_ismod(pp)) { 5067 page_unlock(pp); 5068 status = PGREL_MOD; 5069 } else { 5070 /*LINTED: constant in conditional context*/ 5071 VN_DISPOSE(pp, B_FREE, 0, kcred); 5072 status = PGREL_CLEAN; 5073 } 5074 } else { 5075 page_unlock(pp); 5076 status = PGREL_NOTREL; 5077 } 5078 return (status); 5079 } 5080 5081 /* 5082 * Given a constituent page, try to demote the large page on the freelist. 5083 * 5084 * Returns nonzero if the page could be demoted successfully. Returns with 5085 * the constituent page still locked. 5086 */ 5087 int 5088 page_try_demote_free_pages(page_t *pp) 5089 { 5090 page_t *rootpp = pp; 5091 pfn_t pfn = page_pptonum(pp); 5092 spgcnt_t npgs; 5093 uint_t szc = pp->p_szc; 5094 5095 ASSERT(PP_ISFREE(pp)); 5096 ASSERT(PAGE_EXCL(pp)); 5097 5098 /* 5099 * Adjust rootpp and lock it, if `pp' is not the base 5100 * constituent page. 5101 */ 5102 npgs = page_get_pagecnt(pp->p_szc); 5103 if (npgs == 1) { 5104 return (0); 5105 } 5106 5107 if (!IS_P2ALIGNED(pfn, npgs)) { 5108 pfn = P2ALIGN(pfn, npgs); 5109 rootpp = page_numtopp_nolock(pfn); 5110 } 5111 5112 if (pp != rootpp && !page_trylock(rootpp, SE_EXCL)) { 5113 return (0); 5114 } 5115 5116 if (rootpp->p_szc != szc) { 5117 if (pp != rootpp) 5118 page_unlock(rootpp); 5119 return (0); 5120 } 5121 5122 page_demote_free_pages(rootpp); 5123 5124 if (pp != rootpp) 5125 page_unlock(rootpp); 5126 5127 ASSERT(PP_ISFREE(pp)); 5128 ASSERT(PAGE_EXCL(pp)); 5129 return (1); 5130 } 5131 5132 /* 5133 * Given a constituent page, try to demote the large page. 5134 * 5135 * Returns nonzero if the page could be demoted successfully. Returns with 5136 * the constituent page still locked. 5137 */ 5138 int 5139 page_try_demote_pages(page_t *pp) 5140 { 5141 page_t *tpp, *rootpp = pp; 5142 pfn_t pfn = page_pptonum(pp); 5143 spgcnt_t i, npgs; 5144 uint_t szc = pp->p_szc; 5145 vnode_t *vp = pp->p_vnode; 5146 5147 ASSERT(PAGE_EXCL(pp)); 5148 5149 VM_STAT_ADD(pagecnt.pc_try_demote_pages[0]); 5150 5151 if (pp->p_szc == 0) { 5152 VM_STAT_ADD(pagecnt.pc_try_demote_pages[1]); 5153 return (1); 5154 } 5155 5156 if (vp != NULL && !IS_SWAPFSVP(vp) && !VN_ISKAS(vp)) { 5157 VM_STAT_ADD(pagecnt.pc_try_demote_pages[2]); 5158 page_demote_vp_pages(pp); 5159 ASSERT(pp->p_szc == 0); 5160 return (1); 5161 } 5162 5163 /* 5164 * Adjust rootpp if passed in is not the base 5165 * constituent page. 5166 */ 5167 npgs = page_get_pagecnt(pp->p_szc); 5168 ASSERT(npgs > 1); 5169 if (!IS_P2ALIGNED(pfn, npgs)) { 5170 pfn = P2ALIGN(pfn, npgs); 5171 rootpp = page_numtopp_nolock(pfn); 5172 VM_STAT_ADD(pagecnt.pc_try_demote_pages[3]); 5173 ASSERT(rootpp->p_vnode != NULL); 5174 ASSERT(rootpp->p_szc == szc); 5175 } 5176 5177 /* 5178 * We can't demote kernel pages since we can't hat_unload() 5179 * the mappings. 5180 */ 5181 if (VN_ISKAS(rootpp->p_vnode)) 5182 return (0); 5183 5184 /* 5185 * Attempt to lock all constituent pages except the page passed 5186 * in since it's already locked. 5187 */ 5188 for (tpp = rootpp, i = 0; i < npgs; i++, tpp++) { 5189 ASSERT(!PP_ISFREE(tpp)); 5190 ASSERT(tpp->p_vnode != NULL); 5191 5192 if (tpp != pp && !page_trylock(tpp, SE_EXCL)) 5193 break; 5194 ASSERT(tpp->p_szc == rootpp->p_szc); 5195 ASSERT(page_pptonum(tpp) == page_pptonum(rootpp) + i); 5196 } 5197 5198 /* 5199 * If we failed to lock them all then unlock what we have 5200 * locked so far and bail. 5201 */ 5202 if (i < npgs) { 5203 tpp = rootpp; 5204 while (i-- > 0) { 5205 if (tpp != pp) 5206 page_unlock(tpp); 5207 tpp++; 5208 } 5209 VM_STAT_ADD(pagecnt.pc_try_demote_pages[4]); 5210 return (0); 5211 } 5212 5213 for (tpp = rootpp, i = 0; i < npgs; i++, tpp++) { 5214 ASSERT(PAGE_EXCL(tpp)); 5215 ASSERT(tpp->p_slckcnt == 0); 5216 (void) hat_pageunload(tpp, HAT_FORCE_PGUNLOAD); 5217 tpp->p_szc = 0; 5218 } 5219 5220 /* 5221 * Unlock all pages except the page passed in. 5222 */ 5223 for (tpp = rootpp, i = 0; i < npgs; i++, tpp++) { 5224 ASSERT(!hat_page_is_mapped(tpp)); 5225 if (tpp != pp) 5226 page_unlock(tpp); 5227 } 5228 5229 VM_STAT_ADD(pagecnt.pc_try_demote_pages[5]); 5230 return (1); 5231 } 5232 5233 /* 5234 * Called by page_free() and page_destroy() to demote the page size code 5235 * (p_szc) to 0 (since we can't just put a single PAGESIZE page with non zero 5236 * p_szc on free list, neither can we just clear p_szc of a single page_t 5237 * within a large page since it will break other code that relies on p_szc 5238 * being the same for all page_t's of a large page). Anonymous pages should 5239 * never end up here because anon_map_getpages() cannot deal with p_szc 5240 * changes after a single constituent page is locked. While anonymous or 5241 * kernel large pages are demoted or freed the entire large page at a time 5242 * with all constituent pages locked EXCL for the file system pages we 5243 * have to be able to demote a large page (i.e. decrease all constituent pages 5244 * p_szc) with only just an EXCL lock on one of constituent pages. The reason 5245 * we can easily deal with anonymous page demotion the entire large page at a 5246 * time is that those operation originate at address space level and concern 5247 * the entire large page region with actual demotion only done when pages are 5248 * not shared with any other processes (therefore we can always get EXCL lock 5249 * on all anonymous constituent pages after clearing segment page 5250 * cache). However file system pages can be truncated or invalidated at a 5251 * PAGESIZE level from the file system side and end up in page_free() or 5252 * page_destroy() (we also allow only part of the large page to be SOFTLOCKed 5253 * and therefore pageout should be able to demote a large page by EXCL locking 5254 * any constituent page that is not under SOFTLOCK). In those cases we cannot 5255 * rely on being able to lock EXCL all constituent pages. 5256 * 5257 * To prevent szc changes on file system pages one has to lock all constituent 5258 * pages at least SHARED (or call page_szc_lock()). The only subsystem that 5259 * doesn't rely on locking all constituent pages (or using page_szc_lock()) to 5260 * prevent szc changes is hat layer that uses its own page level mlist 5261 * locks. hat assumes that szc doesn't change after mlist lock for a page is 5262 * taken. Therefore we need to change szc under hat level locks if we only 5263 * have an EXCL lock on a single constituent page and hat still references any 5264 * of constituent pages. (Note we can't "ignore" hat layer by simply 5265 * hat_pageunload() all constituent pages without having EXCL locks on all of 5266 * constituent pages). We use hat_page_demote() call to safely demote szc of 5267 * all constituent pages under hat locks when we only have an EXCL lock on one 5268 * of constituent pages. 5269 * 5270 * This routine calls page_szc_lock() before calling hat_page_demote() to 5271 * allow segvn in one special case not to lock all constituent pages SHARED 5272 * before calling hat_memload_array() that relies on p_szc not changing even 5273 * before hat level mlist lock is taken. In that case segvn uses 5274 * page_szc_lock() to prevent hat_page_demote() changing p_szc values. 5275 * 5276 * Anonymous or kernel page demotion still has to lock all pages exclusively 5277 * and do hat_pageunload() on all constituent pages before demoting the page 5278 * therefore there's no need for anonymous or kernel page demotion to use 5279 * hat_page_demote() mechanism. 5280 * 5281 * hat_page_demote() removes all large mappings that map pp and then decreases 5282 * p_szc starting from the last constituent page of the large page. By working 5283 * from the tail of a large page in pfn decreasing order allows one looking at 5284 * the root page to know that hat_page_demote() is done for root's szc area. 5285 * e.g. if a root page has szc 1 one knows it only has to lock all constituent 5286 * pages within szc 1 area to prevent szc changes because hat_page_demote() 5287 * that started on this page when it had szc > 1 is done for this szc 1 area. 5288 * 5289 * We are guaranteed that all constituent pages of pp's large page belong to 5290 * the same vnode with the consecutive offsets increasing in the direction of 5291 * the pfn i.e. the identity of constituent pages can't change until their 5292 * p_szc is decreased. Therefore it's safe for hat_page_demote() to remove 5293 * large mappings to pp even though we don't lock any constituent page except 5294 * pp (i.e. we won't unload e.g. kernel locked page). 5295 */ 5296 static void 5297 page_demote_vp_pages(page_t *pp) 5298 { 5299 kmutex_t *mtx; 5300 5301 ASSERT(PAGE_EXCL(pp)); 5302 ASSERT(!PP_ISFREE(pp)); 5303 ASSERT(pp->p_vnode != NULL); 5304 ASSERT(!IS_SWAPFSVP(pp->p_vnode)); 5305 ASSERT(!PP_ISKAS(pp)); 5306 5307 VM_STAT_ADD(pagecnt.pc_demote_pages[0]); 5308 5309 mtx = page_szc_lock(pp); 5310 if (mtx != NULL) { 5311 hat_page_demote(pp); 5312 mutex_exit(mtx); 5313 } 5314 ASSERT(pp->p_szc == 0); 5315 } 5316 5317 /* 5318 * Mark any existing pages for migration in the given range 5319 */ 5320 void 5321 page_mark_migrate(struct seg *seg, caddr_t addr, size_t len, 5322 struct anon_map *amp, ulong_t anon_index, vnode_t *vp, 5323 u_offset_t vnoff, int rflag) 5324 { 5325 struct anon *ap; 5326 vnode_t *curvp; 5327 lgrp_t *from; 5328 pgcnt_t i; 5329 pgcnt_t nlocked; 5330 u_offset_t off; 5331 pfn_t pfn; 5332 size_t pgsz; 5333 size_t segpgsz; 5334 pgcnt_t pages; 5335 uint_t pszc; 5336 page_t **ppa; 5337 pgcnt_t ppa_nentries; 5338 page_t *pp; 5339 caddr_t va; 5340 ulong_t an_idx; 5341 anon_sync_obj_t cookie; 5342 5343 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 5344 5345 /* 5346 * Don't do anything if don't need to do lgroup optimizations 5347 * on this system 5348 */ 5349 if (!lgrp_optimizations()) 5350 return; 5351 5352 /* 5353 * Align address and length to (potentially large) page boundary 5354 */ 5355 segpgsz = page_get_pagesize(seg->s_szc); 5356 addr = (caddr_t)P2ALIGN((uintptr_t)addr, segpgsz); 5357 if (rflag) 5358 len = P2ROUNDUP(len, segpgsz); 5359 5360 /* 5361 * Allocate page array to accommodate largest page size 5362 */ 5363 pgsz = page_get_pagesize(page_num_pagesizes() - 1); 5364 ppa_nentries = btop(pgsz); 5365 ppa = kmem_zalloc(ppa_nentries * sizeof (page_t *), KM_SLEEP); 5366 5367 /* 5368 * Do one (large) page at a time 5369 */ 5370 va = addr; 5371 while (va < addr + len) { 5372 /* 5373 * Lookup (root) page for vnode and offset corresponding to 5374 * this virtual address 5375 * Try anonmap first since there may be copy-on-write 5376 * pages, but initialize vnode pointer and offset using 5377 * vnode arguments just in case there isn't an amp. 5378 */ 5379 curvp = vp; 5380 off = vnoff + va - seg->s_base; 5381 if (amp) { 5382 ANON_LOCK_ENTER(&->a_rwlock, RW_READER); 5383 an_idx = anon_index + seg_page(seg, va); 5384 anon_array_enter(amp, an_idx, &cookie); 5385 ap = anon_get_ptr(amp->ahp, an_idx); 5386 if (ap) 5387 swap_xlate(ap, &curvp, &off); 5388 anon_array_exit(&cookie); 5389 ANON_LOCK_EXIT(&->a_rwlock); 5390 } 5391 5392 pp = NULL; 5393 if (curvp) 5394 pp = page_lookup(curvp, off, SE_SHARED); 5395 5396 /* 5397 * If there isn't a page at this virtual address, 5398 * skip to next page 5399 */ 5400 if (pp == NULL) { 5401 va += PAGESIZE; 5402 continue; 5403 } 5404 5405 /* 5406 * Figure out which lgroup this page is in for kstats 5407 */ 5408 pfn = page_pptonum(pp); 5409 from = lgrp_pfn_to_lgrp(pfn); 5410 5411 /* 5412 * Get page size, and round up and skip to next page boundary 5413 * if unaligned address 5414 */ 5415 pszc = pp->p_szc; 5416 pgsz = page_get_pagesize(pszc); 5417 pages = btop(pgsz); 5418 if (!IS_P2ALIGNED(va, pgsz) || 5419 !IS_P2ALIGNED(pfn, pages) || 5420 pgsz > segpgsz) { 5421 pgsz = MIN(pgsz, segpgsz); 5422 page_unlock(pp); 5423 i = btop(P2END((uintptr_t)va, pgsz) - 5424 (uintptr_t)va); 5425 va = (caddr_t)P2END((uintptr_t)va, pgsz); 5426 lgrp_stat_add(from->lgrp_id, LGRP_PMM_FAIL_PGS, i); 5427 continue; 5428 } 5429 5430 /* 5431 * Upgrade to exclusive lock on page 5432 */ 5433 if (!page_tryupgrade(pp)) { 5434 page_unlock(pp); 5435 va += pgsz; 5436 lgrp_stat_add(from->lgrp_id, LGRP_PMM_FAIL_PGS, 5437 btop(pgsz)); 5438 continue; 5439 } 5440 5441 /* 5442 * Remember pages locked exclusively and how many 5443 */ 5444 ppa[0] = pp; 5445 nlocked = 1; 5446 5447 /* 5448 * Lock constituent pages if this is large page 5449 */ 5450 if (pages > 1) { 5451 /* 5452 * Lock all constituents except root page, since it 5453 * should be locked already. 5454 */ 5455 for (i = 1; i < pages; i++) { 5456 pp++; 5457 if (!page_trylock(pp, SE_EXCL)) { 5458 break; 5459 } 5460 if (PP_ISFREE(pp) || 5461 pp->p_szc != pszc) { 5462 /* 5463 * hat_page_demote() raced in with us. 5464 */ 5465 ASSERT(!IS_SWAPFSVP(curvp)); 5466 page_unlock(pp); 5467 break; 5468 } 5469 ppa[nlocked] = pp; 5470 nlocked++; 5471 } 5472 } 5473 5474 /* 5475 * If all constituent pages couldn't be locked, 5476 * unlock pages locked so far and skip to next page. 5477 */ 5478 if (nlocked != pages) { 5479 for (i = 0; i < nlocked; i++) 5480 page_unlock(ppa[i]); 5481 va += pgsz; 5482 lgrp_stat_add(from->lgrp_id, LGRP_PMM_FAIL_PGS, 5483 btop(pgsz)); 5484 continue; 5485 } 5486 5487 /* 5488 * hat_page_demote() can no longer happen 5489 * since last cons page had the right p_szc after 5490 * all cons pages were locked. all cons pages 5491 * should now have the same p_szc. 5492 */ 5493 5494 /* 5495 * All constituent pages locked successfully, so mark 5496 * large page for migration and unload the mappings of 5497 * constituent pages, so a fault will occur on any part of the 5498 * large page 5499 */ 5500 PP_SETMIGRATE(ppa[0]); 5501 for (i = 0; i < nlocked; i++) { 5502 pp = ppa[i]; 5503 (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD); 5504 ASSERT(hat_page_getshare(pp) == 0); 5505 page_unlock(pp); 5506 } 5507 lgrp_stat_add(from->lgrp_id, LGRP_PMM_PGS, nlocked); 5508 5509 va += pgsz; 5510 } 5511 kmem_free(ppa, ppa_nentries * sizeof (page_t *)); 5512 } 5513 5514 /* 5515 * Migrate any pages that have been marked for migration in the given range 5516 */ 5517 void 5518 page_migrate( 5519 struct seg *seg, 5520 caddr_t addr, 5521 page_t **ppa, 5522 pgcnt_t npages) 5523 { 5524 lgrp_t *from; 5525 lgrp_t *to; 5526 page_t *newpp; 5527 page_t *pp; 5528 pfn_t pfn; 5529 size_t pgsz; 5530 spgcnt_t page_cnt; 5531 spgcnt_t i; 5532 uint_t pszc; 5533 5534 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 5535 5536 while (npages > 0) { 5537 pp = *ppa; 5538 pszc = pp->p_szc; 5539 pgsz = page_get_pagesize(pszc); 5540 page_cnt = btop(pgsz); 5541 5542 /* 5543 * Check to see whether this page is marked for migration 5544 * 5545 * Assume that root page of large page is marked for 5546 * migration and none of the other constituent pages 5547 * are marked. This really simplifies clearing the 5548 * migrate bit by not having to clear it from each 5549 * constituent page. 5550 * 5551 * note we don't want to relocate an entire large page if 5552 * someone is only using one subpage. 5553 */ 5554 if (npages < page_cnt) 5555 break; 5556 5557 /* 5558 * Is it marked for migration? 5559 */ 5560 if (!PP_ISMIGRATE(pp)) 5561 goto next; 5562 5563 /* 5564 * Determine lgroups that page is being migrated between 5565 */ 5566 pfn = page_pptonum(pp); 5567 if (!IS_P2ALIGNED(pfn, page_cnt)) { 5568 break; 5569 } 5570 from = lgrp_pfn_to_lgrp(pfn); 5571 to = lgrp_mem_choose(seg, addr, pgsz); 5572 5573 /* 5574 * Need to get exclusive lock's to migrate 5575 */ 5576 for (i = 0; i < page_cnt; i++) { 5577 ASSERT(PAGE_LOCKED(ppa[i])); 5578 if (page_pptonum(ppa[i]) != pfn + i || 5579 ppa[i]->p_szc != pszc) { 5580 break; 5581 } 5582 if (!page_tryupgrade(ppa[i])) { 5583 lgrp_stat_add(from->lgrp_id, 5584 LGRP_PM_FAIL_LOCK_PGS, 5585 page_cnt); 5586 break; 5587 } 5588 5589 /* 5590 * Check to see whether we are trying to migrate 5591 * page to lgroup where it is allocated already. 5592 * If so, clear the migrate bit and skip to next 5593 * page. 5594 */ 5595 if (i == 0 && to == from) { 5596 PP_CLRMIGRATE(ppa[0]); 5597 page_downgrade(ppa[0]); 5598 goto next; 5599 } 5600 } 5601 5602 /* 5603 * If all constituent pages couldn't be locked, 5604 * unlock pages locked so far and skip to next page. 5605 */ 5606 if (i != page_cnt) { 5607 while (--i != -1) { 5608 page_downgrade(ppa[i]); 5609 } 5610 goto next; 5611 } 5612 5613 (void) page_create_wait(page_cnt, PG_WAIT); 5614 newpp = page_get_replacement_page(pp, to, PGR_SAMESZC); 5615 if (newpp == NULL) { 5616 page_create_putback(page_cnt); 5617 for (i = 0; i < page_cnt; i++) { 5618 page_downgrade(ppa[i]); 5619 } 5620 lgrp_stat_add(to->lgrp_id, LGRP_PM_FAIL_ALLOC_PGS, 5621 page_cnt); 5622 goto next; 5623 } 5624 ASSERT(newpp->p_szc == pszc); 5625 /* 5626 * Clear migrate bit and relocate page 5627 */ 5628 PP_CLRMIGRATE(pp); 5629 if (page_relocate(&pp, &newpp, 0, 1, &page_cnt, to)) { 5630 panic("page_migrate: page_relocate failed"); 5631 } 5632 ASSERT(page_cnt * PAGESIZE == pgsz); 5633 5634 /* 5635 * Keep stats for number of pages migrated from and to 5636 * each lgroup 5637 */ 5638 lgrp_stat_add(from->lgrp_id, LGRP_PM_SRC_PGS, page_cnt); 5639 lgrp_stat_add(to->lgrp_id, LGRP_PM_DEST_PGS, page_cnt); 5640 /* 5641 * update the page_t array we were passed in and 5642 * unlink constituent pages of a large page. 5643 */ 5644 for (i = 0; i < page_cnt; ++i, ++pp) { 5645 ASSERT(PAGE_EXCL(newpp)); 5646 ASSERT(newpp->p_szc == pszc); 5647 ppa[i] = newpp; 5648 pp = newpp; 5649 page_sub(&newpp, pp); 5650 page_downgrade(pp); 5651 } 5652 ASSERT(newpp == NULL); 5653 next: 5654 addr += pgsz; 5655 ppa += page_cnt; 5656 npages -= page_cnt; 5657 } 5658 } 5659 5660 ulong_t mem_waiters = 0; 5661 ulong_t max_count = 20; 5662 #define MAX_DELAY 0x1ff 5663 5664 /* 5665 * Check if enough memory is available to proceed. 5666 * Depending on system configuration and how much memory is 5667 * reserved for swap we need to check against two variables. 5668 * e.g. on systems with little physical swap availrmem can be 5669 * more reliable indicator of how much memory is available. 5670 * On systems with large phys swap freemem can be better indicator. 5671 * If freemem drops below threshold level don't return an error 5672 * immediately but wake up pageout to free memory and block. 5673 * This is done number of times. If pageout is not able to free 5674 * memory within certain time return an error. 5675 * The same applies for availrmem but kmem_reap is used to 5676 * free memory. 5677 */ 5678 int 5679 page_mem_avail(pgcnt_t npages) 5680 { 5681 ulong_t count; 5682 5683 #if defined(__i386) 5684 if (freemem > desfree + npages && 5685 availrmem > swapfs_reserve + npages && 5686 btop(vmem_size(heap_arena, VMEM_FREE)) > tune.t_minarmem + 5687 npages) 5688 return (1); 5689 #else 5690 if (freemem > desfree + npages && 5691 availrmem > swapfs_reserve + npages) 5692 return (1); 5693 #endif 5694 5695 count = max_count; 5696 atomic_add_long(&mem_waiters, 1); 5697 5698 while (freemem < desfree + npages && --count) { 5699 cv_signal(&proc_pageout->p_cv); 5700 if (delay_sig(hz + (mem_waiters & MAX_DELAY))) { 5701 atomic_add_long(&mem_waiters, -1); 5702 return (0); 5703 } 5704 } 5705 if (count == 0) { 5706 atomic_add_long(&mem_waiters, -1); 5707 return (0); 5708 } 5709 5710 count = max_count; 5711 while (availrmem < swapfs_reserve + npages && --count) { 5712 kmem_reap(); 5713 if (delay_sig(hz + (mem_waiters & MAX_DELAY))) { 5714 atomic_add_long(&mem_waiters, -1); 5715 return (0); 5716 } 5717 } 5718 atomic_add_long(&mem_waiters, -1); 5719 if (count == 0) 5720 return (0); 5721 5722 #if defined(__i386) 5723 if (btop(vmem_size(heap_arena, VMEM_FREE)) < 5724 tune.t_minarmem + npages) 5725 return (0); 5726 #endif 5727 return (1); 5728 } 5729 5730 #define MAX_CNT 60 /* max num of iterations */ 5731 /* 5732 * Reclaim/reserve availrmem for npages. 5733 * If there is not enough memory start reaping seg, kmem caches. 5734 * Start pageout scanner (via page_needfree()). 5735 * Exit after ~ MAX_CNT s regardless of how much memory has been released. 5736 * Note: There is no guarantee that any availrmem will be freed as 5737 * this memory typically is locked (kernel heap) or reserved for swap. 5738 * Also due to memory fragmentation kmem allocator may not be able 5739 * to free any memory (single user allocated buffer will prevent 5740 * freeing slab or a page). 5741 */ 5742 int 5743 page_reclaim_mem(pgcnt_t npages, pgcnt_t epages, int adjust) 5744 { 5745 int i = 0; 5746 int ret = 0; 5747 pgcnt_t deficit; 5748 pgcnt_t old_availrmem; 5749 5750 mutex_enter(&freemem_lock); 5751 old_availrmem = availrmem - 1; 5752 while ((availrmem < tune.t_minarmem + npages + epages) && 5753 (old_availrmem < availrmem) && (i++ < MAX_CNT)) { 5754 old_availrmem = availrmem; 5755 deficit = tune.t_minarmem + npages + epages - availrmem; 5756 mutex_exit(&freemem_lock); 5757 page_needfree(deficit); 5758 kmem_reap(); 5759 delay(hz); 5760 page_needfree(-(spgcnt_t)deficit); 5761 mutex_enter(&freemem_lock); 5762 } 5763 5764 if (adjust && (availrmem >= tune.t_minarmem + npages + epages)) { 5765 availrmem -= npages; 5766 ret = 1; 5767 } 5768 5769 mutex_exit(&freemem_lock); 5770 5771 return (ret); 5772 } 5773 5774 /* 5775 * Search the memory segments to locate the desired page. Within a 5776 * segment, pages increase linearly with one page structure per 5777 * physical page frame (size PAGESIZE). The search begins 5778 * with the segment that was accessed last, to take advantage of locality. 5779 * If the hint misses, we start from the beginning of the sorted memseg list 5780 */ 5781 5782 5783 /* 5784 * Some data structures for pfn to pp lookup. 5785 */ 5786 ulong_t mhash_per_slot; 5787 struct memseg *memseg_hash[N_MEM_SLOTS]; 5788 5789 page_t * 5790 page_numtopp_nolock(pfn_t pfnum) 5791 { 5792 struct memseg *seg; 5793 page_t *pp; 5794 vm_cpu_data_t *vc; 5795 5796 /* 5797 * We need to disable kernel preemption while referencing the 5798 * cpu_vm_data field in order to prevent us from being switched to 5799 * another cpu and trying to reference it after it has been freed. 5800 * This will keep us on cpu and prevent it from being removed while 5801 * we are still on it. 5802 */ 5803 kpreempt_disable(); 5804 vc = CPU->cpu_vm_data; 5805 ASSERT(vc != NULL); 5806 5807 MEMSEG_STAT_INCR(nsearch); 5808 5809 /* Try last winner first */ 5810 if (((seg = vc->vc_pnum_memseg) != NULL) && 5811 (pfnum >= seg->pages_base) && (pfnum < seg->pages_end)) { 5812 MEMSEG_STAT_INCR(nlastwon); 5813 pp = seg->pages + (pfnum - seg->pages_base); 5814 if (pp->p_pagenum == pfnum) { 5815 kpreempt_enable(); 5816 return ((page_t *)pp); 5817 } 5818 } 5819 5820 /* Else Try hash */ 5821 if (((seg = memseg_hash[MEMSEG_PFN_HASH(pfnum)]) != NULL) && 5822 (pfnum >= seg->pages_base) && (pfnum < seg->pages_end)) { 5823 MEMSEG_STAT_INCR(nhashwon); 5824 vc->vc_pnum_memseg = seg; 5825 pp = seg->pages + (pfnum - seg->pages_base); 5826 if (pp->p_pagenum == pfnum) { 5827 kpreempt_enable(); 5828 return ((page_t *)pp); 5829 } 5830 } 5831 5832 /* Else Brute force */ 5833 for (seg = memsegs; seg != NULL; seg = seg->next) { 5834 if (pfnum >= seg->pages_base && pfnum < seg->pages_end) { 5835 vc->vc_pnum_memseg = seg; 5836 pp = seg->pages + (pfnum - seg->pages_base); 5837 kpreempt_enable(); 5838 return ((page_t *)pp); 5839 } 5840 } 5841 vc->vc_pnum_memseg = NULL; 5842 kpreempt_enable(); 5843 MEMSEG_STAT_INCR(nnotfound); 5844 return ((page_t *)NULL); 5845 5846 } 5847 5848 struct memseg * 5849 page_numtomemseg_nolock(pfn_t pfnum) 5850 { 5851 struct memseg *seg; 5852 page_t *pp; 5853 5854 /* Try hash */ 5855 if (((seg = memseg_hash[MEMSEG_PFN_HASH(pfnum)]) != NULL) && 5856 (pfnum >= seg->pages_base) && (pfnum < seg->pages_end)) { 5857 pp = seg->pages + (pfnum - seg->pages_base); 5858 if (pp->p_pagenum == pfnum) 5859 return (seg); 5860 } 5861 5862 /* Else Brute force */ 5863 for (seg = memsegs; seg != NULL; seg = seg->next) { 5864 if (pfnum >= seg->pages_base && pfnum < seg->pages_end) { 5865 return (seg); 5866 } 5867 } 5868 return ((struct memseg *)NULL); 5869 } 5870 5871 /* 5872 * Given a page and a count return the page struct that is 5873 * n structs away from the current one in the global page 5874 * list. 5875 * 5876 * This function wraps to the first page upon 5877 * reaching the end of the memseg list. 5878 */ 5879 page_t * 5880 page_nextn(page_t *pp, ulong_t n) 5881 { 5882 struct memseg *seg; 5883 page_t *ppn; 5884 vm_cpu_data_t *vc; 5885 5886 /* 5887 * We need to disable kernel preemption while referencing the 5888 * cpu_vm_data field in order to prevent us from being switched to 5889 * another cpu and trying to reference it after it has been freed. 5890 * This will keep us on cpu and prevent it from being removed while 5891 * we are still on it. 5892 */ 5893 kpreempt_disable(); 5894 vc = (vm_cpu_data_t *)CPU->cpu_vm_data; 5895 5896 ASSERT(vc != NULL); 5897 5898 if (((seg = vc->vc_pnext_memseg) == NULL) || 5899 (seg->pages_base == seg->pages_end) || 5900 !(pp >= seg->pages && pp < seg->epages)) { 5901 5902 for (seg = memsegs; seg; seg = seg->next) { 5903 if (pp >= seg->pages && pp < seg->epages) 5904 break; 5905 } 5906 5907 if (seg == NULL) { 5908 /* Memory delete got in, return something valid. */ 5909 /* TODO: fix me. */ 5910 seg = memsegs; 5911 pp = seg->pages; 5912 } 5913 } 5914 5915 /* check for wraparound - possible if n is large */ 5916 while ((ppn = (pp + n)) >= seg->epages || ppn < pp) { 5917 n -= seg->epages - pp; 5918 seg = seg->next; 5919 if (seg == NULL) 5920 seg = memsegs; 5921 pp = seg->pages; 5922 } 5923 vc->vc_pnext_memseg = seg; 5924 kpreempt_enable(); 5925 return (ppn); 5926 } 5927 5928 /* 5929 * Initialize for a loop using page_next_scan_large(). 5930 */ 5931 page_t * 5932 page_next_scan_init(void **cookie) 5933 { 5934 ASSERT(cookie != NULL); 5935 *cookie = (void *)memsegs; 5936 return ((page_t *)memsegs->pages); 5937 } 5938 5939 /* 5940 * Return the next page in a scan of page_t's, assuming we want 5941 * to skip over sub-pages within larger page sizes. 5942 * 5943 * The cookie is used to keep track of the current memseg. 5944 */ 5945 page_t * 5946 page_next_scan_large( 5947 page_t *pp, 5948 ulong_t *n, 5949 void **cookie) 5950 { 5951 struct memseg *seg = (struct memseg *)*cookie; 5952 page_t *new_pp; 5953 ulong_t cnt; 5954 pfn_t pfn; 5955 5956 5957 /* 5958 * get the count of page_t's to skip based on the page size 5959 */ 5960 ASSERT(pp != NULL); 5961 if (pp->p_szc == 0) { 5962 cnt = 1; 5963 } else { 5964 pfn = page_pptonum(pp); 5965 cnt = page_get_pagecnt(pp->p_szc); 5966 cnt -= pfn & (cnt - 1); 5967 } 5968 *n += cnt; 5969 new_pp = pp + cnt; 5970 5971 /* 5972 * Catch if we went past the end of the current memory segment. If so, 5973 * just move to the next segment with pages. 5974 */ 5975 if (new_pp >= seg->epages) { 5976 do { 5977 seg = seg->next; 5978 if (seg == NULL) 5979 seg = memsegs; 5980 } while (seg->pages == seg->epages); 5981 new_pp = seg->pages; 5982 *cookie = (void *)seg; 5983 } 5984 5985 return (new_pp); 5986 } 5987 5988 5989 /* 5990 * Returns next page in list. Note: this function wraps 5991 * to the first page in the list upon reaching the end 5992 * of the list. Callers should be aware of this fact. 5993 */ 5994 5995 /* We should change this be a #define */ 5996 5997 page_t * 5998 page_next(page_t *pp) 5999 { 6000 return (page_nextn(pp, 1)); 6001 } 6002 6003 page_t * 6004 page_first() 6005 { 6006 return ((page_t *)memsegs->pages); 6007 } 6008 6009 6010 /* 6011 * This routine is called at boot with the initial memory configuration 6012 * and when memory is added or removed. 6013 */ 6014 void 6015 build_pfn_hash() 6016 { 6017 pfn_t cur; 6018 pgcnt_t index; 6019 struct memseg *pseg; 6020 int i; 6021 6022 /* 6023 * Clear memseg_hash array. 6024 * Since memory add/delete is designed to operate concurrently 6025 * with normal operation, the hash rebuild must be able to run 6026 * concurrently with page_numtopp_nolock(). To support this 6027 * functionality, assignments to memseg_hash array members must 6028 * be done atomically. 6029 * 6030 * NOTE: bzero() does not currently guarantee this for kernel 6031 * threads, and cannot be used here. 6032 */ 6033 for (i = 0; i < N_MEM_SLOTS; i++) 6034 memseg_hash[i] = NULL; 6035 6036 hat_kpm_mseghash_clear(N_MEM_SLOTS); 6037 6038 /* 6039 * Physmax is the last valid pfn. 6040 */ 6041 mhash_per_slot = (physmax + 1) >> MEM_HASH_SHIFT; 6042 for (pseg = memsegs; pseg != NULL; pseg = pseg->next) { 6043 index = MEMSEG_PFN_HASH(pseg->pages_base); 6044 cur = pseg->pages_base; 6045 do { 6046 if (index >= N_MEM_SLOTS) 6047 index = MEMSEG_PFN_HASH(cur); 6048 6049 if (memseg_hash[index] == NULL || 6050 memseg_hash[index]->pages_base > pseg->pages_base) { 6051 memseg_hash[index] = pseg; 6052 hat_kpm_mseghash_update(index, pseg); 6053 } 6054 cur += mhash_per_slot; 6055 index++; 6056 } while (cur < pseg->pages_end); 6057 } 6058 } 6059 6060 /* 6061 * Return the pagenum for the pp 6062 */ 6063 pfn_t 6064 page_pptonum(page_t *pp) 6065 { 6066 return (pp->p_pagenum); 6067 } 6068 6069 /* 6070 * interface to the referenced and modified etc bits 6071 * in the PSM part of the page struct 6072 * when no locking is desired. 6073 */ 6074 void 6075 page_set_props(page_t *pp, uint_t flags) 6076 { 6077 ASSERT((flags & ~(P_MOD | P_REF | P_RO)) == 0); 6078 pp->p_nrm |= (uchar_t)flags; 6079 } 6080 6081 extern void mach_sync_icache_pp(page_t *); 6082 #pragma weak mach_sync_icache_pp 6083 6084 /* 6085 * Flush I-cache if the page is being reassigned. The hashout flag is 6086 * set when a page has been removed from a hash chain (i.e. vnode 6087 * pages). If the page stays on the hash chain there is a chance it 6088 * will be re-used, therefore there is no need to flush the 6089 * I-cache. However, if the page is being removed from a hash chain 6090 * then the page can be used for any new purpose, and the I-cache must 6091 * be flushed. 6092 */ 6093 /* ARGSUSED */ 6094 void 6095 page_clr_all_props(page_t *pp, int hashout) 6096 { 6097 if (&mach_sync_icache_pp != NULL && hashout) { 6098 mach_sync_icache_pp(pp); 6099 } 6100 pp->p_nrm = 0; 6101 } 6102 6103 /* 6104 * Clear p_lckcnt and p_cowcnt, adjusting freemem if required. 6105 */ 6106 int 6107 page_clear_lck_cow(page_t *pp, int adjust) 6108 { 6109 int f_amount; 6110 6111 ASSERT(PAGE_EXCL(pp)); 6112 6113 /* 6114 * The page_struct_lock need not be acquired here since 6115 * we require the caller hold the page exclusively locked. 6116 */ 6117 f_amount = 0; 6118 if (pp->p_lckcnt) { 6119 f_amount = 1; 6120 pp->p_lckcnt = 0; 6121 } 6122 if (pp->p_cowcnt) { 6123 f_amount += pp->p_cowcnt; 6124 pp->p_cowcnt = 0; 6125 } 6126 6127 if (adjust && f_amount) { 6128 mutex_enter(&freemem_lock); 6129 availrmem += f_amount; 6130 mutex_exit(&freemem_lock); 6131 } 6132 6133 return (f_amount); 6134 } 6135 6136 /* 6137 * The following functions is called from free_vp_pages() 6138 * for an inexact estimate of a newly free'd page... 6139 */ 6140 ulong_t 6141 page_share_cnt(page_t *pp) 6142 { 6143 return (hat_page_getshare(pp)); 6144 } 6145 6146 int 6147 page_isshared(page_t *pp) 6148 { 6149 return (hat_page_checkshare(pp, 1)); 6150 } 6151 6152 int 6153 page_isfree(page_t *pp) 6154 { 6155 return (PP_ISFREE(pp)); 6156 } 6157 6158 int 6159 page_isref(page_t *pp) 6160 { 6161 return (hat_page_getattr(pp, P_REF)); 6162 } 6163 6164 int 6165 page_ismod(page_t *pp) 6166 { 6167 return (hat_page_getattr(pp, P_MOD)); 6168 } 6169 6170 /* 6171 * The following code all currently relates to the page capture logic: 6172 * 6173 * This logic is used for cases where there is a desire to claim a certain 6174 * physical page in the system for the caller. As it may not be possible 6175 * to capture the page immediately, the p_toxic bits are used in the page 6176 * structure to indicate that someone wants to capture this page. When the 6177 * page gets unlocked, the toxic flag will be noted and an attempt to capture 6178 * the page will be made. If it is successful, the original callers callback 6179 * will be called with the page to do with it what they please. 6180 * 6181 * There is also an async thread which wakes up to attempt to capture 6182 * pages occasionally which have the capture bit set. All of the pages which 6183 * need to be captured asynchronously have been inserted into the 6184 * page_capture_hash and thus this thread walks that hash list. Items in the 6185 * hash have an expiration time so this thread handles that as well by removing 6186 * the item from the hash if it has expired. 6187 * 6188 * Some important things to note are: 6189 * - if the PR_CAPTURE bit is set on a page, then the page is in the 6190 * page_capture_hash. The page_capture_hash_head.pchh_mutex is needed 6191 * to set and clear this bit, and while the lock is held is the only time 6192 * you can add or remove an entry from the hash. 6193 * - the PR_CAPTURE bit can only be set and cleared while holding the 6194 * page_capture_hash_head.pchh_mutex 6195 * - the t_flag field of the thread struct is used with the T_CAPTURING 6196 * flag to prevent recursion while dealing with large pages. 6197 * - pages which need to be retired never expire on the page_capture_hash. 6198 */ 6199 6200 static void page_capture_thread(void); 6201 static kthread_t *pc_thread_id; 6202 kcondvar_t pc_cv; 6203 static kmutex_t pc_thread_mutex; 6204 static clock_t pc_thread_shortwait; 6205 static clock_t pc_thread_longwait; 6206 static int pc_thread_retry; 6207 6208 struct page_capture_callback pc_cb[PC_NUM_CALLBACKS]; 6209 6210 /* Note that this is a circular linked list */ 6211 typedef struct page_capture_hash_bucket { 6212 page_t *pp; 6213 uint_t szc; 6214 uint_t flags; 6215 clock_t expires; /* lbolt at which this request expires. */ 6216 void *datap; /* Cached data passed in for callback */ 6217 struct page_capture_hash_bucket *next; 6218 struct page_capture_hash_bucket *prev; 6219 } page_capture_hash_bucket_t; 6220 6221 /* 6222 * Each hash bucket will have it's own mutex and two lists which are: 6223 * active (0): represents requests which have not been processed by 6224 * the page_capture async thread yet. 6225 * walked (1): represents requests which have been processed by the 6226 * page_capture async thread within it's given walk of this bucket. 6227 * 6228 * These are all needed so that we can synchronize all async page_capture 6229 * events. When the async thread moves to a new bucket, it will append the 6230 * walked list to the active list and walk each item one at a time, moving it 6231 * from the active list to the walked list. Thus if there is an async request 6232 * outstanding for a given page, it will always be in one of the two lists. 6233 * New requests will always be added to the active list. 6234 * If we were not able to capture a page before the request expired, we'd free 6235 * up the request structure which would indicate to page_capture that there is 6236 * no longer a need for the given page, and clear the PR_CAPTURE flag if 6237 * possible. 6238 */ 6239 typedef struct page_capture_hash_head { 6240 kmutex_t pchh_mutex; 6241 uint_t num_pages; 6242 page_capture_hash_bucket_t lists[2]; /* sentinel nodes */ 6243 } page_capture_hash_head_t; 6244 6245 #ifdef DEBUG 6246 #define NUM_PAGE_CAPTURE_BUCKETS 4 6247 #else 6248 #define NUM_PAGE_CAPTURE_BUCKETS 64 6249 #endif 6250 6251 page_capture_hash_head_t page_capture_hash[NUM_PAGE_CAPTURE_BUCKETS]; 6252 6253 /* for now use a very simple hash based upon the size of a page struct */ 6254 #define PAGE_CAPTURE_HASH(pp) \ 6255 ((int)(((uintptr_t)pp >> 7) & (NUM_PAGE_CAPTURE_BUCKETS - 1))) 6256 6257 extern pgcnt_t swapfs_minfree; 6258 6259 int page_trycapture(page_t *pp, uint_t szc, uint_t flags, void *datap); 6260 6261 /* 6262 * a callback function is required for page capture requests. 6263 */ 6264 void 6265 page_capture_register_callback(uint_t index, clock_t duration, 6266 int (*cb_func)(page_t *, void *, uint_t)) 6267 { 6268 ASSERT(pc_cb[index].cb_active == 0); 6269 ASSERT(cb_func != NULL); 6270 rw_enter(&pc_cb[index].cb_rwlock, RW_WRITER); 6271 pc_cb[index].duration = duration; 6272 pc_cb[index].cb_func = cb_func; 6273 pc_cb[index].cb_active = 1; 6274 rw_exit(&pc_cb[index].cb_rwlock); 6275 } 6276 6277 void 6278 page_capture_unregister_callback(uint_t index) 6279 { 6280 int i, j; 6281 struct page_capture_hash_bucket *bp1; 6282 struct page_capture_hash_bucket *bp2; 6283 struct page_capture_hash_bucket *head = NULL; 6284 uint_t flags = (1 << index); 6285 6286 rw_enter(&pc_cb[index].cb_rwlock, RW_WRITER); 6287 ASSERT(pc_cb[index].cb_active == 1); 6288 pc_cb[index].duration = 0; /* Paranoia */ 6289 pc_cb[index].cb_func = NULL; /* Paranoia */ 6290 pc_cb[index].cb_active = 0; 6291 rw_exit(&pc_cb[index].cb_rwlock); 6292 6293 /* 6294 * Just move all the entries to a private list which we can walk 6295 * through without the need to hold any locks. 6296 * No more requests can get added to the hash lists for this consumer 6297 * as the cb_active field for the callback has been cleared. 6298 */ 6299 for (i = 0; i < NUM_PAGE_CAPTURE_BUCKETS; i++) { 6300 mutex_enter(&page_capture_hash[i].pchh_mutex); 6301 for (j = 0; j < 2; j++) { 6302 bp1 = page_capture_hash[i].lists[j].next; 6303 /* walk through all but first (sentinel) element */ 6304 while (bp1 != &page_capture_hash[i].lists[j]) { 6305 bp2 = bp1; 6306 if (bp2->flags & flags) { 6307 bp1 = bp2->next; 6308 bp1->prev = bp2->prev; 6309 bp2->prev->next = bp1; 6310 bp2->next = head; 6311 head = bp2; 6312 /* 6313 * Clear the PR_CAPTURE bit as we 6314 * hold appropriate locks here. 6315 */ 6316 page_clrtoxic(head->pp, PR_CAPTURE); 6317 page_capture_hash[i].num_pages--; 6318 continue; 6319 } 6320 bp1 = bp1->next; 6321 } 6322 } 6323 mutex_exit(&page_capture_hash[i].pchh_mutex); 6324 } 6325 6326 while (head != NULL) { 6327 bp1 = head; 6328 head = head->next; 6329 kmem_free(bp1, sizeof (*bp1)); 6330 } 6331 } 6332 6333 6334 /* 6335 * Find pp in the active list and move it to the walked list if it 6336 * exists. 6337 * Note that most often pp should be at the front of the active list 6338 * as it is currently used and thus there is no other sort of optimization 6339 * being done here as this is a linked list data structure. 6340 * Returns 1 on successful move or 0 if page could not be found. 6341 */ 6342 static int 6343 page_capture_move_to_walked(page_t *pp) 6344 { 6345 page_capture_hash_bucket_t *bp; 6346 int index; 6347 6348 index = PAGE_CAPTURE_HASH(pp); 6349 6350 mutex_enter(&page_capture_hash[index].pchh_mutex); 6351 bp = page_capture_hash[index].lists[0].next; 6352 while (bp != &page_capture_hash[index].lists[0]) { 6353 if (bp->pp == pp) { 6354 /* Remove from old list */ 6355 bp->next->prev = bp->prev; 6356 bp->prev->next = bp->next; 6357 6358 /* Add to new list */ 6359 bp->next = page_capture_hash[index].lists[1].next; 6360 bp->prev = &page_capture_hash[index].lists[1]; 6361 page_capture_hash[index].lists[1].next = bp; 6362 bp->next->prev = bp; 6363 mutex_exit(&page_capture_hash[index].pchh_mutex); 6364 6365 return (1); 6366 } 6367 bp = bp->next; 6368 } 6369 mutex_exit(&page_capture_hash[index].pchh_mutex); 6370 return (0); 6371 } 6372 6373 /* 6374 * Add a new entry to the page capture hash. The only case where a new 6375 * entry is not added is when the page capture consumer is no longer registered. 6376 * In this case, we'll silently not add the page to the hash. We know that 6377 * page retire will always be registered for the case where we are currently 6378 * unretiring a page and thus there are no conflicts. 6379 */ 6380 static void 6381 page_capture_add_hash(page_t *pp, uint_t szc, uint_t flags, void *datap) 6382 { 6383 page_capture_hash_bucket_t *bp1; 6384 page_capture_hash_bucket_t *bp2; 6385 int index; 6386 int cb_index; 6387 int i; 6388 #ifdef DEBUG 6389 page_capture_hash_bucket_t *tp1; 6390 int l; 6391 #endif 6392 6393 ASSERT(!(flags & CAPTURE_ASYNC)); 6394 6395 bp1 = kmem_alloc(sizeof (struct page_capture_hash_bucket), KM_SLEEP); 6396 6397 bp1->pp = pp; 6398 bp1->szc = szc; 6399 bp1->flags = flags; 6400 bp1->datap = datap; 6401 6402 for (cb_index = 0; cb_index < PC_NUM_CALLBACKS; cb_index++) { 6403 if ((flags >> cb_index) & 1) { 6404 break; 6405 } 6406 } 6407 6408 ASSERT(cb_index != PC_NUM_CALLBACKS); 6409 6410 rw_enter(&pc_cb[cb_index].cb_rwlock, RW_READER); 6411 if (pc_cb[cb_index].cb_active) { 6412 if (pc_cb[cb_index].duration == -1) { 6413 bp1->expires = (clock_t)-1; 6414 } else { 6415 bp1->expires = lbolt + pc_cb[cb_index].duration; 6416 } 6417 } else { 6418 /* There's no callback registered so don't add to the hash */ 6419 rw_exit(&pc_cb[cb_index].cb_rwlock); 6420 kmem_free(bp1, sizeof (*bp1)); 6421 return; 6422 } 6423 6424 index = PAGE_CAPTURE_HASH(pp); 6425 6426 /* 6427 * Only allow capture flag to be modified under this mutex. 6428 * Prevents multiple entries for same page getting added. 6429 */ 6430 mutex_enter(&page_capture_hash[index].pchh_mutex); 6431 6432 /* 6433 * if not already on the hash, set capture bit and add to the hash 6434 */ 6435 if (!(pp->p_toxic & PR_CAPTURE)) { 6436 #ifdef DEBUG 6437 /* Check for duplicate entries */ 6438 for (l = 0; l < 2; l++) { 6439 tp1 = page_capture_hash[index].lists[l].next; 6440 while (tp1 != &page_capture_hash[index].lists[l]) { 6441 if (tp1->pp == pp) { 6442 panic("page pp 0x%p already on hash " 6443 "at 0x%p\n", 6444 (void *)pp, (void *)tp1); 6445 } 6446 tp1 = tp1->next; 6447 } 6448 } 6449 6450 #endif 6451 page_settoxic(pp, PR_CAPTURE); 6452 bp1->next = page_capture_hash[index].lists[0].next; 6453 bp1->prev = &page_capture_hash[index].lists[0]; 6454 bp1->next->prev = bp1; 6455 page_capture_hash[index].lists[0].next = bp1; 6456 page_capture_hash[index].num_pages++; 6457 if (flags & CAPTURE_RETIRE) { 6458 page_retire_incr_pend_count(datap); 6459 } 6460 mutex_exit(&page_capture_hash[index].pchh_mutex); 6461 rw_exit(&pc_cb[cb_index].cb_rwlock); 6462 cv_signal(&pc_cv); 6463 return; 6464 } 6465 6466 /* 6467 * A page retire request will replace any other request. 6468 * A second physmem request which is for a different process than 6469 * the currently registered one will be dropped as there is 6470 * no way to hold the private data for both calls. 6471 * In the future, once there are more callers, this will have to 6472 * be worked out better as there needs to be private storage for 6473 * at least each type of caller (maybe have datap be an array of 6474 * *void's so that we can index based upon callers index). 6475 */ 6476 6477 /* walk hash list to update expire time */ 6478 for (i = 0; i < 2; i++) { 6479 bp2 = page_capture_hash[index].lists[i].next; 6480 while (bp2 != &page_capture_hash[index].lists[i]) { 6481 if (bp2->pp == pp) { 6482 if (flags & CAPTURE_RETIRE) { 6483 if (!(bp2->flags & CAPTURE_RETIRE)) { 6484 page_retire_incr_pend_count( 6485 datap); 6486 bp2->flags = flags; 6487 bp2->expires = bp1->expires; 6488 bp2->datap = datap; 6489 } 6490 } else { 6491 ASSERT(flags & CAPTURE_PHYSMEM); 6492 if (!(bp2->flags & CAPTURE_RETIRE) && 6493 (datap == bp2->datap)) { 6494 bp2->expires = bp1->expires; 6495 } 6496 } 6497 mutex_exit(&page_capture_hash[index]. 6498 pchh_mutex); 6499 rw_exit(&pc_cb[cb_index].cb_rwlock); 6500 kmem_free(bp1, sizeof (*bp1)); 6501 return; 6502 } 6503 bp2 = bp2->next; 6504 } 6505 } 6506 6507 /* 6508 * the PR_CAPTURE flag is protected by the page_capture_hash mutexes 6509 * and thus it either has to be set or not set and can't change 6510 * while holding the mutex above. 6511 */ 6512 panic("page_capture_add_hash, PR_CAPTURE flag set on pp %p\n", 6513 (void *)pp); 6514 } 6515 6516 /* 6517 * We have a page in our hands, lets try and make it ours by turning 6518 * it into a clean page like it had just come off the freelists. 6519 * 6520 * Returns 0 on success, with the page still EXCL locked. 6521 * On failure, the page will be unlocked, and returns EAGAIN 6522 */ 6523 static int 6524 page_capture_clean_page(page_t *pp) 6525 { 6526 page_t *newpp; 6527 int skip_unlock = 0; 6528 spgcnt_t count; 6529 page_t *tpp; 6530 int ret = 0; 6531 int extra; 6532 6533 ASSERT(PAGE_EXCL(pp)); 6534 ASSERT(!PP_RETIRED(pp)); 6535 ASSERT(curthread->t_flag & T_CAPTURING); 6536 6537 if (PP_ISFREE(pp)) { 6538 if (!page_reclaim(pp, NULL)) { 6539 skip_unlock = 1; 6540 ret = EAGAIN; 6541 goto cleanup; 6542 } 6543 ASSERT(pp->p_szc == 0); 6544 if (pp->p_vnode != NULL) { 6545 /* 6546 * Since this page came from the 6547 * cachelist, we must destroy the 6548 * old vnode association. 6549 */ 6550 page_hashout(pp, NULL); 6551 } 6552 goto cleanup; 6553 } 6554 6555 /* 6556 * If we know page_relocate will fail, skip it 6557 * It could still fail due to a UE on another page but we 6558 * can't do anything about that. 6559 */ 6560 if (pp->p_toxic & PR_UE) { 6561 goto skip_relocate; 6562 } 6563 6564 /* 6565 * It's possible that pages can not have a vnode as fsflush comes 6566 * through and cleans up these pages. It's ugly but that's how it is. 6567 */ 6568 if (pp->p_vnode == NULL) { 6569 goto skip_relocate; 6570 } 6571 6572 /* 6573 * Page was not free, so lets try to relocate it. 6574 * page_relocate only works with root pages, so if this is not a root 6575 * page, we need to demote it to try and relocate it. 6576 * Unfortunately this is the best we can do right now. 6577 */ 6578 newpp = NULL; 6579 if ((pp->p_szc > 0) && (pp != PP_PAGEROOT(pp))) { 6580 if (page_try_demote_pages(pp) == 0) { 6581 ret = EAGAIN; 6582 goto cleanup; 6583 } 6584 } 6585 ret = page_relocate(&pp, &newpp, 1, 0, &count, NULL); 6586 if (ret == 0) { 6587 page_t *npp; 6588 /* unlock the new page(s) */ 6589 while (count-- > 0) { 6590 ASSERT(newpp != NULL); 6591 npp = newpp; 6592 page_sub(&newpp, npp); 6593 page_unlock(npp); 6594 } 6595 ASSERT(newpp == NULL); 6596 /* 6597 * Check to see if the page we have is too large. 6598 * If so, demote it freeing up the extra pages. 6599 */ 6600 if (pp->p_szc > 0) { 6601 /* For now demote extra pages to szc == 0 */ 6602 extra = page_get_pagecnt(pp->p_szc) - 1; 6603 while (extra > 0) { 6604 tpp = pp->p_next; 6605 page_sub(&pp, tpp); 6606 tpp->p_szc = 0; 6607 page_free(tpp, 1); 6608 extra--; 6609 } 6610 /* Make sure to set our page to szc 0 as well */ 6611 ASSERT(pp->p_next == pp && pp->p_prev == pp); 6612 pp->p_szc = 0; 6613 } 6614 goto cleanup; 6615 } else if (ret == EIO) { 6616 ret = EAGAIN; 6617 goto cleanup; 6618 } else { 6619 /* 6620 * Need to reset return type as we failed to relocate the page 6621 * but that does not mean that some of the next steps will not 6622 * work. 6623 */ 6624 ret = 0; 6625 } 6626 6627 skip_relocate: 6628 6629 if (pp->p_szc > 0) { 6630 if (page_try_demote_pages(pp) == 0) { 6631 ret = EAGAIN; 6632 goto cleanup; 6633 } 6634 } 6635 6636 ASSERT(pp->p_szc == 0); 6637 6638 if (hat_ismod(pp)) { 6639 ret = EAGAIN; 6640 goto cleanup; 6641 } 6642 if (PP_ISKAS(pp)) { 6643 ret = EAGAIN; 6644 goto cleanup; 6645 } 6646 if (pp->p_lckcnt || pp->p_cowcnt) { 6647 ret = EAGAIN; 6648 goto cleanup; 6649 } 6650 6651 (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD); 6652 ASSERT(!hat_page_is_mapped(pp)); 6653 6654 if (hat_ismod(pp)) { 6655 /* 6656 * This is a semi-odd case as the page is now modified but not 6657 * mapped as we just unloaded the mappings above. 6658 */ 6659 ret = EAGAIN; 6660 goto cleanup; 6661 } 6662 if (pp->p_vnode != NULL) { 6663 page_hashout(pp, NULL); 6664 } 6665 6666 /* 6667 * At this point, the page should be in a clean state and 6668 * we can do whatever we want with it. 6669 */ 6670 6671 cleanup: 6672 if (ret != 0) { 6673 if (!skip_unlock) { 6674 page_unlock(pp); 6675 } 6676 } else { 6677 ASSERT(pp->p_szc == 0); 6678 ASSERT(PAGE_EXCL(pp)); 6679 6680 pp->p_next = pp; 6681 pp->p_prev = pp; 6682 } 6683 return (ret); 6684 } 6685 6686 /* 6687 * Various callers of page_trycapture() can have different restrictions upon 6688 * what memory they have access to. 6689 * Returns 0 on success, with the following error codes on failure: 6690 * EPERM - The requested page is long term locked, and thus repeated 6691 * requests to capture this page will likely fail. 6692 * ENOMEM - There was not enough free memory in the system to safely 6693 * map the requested page. 6694 * ENOENT - The requested page was inside the kernel cage, and the 6695 * PHYSMEM_CAGE flag was not set. 6696 */ 6697 int 6698 page_capture_pre_checks(page_t *pp, uint_t flags) 6699 { 6700 #if defined(__sparc) 6701 extern struct vnode prom_ppages; 6702 #endif /* __sparc */ 6703 6704 ASSERT(pp != NULL); 6705 6706 #if defined(__sparc) 6707 if (pp->p_vnode == &prom_ppages) { 6708 return (EPERM); 6709 } 6710 6711 if (PP_ISNORELOC(pp) && !(flags & CAPTURE_GET_CAGE) && 6712 (flags & CAPTURE_PHYSMEM)) { 6713 return (ENOENT); 6714 } 6715 6716 if (PP_ISNORELOCKERNEL(pp)) { 6717 return (EPERM); 6718 } 6719 #else 6720 if (PP_ISKAS(pp)) { 6721 return (EPERM); 6722 } 6723 #endif /* __sparc */ 6724 6725 /* only physmem currently has the restrictions checked below */ 6726 if (!(flags & CAPTURE_PHYSMEM)) { 6727 return (0); 6728 } 6729 6730 if (availrmem < swapfs_minfree) { 6731 /* 6732 * We won't try to capture this page as we are 6733 * running low on memory. 6734 */ 6735 return (ENOMEM); 6736 } 6737 return (0); 6738 } 6739 6740 /* 6741 * Once we have a page in our mits, go ahead and complete the capture 6742 * operation. 6743 * Returns 1 on failure where page is no longer needed 6744 * Returns 0 on success 6745 * Returns -1 if there was a transient failure. 6746 * Failure cases must release the SE_EXCL lock on pp (usually via page_free). 6747 */ 6748 int 6749 page_capture_take_action(page_t *pp, uint_t flags, void *datap) 6750 { 6751 int cb_index; 6752 int ret = 0; 6753 page_capture_hash_bucket_t *bp1; 6754 page_capture_hash_bucket_t *bp2; 6755 int index; 6756 int found = 0; 6757 int i; 6758 6759 ASSERT(PAGE_EXCL(pp)); 6760 ASSERT(curthread->t_flag & T_CAPTURING); 6761 6762 for (cb_index = 0; cb_index < PC_NUM_CALLBACKS; cb_index++) { 6763 if ((flags >> cb_index) & 1) { 6764 break; 6765 } 6766 } 6767 ASSERT(cb_index < PC_NUM_CALLBACKS); 6768 6769 /* 6770 * Remove the entry from the page_capture hash, but don't free it yet 6771 * as we may need to put it back. 6772 * Since we own the page at this point in time, we should find it 6773 * in the hash if this is an ASYNC call. If we don't it's likely 6774 * that the page_capture_async() thread decided that this request 6775 * had expired, in which case we just continue on. 6776 */ 6777 if (flags & CAPTURE_ASYNC) { 6778 6779 index = PAGE_CAPTURE_HASH(pp); 6780 6781 mutex_enter(&page_capture_hash[index].pchh_mutex); 6782 for (i = 0; i < 2 && !found; i++) { 6783 bp1 = page_capture_hash[index].lists[i].next; 6784 while (bp1 != &page_capture_hash[index].lists[i]) { 6785 if (bp1->pp == pp) { 6786 bp1->next->prev = bp1->prev; 6787 bp1->prev->next = bp1->next; 6788 page_capture_hash[index].num_pages--; 6789 page_clrtoxic(pp, PR_CAPTURE); 6790 found = 1; 6791 break; 6792 } 6793 bp1 = bp1->next; 6794 } 6795 } 6796 mutex_exit(&page_capture_hash[index].pchh_mutex); 6797 } 6798 6799 /* Synchronize with the unregister func. */ 6800 rw_enter(&pc_cb[cb_index].cb_rwlock, RW_READER); 6801 if (!pc_cb[cb_index].cb_active) { 6802 page_free(pp, 1); 6803 rw_exit(&pc_cb[cb_index].cb_rwlock); 6804 if (found) { 6805 kmem_free(bp1, sizeof (*bp1)); 6806 } 6807 return (1); 6808 } 6809 6810 /* 6811 * We need to remove the entry from the page capture hash and turn off 6812 * the PR_CAPTURE bit before calling the callback. We'll need to cache 6813 * the entry here, and then based upon the return value, cleanup 6814 * appropriately or re-add it to the hash, making sure that someone else 6815 * hasn't already done so. 6816 * It should be rare for the callback to fail and thus it's ok for 6817 * the failure path to be a bit complicated as the success path is 6818 * cleaner and the locking rules are easier to follow. 6819 */ 6820 6821 ret = pc_cb[cb_index].cb_func(pp, datap, flags); 6822 6823 rw_exit(&pc_cb[cb_index].cb_rwlock); 6824 6825 /* 6826 * If this was an ASYNC request, we need to cleanup the hash if the 6827 * callback was successful or if the request was no longer valid. 6828 * For non-ASYNC requests, we return failure to map and the caller 6829 * will take care of adding the request to the hash. 6830 * Note also that the callback itself is responsible for the page 6831 * at this point in time in terms of locking ... The most common 6832 * case for the failure path should just be a page_free. 6833 */ 6834 if (ret >= 0) { 6835 if (found) { 6836 if (bp1->flags & CAPTURE_RETIRE) { 6837 page_retire_decr_pend_count(datap); 6838 } 6839 kmem_free(bp1, sizeof (*bp1)); 6840 } 6841 return (ret); 6842 } 6843 if (!found) { 6844 return (ret); 6845 } 6846 6847 ASSERT(flags & CAPTURE_ASYNC); 6848 6849 /* 6850 * Check for expiration time first as we can just free it up if it's 6851 * expired. 6852 */ 6853 if (lbolt > bp1->expires && bp1->expires != -1) { 6854 kmem_free(bp1, sizeof (*bp1)); 6855 return (ret); 6856 } 6857 6858 /* 6859 * The callback failed and there used to be an entry in the hash for 6860 * this page, so we need to add it back to the hash. 6861 */ 6862 mutex_enter(&page_capture_hash[index].pchh_mutex); 6863 if (!(pp->p_toxic & PR_CAPTURE)) { 6864 /* just add bp1 back to head of walked list */ 6865 page_settoxic(pp, PR_CAPTURE); 6866 bp1->next = page_capture_hash[index].lists[1].next; 6867 bp1->prev = &page_capture_hash[index].lists[1]; 6868 bp1->next->prev = bp1; 6869 page_capture_hash[index].lists[1].next = bp1; 6870 page_capture_hash[index].num_pages++; 6871 mutex_exit(&page_capture_hash[index].pchh_mutex); 6872 return (ret); 6873 } 6874 6875 /* 6876 * Otherwise there was a new capture request added to list 6877 * Need to make sure that our original data is represented if 6878 * appropriate. 6879 */ 6880 for (i = 0; i < 2; i++) { 6881 bp2 = page_capture_hash[index].lists[i].next; 6882 while (bp2 != &page_capture_hash[index].lists[i]) { 6883 if (bp2->pp == pp) { 6884 if (bp1->flags & CAPTURE_RETIRE) { 6885 if (!(bp2->flags & CAPTURE_RETIRE)) { 6886 bp2->szc = bp1->szc; 6887 bp2->flags = bp1->flags; 6888 bp2->expires = bp1->expires; 6889 bp2->datap = bp1->datap; 6890 } 6891 } else { 6892 ASSERT(bp1->flags & CAPTURE_PHYSMEM); 6893 if (!(bp2->flags & CAPTURE_RETIRE)) { 6894 bp2->szc = bp1->szc; 6895 bp2->flags = bp1->flags; 6896 bp2->expires = bp1->expires; 6897 bp2->datap = bp1->datap; 6898 } 6899 } 6900 mutex_exit(&page_capture_hash[index]. 6901 pchh_mutex); 6902 kmem_free(bp1, sizeof (*bp1)); 6903 return (ret); 6904 } 6905 bp2 = bp2->next; 6906 } 6907 } 6908 panic("PR_CAPTURE set but not on hash for pp 0x%p\n", (void *)pp); 6909 /*NOTREACHED*/ 6910 } 6911 6912 /* 6913 * Try to capture the given page for the caller specified in the flags 6914 * parameter. The page will either be captured and handed over to the 6915 * appropriate callback, or will be queued up in the page capture hash 6916 * to be captured asynchronously. 6917 * If the current request is due to an async capture, the page must be 6918 * exclusively locked before calling this function. 6919 * Currently szc must be 0 but in the future this should be expandable to 6920 * other page sizes. 6921 * Returns 0 on success, with the following error codes on failure: 6922 * EPERM - The requested page is long term locked, and thus repeated 6923 * requests to capture this page will likely fail. 6924 * ENOMEM - There was not enough free memory in the system to safely 6925 * map the requested page. 6926 * ENOENT - The requested page was inside the kernel cage, and the 6927 * CAPTURE_GET_CAGE flag was not set. 6928 * EAGAIN - The requested page could not be capturead at this point in 6929 * time but future requests will likely work. 6930 * EBUSY - The requested page is retired and the CAPTURE_GET_RETIRED flag 6931 * was not set. 6932 */ 6933 int 6934 page_itrycapture(page_t *pp, uint_t szc, uint_t flags, void *datap) 6935 { 6936 int ret; 6937 int cb_index; 6938 6939 if (flags & CAPTURE_ASYNC) { 6940 ASSERT(PAGE_EXCL(pp)); 6941 goto async; 6942 } 6943 6944 /* Make sure there's enough availrmem ... */ 6945 ret = page_capture_pre_checks(pp, flags); 6946 if (ret != 0) { 6947 return (ret); 6948 } 6949 6950 if (!page_trylock(pp, SE_EXCL)) { 6951 for (cb_index = 0; cb_index < PC_NUM_CALLBACKS; cb_index++) { 6952 if ((flags >> cb_index) & 1) { 6953 break; 6954 } 6955 } 6956 ASSERT(cb_index < PC_NUM_CALLBACKS); 6957 ret = EAGAIN; 6958 /* Special case for retired pages */ 6959 if (PP_RETIRED(pp)) { 6960 if (flags & CAPTURE_GET_RETIRED) { 6961 if (!page_unretire_pp(pp, PR_UNR_TEMP)) { 6962 /* 6963 * Need to set capture bit and add to 6964 * hash so that the page will be 6965 * retired when freed. 6966 */ 6967 page_capture_add_hash(pp, szc, 6968 CAPTURE_RETIRE, NULL); 6969 ret = 0; 6970 goto own_page; 6971 } 6972 } else { 6973 return (EBUSY); 6974 } 6975 } 6976 page_capture_add_hash(pp, szc, flags, datap); 6977 return (ret); 6978 } 6979 6980 async: 6981 ASSERT(PAGE_EXCL(pp)); 6982 6983 /* Need to check for physmem async requests that availrmem is sane */ 6984 if ((flags & (CAPTURE_ASYNC | CAPTURE_PHYSMEM)) == 6985 (CAPTURE_ASYNC | CAPTURE_PHYSMEM) && 6986 (availrmem < swapfs_minfree)) { 6987 page_unlock(pp); 6988 return (ENOMEM); 6989 } 6990 6991 ret = page_capture_clean_page(pp); 6992 6993 if (ret != 0) { 6994 /* We failed to get the page, so lets add it to the hash */ 6995 if (!(flags & CAPTURE_ASYNC)) { 6996 page_capture_add_hash(pp, szc, flags, datap); 6997 } 6998 return (ret); 6999 } 7000 7001 own_page: 7002 ASSERT(PAGE_EXCL(pp)); 7003 ASSERT(pp->p_szc == 0); 7004 7005 /* Call the callback */ 7006 ret = page_capture_take_action(pp, flags, datap); 7007 7008 if (ret == 0) { 7009 return (0); 7010 } 7011 7012 /* 7013 * Note that in the failure cases from page_capture_take_action, the 7014 * EXCL lock will have already been dropped. 7015 */ 7016 if ((ret == -1) && (!(flags & CAPTURE_ASYNC))) { 7017 page_capture_add_hash(pp, szc, flags, datap); 7018 } 7019 return (EAGAIN); 7020 } 7021 7022 int 7023 page_trycapture(page_t *pp, uint_t szc, uint_t flags, void *datap) 7024 { 7025 int ret; 7026 7027 curthread->t_flag |= T_CAPTURING; 7028 ret = page_itrycapture(pp, szc, flags, datap); 7029 curthread->t_flag &= ~T_CAPTURING; /* xor works as we know its set */ 7030 return (ret); 7031 } 7032 7033 /* 7034 * When unlocking a page which has the PR_CAPTURE bit set, this routine 7035 * gets called to try and capture the page. 7036 */ 7037 void 7038 page_unlock_capture(page_t *pp) 7039 { 7040 page_capture_hash_bucket_t *bp; 7041 int index; 7042 int i; 7043 uint_t szc; 7044 uint_t flags = 0; 7045 void *datap; 7046 kmutex_t *mp; 7047 extern vnode_t retired_pages; 7048 7049 /* 7050 * We need to protect against a possible deadlock here where we own 7051 * the vnode page hash mutex and want to acquire it again as there 7052 * are locations in the code, where we unlock a page while holding 7053 * the mutex which can lead to the page being captured and eventually 7054 * end up here. As we may be hashing out the old page and hashing into 7055 * the retire vnode, we need to make sure we don't own them. 7056 * Other callbacks who do hash operations also need to make sure that 7057 * before they hashin to a vnode that they do not currently own the 7058 * vphm mutex otherwise there will be a panic. 7059 */ 7060 if (mutex_owned(page_vnode_mutex(&retired_pages))) { 7061 page_unlock_nocapture(pp); 7062 return; 7063 } 7064 if (pp->p_vnode != NULL && mutex_owned(page_vnode_mutex(pp->p_vnode))) { 7065 page_unlock_nocapture(pp); 7066 return; 7067 } 7068 7069 index = PAGE_CAPTURE_HASH(pp); 7070 7071 mp = &page_capture_hash[index].pchh_mutex; 7072 mutex_enter(mp); 7073 for (i = 0; i < 2; i++) { 7074 bp = page_capture_hash[index].lists[i].next; 7075 while (bp != &page_capture_hash[index].lists[i]) { 7076 if (bp->pp == pp) { 7077 szc = bp->szc; 7078 flags = bp->flags | CAPTURE_ASYNC; 7079 datap = bp->datap; 7080 mutex_exit(mp); 7081 (void) page_trycapture(pp, szc, flags, datap); 7082 return; 7083 } 7084 bp = bp->next; 7085 } 7086 } 7087 7088 /* Failed to find page in hash so clear flags and unlock it. */ 7089 page_clrtoxic(pp, PR_CAPTURE); 7090 page_unlock(pp); 7091 7092 mutex_exit(mp); 7093 } 7094 7095 void 7096 page_capture_init() 7097 { 7098 int i; 7099 for (i = 0; i < NUM_PAGE_CAPTURE_BUCKETS; i++) { 7100 page_capture_hash[i].lists[0].next = 7101 &page_capture_hash[i].lists[0]; 7102 page_capture_hash[i].lists[0].prev = 7103 &page_capture_hash[i].lists[0]; 7104 page_capture_hash[i].lists[1].next = 7105 &page_capture_hash[i].lists[1]; 7106 page_capture_hash[i].lists[1].prev = 7107 &page_capture_hash[i].lists[1]; 7108 } 7109 7110 pc_thread_shortwait = 23 * hz; 7111 pc_thread_longwait = 1201 * hz; 7112 pc_thread_retry = 3; 7113 mutex_init(&pc_thread_mutex, NULL, MUTEX_DEFAULT, NULL); 7114 cv_init(&pc_cv, NULL, CV_DEFAULT, NULL); 7115 pc_thread_id = thread_create(NULL, 0, page_capture_thread, NULL, 0, &p0, 7116 TS_RUN, minclsyspri); 7117 } 7118 7119 /* 7120 * It is necessary to scrub any failing pages prior to reboot in order to 7121 * prevent a latent error trap from occurring on the next boot. 7122 */ 7123 void 7124 page_retire_mdboot() 7125 { 7126 page_t *pp; 7127 int i, j; 7128 page_capture_hash_bucket_t *bp; 7129 7130 /* walk lists looking for pages to scrub */ 7131 for (i = 0; i < NUM_PAGE_CAPTURE_BUCKETS; i++) { 7132 if (page_capture_hash[i].num_pages == 0) 7133 continue; 7134 7135 mutex_enter(&page_capture_hash[i].pchh_mutex); 7136 7137 for (j = 0; j < 2; j++) { 7138 bp = page_capture_hash[i].lists[j].next; 7139 while (bp != &page_capture_hash[i].lists[j]) { 7140 pp = bp->pp; 7141 if (PP_TOXIC(pp)) { 7142 if (page_trylock(pp, SE_EXCL)) { 7143 PP_CLRFREE(pp); 7144 pagescrub(pp, 0, PAGESIZE); 7145 page_unlock(pp); 7146 } 7147 } 7148 bp = bp->next; 7149 } 7150 } 7151 mutex_exit(&page_capture_hash[i].pchh_mutex); 7152 } 7153 } 7154 7155 /* 7156 * Walk the page_capture_hash trying to capture pages and also cleanup old 7157 * entries which have expired. 7158 */ 7159 void 7160 page_capture_async() 7161 { 7162 page_t *pp; 7163 int i; 7164 int ret; 7165 page_capture_hash_bucket_t *bp1, *bp2; 7166 uint_t szc; 7167 uint_t flags; 7168 void *datap; 7169 7170 /* If there are outstanding pages to be captured, get to work */ 7171 for (i = 0; i < NUM_PAGE_CAPTURE_BUCKETS; i++) { 7172 if (page_capture_hash[i].num_pages == 0) 7173 continue; 7174 /* Append list 1 to list 0 and then walk through list 0 */ 7175 mutex_enter(&page_capture_hash[i].pchh_mutex); 7176 bp1 = &page_capture_hash[i].lists[1]; 7177 bp2 = bp1->next; 7178 if (bp1 != bp2) { 7179 bp1->prev->next = page_capture_hash[i].lists[0].next; 7180 bp2->prev = &page_capture_hash[i].lists[0]; 7181 page_capture_hash[i].lists[0].next->prev = bp1->prev; 7182 page_capture_hash[i].lists[0].next = bp2; 7183 bp1->next = bp1; 7184 bp1->prev = bp1; 7185 } 7186 7187 /* list[1] will be empty now */ 7188 7189 bp1 = page_capture_hash[i].lists[0].next; 7190 while (bp1 != &page_capture_hash[i].lists[0]) { 7191 /* Check expiration time */ 7192 if ((lbolt > bp1->expires && bp1->expires != -1) || 7193 page_deleted(bp1->pp)) { 7194 page_capture_hash[i].lists[0].next = bp1->next; 7195 bp1->next->prev = 7196 &page_capture_hash[i].lists[0]; 7197 page_capture_hash[i].num_pages--; 7198 7199 /* 7200 * We can safely remove the PR_CAPTURE bit 7201 * without holding the EXCL lock on the page 7202 * as the PR_CAPTURE bit requres that the 7203 * page_capture_hash[].pchh_mutex be held 7204 * to modify it. 7205 */ 7206 page_clrtoxic(bp1->pp, PR_CAPTURE); 7207 mutex_exit(&page_capture_hash[i].pchh_mutex); 7208 kmem_free(bp1, sizeof (*bp1)); 7209 mutex_enter(&page_capture_hash[i].pchh_mutex); 7210 bp1 = page_capture_hash[i].lists[0].next; 7211 continue; 7212 } 7213 pp = bp1->pp; 7214 szc = bp1->szc; 7215 flags = bp1->flags; 7216 datap = bp1->datap; 7217 mutex_exit(&page_capture_hash[i].pchh_mutex); 7218 if (page_trylock(pp, SE_EXCL)) { 7219 ret = page_trycapture(pp, szc, 7220 flags | CAPTURE_ASYNC, datap); 7221 } else { 7222 ret = 1; /* move to walked hash */ 7223 } 7224 7225 if (ret != 0) { 7226 /* Move to walked hash */ 7227 (void) page_capture_move_to_walked(pp); 7228 } 7229 mutex_enter(&page_capture_hash[i].pchh_mutex); 7230 bp1 = page_capture_hash[i].lists[0].next; 7231 } 7232 7233 mutex_exit(&page_capture_hash[i].pchh_mutex); 7234 } 7235 } 7236 7237 /* 7238 * This function is called by the page_capture_thread, and is needed in 7239 * in order to initiate aio cleanup, so that pages used in aio 7240 * will be unlocked and subsequently retired by page_capture_thread. 7241 */ 7242 static int 7243 do_aio_cleanup(void) 7244 { 7245 proc_t *procp; 7246 int (*aio_cleanup_dr_delete_memory)(proc_t *); 7247 int cleaned = 0; 7248 7249 if (modload("sys", "kaio") == -1) { 7250 cmn_err(CE_WARN, "do_aio_cleanup: cannot load kaio"); 7251 return (0); 7252 } 7253 /* 7254 * We use the aio_cleanup_dr_delete_memory function to 7255 * initiate the actual clean up; this function will wake 7256 * up the per-process aio_cleanup_thread. 7257 */ 7258 aio_cleanup_dr_delete_memory = (int (*)(proc_t *)) 7259 modgetsymvalue("aio_cleanup_dr_delete_memory", 0); 7260 if (aio_cleanup_dr_delete_memory == NULL) { 7261 cmn_err(CE_WARN, 7262 "aio_cleanup_dr_delete_memory not found in kaio"); 7263 return (0); 7264 } 7265 mutex_enter(&pidlock); 7266 for (procp = practive; (procp != NULL); procp = procp->p_next) { 7267 mutex_enter(&procp->p_lock); 7268 if (procp->p_aio != NULL) { 7269 /* cleanup proc's outstanding kaio */ 7270 cleaned += (*aio_cleanup_dr_delete_memory)(procp); 7271 } 7272 mutex_exit(&procp->p_lock); 7273 } 7274 mutex_exit(&pidlock); 7275 return (cleaned); 7276 } 7277 7278 /* 7279 * helper function for page_capture_thread 7280 */ 7281 static void 7282 page_capture_handle_outstanding(void) 7283 { 7284 int ntry; 7285 7286 /* Reap pages before attempting capture pages */ 7287 kmem_reap(); 7288 7289 if ((page_retire_pend_count() > page_retire_pend_kas_count()) && 7290 hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) { 7291 /* 7292 * Note: Purging only for platforms that support 7293 * ISM hat_pageunload() - mainly SPARC. On x86/x64 7294 * platforms ISM pages SE_SHARED locked until destroyed. 7295 */ 7296 7297 /* disable and purge seg_pcache */ 7298 (void) seg_p_disable(); 7299 for (ntry = 0; ntry < pc_thread_retry; ntry++) { 7300 if (!page_retire_pend_count()) 7301 break; 7302 if (do_aio_cleanup()) { 7303 /* 7304 * allow the apps cleanup threads 7305 * to run 7306 */ 7307 delay(pc_thread_shortwait); 7308 } 7309 page_capture_async(); 7310 } 7311 /* reenable seg_pcache */ 7312 seg_p_enable(); 7313 7314 /* completed what can be done. break out */ 7315 return; 7316 } 7317 7318 /* 7319 * For kernel pages and/or unsupported HAT_DYNAMIC_ISM_UNMAP, reap 7320 * and then attempt to capture. 7321 */ 7322 seg_preap(); 7323 page_capture_async(); 7324 } 7325 7326 /* 7327 * The page_capture_thread loops forever, looking to see if there are 7328 * pages still waiting to be captured. 7329 */ 7330 static void 7331 page_capture_thread(void) 7332 { 7333 callb_cpr_t c; 7334 int outstanding; 7335 int i; 7336 7337 CALLB_CPR_INIT(&c, &pc_thread_mutex, callb_generic_cpr, "page_capture"); 7338 7339 mutex_enter(&pc_thread_mutex); 7340 for (;;) { 7341 outstanding = 0; 7342 for (i = 0; i < NUM_PAGE_CAPTURE_BUCKETS; i++) 7343 outstanding += page_capture_hash[i].num_pages; 7344 if (outstanding) { 7345 page_capture_handle_outstanding(); 7346 CALLB_CPR_SAFE_BEGIN(&c); 7347 (void) cv_timedwait(&pc_cv, &pc_thread_mutex, 7348 lbolt + pc_thread_shortwait); 7349 CALLB_CPR_SAFE_END(&c, &pc_thread_mutex); 7350 } else { 7351 CALLB_CPR_SAFE_BEGIN(&c); 7352 (void) cv_timedwait(&pc_cv, &pc_thread_mutex, 7353 lbolt + pc_thread_longwait); 7354 CALLB_CPR_SAFE_END(&c, &pc_thread_mutex); 7355 } 7356 } 7357 /*NOTREACHED*/ 7358 } 7359 /* 7360 * Attempt to locate a bucket that has enough pages to satisfy the request. 7361 * The initial check is done without the lock to avoid unneeded contention. 7362 * The function returns 1 if enough pages were found, else 0 if it could not 7363 * find enough pages in a bucket. 7364 */ 7365 static int 7366 pcf_decrement_bucket(pgcnt_t npages) 7367 { 7368 struct pcf *p; 7369 struct pcf *q; 7370 int i; 7371 7372 p = &pcf[PCF_INDEX()]; 7373 q = &pcf[pcf_fanout]; 7374 for (i = 0; i < pcf_fanout; i++) { 7375 if (p->pcf_count > npages) { 7376 /* 7377 * a good one to try. 7378 */ 7379 mutex_enter(&p->pcf_lock); 7380 if (p->pcf_count > npages) { 7381 p->pcf_count -= (uint_t)npages; 7382 /* 7383 * freemem is not protected by any lock. 7384 * Thus, we cannot have any assertion 7385 * containing freemem here. 7386 */ 7387 freemem -= npages; 7388 mutex_exit(&p->pcf_lock); 7389 return (1); 7390 } 7391 mutex_exit(&p->pcf_lock); 7392 } 7393 p++; 7394 if (p >= q) { 7395 p = pcf; 7396 } 7397 } 7398 return (0); 7399 } 7400 7401 /* 7402 * Arguments: 7403 * pcftotal_ret: If the value is not NULL and we have walked all the 7404 * buckets but did not find enough pages then it will 7405 * be set to the total number of pages in all the pcf 7406 * buckets. 7407 * npages: Is the number of pages we have been requested to 7408 * find. 7409 * unlock: If set to 0 we will leave the buckets locked if the 7410 * requested number of pages are not found. 7411 * 7412 * Go and try to satisfy the page request from any number of buckets. 7413 * This can be a very expensive operation as we have to lock the buckets 7414 * we are checking (and keep them locked), starting at bucket 0. 7415 * 7416 * The function returns 1 if enough pages were found, else 0 if it could not 7417 * find enough pages in the buckets. 7418 * 7419 */ 7420 static int 7421 pcf_decrement_multiple(pgcnt_t *pcftotal_ret, pgcnt_t npages, int unlock) 7422 { 7423 struct pcf *p; 7424 pgcnt_t pcftotal; 7425 int i; 7426 7427 p = pcf; 7428 /* try to collect pages from several pcf bins */ 7429 for (pcftotal = 0, i = 0; i < pcf_fanout; i++) { 7430 mutex_enter(&p->pcf_lock); 7431 pcftotal += p->pcf_count; 7432 if (pcftotal >= npages) { 7433 /* 7434 * Wow! There are enough pages laying around 7435 * to satisfy the request. Do the accounting, 7436 * drop the locks we acquired, and go back. 7437 * 7438 * freemem is not protected by any lock. So, 7439 * we cannot have any assertion containing 7440 * freemem. 7441 */ 7442 freemem -= npages; 7443 while (p >= pcf) { 7444 if (p->pcf_count <= npages) { 7445 npages -= p->pcf_count; 7446 p->pcf_count = 0; 7447 } else { 7448 p->pcf_count -= (uint_t)npages; 7449 npages = 0; 7450 } 7451 mutex_exit(&p->pcf_lock); 7452 p--; 7453 } 7454 ASSERT(npages == 0); 7455 return (1); 7456 } 7457 p++; 7458 } 7459 if (unlock) { 7460 /* failed to collect pages - release the locks */ 7461 while (--p >= pcf) { 7462 mutex_exit(&p->pcf_lock); 7463 } 7464 } 7465 if (pcftotal_ret != NULL) 7466 *pcftotal_ret = pcftotal; 7467 return (0); 7468 } 7469