Lines Matching refs:pp
220 #define PP_CTR_LOCK_INDX(pp) \ argument
221 (((pp)->p_pagenum >> \
242 static int page_trylock_cons(page_t *pp, se_t se);
807 page_ctr_add_internal(int mnode, int mtype, page_t *pp, int flags) in page_ctr_add_internal() argument
814 ASSERT(mnode == PP_2_MEM_NODE(pp)); in page_ctr_add_internal()
815 ASSERT(mtype == PP_2_MTYPE(pp)); in page_ctr_add_internal()
817 ASSERT(pp->p_szc < mmu_page_sizes); in page_ctr_add_internal()
819 PLCNT_INCR(pp, mnode, mtype, pp->p_szc, flags); in page_ctr_add_internal()
822 if (pp->p_szc >= mmu_page_sizes - 1) { in page_ctr_add_internal()
826 r = pp->p_szc + 1; in page_ctr_add_internal()
827 pfnum = pp->p_pagenum; in page_ctr_add_internal()
828 lckidx = PP_CTR_LOCK_INDX(pp); in page_ctr_add_internal()
844 int root_mtype = PP_2_MTYPE(PP_GROUPLEADER(pp, r)); in page_ctr_add_internal()
849 cand->pcc_color_free[PP_2_BIN_SZC(pp, r)]++; in page_ctr_add_internal()
856 page_ctr_add(int mnode, int mtype, page_t *pp, int flags) in page_ctr_add() argument
858 int lckidx = PP_CTR_LOCK_INDX(pp); in page_ctr_add()
862 page_ctr_add_internal(mnode, mtype, pp, flags); in page_ctr_add()
867 page_ctr_sub_internal(int mnode, int mtype, page_t *pp, int flags) in page_ctr_sub_internal() argument
874 ASSERT(mnode == PP_2_MEM_NODE(pp)); in page_ctr_sub_internal()
875 ASSERT(mtype == PP_2_MTYPE(pp)); in page_ctr_sub_internal()
877 ASSERT(pp->p_szc < mmu_page_sizes); in page_ctr_sub_internal()
879 PLCNT_DECR(pp, mnode, mtype, pp->p_szc, flags); in page_ctr_sub_internal()
882 if (pp->p_szc >= mmu_page_sizes - 1) { in page_ctr_sub_internal()
886 r = pp->p_szc + 1; in page_ctr_sub_internal()
887 pfnum = pp->p_pagenum; in page_ctr_sub_internal()
888 lckidx = PP_CTR_LOCK_INDX(pp); in page_ctr_sub_internal()
904 int root_mtype = PP_2_MTYPE(PP_GROUPLEADER(pp, r)); in page_ctr_sub_internal()
909 ASSERT(cand->pcc_color_free[PP_2_BIN_SZC(pp, r)] != 0); in page_ctr_sub_internal()
912 cand->pcc_color_free[PP_2_BIN_SZC(pp, r)]--; in page_ctr_sub_internal()
919 page_ctr_sub(int mnode, int mtype, page_t *pp, int flags) in page_ctr_sub() argument
921 int lckidx = PP_CTR_LOCK_INDX(pp); in page_ctr_sub()
925 page_ctr_sub_internal(mnode, mtype, pp, flags); in page_ctr_sub()
1317 chk_lpg(page_t *pp, uchar_t szc) in chk_lpg() argument
1319 spgcnt_t npgs = page_get_pagecnt(pp->p_szc); in chk_lpg()
1323 ASSERT(pp->p_szc == 0); in chk_lpg()
1324 ASSERT(pp->p_next == pp); in chk_lpg()
1325 ASSERT(pp->p_prev == pp); in chk_lpg()
1329 ASSERT(pp->p_vpnext == pp || pp->p_vpnext == NULL); in chk_lpg()
1330 ASSERT(pp->p_vpprev == pp || pp->p_vpprev == NULL); in chk_lpg()
1332 ASSERT(IS_P2ALIGNED(pp->p_pagenum, npgs)); in chk_lpg()
1333 ASSERT(pp->p_pagenum == (pp->p_next->p_pagenum - 1)); in chk_lpg()
1334 ASSERT(pp->p_prev->p_pagenum == (pp->p_pagenum + (npgs - 1))); in chk_lpg()
1335 ASSERT(pp->p_prev == (pp + (npgs - 1))); in chk_lpg()
1340 noreloc = PP_ISNORELOC(pp); in chk_lpg()
1343 ASSERT(pp->p_pagenum == pp->p_next->p_pagenum - 1); in chk_lpg()
1344 ASSERT(pp->p_next == (pp + 1)); in chk_lpg()
1346 ASSERT(pp->p_szc == szc); in chk_lpg()
1347 ASSERT(PP_ISFREE(pp)); in chk_lpg()
1348 ASSERT(PP_ISAGED(pp)); in chk_lpg()
1349 ASSERT(pp->p_vpnext == pp || pp->p_vpnext == NULL); in chk_lpg()
1350 ASSERT(pp->p_vpprev == pp || pp->p_vpprev == NULL); in chk_lpg()
1351 ASSERT(pp->p_vnode == NULL); in chk_lpg()
1352 ASSERT(PP_ISNORELOC(pp) == noreloc); in chk_lpg()
1354 pp = pp->p_next; in chk_lpg()
1384 page_list_add(page_t *pp, int flags) in page_list_add() argument
1391 ASSERT(PAGE_EXCL(pp) || (flags & PG_LIST_ISINIT)); in page_list_add()
1392 ASSERT(PP_ISFREE(pp)); in page_list_add()
1393 ASSERT(!hat_page_is_mapped(pp)); in page_list_add()
1394 ASSERT(hat_page_getshare(pp) == 0); in page_list_add()
1399 ASSERT(pp->p_szc == 0); in page_list_add()
1407 bin = PP_2_BIN(pp); in page_list_add()
1408 mnode = PP_2_MEM_NODE(pp); in page_list_add()
1409 mtype = PP_2_MTYPE(pp); in page_list_add()
1421 pp->p_next = *ppp; in page_list_add()
1422 pp->p_prev = (*ppp)->p_prev; in page_list_add()
1423 (*ppp)->p_prev = pp; in page_list_add()
1424 pp->p_prev->p_next = pp; in page_list_add()
1426 *ppp = pp; in page_list_add()
1428 page_ctr_add_internal(mnode, mtype, pp, flags); in page_list_add()
1435 ASSERT(PP_ISAGED(pp)); in page_list_add()
1440 ASSERT(pp->p_vnode); in page_list_add()
1441 ASSERT((pp->p_offset & PAGEOFFSET) == 0); in page_list_add()
1445 page_add(ppp, pp); in page_list_add()
1453 page_ctr_add(mnode, mtype, pp, flags); in page_list_add()
1459 if (PP_ISNORELOC(pp)) { in page_list_add()
1466 ASSERT(PAGE_EXCL(pp) || (flags & PG_LIST_ISINIT)); in page_list_add()
1477 page_list_noreloc_startup(page_t *pp) in page_list_noreloc_startup() argument
1489 if (pp->p_szc != 0) in page_list_noreloc_startup()
1490 page_boot_demote(pp); in page_list_noreloc_startup()
1495 bin = PP_2_BIN(pp); in page_list_noreloc_startup()
1496 mnode = PP_2_MEM_NODE(pp); in page_list_noreloc_startup()
1497 mtype = PP_2_MTYPE(pp); in page_list_noreloc_startup()
1499 ASSERT(pp->p_szc == 0); in page_list_noreloc_startup()
1501 if (PP_ISAGED(pp)) { in page_list_noreloc_startup()
1514 if (*ppp == pp) in page_list_noreloc_startup()
1515 *ppp = pp->p_next; /* go to next page */ in page_list_noreloc_startup()
1516 if (*ppp == pp) { in page_list_noreloc_startup()
1519 pp->p_prev->p_next = pp->p_next; in page_list_noreloc_startup()
1520 pp->p_next->p_prev = pp->p_prev; in page_list_noreloc_startup()
1526 page_ctr_sub_internal(mnode, mtype, pp, flags); in page_list_noreloc_startup()
1531 PP_SETNORELOC(pp); in page_list_noreloc_startup()
1533 mtype = PP_2_MTYPE(pp); in page_list_noreloc_startup()
1539 if (PP_ISAGED(pp)) { in page_list_noreloc_startup()
1549 *ppp = pp; in page_list_noreloc_startup()
1550 pp->p_next = pp->p_prev = pp; in page_list_noreloc_startup()
1552 pp->p_next = *ppp; in page_list_noreloc_startup()
1553 pp->p_prev = (*ppp)->p_prev; in page_list_noreloc_startup()
1554 (*ppp)->p_prev = pp; in page_list_noreloc_startup()
1555 pp->p_prev->p_next = pp; in page_list_noreloc_startup()
1561 page_ctr_add_internal(mnode, mtype, pp, flags); in page_list_noreloc_startup()
1572 page_list_noreloc_startup(page_t *pp) in page_list_noreloc_startup() argument
1579 page_list_add_pages(page_t *pp, int flags) in page_list_add_pages() argument
1589 CHK_LPG(pp, pp->p_szc); in page_list_add_pages()
1590 VM_STAT_ADD(vmm_vmstats.pladd_free[pp->p_szc]); in page_list_add_pages()
1592 bin = PP_2_BIN(pp); in page_list_add_pages()
1593 mnode = PP_2_MEM_NODE(pp); in page_list_add_pages()
1594 mtype = PP_2_MTYPE(pp); in page_list_add_pages()
1597 ASSERT(pp->p_szc == mmu_page_sizes - 1); in page_list_add_pages()
1598 page_vpadd(&PAGE_FREELISTS(mnode, pp->p_szc, bin, mtype), pp); in page_list_add_pages()
1599 ASSERT(!PP_ISNORELOC(pp)); in page_list_add_pages()
1600 PLCNT_INCR(pp, mnode, mtype, pp->p_szc, flags); in page_list_add_pages()
1603 ASSERT(pp->p_szc != 0 && pp->p_szc < mmu_page_sizes); in page_list_add_pages()
1608 page_vpadd(&PAGE_FREELISTS(mnode, pp->p_szc, bin, mtype), pp); in page_list_add_pages()
1609 page_ctr_add(mnode, mtype, pp, PG_FREE_LIST); in page_list_add_pages()
1612 pgcnt = page_get_pagecnt(pp->p_szc); in page_list_add_pages()
1614 if (PP_ISNORELOC(pp)) in page_list_add_pages()
1617 for (i = 0; i < pgcnt; i++, pp++) in page_list_add_pages()
1618 page_unlock_nocapture(pp); in page_list_add_pages()
1627 page_boot_demote(page_t *pp) in page_boot_demote() argument
1629 ASSERT(pp->p_szc != 0); in page_boot_demote()
1630 ASSERT(PP_ISFREE(pp)); in page_boot_demote()
1631 ASSERT(PP_ISAGED(pp)); in page_boot_demote()
1633 (void) page_demote(PP_2_MEM_NODE(pp), in page_boot_demote()
1634 PFN_BASE(pp->p_pagenum, pp->p_szc), 0, pp->p_szc, 0, PC_NO_COLOR, in page_boot_demote()
1637 ASSERT(PP_ISFREE(pp)); in page_boot_demote()
1638 ASSERT(PP_ISAGED(pp)); in page_boot_demote()
1639 ASSERT(pp->p_szc == 0); in page_boot_demote()
1649 page_list_sub(page_t *pp, int flags) in page_list_sub() argument
1657 ASSERT(PAGE_EXCL(pp)); in page_list_sub()
1658 ASSERT(PP_ISFREE(pp)); in page_list_sub()
1672 bin = PP_2_BIN(pp); in page_list_sub()
1673 mnode = PP_2_MEM_NODE(pp); in page_list_sub()
1676 if (PP_2_BIN(pp) != bin) { in page_list_sub()
1680 mtype = PP_2_MTYPE(pp); in page_list_sub()
1684 ASSERT(PP_ISAGED(pp)); in page_list_sub()
1685 ppp = &PAGE_FREELISTS(mnode, pp->p_szc, bin, mtype); in page_list_sub()
1688 ASSERT(!PP_ISAGED(pp)); in page_list_sub()
1699 if (pp->p_szc == 0) { in page_list_sub()
1700 page_sub(ppp, pp); in page_list_sub()
1705 page_ctr_sub(mnode, mtype, pp, flags); in page_list_sub()
1709 if (PP_ISNORELOC(pp)) { in page_list_sub()
1736 if (pp->p_szc != 0) { in page_list_sub()
1740 (void) page_demote(mnode, PFN_BASE(pp->p_pagenum, pp->p_szc), in page_list_sub()
1741 0, pp->p_szc, 0, PC_NO_COLOR, PC_FREE); in page_list_sub()
1743 ASSERT(PP_ISFREE(pp)); in page_list_sub()
1744 ASSERT(PP_ISAGED(pp)); in page_list_sub()
1745 ASSERT(pp->p_szc == 0); in page_list_sub()
1751 bin = PP_2_BIN(pp); in page_list_sub()
1752 mtype = PP_2_MTYPE(pp); in page_list_sub()
1753 ppp = &PAGE_FREELISTS(mnode, pp->p_szc, bin, mtype); in page_list_sub()
1755 page_sub(ppp, pp); in page_list_sub()
1756 page_ctr_sub(mnode, mtype, pp, flags); in page_list_sub()
1760 if (PP_ISNORELOC(pp)) { in page_list_sub()
1767 page_list_sub_pages(page_t *pp, uint_t szc) in page_list_sub_pages() argument
1773 ASSERT(PAGE_EXCL(pp)); in page_list_sub_pages()
1774 ASSERT(PP_ISFREE(pp)); in page_list_sub_pages()
1775 ASSERT(PP_ISAGED(pp)); in page_list_sub_pages()
1781 bin = PP_2_BIN(pp); in page_list_sub_pages()
1782 mnode = PP_2_MEM_NODE(pp); in page_list_sub_pages()
1785 if (PP_2_BIN(pp) != bin) { in page_list_sub_pages()
1796 if (pp->p_szc > szc) { in page_list_sub_pages()
1800 if (pp->p_szc > szc) { in page_list_sub_pages()
1803 PFN_BASE(pp->p_pagenum, pp->p_szc), 0, in page_list_sub_pages()
1804 pp->p_szc, szc, PC_NO_COLOR, PC_FREE); in page_list_sub_pages()
1806 bin = PP_2_BIN(pp); in page_list_sub_pages()
1808 ASSERT(PP_ISFREE(pp)); in page_list_sub_pages()
1809 ASSERT(PP_ISAGED(pp)); in page_list_sub_pages()
1810 ASSERT(pp->p_szc <= szc); in page_list_sub_pages()
1811 ASSERT(pp == PP_PAGEROOT(pp)); in page_list_sub_pages()
1813 VM_STAT_ADD(vmm_vmstats.plsub_free[pp->p_szc]); in page_list_sub_pages()
1815 mtype = PP_2_MTYPE(pp); in page_list_sub_pages()
1816 if (pp->p_szc != 0) { in page_list_sub_pages()
1817 page_vpsub(&PAGE_FREELISTS(mnode, pp->p_szc, bin, mtype), pp); in page_list_sub_pages()
1818 CHK_LPG(pp, pp->p_szc); in page_list_sub_pages()
1821 page_sub(&PAGE_FREELISTS(mnode, pp->p_szc, bin, mtype), pp); in page_list_sub_pages()
1823 page_ctr_sub(mnode, mtype, pp, PG_FREE_LIST); in page_list_sub_pages()
1832 if (PP_ISNORELOC(pp)) { in page_list_sub_pages()
1835 pgcnt = page_get_pagecnt(pp->p_szc); in page_list_sub_pages()
1847 mach_page_add(page_t **ppp, page_t *pp) in mach_page_add() argument
1850 pp->p_next = pp->p_prev = pp; in mach_page_add()
1852 pp->p_next = *ppp; in mach_page_add()
1853 pp->p_prev = (*ppp)->p_prev; in mach_page_add()
1854 (*ppp)->p_prev = pp; in mach_page_add()
1855 pp->p_prev->p_next = pp; in mach_page_add()
1857 *ppp = pp; in mach_page_add()
1867 mach_page_sub(page_t **ppp, page_t *pp) in mach_page_sub() argument
1869 ASSERT(pp != NULL && PP_ISFREE(pp)); in mach_page_sub()
1871 if (*ppp == NULL || pp == NULL) in mach_page_sub()
1874 if (*ppp == pp) in mach_page_sub()
1875 *ppp = pp->p_next; /* go to next page */ in mach_page_sub()
1877 if (*ppp == pp) in mach_page_sub()
1880 pp->p_prev->p_next = pp->p_next; in mach_page_sub()
1881 pp->p_next->p_prev = pp->p_prev; in mach_page_sub()
1883 pp->p_prev = pp->p_next = pp; /* make pp a list of one */ in mach_page_sub()
1890 page_promote_size(page_t *pp, uint_t cur_szc) in page_promote_size() argument
1898 pfn = page_pptonum(pp); in page_promote_size()
1968 page_t *pp, *pplist, *tpp, *start_pp; in page_promote() local
2005 for (pp = start_pp + new_npgs; --pp > start_pp; ) { in page_promote()
2006 if (noreloc != PP_ISNORELOC(pp)) { in page_promote()
2015 pp = start_pp; in page_promote()
2022 ASSERT(PP_ISFREE(pp)); in page_promote()
2023 bin = PP_2_BIN(pp); in page_promote()
2024 ASSERT(mnode == PP_2_MEM_NODE(pp)); in page_promote()
2025 mtype = PP_2_MTYPE(pp); in page_promote()
2026 if (PP_ISAGED(pp)) { in page_promote()
2031 if (pp->p_szc) { in page_promote()
2033 pp->p_szc, bin, mtype), pp); in page_promote()
2036 bin, mtype), pp); in page_promote()
2040 ASSERT(pp->p_szc == 0); in page_promote()
2049 if (!page_trylock(pp, SE_EXCL)) { in page_promote()
2063 index = PAGE_HASH_FUNC(pp->p_vnode, pp->p_offset); in page_promote()
2066 page_unlock_nocapture(pp); in page_promote()
2070 mach_page_sub(&PAGE_CACHELISTS(mnode, bin, mtype), pp); in page_promote()
2071 page_hashout(pp, phm); in page_promote()
2073 PP_SETAGED(pp); in page_promote()
2074 page_unlock_nocapture(pp); in page_promote()
2077 page_ctr_sub(mnode, mtype, pp, which_list); in page_promote()
2083 tmpnpgs = npgs = page_get_pagecnt(pp->p_szc); in page_promote()
2085 tpp = pp; in page_promote()
2090 page_list_concat(&pplist, &pp); in page_promote()
2091 pp += tmpnpgs; in page_promote()
2125 pp = pplist; in page_promote()
2126 mach_page_sub(&pplist, pp); in page_promote()
2127 pp->p_szc = 0; in page_promote()
2128 bin = PP_2_BIN(pp); in page_promote()
2129 mtype = PP_2_MTYPE(pp); in page_promote()
2130 mach_page_add(&PAGE_FREELISTS(mnode, 0, bin, mtype), pp); in page_promote()
2131 page_ctr_add(mnode, mtype, pp, PG_FREE_LIST); in page_promote()
2153 page_t *pp, *pplist, *npplist; in page_demote() local
2182 pp = pplist; in page_demote()
2184 ASSERT(pp->p_szc == cur_szc); in page_demote()
2190 mach_page_sub(&pplist, pp); in page_demote()
2191 ASSERT(pp->p_szc == cur_szc); in page_demote()
2193 ASSERT(mnode == PP_2_MEM_NODE(pp)); in page_demote()
2194 pp->p_szc = new_szc; in page_demote()
2195 bin = PP_2_BIN(pp); in page_demote()
2198 pp->p_pagenum < pfnmax) && in page_demote()
2199 page_trylock_cons(pp, SE_EXCL)) { in page_demote()
2200 ret_pp = pp; in page_demote()
2202 mtype = PP_2_MTYPE(pp); in page_demote()
2204 mtype), pp); in page_demote()
2205 page_ctr_add(mnode, mtype, pp, PG_FREE_LIST); in page_demote()
2216 pp = pplist; in page_demote()
2219 ASSERT(pp->p_szc == cur_szc); in page_demote()
2224 if (pfnmax == 0 || pp->p_pagenum < pfnmax) { in page_demote()
2227 pp->p_szc = new_szc; in page_demote()
2228 pp = pp->p_next; in page_demote()
2232 (pfnmax == 0 || pp->p_pagenum < pfnmax)) { in page_demote()
2233 try_to_return_this_page = pp; in page_demote()
2248 mtype = PP_2_MTYPE(pp); in page_demote()
2542 page_t *pp, *firstpp; in page_freelist_split() local
2568 firstpp = pp = PAGE_FREELISTS(mnode, nszc, bin, mtype); in page_freelist_split()
2574 if (pp && in page_freelist_split()
2575 ((pfnhi != PFNNULL && pp->p_pagenum >= pfnhi) || in page_freelist_split()
2576 (pfnlo != PFNNULL && pp->p_pagenum < pfnlo))) { in page_freelist_split()
2578 pp = pp->p_vpnext; in page_freelist_split()
2579 if (pp == firstpp) { in page_freelist_split()
2580 pp = NULL; in page_freelist_split()
2584 pp->p_pagenum >= pfnhi) || in page_freelist_split()
2586 pp->p_pagenum < pfnlo)); in page_freelist_split()
2588 if (pfnhi != PFNNULL && pp != NULL) in page_freelist_split()
2589 ASSERT(pp->p_pagenum < pfnhi); in page_freelist_split()
2591 if (pfnlo != PFNNULL && pp != NULL) in page_freelist_split()
2592 ASSERT(pp->p_pagenum >= pfnlo); in page_freelist_split()
2594 if (pp) { in page_freelist_split()
2598 ASSERT(pp->p_szc == nszc); in page_freelist_split()
2600 ret_pp = page_demote(mnode, pp->p_pagenum, in page_freelist_split()
2601 pfnhi, pp->p_szc, szc, ccolor, PC_ALLOC); in page_freelist_split()
2655 page_trylock_cons(page_t *pp, se_t se) in page_trylock_cons() argument
2657 page_t *tpp, *first_pp = pp; in page_trylock_cons()
2662 if (!page_trylock(pp, se)) { in page_trylock_cons()
2669 if (pp->p_szc == 0) { in page_trylock_cons()
2676 tpp = pp->p_next; in page_trylock_cons()
2677 while (tpp != pp) { in page_trylock_cons()
2876 page_t *pp, *first_pp; in page_get_mnode_freelist() local
2909 pp = PAGE_FREELISTS(mnode, szc, bin, mtype); in page_get_mnode_freelist()
2910 if (pp == NULL) in page_get_mnode_freelist()
2918 ASSERT(PP_ISFREE(pp)); in page_get_mnode_freelist()
2919 ASSERT(PP_ISAGED(pp)); in page_get_mnode_freelist()
2920 ASSERT(pp->p_vnode == NULL); in page_get_mnode_freelist()
2921 ASSERT(pp->p_hash == NULL); in page_get_mnode_freelist()
2922 ASSERT(pp->p_offset == (u_offset_t)-1); in page_get_mnode_freelist()
2923 ASSERT(pp->p_szc == szc); in page_get_mnode_freelist()
2924 ASSERT(PFN_2_MEM_NODE(pp->p_pagenum) == mnode); in page_get_mnode_freelist()
2938 first_pp = pp; in page_get_mnode_freelist()
2939 while (IS_DUMP_PAGE(pp) || !page_trylock_cons(pp, in page_get_mnode_freelist()
2942 pp = pp->p_next; in page_get_mnode_freelist()
2944 pp = pp->p_vpnext; in page_get_mnode_freelist()
2947 ASSERT(PP_ISFREE(pp)); in page_get_mnode_freelist()
2948 ASSERT(PP_ISAGED(pp)); in page_get_mnode_freelist()
2949 ASSERT(pp->p_vnode == NULL); in page_get_mnode_freelist()
2950 ASSERT(pp->p_hash == NULL); in page_get_mnode_freelist()
2951 ASSERT(pp->p_offset == (u_offset_t)-1); in page_get_mnode_freelist()
2952 ASSERT(pp->p_szc == szc); in page_get_mnode_freelist()
2953 ASSERT(PFN_2_MEM_NODE(pp->p_pagenum) == mnode); in page_get_mnode_freelist()
2955 if (pp == first_pp) in page_get_mnode_freelist()
2959 ASSERT(pp != NULL); in page_get_mnode_freelist()
2960 ASSERT(mtype == PP_2_MTYPE(pp)); in page_get_mnode_freelist()
2961 ASSERT(pp->p_szc == szc); in page_get_mnode_freelist()
2964 szc, bin, mtype), pp); in page_get_mnode_freelist()
2967 szc, bin, mtype), pp); in page_get_mnode_freelist()
2968 CHK_LPG(pp, szc); in page_get_mnode_freelist()
2970 page_ctr_sub(mnode, mtype, pp, PG_FREE_LIST); in page_get_mnode_freelist()
2972 if ((PP_ISFREE(pp) == 0) || (PP_ISAGED(pp) == 0)) in page_get_mnode_freelist()
2973 panic("free page is not. pp %p", (void *)pp); in page_get_mnode_freelist()
2977 ASSERT(!kcage_on || PP_ISNORELOC(pp) || in page_get_mnode_freelist()
2980 if (PP_ISNORELOC(pp)) in page_get_mnode_freelist()
2984 return (pp); in page_get_mnode_freelist()
3016 (pp = page_freelist_split(szc, bin, mnode, in page_get_mnode_freelist()
3018 return (pp); in page_get_mnode_freelist()
3020 if (szc > 0 && (pp = page_freelist_coalesce(mnode, szc, in page_get_mnode_freelist()
3022 return (pp); in page_get_mnode_freelist()
3045 page_freecnt(int mnode, page_t *pp, uchar_t szc) in page_freecnt() argument
3055 ASSERT((pp->p_pagenum & (PNUM_SIZE(szc) - 1)) == 0); in page_freecnt()
3060 idx = PNUM_TO_IDX(mnode, r, pp->p_pagenum); in page_freecnt()
3072 idx = PNUM_TO_IDX(mnode, r, pp->p_pagenum); in page_freecnt()
3102 page_t *pp; in page_trylock_contig_pages() local
3124 pp = &spp[i]; in page_trylock_contig_pages()
3125 if (!page_trylock(pp, SE_EXCL)) { in page_trylock_contig_pages()
3128 pp = &spp[i]; in page_trylock_contig_pages()
3129 ASSERT(PAGE_EXCL(pp)); in page_trylock_contig_pages()
3130 page_unlock_nocapture(pp); in page_trylock_contig_pages()
3135 if ((pp->p_szc > szc || (szc && pp->p_szc == szc)) && in page_trylock_contig_pages()
3136 !PP_ISFREE(pp)) { in page_trylock_contig_pages()
3139 page_unlock_nocapture(pp); in page_trylock_contig_pages()
3148 if (PP_ISNORELOC(pp) || in page_trylock_contig_pages()
3149 pp->p_lckcnt != 0 || pp->p_cowcnt != 0) { in page_trylock_contig_pages()
3152 pp = &spp[i]; in page_trylock_contig_pages()
3153 ASSERT(PAGE_EXCL(pp)); in page_trylock_contig_pages()
3154 page_unlock_nocapture(pp); in page_trylock_contig_pages()
3170 page_claim_contig_pages(page_t *pp, uchar_t szc, int flags) in page_claim_contig_pages() argument
3177 ASSERT(pp != NULL); in page_claim_contig_pages()
3181 ASSERT(PAGE_EXCL(pp)); in page_claim_contig_pages()
3182 ASSERT(!PP_ISNORELOC(pp)); in page_claim_contig_pages()
3183 if (PP_ISFREE(pp)) { in page_claim_contig_pages()
3191 if (PP_ISAGED(pp)) { in page_claim_contig_pages()
3192 page_list_sub_pages(pp, szc); in page_claim_contig_pages()
3193 if (pp->p_szc == szc) { in page_claim_contig_pages()
3194 return (pp); in page_claim_contig_pages()
3196 ASSERT(pp->p_szc < szc); in page_claim_contig_pages()
3197 npgs = page_get_pagecnt(pp->p_szc); in page_claim_contig_pages()
3198 hpp = pp; in page_claim_contig_pages()
3199 for (i = 0; i < npgs; i++, pp++) { in page_claim_contig_pages()
3200 pp->p_szc = szc; in page_claim_contig_pages()
3206 ASSERT(!PP_ISAGED(pp)); in page_claim_contig_pages()
3207 ASSERT(pp->p_szc == 0); in page_claim_contig_pages()
3208 page_list_sub(pp, PG_CACHE_LIST); in page_claim_contig_pages()
3209 page_hashout(pp, NULL); in page_claim_contig_pages()
3210 PP_SETAGED(pp); in page_claim_contig_pages()
3211 pp->p_szc = szc; in page_claim_contig_pages()
3212 page_list_concat(&pplist, &pp); in page_claim_contig_pages()
3213 pp++; in page_claim_contig_pages()
3217 npgs = page_get_pagecnt(pp->p_szc); in page_claim_contig_pages()
3229 if (pp->p_szc < szc || (szc == 0 && (flags & PGI_PGCPSZC0))) { in page_claim_contig_pages()
3230 replpp = page_get_replacement_page(pp, NULL, 0); in page_claim_contig_pages()
3232 npgs = page_get_pagecnt(pp->p_szc); in page_claim_contig_pages()
3234 targpp = pp; in page_claim_contig_pages()
3248 ASSERT(PAGE_EXCL(pp)); in page_claim_contig_pages()
3249 page_unlock_nocapture(pp); in page_claim_contig_pages()
3250 pp++; in page_claim_contig_pages()
3256 pp = pplist; in page_claim_contig_pages()
3257 page_sub(&pplist, pp); in page_claim_contig_pages()
3258 ASSERT(PAGE_EXCL(pp)); in page_claim_contig_pages()
3259 ASSERT(pp->p_szc == szc); in page_claim_contig_pages()
3260 ASSERT(PP_ISFREE(pp)); in page_claim_contig_pages()
3261 ASSERT(PP_ISAGED(pp)); in page_claim_contig_pages()
3262 pp->p_szc = 0; in page_claim_contig_pages()
3263 page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL); in page_claim_contig_pages()
3264 page_unlock_nocapture(pp); in page_claim_contig_pages()
3272 ASSERT(pp == targpp); in page_claim_contig_pages()
3275 ASSERT(hpp = pp); /* That's right, it's an assignment */ in page_claim_contig_pages()
3277 pp += npgs; in page_claim_contig_pages()
3381 page_t *pp, *randpp, *endpp; in page_geti_contig_pages() local
3530 pp = randpp; in page_geti_contig_pages()
3536 ASSERT(!(pp->p_pagenum & szcpgmask)); in page_geti_contig_pages()
3537 ASSERT(((PP_2_BIN(pp) ^ bin) & ceq_mask) == 0); in page_geti_contig_pages()
3539 if (page_trylock_contig_pages(mnode, pp, szc, flags)) { in page_geti_contig_pages()
3541 if (page_claim_contig_pages(pp, szc, flags)) { in page_geti_contig_pages()
3543 return (pp); in page_geti_contig_pages()
3548 pp += skip; in page_geti_contig_pages()
3550 pfn_t pfn = pp->p_pagenum; in page_geti_contig_pages()
3555 pp = endpp; in page_geti_contig_pages()
3557 pp = mseg->pages + in page_geti_contig_pages()
3561 if (pp >= endpp) { in page_geti_contig_pages()
3564 pp = mseg->pages + (lo - mseg->pages_base); in page_geti_contig_pages()
3565 ASSERT(pp->p_pagenum == lo); in page_geti_contig_pages()
3566 ASSERT(pp + szcpgcnt <= endpp); in page_geti_contig_pages()
3568 } while (pp != randpp); in page_geti_contig_pages()
3595 page_t *pp; in page_get_contig_pages() local
3628 pp = page_geti_contig_pages(mnode, bin, szc, flags, in page_get_contig_pages()
3631 if (pp != NULL) { in page_get_contig_pages()
3638 return (pp); in page_get_contig_pages()
3690 page_t *pp = NULL; in page_get_freelist() local
3750 pp = page_get_func(mnode, bin, mtype, szc, flags); in page_get_freelist()
3751 if (pp != NULL) { in page_get_freelist()
3758 return (pp); in page_get_freelist()
3761 ASSERT(pp == NULL); in page_get_freelist()
3783 pp = page_get_func(mnode, bin, mtype, szc, flags); in page_get_freelist()
3784 if (pp != NULL) { in page_get_freelist()
3791 return (pp); in page_get_freelist()
3794 ASSERT(pp == NULL); in page_get_freelist()
3840 page_t *pp; in page_get_cachelist() local
3887 pp = page_get_mnode_cachelist(bin, flags, mnode, mtype); in page_get_cachelist()
3888 if (pp != NULL) { in page_get_cachelist()
3895 return (pp); in page_get_cachelist()
3908 pp = page_get_mnode_freelist(mnode, bin, mtype, in page_get_cachelist()
3910 if (pp != NULL) { in page_get_cachelist()
3917 return (pp); in page_get_cachelist()
3919 pp = page_get_mnode_cachelist(bin, flags, mnode, mtype); in page_get_cachelist()
3920 if (pp != NULL) { in page_get_cachelist()
3927 return (pp); in page_get_cachelist()
3939 page_t *pp, *first_pp; in page_get_mnode_cachelist() local
3973 pp = PAGE_CACHELISTS(mnode, bin, mtype); in page_get_mnode_cachelist()
3974 if (pp == NULL) in page_get_mnode_cachelist()
3977 first_pp = pp; in page_get_mnode_cachelist()
3978 ASSERT(pp->p_vnode); in page_get_mnode_cachelist()
3979 ASSERT(PP_ISAGED(pp) == 0); in page_get_mnode_cachelist()
3980 ASSERT(pp->p_szc == 0); in page_get_mnode_cachelist()
3981 ASSERT(PFN_2_MEM_NODE(pp->p_pagenum) == mnode); in page_get_mnode_cachelist()
3982 while (IS_DUMP_PAGE(pp) || !page_trylock(pp, SE_EXCL)) { in page_get_mnode_cachelist()
3983 pp = pp->p_next; in page_get_mnode_cachelist()
3984 ASSERT(pp->p_szc == 0); in page_get_mnode_cachelist()
3985 if (pp == first_pp) { in page_get_mnode_cachelist()
3996 pp = NULL; in page_get_mnode_cachelist()
3999 ASSERT(pp->p_vnode); in page_get_mnode_cachelist()
4000 ASSERT(PP_ISFREE(pp)); in page_get_mnode_cachelist()
4001 ASSERT(PP_ISAGED(pp) == 0); in page_get_mnode_cachelist()
4002 ASSERT(PFN_2_MEM_NODE(pp->p_pagenum) == in page_get_mnode_cachelist()
4006 if (pp) { in page_get_mnode_cachelist()
4012 ASSERT(mtype == PP_2_MTYPE(pp)); in page_get_mnode_cachelist()
4014 page_sub(ppp, pp); in page_get_mnode_cachelist()
4020 page_ctr_sub(mnode, mtype, pp, PG_CACHE_LIST); in page_get_mnode_cachelist()
4022 ASSERT(pp->p_vnode); in page_get_mnode_cachelist()
4023 ASSERT(PP_ISAGED(pp) == 0); in page_get_mnode_cachelist()
4027 PP_ISNORELOC(pp)); in page_get_mnode_cachelist()
4028 if (PP_ISNORELOC(pp)) { in page_get_mnode_cachelist()
4033 return (pp); in page_get_mnode_cachelist()
4092 page_t *pp, *pplist; in page_get_replacement_page() local
4333 pp = pplist; in page_get_replacement_page()
4334 page_sub(&pplist, pp); in page_get_replacement_page()
4335 PP_CLRFREE(pp); in page_get_replacement_page()
4336 PP_CLRAGED(pp); in page_get_replacement_page()
4337 page_list_concat(&pl, &pp); in page_get_replacement_page()
4366 page_demote_free_pages(page_t *pp) in page_demote_free_pages() argument
4371 ASSERT(pp != NULL); in page_demote_free_pages()
4372 ASSERT(PAGE_LOCKED(pp)); in page_demote_free_pages()
4373 ASSERT(PP_ISFREE(pp)); in page_demote_free_pages()
4374 ASSERT(pp->p_szc != 0 && pp->p_szc < mmu_page_sizes); in page_demote_free_pages()
4376 mnode = PP_2_MEM_NODE(pp); in page_demote_free_pages()
4378 if (pp->p_szc != 0) { in page_demote_free_pages()
4379 (void) page_demote(mnode, PFN_BASE(pp->p_pagenum, in page_demote_free_pages()
4380 pp->p_szc), 0, pp->p_szc, 0, PC_NO_COLOR, PC_FREE); in page_demote_free_pages()
4383 ASSERT(pp->p_szc == 0); in page_demote_free_pages()