Lines Matching refs:pp
574 page_t *pp, in add_physmem() argument
584 "add_physmem:pp %p num %lu", pp, num); in add_physmem()
602 for (; num; pp++, pnum++, num--) { in add_physmem()
608 add_physmem_cb(pp, pnum); in add_physmem()
610 pp->p_lckcnt = 0; in add_physmem()
611 pp->p_cowcnt = 0; in add_physmem()
612 pp->p_slckcnt = 0; in add_physmem()
618 pp->p_selock = 0; in add_physmem()
623 page_iolock_init(pp); in add_physmem()
628 PP_SETFREE(pp); in add_physmem()
629 page_clr_all_props(pp); in add_physmem()
630 PP_SETAGED(pp); in add_physmem()
631 pp->p_offset = (u_offset_t)-1; in add_physmem()
632 pp->p_next = pp; in add_physmem()
633 pp->p_prev = pp; in add_physmem()
639 pp->p_szc = 0; in add_physmem()
640 page_free_at_startup(pp); in add_physmem()
655 pp->p_szc = 0; in add_physmem()
656 page_free_at_startup(pp); in add_physmem()
663 pp->p_szc = szc; in add_physmem()
664 page_list_concat(&root, &pp); in add_physmem()
691 pp->p_szc = 0; in add_physmem()
692 page_free_at_startup(pp); in add_physmem()
699 pp->p_szc = szc; in add_physmem()
701 root = pp; in add_physmem()
742 page_t *pp; in page_lookup_create() local
762 pp = page_hash_search(index, vp, off); in page_lookup_create()
763 if (pp != NULL) { in page_lookup_create()
769 if (!page_try_reclaim_lock(pp, se, es)) { in page_lookup_create()
785 if (!page_lock_es(pp, se, phm, P_RECLAIM, es)) { in page_lookup_create()
806 if (((volatile struct vnode *)(pp->p_vnode) != vp) || in page_lookup_create()
807 ((volatile u_offset_t)(pp->p_offset) != off)) { in page_lookup_create()
811 (void *)pp); in page_lookup_create()
814 page_unlock(pp); in page_lookup_create()
834 if ((!hash_locked) && (PP_ISFREE(pp))) { in page_lookup_create()
835 ASSERT(PP_ISAGED(pp) == 0); in page_lookup_create()
843 if (!page_reclaim(pp, NULL)) { in page_lookup_create()
855 page_downgrade(pp); in page_lookup_create()
863 if (newpp != NULL && pp->p_szc < newpp->p_szc && in page_lookup_create()
864 PAGE_EXCL(pp) && nrelocp != NULL) { in page_lookup_create()
866 (void) page_relocate(&pp, &newpp, 1, 1, nrelocp, in page_lookup_create()
873 pp = newpp; in page_lookup_create()
877 page_downgrade(pp); in page_lookup_create()
882 if (PAGE_EXCL(pp) && se == SE_SHARED) { in page_lookup_create()
883 page_downgrade(pp); in page_lookup_create()
885 VM_STAT_COND_ADD(pp->p_szc < newpp->p_szc, in page_lookup_create()
887 VM_STAT_COND_ADD(pp->p_szc == newpp->p_szc, in page_lookup_create()
889 VM_STAT_COND_ADD(pp->p_szc > newpp->p_szc, in page_lookup_create()
891 } else if (newpp != NULL && PAGE_EXCL(pp)) { in page_lookup_create()
928 pp = newpp; in page_lookup_create()
935 ASSERT(pp ? PAGE_LOCKED_SE(pp, se) : 1); in page_lookup_create()
937 ASSERT(pp ? ((PP_ISFREE(pp) == 0) && (PP_ISAGED(pp) == 0)) : 1); in page_lookup_create()
939 return (pp); in page_lookup_create()
951 page_t *pp; in page_lookup_nowait() local
960 pp = page_hash_search(index, vp, off); in page_lookup_nowait()
962 if (pp == NULL) { in page_lookup_nowait()
968 pp = page_hash_search(index, vp, off); in page_lookup_nowait()
971 if (pp == NULL || PP_ISFREE(pp)) { in page_lookup_nowait()
973 pp = NULL; in page_lookup_nowait()
975 if (!page_trylock(pp, se)) { in page_lookup_nowait()
977 pp = NULL; in page_lookup_nowait()
983 if (((volatile struct vnode *)(pp->p_vnode) != vp) || in page_lookup_nowait()
984 ((u_offset_t)(pp->p_offset) != off)) { in page_lookup_nowait()
988 (void *)pp); in page_lookup_nowait()
991 page_unlock(pp); in page_lookup_nowait()
994 if (PP_ISFREE(pp)) { in page_lookup_nowait()
996 page_unlock(pp); in page_lookup_nowait()
997 pp = NULL; in page_lookup_nowait()
1006 ASSERT(pp ? PAGE_LOCKED_SE(pp, se) : 1); in page_lookup_nowait()
1008 return (pp); in page_lookup_nowait()
1019 page_t *pp; in page_find() local
1030 pp = page_hash_search(index, vp, off); in page_find()
1033 ASSERT(pp == NULL || PAGE_LOCKED(pp) || panicstr); in page_find()
1034 return (pp); in page_find()
1083 page_t *pp; in page_exists_physcontig() local
1102 pp = page_hash_search(index, vp, off); in page_exists_physcontig()
1107 if (pp == NULL) { in page_exists_physcontig()
1113 rootpp = pp; in page_exists_physcontig()
1116 if ((pszc = pp->p_szc) >= szc && ppa != NULL) { in page_exists_physcontig()
1118 if (!page_trylock(pp, SE_SHARED)) { in page_exists_physcontig()
1125 if (pp->p_szc != pszc || pp->p_vnode != vp || in page_exists_physcontig()
1126 pp->p_offset != off || pp->p_pagenum != pfn) { in page_exists_physcontig()
1128 page_unlock(pp); in page_exists_physcontig()
1136 ASSERT(!PP_ISFREE(pp)); in page_exists_physcontig()
1138 page_unlock(pp); in page_exists_physcontig()
1141 ppa[0] = pp; in page_exists_physcontig()
1142 pp++; in page_exists_physcontig()
1145 for (i = 1; i < pages; i++, pp++, off += PAGESIZE, pfn++) { in page_exists_physcontig()
1146 if (!page_trylock(pp, SE_SHARED)) { in page_exists_physcontig()
1148 pp--; in page_exists_physcontig()
1150 page_unlock(pp); in page_exists_physcontig()
1151 pp--; in page_exists_physcontig()
1156 if (pp->p_szc != pszc) { in page_exists_physcontig()
1158 page_unlock(pp); in page_exists_physcontig()
1159 pp--; in page_exists_physcontig()
1161 page_unlock(pp); in page_exists_physcontig()
1162 pp--; in page_exists_physcontig()
1174 ASSERT(!PP_ISFREE(pp)); in page_exists_physcontig()
1175 if (pp->p_vnode != vp || in page_exists_physcontig()
1176 pp->p_offset != off) { in page_exists_physcontig()
1180 ppa[i] = pp; in page_exists_physcontig()
1181 ASSERT(pp->p_pagenum == pfn); in page_exists_physcontig()
1212 for (i = 0; i < pages; i++, pp++, off += PAGESIZE, pfn++) { in page_exists_physcontig()
1213 if (!page_trylock(pp, SE_EXCL)) { in page_exists_physcontig()
1220 if (pp->p_pagenum != pfn) { in page_exists_physcontig()
1221 page_unlock(pp); in page_exists_physcontig()
1224 if (pp->p_vnode != vp || in page_exists_physcontig()
1225 pp->p_offset != off) { in page_exists_physcontig()
1227 page_unlock(pp); in page_exists_physcontig()
1230 if (pp->p_szc >= szc) { in page_exists_physcontig()
1232 page_unlock(pp); in page_exists_physcontig()
1240 --pp; in page_exists_physcontig()
1242 page_unlock(pp); in page_exists_physcontig()
1243 --pp; in page_exists_physcontig()
1248 pp = rootpp; in page_exists_physcontig()
1249 for (i = 0; i < pages; i++, pp++) { in page_exists_physcontig()
1250 if (PP_ISFREE(pp)) { in page_exists_physcontig()
1252 ASSERT(!PP_ISAGED(pp)); in page_exists_physcontig()
1253 ASSERT(pp->p_szc == 0); in page_exists_physcontig()
1254 if (!page_reclaim(pp, NULL)) { in page_exists_physcontig()
1258 ASSERT(pp->p_szc < szc); in page_exists_physcontig()
1260 (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD); in page_exists_physcontig()
1270 pp = rootpp; in page_exists_physcontig()
1271 for (j = 0; j < pages; j++, pp++) { in page_exists_physcontig()
1273 page_unlock(pp); in page_exists_physcontig()
1280 pp = rootpp; in page_exists_physcontig()
1281 for (i = 0; i < pages; i++, pp++, off += PAGESIZE) { in page_exists_physcontig()
1282 ASSERT(PAGE_EXCL(pp)); in page_exists_physcontig()
1283 ASSERT(!PP_ISFREE(pp)); in page_exists_physcontig()
1284 ASSERT(!hat_page_is_mapped(pp)); in page_exists_physcontig()
1285 ASSERT(pp->p_vnode == vp); in page_exists_physcontig()
1286 ASSERT(pp->p_offset == off); in page_exists_physcontig()
1287 pp->p_szc = szc; in page_exists_physcontig()
1289 pp = rootpp; in page_exists_physcontig()
1290 for (i = 0; i < pages; i++, pp++) { in page_exists_physcontig()
1292 page_unlock(pp); in page_exists_physcontig()
1294 ppa[i] = pp; in page_exists_physcontig()
1316 page_t *pp; in page_exists_forreal() local
1329 pp = page_hash_search(index, vp, off); in page_exists_forreal()
1330 if (pp != NULL) { in page_exists_forreal()
1331 *szc = pp->p_szc; in page_exists_forreal()
1777 page_t *pp; in page_create_get_something() local
1869 pp = page_get_freelist(vp, off, seg, vaddr, PAGESIZE, in page_create_get_something()
1871 if (pp == NULL) { in page_create_get_something()
1872 pp = page_get_cachelist(vp, off, seg, vaddr, in page_create_get_something()
1875 if (pp == NULL) { in page_create_get_something()
1934 return (pp); in page_create_get_something()
2010 page_t *pplist = NULL, *pp; in page_alloc_pages() local
2058 pp = page_get_freelist(vp, 0, seg, addr, pgsz, in page_alloc_pages()
2060 if (pp == NULL) { in page_alloc_pages()
2061 pp = page_get_freelist(vp, 0, seg, addr, pgsz, in page_alloc_pages()
2065 pp = page_get_freelist(vp, 0, seg, addr, pgsz, in page_alloc_pages()
2068 if (pp != NULL) { in page_alloc_pages()
2070 page_list_concat(&pplist, &pp); in page_alloc_pages()
2097 pp = pplist; in page_alloc_pages()
2105 ASSERT(PP_ISFREE(pp)); in page_alloc_pages()
2106 ASSERT(PP_ISAGED(pp)); in page_alloc_pages()
2110 PP_CLRFREE(pp); in page_alloc_pages()
2111 PP_CLRAGED(pp); in page_alloc_pages()
2112 page_sub(&pplist, pp); in page_alloc_pages()
2113 *ppa++ = pp; in page_alloc_pages()
2117 ASSERT(pp->p_szc != 0); in page_alloc_pages()
2118 curnpgs = page_get_pagecnt(pp->p_szc); in page_alloc_pages()
2119 page_list_break(&pp, &pplist, curnpgs); in page_alloc_pages()
2120 page_list_add_pages(pp, 0); in page_alloc_pages()
2125 pp = pplist; in page_alloc_pages()
2128 PP_CLRFREE(pp); in page_alloc_pages()
2129 PP_CLRAGED(pp); in page_alloc_pages()
2130 pp = pp->p_next; in page_alloc_pages()
2151 page_t *pp; in page_create_va_large() local
2252 pp = rootpp; in page_create_va_large()
2254 ASSERT(PAGE_EXCL(pp)); in page_create_va_large()
2255 ASSERT(pp->p_vnode == NULL); in page_create_va_large()
2256 ASSERT(!hat_page_is_mapped(pp)); in page_create_va_large()
2257 PP_CLRFREE(pp); in page_create_va_large()
2258 PP_CLRAGED(pp); in page_create_va_large()
2259 if (!page_hashin(pp, vp, off, NULL)) in page_create_va_large()
2261 (void *)pp); in page_create_va_large()
2262 page_io_lock(pp); in page_create_va_large()
2264 pp = pp->p_next; in page_create_va_large()
2390 page_t *pp; in page_create_va() local
2459 pp = page_hash_search(index, vp, off); in page_create_va()
2460 if (pp == NULL) { in page_create_va()
2462 pp = npp; in page_create_va()
2464 if (!page_hashin(pp, vp, off, phm)) { in page_create_va()
2477 (void *)pp, (void *)vp, off, (void *)phm); in page_create_va()
2495 page_set_props(pp, P_REF); in page_create_va()
2508 pp = plist; in page_create_va()
2509 page_sub(&plist, pp); in page_create_va()
2510 page_io_unlock(pp); in page_create_va()
2512 ASSERT(pp->p_szc == 0); in page_create_va()
2514 VN_DISPOSE(pp, B_INVAL, 0, kcred); in page_create_va()
2520 if (!page_lock(pp, SE_EXCL, phm, P_NO_RECLAIM)) { in page_create_va()
2533 if (PP_ISFREE(pp)) { in page_create_va()
2534 ASSERT(PP_ISAGED(pp) == 0); in page_create_va()
2536 page_list_sub(pp, PG_CACHE_LIST); in page_create_va()
2537 PP_CLRFREE(pp); in page_create_va()
2547 page_io_lock(pp); in page_create_va()
2548 page_add(&plist, pp); in page_create_va()
2640 page_free(page_t *pp, int dontneed) in page_free() argument
2645 ASSERT((PAGE_EXCL(pp) && in page_free()
2646 !page_iolock_assert(pp)) || panicstr); in page_free()
2648 if (PP_ISFREE(pp)) { in page_free()
2649 panic("page_free: page %p is free", (void *)pp); in page_free()
2652 if (pp->p_szc != 0) { in page_free()
2653 if (pp->p_vnode == NULL || IS_SWAPFSVP(pp->p_vnode) || in page_free()
2654 PP_ISKAS(pp)) { in page_free()
2656 "or no vnode large page %p", (void *)pp); in page_free()
2658 page_demote_vp_pages(pp); in page_free()
2659 ASSERT(pp->p_szc == 0); in page_free()
2666 if (hat_page_is_mapped(pp) || pp->p_lckcnt != 0 || pp->p_cowcnt != 0 || in page_free()
2667 pp->p_slckcnt != 0) { in page_free()
2669 "slckcnt = %d", (void *)pp, page_pptonum(pp), pp->p_lckcnt, in page_free()
2670 pp->p_cowcnt, pp->p_slckcnt); in page_free()
2674 ASSERT(!hat_page_getshare(pp)); in page_free()
2676 PP_SETFREE(pp); in page_free()
2677 ASSERT(pp->p_vnode == NULL || !IS_VMODSORT(pp->p_vnode) || in page_free()
2678 !hat_ismod(pp)); in page_free()
2679 page_clr_all_props(pp); in page_free()
2680 ASSERT(!hat_page_getshare(pp)); in page_free()
2688 if (pp->p_vnode == NULL) { in page_free()
2692 PP_SETAGED(pp); in page_free()
2693 pp->p_offset = (u_offset_t)-1; in page_free()
2694 page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL); in page_free()
2697 "page_free_free:pp %p", pp); in page_free()
2699 PP_CLRAGED(pp); in page_free()
2703 page_list_add(pp, PG_CACHE_LIST | PG_LIST_TAIL); in page_free()
2707 "page_free_cache_tail:pp %p", pp); in page_free()
2709 page_list_add(pp, PG_CACHE_LIST | PG_LIST_HEAD); in page_free()
2713 "page_free_cache_head:pp %p", pp); in page_free()
2716 page_unlock(pp); in page_free()
2762 page_free_at_startup(page_t *pp) in page_free_at_startup() argument
2767 page_list_add(pp, PG_FREE_LIST | PG_LIST_HEAD | PG_LIST_ISINIT); in page_free_at_startup()
2785 page_free_pages(page_t *pp) in page_free_pages() argument
2788 pgcnt_t pgcnt = page_get_pagecnt(pp->p_szc); in page_free_pages()
2790 uint_t szc = pp->p_szc; in page_free_pages()
2794 "page_free_free:pp %p", pp); in page_free_pages()
2796 ASSERT(pp->p_szc != 0 && pp->p_szc < page_num_pagesizes()); in page_free_pages()
2797 if ((page_pptonum(pp) & (pgcnt - 1)) != 0) { in page_free_pages()
2798 panic("page_free_pages: not root page %p", (void *)pp); in page_free_pages()
2802 for (i = 0, tpp = pp; i < pgcnt; i++, tpp++) { in page_free_pages()
2827 ASSERT(rootpp == pp); in page_free_pages()
2848 page_t *pp; in free_vp_pages() local
2865 pp = page_exists(vp, off); in free_vp_pages()
2872 if (pp == NULL || in free_vp_pages()
2873 PP_ISFREE(pp) || in free_vp_pages()
2874 page_share_cnt(pp) > 0 || in free_vp_pages()
2875 !page_trylock(pp, SE_EXCL)) in free_vp_pages()
2882 ASSERT(PAGE_LOCKED_SE(pp, SE_EXCL)); in free_vp_pages()
2883 if (pp->p_vnode != vp || pp->p_offset != off || PP_ISFREE(pp)) { in free_vp_pages()
2884 page_unlock(pp); in free_vp_pages()
2891 (void) page_release(pp, 1); in free_vp_pages()
2912 page_reclaim(page_t *pp, kmutex_t *lock) in page_reclaim() argument
2920 ASSERT(PAGE_EXCL(pp) && PP_ISFREE(pp)); in page_reclaim()
2993 page_unlock(pp); in page_reclaim()
3037 if (PP_ISAGED(pp)) { in page_reclaim()
3038 page_list_sub(pp, PG_FREE_LIST); in page_reclaim()
3040 "page_reclaim_free:pp %p", pp); in page_reclaim()
3042 page_list_sub(pp, PG_CACHE_LIST); in page_reclaim()
3044 "page_reclaim_cache:pp %p", pp); in page_reclaim()
3054 PP_CLRFREE(pp); in page_reclaim()
3055 PP_CLRAGED(pp); in page_reclaim()
3056 page_set_props(pp, P_REF); in page_reclaim()
3063 ASSERT(pp->p_szc == 0); in page_reclaim()
3074 page_destroy(page_t *pp, int dontfree) in page_destroy() argument
3076 ASSERT((PAGE_EXCL(pp) && in page_destroy()
3077 !page_iolock_assert(pp)) || panicstr); in page_destroy()
3078 ASSERT(pp->p_slckcnt == 0 || panicstr); in page_destroy()
3080 if (pp->p_szc != 0) { in page_destroy()
3081 if (pp->p_vnode == NULL || IS_SWAPFSVP(pp->p_vnode) || in page_destroy()
3082 PP_ISKAS(pp)) { in page_destroy()
3084 "large page %p", (void *)pp); in page_destroy()
3086 page_demote_vp_pages(pp); in page_destroy()
3087 ASSERT(pp->p_szc == 0); in page_destroy()
3090 TRACE_1(TR_FAC_VM, TR_PAGE_DESTROY, "page_destroy:pp %p", pp); in page_destroy()
3096 (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD); in page_destroy()
3097 page_hashout(pp, NULL); in page_destroy()
3106 if ((pp->p_lckcnt != 0) || (pp->p_cowcnt != 0)) { in page_destroy()
3108 if (pp->p_lckcnt != 0) { in page_destroy()
3111 pp->p_lckcnt = 0; in page_destroy()
3113 if (pp->p_cowcnt != 0) { in page_destroy()
3114 availrmem += pp->p_cowcnt; in page_destroy()
3115 pages_locked -= pp->p_cowcnt; in page_destroy()
3116 pp->p_cowcnt = 0; in page_destroy()
3123 page_free(pp, 0); in page_destroy()
3128 page_destroy_pages(page_t *pp) in page_destroy_pages() argument
3132 pgcnt_t pgcnt = page_get_pagecnt(pp->p_szc); in page_destroy_pages()
3134 uint_t szc = pp->p_szc; in page_destroy_pages()
3136 ASSERT(pp->p_szc != 0 && pp->p_szc < page_num_pagesizes()); in page_destroy_pages()
3140 TRACE_1(TR_FAC_VM, TR_PAGE_DESTROY, "page_destroy_pages:pp %p", pp); in page_destroy_pages()
3142 if ((page_pptonum(pp) & (pgcnt - 1)) != 0) { in page_destroy_pages()
3143 panic("page_destroy_pages: not root page %p", (void *)pp); in page_destroy_pages()
3147 for (i = 0, tpp = pp; i < pgcnt; i++, tpp++) { in page_destroy_pages()
3173 ASSERT(rootpp == pp); in page_destroy_pages()
3193 page_destroy_free(page_t *pp) in page_destroy_free() argument
3195 ASSERT(PAGE_EXCL(pp)); in page_destroy_free()
3196 ASSERT(PP_ISFREE(pp)); in page_destroy_free()
3197 ASSERT(pp->p_vnode); in page_destroy_free()
3198 ASSERT(hat_page_getattr(pp, P_MOD | P_REF | P_RO) == 0); in page_destroy_free()
3199 ASSERT(!hat_page_is_mapped(pp)); in page_destroy_free()
3200 ASSERT(PP_ISAGED(pp) == 0); in page_destroy_free()
3201 ASSERT(pp->p_szc == 0); in page_destroy_free()
3204 page_list_sub(pp, PG_CACHE_LIST); in page_destroy_free()
3206 page_hashout(pp, NULL); in page_destroy_free()
3207 ASSERT(pp->p_vnode == NULL); in page_destroy_free()
3208 ASSERT(pp->p_offset == (u_offset_t)-1); in page_destroy_free()
3209 ASSERT(pp->p_hash == NULL); in page_destroy_free()
3211 PP_SETAGED(pp); in page_destroy_free()
3212 page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL); in page_destroy_free()
3213 page_unlock(pp); in page_destroy_free()
3238 page_t *pp; in page_rename() local
3289 pp = page_hash_search(index, vp, off); in page_rename()
3290 if (pp != NULL) { in page_rename()
3301 if (!page_lock(pp, SE_EXCL, phm, P_RECLAIM)) { in page_rename()
3319 if (hat_page_is_mapped(pp)) { in page_rename()
3328 (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD); in page_rename()
3329 if (pp->p_szc != 0) { in page_rename()
3332 page_demote_vp_pages(pp); in page_rename()
3333 ASSERT(pp->p_szc == 0); in page_rename()
3336 } else if (pp->p_szc != 0) { in page_rename()
3340 page_demote_vp_pages(pp); in page_rename()
3341 ASSERT(pp->p_szc == 0); in page_rename()
3344 page_hashout(pp, phm); in page_rename()
3356 panic("page_rename: Can't hash in page: %p", (void *)pp); in page_rename()
3367 if (pp != NULL) { in page_rename()
3368 ASSERT(!hat_page_is_mapped(pp)); in page_rename()
3370 ASSERT(pp->p_szc == 0); in page_rename()
3377 olckcnt = pp->p_lckcnt; in page_rename()
3378 ocowcnt = pp->p_cowcnt; in page_rename()
3379 pp->p_lckcnt = pp->p_cowcnt = 0; in page_rename()
3386 VN_DISPOSE(pp, B_FREE, 0, kcred); in page_rename()
3408 page_do_hashin(page_t *pp, vnode_t *vp, u_offset_t offset) in page_do_hashin() argument
3414 ASSERT(PAGE_EXCL(pp)); in page_do_hashin()
3424 pp->p_vnode = vp; in page_do_hashin()
3425 pp->p_offset = offset; in page_do_hashin()
3431 PP_SETSWAP(pp); in page_do_hashin()
3442 pp->p_vnode = NULL; in page_do_hashin()
3443 pp->p_offset = (u_offset_t)(-1); in page_do_hashin()
3447 pp->p_hash = *listp; in page_do_hashin()
3448 *listp = pp; in page_do_hashin()
3453 if (vp->v_pages != NULL && IS_VMODSORT(vp) && hat_ismod(pp)) in page_do_hashin()
3458 page_vpadd(listp, pp); in page_do_hashin()
3470 page_hashin(page_t *pp, vnode_t *vp, u_offset_t offset, kmutex_t *hold) in page_hashin() argument
3477 ASSERT(pp->p_fsdata == 0 || panicstr); in page_hashin()
3481 pp, vp, offset); in page_hashin()
3495 rc = page_do_hashin(pp, vp, offset); in page_hashin()
3509 page_do_hashout(page_t *pp) in page_do_hashout() argument
3513 vnode_t *vp = pp->p_vnode; in page_do_hashout()
3521 hpp = &page_hash[PAGE_HASH_FUNC(vp, pp->p_offset)]; in page_do_hashout()
3525 if (hp == pp) in page_do_hashout()
3533 *hpp = pp->p_hash; in page_do_hashout()
3539 page_vpsub(&vp->v_pages, pp); in page_do_hashout()
3541 pp->p_hash = NULL; in page_do_hashout()
3542 page_clr_all_props(pp); in page_do_hashout()
3543 PP_CLRSWAP(pp); in page_do_hashout()
3544 pp->p_vnode = NULL; in page_do_hashout()
3545 pp->p_offset = (u_offset_t)-1; in page_do_hashout()
3546 pp->p_fsdata = 0; in page_do_hashout()
3556 page_hashout(page_t *pp, kmutex_t *phm) in page_hashout() argument
3565 ASSERT(pp->p_vnode != NULL); in page_hashout()
3566 ASSERT((PAGE_EXCL(pp) && !page_iolock_assert(pp)) || panicstr); in page_hashout()
3567 ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(pp->p_vnode))); in page_hashout()
3569 vp = pp->p_vnode; in page_hashout()
3572 "page_hashout:pp %p vp %p", pp, vp); in page_hashout()
3578 index = PAGE_HASH_FUNC(vp, pp->p_offset); in page_hashout()
3593 page_do_hashout(pp); in page_hashout()
3604 sep = page_se_mutex(pp); in page_hashout()
3606 pp->p_selock &= ~SE_EWANTED; in page_hashout()
3607 if (CV_HAS_WAITERS(&pp->p_cv)) in page_hashout()
3608 cv_broadcast(&pp->p_cv); in page_hashout()
3618 page_add(page_t **ppp, page_t *pp) in page_add() argument
3620 ASSERT(PAGE_EXCL(pp) || (PAGE_SHARED(pp) && page_iolock_assert(pp))); in page_add()
3622 page_add_common(ppp, pp); in page_add()
3631 page_add_common(page_t **ppp, page_t *pp) in page_add_common() argument
3634 pp->p_next = pp->p_prev = pp; in page_add_common()
3636 pp->p_next = *ppp; in page_add_common()
3637 pp->p_prev = (*ppp)->p_prev; in page_add_common()
3638 (*ppp)->p_prev = pp; in page_add_common()
3639 pp->p_prev->p_next = pp; in page_add_common()
3641 *ppp = pp; in page_add_common()
3652 page_sub(page_t **ppp, page_t *pp) in page_sub() argument
3654 ASSERT(pp != NULL && (PP_ISFREE(pp)) ? 1 : in page_sub()
3655 (PAGE_EXCL(pp)) || (PAGE_SHARED(pp) && page_iolock_assert(pp))); in page_sub()
3657 if (*ppp == NULL || pp == NULL) { in page_sub()
3659 (void *)pp, (void *)(*ppp)); in page_sub()
3663 page_sub_common(ppp, pp); in page_sub()
3671 page_sub_common(page_t **ppp, page_t *pp) in page_sub_common() argument
3673 if (*ppp == pp) in page_sub_common()
3674 *ppp = pp->p_next; /* go to next page */ in page_sub_common()
3676 if (*ppp == pp) in page_sub_common()
3679 pp->p_prev->p_next = pp->p_next; in page_sub_common()
3680 pp->p_next->p_prev = pp->p_prev; in page_sub_common()
3682 pp->p_prev = pp->p_next = pp; /* make pp a list of one */ in page_sub_common()
3757 page_list_next(page_t *pp) in page_list_next() argument
3759 return (pp->p_next); in page_list_next()
3770 page_vpadd(page_t **ppp, page_t *pp) in page_vpadd() argument
3773 pp->p_vpnext = pp->p_vpprev = pp; in page_vpadd()
3775 pp->p_vpnext = *ppp; in page_vpadd()
3776 pp->p_vpprev = (*ppp)->p_vpprev; in page_vpadd()
3777 (*ppp)->p_vpprev = pp; in page_vpadd()
3778 pp->p_vpprev->p_vpnext = pp; in page_vpadd()
3780 *ppp = pp; in page_vpadd()
3790 page_vpsub(page_t **ppp, page_t *pp) in page_vpsub() argument
3792 if (*ppp == NULL || pp == NULL) { in page_vpsub()
3794 (void *)pp, (void *)(*ppp)); in page_vpsub()
3798 if (*ppp == pp) in page_vpsub()
3799 *ppp = pp->p_vpnext; /* go to next page */ in page_vpsub()
3801 if (*ppp == pp) in page_vpsub()
3804 pp->p_vpprev->p_vpnext = pp->p_vpnext; in page_vpsub()
3805 pp->p_vpnext->p_vpprev = pp->p_vpprev; in page_vpsub()
3807 pp->p_vpprev = pp->p_vpnext = pp; /* make pp a list of one */ in page_vpsub()
3818 page_t *pp, /* page to be locked */ in page_pp_lock() argument
3824 ASSERT(PAGE_LOCKED(pp)); in page_pp_lock()
3826 page_struct_lock(pp); in page_pp_lock()
3833 (pp->p_cowcnt < (ushort_t)PAGE_LOCK_MAXIMUM)) { in page_pp_lock()
3838 if (++pp->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) { in page_pp_lock()
3841 page_pptonum(pp)); in page_pp_lock()
3846 if (pp->p_lckcnt) { in page_pp_lock()
3847 if (pp->p_lckcnt < (ushort_t)PAGE_LOCK_MAXIMUM) { in page_pp_lock()
3849 if (++pp->p_lckcnt == in page_pp_lock()
3853 page_pptonum(pp)); in page_pp_lock()
3859 ++pp->p_lckcnt; in page_pp_lock()
3866 ++pp->p_lckcnt; in page_pp_lock()
3873 page_struct_unlock(pp); in page_pp_lock()
3883 page_t *pp, /* page to be unlocked */ in page_pp_unlock() argument
3887 ASSERT(PAGE_LOCKED(pp)); in page_pp_unlock()
3889 page_struct_lock(pp); in page_pp_unlock()
3899 if (pp->p_cowcnt) { in page_pp_unlock()
3901 pp->p_cowcnt--; in page_pp_unlock()
3907 if (pp->p_lckcnt && --pp->p_lckcnt == 0) { in page_pp_unlock()
3916 page_struct_unlock(pp); in page_pp_unlock()
4094 page_addclaim(page_t *pp) in page_addclaim() argument
4098 ASSERT(PAGE_LOCKED(pp)); in page_addclaim()
4100 page_struct_lock(pp); in page_addclaim()
4101 ASSERT(pp->p_lckcnt != 0); in page_addclaim()
4103 if (pp->p_lckcnt == 1) { in page_addclaim()
4104 if (pp->p_cowcnt < (ushort_t)PAGE_LOCK_MAXIMUM) { in page_addclaim()
4105 --pp->p_lckcnt; in page_addclaim()
4107 if (++pp->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) { in page_addclaim()
4110 page_pptonum(pp)); in page_addclaim()
4116 (pp->p_cowcnt < (ushort_t)PAGE_LOCK_MAXIMUM)) { in page_addclaim()
4120 --pp->p_lckcnt; in page_addclaim()
4122 if (++pp->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) { in page_addclaim()
4125 page_pptonum(pp)); in page_addclaim()
4130 page_struct_unlock(pp); in page_addclaim()
4135 page_subclaim(page_t *pp) in page_subclaim() argument
4139 ASSERT(PAGE_LOCKED(pp)); in page_subclaim()
4141 page_struct_lock(pp); in page_subclaim()
4142 ASSERT(pp->p_cowcnt != 0); in page_subclaim()
4144 if (pp->p_lckcnt) { in page_subclaim()
4145 if (pp->p_lckcnt < (ushort_t)PAGE_LOCK_MAXIMUM) { in page_subclaim()
4155 pp->p_cowcnt--; in page_subclaim()
4157 if (++pp->p_lckcnt == (ushort_t)PAGE_LOCK_MAXIMUM) { in page_subclaim()
4160 page_pptonum(pp)); in page_subclaim()
4165 pp->p_cowcnt--; in page_subclaim()
4166 pp->p_lckcnt++; in page_subclaim()
4168 page_struct_unlock(pp); in page_subclaim()
4266 page_t *pp; in page_numtopp() local
4269 pp = page_numtopp_nolock(pfnum); in page_numtopp()
4270 if (pp == NULL) { in page_numtopp()
4277 while (!page_lock(pp, se, (kmutex_t *)NULL, P_RECLAIM)) { in page_numtopp()
4278 if (page_pptonum(pp) != pfnum) in page_numtopp()
4283 if (page_pptonum(pp) != pfnum) { in page_numtopp()
4284 page_unlock(pp); in page_numtopp()
4288 return (pp); in page_numtopp()
4294 page_t *pp; in page_numtopp_noreclaim() local
4297 pp = page_numtopp_nolock(pfnum); in page_numtopp_noreclaim()
4298 if (pp == NULL) { in page_numtopp_noreclaim()
4305 while (!page_lock(pp, se, (kmutex_t *)NULL, P_NO_RECLAIM)) { in page_numtopp_noreclaim()
4306 if (page_pptonum(pp) != pfnum) in page_numtopp_noreclaim()
4311 if (page_pptonum(pp) != pfnum) { in page_numtopp_noreclaim()
4312 page_unlock(pp); in page_numtopp_noreclaim()
4316 return (pp); in page_numtopp_noreclaim()
4326 page_t *pp; in page_numtopp_nowait() local
4329 pp = page_numtopp_nolock(pfnum); in page_numtopp_nowait()
4330 if (pp == NULL) { in page_numtopp_nowait()
4337 if (PP_ISFREE(pp)) in page_numtopp_nowait()
4338 pp = NULL; in page_numtopp_nowait()
4340 if (!page_trylock(pp, se)) in page_numtopp_nowait()
4341 pp = NULL; in page_numtopp_nowait()
4343 if (page_pptonum(pp) != pfnum) { in page_numtopp_nowait()
4344 page_unlock(pp); in page_numtopp_nowait()
4347 if (PP_ISFREE(pp)) { in page_numtopp_nowait()
4348 page_unlock(pp); in page_numtopp_nowait()
4349 pp = NULL; in page_numtopp_nowait()
4353 return (pp); in page_numtopp_nowait()
4364 page_t *pp = page0; in page_busy() local
4369 vnode_t *vp = pp->p_vnode; in page_busy()
4383 if (!PP_ISFREE(pp) && vp != NULL && !VN_ISKAS(vp) && in page_busy()
4384 hat_ismod(pp) && !IS_SWAPVP(vp) && vp->v_vfsp != NULL && in page_busy()
4390 if (!page_trylock(pp, SE_EXCL)) in page_busy()
4393 if (PP_ISFREE(pp) || vp == NULL || IS_SWAPVP(vp) || in page_busy()
4394 pp->p_lckcnt != 0 || pp->p_cowcnt != 0 || in page_busy()
4395 !(hat_pagesync(pp, in page_busy()
4397 page_unlock(pp); in page_busy()
4400 off = pp->p_offset; in page_busy()
4402 page_unlock(pp); in page_busy()
4407 } while ((pp = page_next(pp)) != page0); in page_busy()
4435 page_t *pp; in page_invalidate_pages() local
4446 pp = page0 = page_first(); in page_invalidate_pages()
4456 if ((vp = pp->p_vnode) == NULL || VN_ISKAS(vp)) in page_invalidate_pages()
4462 if (PP_ISFREE(pp) && PP_ISAGED(pp)) in page_invalidate_pages()
4474 if (!page_trylock(pp, SE_EXCL)) { in page_invalidate_pages()
4477 } else if (PP_ISFREE(pp)) { in page_invalidate_pages()
4478 if (!PP_ISAGED(pp)) { in page_invalidate_pages()
4479 page_destroy_free(pp); in page_invalidate_pages()
4481 page_unlock(pp); in page_invalidate_pages()
4492 if (pp->p_lckcnt != 0 || pp->p_cowcnt != 0) { in page_invalidate_pages()
4493 page_unlock(pp); in page_invalidate_pages()
4502 if (!page_try_demote_pages(pp)) { in page_invalidate_pages()
4503 page_unlock(pp); in page_invalidate_pages()
4511 mod = (hat_pagesync(pp, HAT_SYNC_DONTZERO | HAT_SYNC_STOPON_MOD) in page_invalidate_pages()
4514 offset = pp->p_offset; in page_invalidate_pages()
4521 page_unlock(pp); in page_invalidate_pages()
4532 VN_DISPOSE(pp, B_INVAL, 0, kcred); in page_invalidate_pages()
4534 } while ((pp = page_next(pp)) != page0); in page_invalidate_pages()
4708 group_page_trylock(page_t *pp, se_t se) in group_page_trylock() argument
4712 uint_t pszc = pp->p_szc; in group_page_trylock()
4720 if (pp != PP_GROUPLEADER(pp, pszc)) { in group_page_trylock()
4725 ASSERT(PAGE_LOCKED_SE(pp, se)); in group_page_trylock()
4726 ASSERT(!PP_ISFREE(pp)); in group_page_trylock()
4731 tpp = pp + 1; in group_page_trylock()
4734 tpp = pp + 1; in group_page_trylock()
4741 if (pp->p_szc != pszc) { in group_page_trylock()
4742 ASSERT(pp->p_szc < pszc); in group_page_trylock()
4743 ASSERT(pp->p_vnode != NULL && !PP_ISKAS(pp) && in group_page_trylock()
4744 !IS_SWAPFSVP(pp->p_vnode)); in group_page_trylock()
4745 tpp = pp + 1; in group_page_trylock()
4749 pszc = pp->p_szc; in group_page_trylock()
4756 group_page_unlock(page_t *pp) in group_page_unlock() argument
4761 ASSERT(PAGE_LOCKED(pp)); in group_page_unlock()
4762 ASSERT(!PP_ISFREE(pp)); in group_page_unlock()
4763 ASSERT(pp == PP_PAGEROOT(pp)); in group_page_unlock()
4764 npgs = page_get_pagecnt(pp->p_szc); in group_page_unlock()
4765 for (i = 1, tpp = pp + 1; i < npgs; i++, tpp++) { in group_page_unlock()
5052 page_t *pp; in page_free_replacement_page() local
5058 pp = pplist; in page_free_replacement_page()
5059 if (pp->p_szc == 0) { in page_free_replacement_page()
5060 page_sub(&pplist, pp); in page_free_replacement_page()
5061 page_clr_all_props(pp); in page_free_replacement_page()
5062 PP_SETFREE(pp); in page_free_replacement_page()
5063 PP_SETAGED(pp); in page_free_replacement_page()
5064 page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL); in page_free_replacement_page()
5065 page_unlock(pp); in page_free_replacement_page()
5068 spgcnt_t curnpgs = page_get_pagecnt(pp->p_szc); in page_free_replacement_page()
5070 page_list_break(&pp, &pplist, curnpgs); in page_free_replacement_page()
5071 tpp = pp; in page_free_replacement_page()
5078 } while ((tpp = tpp->p_next) != pp); in page_free_replacement_page()
5079 page_list_add_pages(pp, 0); in page_free_replacement_page()
5130 page_release(page_t *pp, int checkmod) in page_release() argument
5134 ASSERT(PAGE_LOCKED(pp) && !PP_ISFREE(pp) && in page_release()
5135 (pp->p_vnode != NULL)); in page_release()
5137 if (!hat_page_is_mapped(pp) && !IS_SWAPVP(pp->p_vnode) && in page_release()
5138 ((PAGE_SHARED(pp) && page_tryupgrade(pp)) || PAGE_EXCL(pp)) && in page_release()
5139 pp->p_lckcnt == 0 && pp->p_cowcnt == 0 && in page_release()
5140 !hat_page_is_mapped(pp)) { in page_release()
5150 if (checkmod && hat_ismod(pp)) { in page_release()
5151 page_unlock(pp); in page_release()
5155 VN_DISPOSE(pp, B_FREE, 0, kcred); in page_release()
5159 page_unlock(pp); in page_release()
5172 page_try_demote_free_pages(page_t *pp) in page_try_demote_free_pages() argument
5174 page_t *rootpp = pp; in page_try_demote_free_pages()
5175 pfn_t pfn = page_pptonum(pp); in page_try_demote_free_pages()
5177 uint_t szc = pp->p_szc; in page_try_demote_free_pages()
5179 ASSERT(PP_ISFREE(pp)); in page_try_demote_free_pages()
5180 ASSERT(PAGE_EXCL(pp)); in page_try_demote_free_pages()
5186 npgs = page_get_pagecnt(pp->p_szc); in page_try_demote_free_pages()
5196 if (pp != rootpp && !page_trylock(rootpp, SE_EXCL)) { in page_try_demote_free_pages()
5201 if (pp != rootpp) in page_try_demote_free_pages()
5208 if (pp != rootpp) in page_try_demote_free_pages()
5211 ASSERT(PP_ISFREE(pp)); in page_try_demote_free_pages()
5212 ASSERT(PAGE_EXCL(pp)); in page_try_demote_free_pages()
5223 page_try_demote_pages(page_t *pp) in page_try_demote_pages() argument
5225 page_t *tpp, *rootpp = pp; in page_try_demote_pages()
5226 pfn_t pfn = page_pptonum(pp); in page_try_demote_pages()
5228 uint_t szc = pp->p_szc; in page_try_demote_pages()
5229 vnode_t *vp = pp->p_vnode; in page_try_demote_pages()
5231 ASSERT(PAGE_EXCL(pp)); in page_try_demote_pages()
5235 if (pp->p_szc == 0) { in page_try_demote_pages()
5242 page_demote_vp_pages(pp); in page_try_demote_pages()
5243 ASSERT(pp->p_szc == 0); in page_try_demote_pages()
5251 npgs = page_get_pagecnt(pp->p_szc); in page_try_demote_pages()
5276 if (tpp != pp && !page_trylock(tpp, SE_EXCL)) in page_try_demote_pages()
5289 if (tpp != pp) in page_try_demote_pages()
5309 if (tpp != pp) in page_try_demote_pages()
5381 page_demote_vp_pages(page_t *pp) in page_demote_vp_pages() argument
5385 ASSERT(PAGE_EXCL(pp)); in page_demote_vp_pages()
5386 ASSERT(!PP_ISFREE(pp)); in page_demote_vp_pages()
5387 ASSERT(pp->p_vnode != NULL); in page_demote_vp_pages()
5388 ASSERT(!IS_SWAPFSVP(pp->p_vnode)); in page_demote_vp_pages()
5389 ASSERT(!PP_ISKAS(pp)); in page_demote_vp_pages()
5393 mtx = page_szc_lock(pp); in page_demote_vp_pages()
5395 hat_page_demote(pp); in page_demote_vp_pages()
5398 ASSERT(pp->p_szc == 0); in page_demote_vp_pages()
5419 page_t *pp0, *pp; in page_mark_migrate() local
5466 pp = NULL; in page_mark_migrate()
5468 pp = page_lookup(curvp, off, SE_SHARED); in page_mark_migrate()
5474 if (pp == NULL) { in page_mark_migrate()
5482 pfn = page_pptonum(pp); in page_mark_migrate()
5489 pszc = pp->p_szc; in page_mark_migrate()
5496 page_unlock(pp); in page_mark_migrate()
5507 if (!page_tryupgrade(pp)) { in page_mark_migrate()
5508 page_unlock(pp); in page_mark_migrate()
5515 pp0 = pp++; in page_mark_migrate()
5527 if (!page_trylock(pp, SE_EXCL)) { in page_mark_migrate()
5530 if (PP_ISFREE(pp) || in page_mark_migrate()
5531 pp->p_szc != pszc) { in page_mark_migrate()
5536 page_unlock(pp); in page_mark_migrate()
5539 pp++; in page_mark_migrate()
5548 while (pp0 < pp) { in page_mark_migrate()
5571 while (pp0 < pp) { in page_mark_migrate()
5595 page_t *pp; in page_migrate() local
5605 pp = *ppa; in page_migrate()
5606 pszc = pp->p_szc; in page_migrate()
5628 if (!PP_ISMIGRATE(pp)) in page_migrate()
5634 pfn = page_pptonum(pp); in page_migrate()
5682 newpp = page_get_replacement_page(pp, to, PGR_SAMESZC); in page_migrate()
5696 PP_CLRMIGRATE(pp); in page_migrate()
5697 if (page_relocate(&pp, &newpp, 0, 1, &page_cnt, to)) { in page_migrate()
5712 for (i = 0; i < page_cnt; ++i, ++pp) { in page_migrate()
5716 pp = newpp; in page_migrate()
5717 page_sub(&newpp, pp); in page_migrate()
5718 page_downgrade(pp); in page_migrate()
5799 page_t *pp; in page_numtopp_nolock() local
5823 pp = seg->pages + (pfnum - seg->pages_base); in page_numtopp_nolock()
5824 if (pp->p_pagenum == pfnum) { in page_numtopp_nolock()
5826 return ((page_t *)pp); in page_numtopp_nolock()
5835 pp = seg->pages + (pfnum - seg->pages_base); in page_numtopp_nolock()
5836 if (pp->p_pagenum == pfnum) { in page_numtopp_nolock()
5838 return ((page_t *)pp); in page_numtopp_nolock()
5846 pp = seg->pages + (pfnum - seg->pages_base); in page_numtopp_nolock()
5847 if (pp->p_pagenum == pfnum) { in page_numtopp_nolock()
5849 return ((page_t *)pp); in page_numtopp_nolock()
5864 page_t *pp; in page_numtomemseg_nolock() local
5875 pp = seg->pages + (pfnum - seg->pages_base); in page_numtomemseg_nolock()
5876 if (pp->p_pagenum == pfnum) { in page_numtomemseg_nolock()
5885 pp = seg->pages + (pfnum - seg->pages_base); in page_numtomemseg_nolock()
5886 if (pp->p_pagenum == pfnum) { in page_numtomemseg_nolock()
5905 page_nextn(page_t *pp, ulong_t n) in page_nextn() argument
5929 !(pp >= seg->pages && pp < seg->epages)) { in page_nextn()
5932 if (pp >= seg->pages && pp < seg->epages) in page_nextn()
5940 pp = seg->pages; in page_nextn()
5945 while ((ppn = (pp + n)) >= seg->epages || ppn < pp) { in page_nextn()
5946 n -= seg->epages - pp; in page_nextn()
5950 pp = seg->pages; in page_nextn()
5976 page_t *pp, in page_next_scan_large() argument
5989 ASSERT(pp != NULL); in page_next_scan_large()
5990 if (pp->p_szc == 0) { in page_next_scan_large()
5993 pfn = page_pptonum(pp); in page_next_scan_large()
5994 cnt = page_get_pagecnt(pp->p_szc); in page_next_scan_large()
5998 new_pp = pp + cnt; in page_next_scan_large()
6027 page_next(page_t *pp) in page_next() argument
6029 return (page_nextn(pp, 1)); in page_next()
6093 page_pptonum(page_t *pp) in page_pptonum() argument
6095 return (pp->p_pagenum); in page_pptonum()
6104 page_set_props(page_t *pp, uint_t flags) in page_set_props() argument
6107 pp->p_nrm |= (uchar_t)flags; in page_set_props()
6111 page_clr_all_props(page_t *pp) in page_clr_all_props() argument
6113 pp->p_nrm = 0; in page_clr_all_props()
6120 page_clear_lck_cow(page_t *pp, int adjust) in page_clear_lck_cow() argument
6124 ASSERT(PAGE_EXCL(pp)); in page_clear_lck_cow()
6131 if (pp->p_lckcnt) { in page_clear_lck_cow()
6133 pp->p_lckcnt = 0; in page_clear_lck_cow()
6135 if (pp->p_cowcnt) { in page_clear_lck_cow()
6136 f_amount += pp->p_cowcnt; in page_clear_lck_cow()
6137 pp->p_cowcnt = 0; in page_clear_lck_cow()
6154 page_share_cnt(page_t *pp) in page_share_cnt() argument
6156 return (hat_page_getshare(pp)); in page_share_cnt()
6160 page_isshared(page_t *pp) in page_isshared() argument
6162 return (hat_page_checkshare(pp, 1)); in page_isshared()
6166 page_isfree(page_t *pp) in page_isfree() argument
6168 return (PP_ISFREE(pp)); in page_isfree()
6172 page_isref(page_t *pp) in page_isref() argument
6174 return (hat_page_getattr(pp, P_REF)); in page_isref()
6178 page_ismod(page_t *pp) in page_ismod() argument
6180 return (hat_page_getattr(pp, P_MOD)); in page_ismod()
6225 page_t *pp; member
6239 #define PAGE_CAPTURE_PRIO(pp) (PP_ISRAF(pp) ? PC_PRI_LO : PC_PRI_HI) argument
6275 #define PAGE_CAPTURE_HASH(pp) \ argument
6276 ((int)(((uintptr_t)pp >> 7) & (NUM_PAGE_CAPTURE_BUCKETS - 1)))
6280 int page_trycapture(page_t *pp, uint_t szc, uint_t flags, void *datap);
6337 page_clrtoxic(head->pp, PR_CAPTURE); in page_capture_unregister_callback()
6365 page_capture_move_to_walked(page_t *pp) in page_capture_move_to_walked() argument
6370 index = PAGE_CAPTURE_HASH(pp); in page_capture_move_to_walked()
6375 if (bp->pp == pp) { in page_capture_move_to_walked()
6396 bp->pri = PAGE_CAPTURE_PRIO(pp); in page_capture_move_to_walked()
6415 page_capture_add_hash(page_t *pp, uint_t szc, uint_t flags, void *datap) in page_capture_add_hash() argument
6432 bp1->pp = pp; in page_capture_add_hash()
6460 index = PAGE_CAPTURE_HASH(pp); in page_capture_add_hash()
6471 if (!(pp->p_toxic & PR_CAPTURE)) { in page_capture_add_hash()
6477 if (tp1->pp == pp) { in page_capture_add_hash()
6480 (void *)pp, (void *)tp1); in page_capture_add_hash()
6487 page_settoxic(pp, PR_CAPTURE); in page_capture_add_hash()
6488 pri = PAGE_CAPTURE_PRIO(pp); in page_capture_add_hash()
6519 if (bp2->pp == pp) { in page_capture_add_hash()
6551 (void *)pp); in page_capture_add_hash()
6562 page_capture_clean_page(page_t *pp) in page_capture_clean_page() argument
6571 ASSERT(PAGE_EXCL(pp)); in page_capture_clean_page()
6572 ASSERT(!PP_RETIRED(pp)); in page_capture_clean_page()
6575 if (PP_ISFREE(pp)) { in page_capture_clean_page()
6576 if (!page_reclaim(pp, NULL)) { in page_capture_clean_page()
6581 ASSERT(pp->p_szc == 0); in page_capture_clean_page()
6582 if (pp->p_vnode != NULL) { in page_capture_clean_page()
6588 page_hashout(pp, NULL); in page_capture_clean_page()
6598 if (pp->p_toxic & PR_UE) { in page_capture_clean_page()
6606 if (pp->p_vnode == NULL) { in page_capture_clean_page()
6617 if ((pp->p_szc > 0) && (pp != PP_PAGEROOT(pp))) { in page_capture_clean_page()
6618 if (page_try_demote_pages(pp) == 0) { in page_capture_clean_page()
6623 ret = page_relocate(&pp, &newpp, 1, 0, &count, NULL); in page_capture_clean_page()
6638 if (pp->p_szc > 0) { in page_capture_clean_page()
6640 extra = page_get_pagecnt(pp->p_szc) - 1; in page_capture_clean_page()
6642 tpp = pp->p_next; in page_capture_clean_page()
6643 page_sub(&pp, tpp); in page_capture_clean_page()
6649 ASSERT(pp->p_next == pp && pp->p_prev == pp); in page_capture_clean_page()
6650 pp->p_szc = 0; in page_capture_clean_page()
6667 if (pp->p_szc > 0) { in page_capture_clean_page()
6668 if (page_try_demote_pages(pp) == 0) { in page_capture_clean_page()
6674 ASSERT(pp->p_szc == 0); in page_capture_clean_page()
6676 if (hat_ismod(pp)) { in page_capture_clean_page()
6680 if (PP_ISKAS(pp)) { in page_capture_clean_page()
6684 if (pp->p_lckcnt || pp->p_cowcnt) { in page_capture_clean_page()
6689 (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD); in page_capture_clean_page()
6690 ASSERT(!hat_page_is_mapped(pp)); in page_capture_clean_page()
6692 if (hat_ismod(pp)) { in page_capture_clean_page()
6700 if (pp->p_vnode != NULL) { in page_capture_clean_page()
6701 page_hashout(pp, NULL); in page_capture_clean_page()
6712 page_unlock(pp); in page_capture_clean_page()
6715 ASSERT(pp->p_szc == 0); in page_capture_clean_page()
6716 ASSERT(PAGE_EXCL(pp)); in page_capture_clean_page()
6718 pp->p_next = pp; in page_capture_clean_page()
6719 pp->p_prev = pp; in page_capture_clean_page()
6736 page_capture_pre_checks(page_t *pp, uint_t flags) in page_capture_pre_checks() argument
6738 ASSERT(pp != NULL); in page_capture_pre_checks()
6741 if (pp->p_vnode == &promvp) { in page_capture_pre_checks()
6745 if (PP_ISNORELOC(pp) && !(flags & CAPTURE_GET_CAGE) && in page_capture_pre_checks()
6750 if (PP_ISNORELOCKERNEL(pp)) { in page_capture_pre_checks()
6754 if (PP_ISKAS(pp)) { in page_capture_pre_checks()
6783 page_capture_take_action(page_t *pp, uint_t flags, void *datap) in page_capture_take_action() argument
6793 ASSERT(PAGE_EXCL(pp)); in page_capture_take_action()
6813 index = PAGE_CAPTURE_HASH(pp); in page_capture_take_action()
6819 if (bp1->pp == pp) { in page_capture_take_action()
6824 page_clrtoxic(pp, PR_CAPTURE); in page_capture_take_action()
6837 page_free(pp, 1); in page_capture_take_action()
6856 ret = pc_cb[cb_index].cb_func(pp, datap, flags); in page_capture_take_action()
6898 if (!(pp->p_toxic & PR_CAPTURE)) { in page_capture_take_action()
6900 page_settoxic(pp, PR_CAPTURE); in page_capture_take_action()
6904 bp1->pri = PAGE_CAPTURE_PRIO(pp); in page_capture_take_action()
6919 if (bp2->pp == pp) { in page_capture_take_action()
6937 bp2->pri = PAGE_CAPTURE_PRIO(pp); in page_capture_take_action()
6947 panic("PR_CAPTURE set but not on hash for pp 0x%p\n", (void *)pp); in page_capture_take_action()
6973 page_itrycapture(page_t *pp, uint_t szc, uint_t flags, void *datap) in page_itrycapture() argument
6979 ASSERT(PAGE_EXCL(pp)); in page_itrycapture()
6984 ret = page_capture_pre_checks(pp, flags); in page_itrycapture()
6989 if (!page_trylock(pp, SE_EXCL)) { in page_itrycapture()
6998 if (PP_RETIRED(pp)) { in page_itrycapture()
7000 if (!page_unretire_pp(pp, PR_UNR_TEMP)) { in page_itrycapture()
7006 page_capture_add_hash(pp, szc, in page_itrycapture()
7015 page_capture_add_hash(pp, szc, flags, datap); in page_itrycapture()
7020 ASSERT(PAGE_EXCL(pp)); in page_itrycapture()
7026 page_unlock(pp); in page_itrycapture()
7030 ret = page_capture_clean_page(pp); in page_itrycapture()
7035 page_capture_add_hash(pp, szc, flags, datap); in page_itrycapture()
7041 ASSERT(PAGE_EXCL(pp)); in page_itrycapture()
7042 ASSERT(pp->p_szc == 0); in page_itrycapture()
7045 ret = page_capture_take_action(pp, flags, datap); in page_itrycapture()
7056 page_capture_add_hash(pp, szc, flags, datap); in page_itrycapture()
7062 page_trycapture(page_t *pp, uint_t szc, uint_t flags, void *datap) in page_trycapture() argument
7067 ret = page_itrycapture(pp, szc, flags, datap); in page_trycapture()
7077 page_unlock_capture(page_t *pp) in page_unlock_capture() argument
7100 page_unlock_nocapture(pp); in page_unlock_capture()
7103 if (pp->p_vnode != NULL && mutex_owned(page_vnode_mutex(pp->p_vnode))) { in page_unlock_capture()
7104 page_unlock_nocapture(pp); in page_unlock_capture()
7108 index = PAGE_CAPTURE_HASH(pp); in page_unlock_capture()
7115 if (bp->pp == pp) { in page_unlock_capture()
7120 (void) page_trycapture(pp, szc, flags, datap); in page_unlock_capture()
7128 page_clrtoxic(pp, PR_CAPTURE); in page_unlock_capture()
7129 page_unlock(pp); in page_unlock_capture()
7165 page_t *pp; in page_retire_mdboot() local
7185 pp = bp->pp; in page_retire_mdboot()
7186 if (PP_TOXIC(pp)) { in page_retire_mdboot()
7187 if (page_trylock(pp, SE_EXCL)) { in page_retire_mdboot()
7188 PP_CLRFREE(pp); in page_retire_mdboot()
7189 pagescrub(pp, 0, PAGESIZE); in page_retire_mdboot()
7190 page_unlock(pp); in page_retire_mdboot()
7207 page_t *pp; in page_capture_async() local
7245 page_deleted(bp1->pp)) { in page_capture_async()
7258 page_clrtoxic(bp1->pp, PR_CAPTURE); in page_capture_async()
7265 pp = bp1->pp; in page_capture_async()
7270 if (page_trylock(pp, SE_EXCL)) { in page_capture_async()
7271 ret = page_trycapture(pp, szc, in page_capture_async()
7279 (void) page_capture_move_to_walked(pp); in page_capture_async()