Lines Matching refs:cp

931 	umem_cache_t *cp;  in umem_cache_applyall()  local
934 for (cp = umem_null_cache.cache_next; cp != &umem_null_cache; in umem_cache_applyall()
935 cp = cp->cache_next) in umem_cache_applyall()
936 func(cp); in umem_cache_applyall()
941 umem_add_update_unlocked(umem_cache_t *cp, int flags) in umem_add_update_unlocked() argument
950 if (cp->cache_uflags & UMU_ACTIVE) { in umem_add_update_unlocked()
951 cp->cache_uflags |= flags; in umem_add_update_unlocked()
953 if (cp->cache_unext != NULL) { in umem_add_update_unlocked()
954 ASSERT(cp->cache_uflags != 0); in umem_add_update_unlocked()
955 cp->cache_uflags |= flags; in umem_add_update_unlocked()
957 ASSERT(cp->cache_uflags == 0); in umem_add_update_unlocked()
958 cp->cache_uflags = flags; in umem_add_update_unlocked()
959 cp->cache_unext = cnext = &umem_null_cache; in umem_add_update_unlocked()
960 cp->cache_uprev = cprev = umem_null_cache.cache_uprev; in umem_add_update_unlocked()
961 cnext->cache_uprev = cp; in umem_add_update_unlocked()
962 cprev->cache_unext = cp; in umem_add_update_unlocked()
968 umem_add_update(umem_cache_t *cp, int flags) in umem_add_update() argument
972 umem_add_update_unlocked(cp, flags); in umem_add_update()
985 umem_remove_updates(umem_cache_t *cp) in umem_remove_updates() argument
992 while (cp->cache_uflags & UMU_ACTIVE) { in umem_remove_updates()
995 ASSERT(cp->cache_unext == NULL); in umem_remove_updates()
997 cp->cache_uflags |= UMU_NOTIFY; in umem_remove_updates()
1014 if (cp->cache_unext != NULL) { in umem_remove_updates()
1015 cp->cache_uprev->cache_unext = cp->cache_unext; in umem_remove_updates()
1016 cp->cache_unext->cache_uprev = cp->cache_uprev; in umem_remove_updates()
1017 cp->cache_uprev = cp->cache_unext = NULL; in umem_remove_updates()
1018 cp->cache_uflags = 0; in umem_remove_updates()
1023 ASSERT(cp->cache_unext == NULL && cp->cache_uflags == 0); in umem_remove_updates()
1030 umem_cache_t *cp; in umem_updateall() local
1040 for (cp = umem_null_cache.cache_next; cp != &umem_null_cache; in umem_updateall()
1041 cp = cp->cache_next) in umem_updateall()
1042 umem_add_update_unlocked(cp, flags); in umem_updateall()
1055 umem_findslab(umem_cache_t *cp, void *buf) in umem_findslab() argument
1059 (void) mutex_lock(&cp->cache_lock); in umem_findslab()
1060 for (sp = cp->cache_nullslab.slab_next; in umem_findslab()
1061 sp != &cp->cache_nullslab; sp = sp->slab_next) { in umem_findslab()
1063 (void) mutex_unlock(&cp->cache_lock); in umem_findslab()
1067 (void) mutex_unlock(&cp->cache_lock); in umem_findslab()
1077 umem_cache_t *cp = cparg; in umem_error() local
1088 sp = umem_findslab(cp, buf); in umem_error()
1090 for (cp = umem_null_cache.cache_prev; cp != &umem_null_cache; in umem_error()
1091 cp = cp->cache_prev) { in umem_error()
1092 if ((sp = umem_findslab(cp, buf)) != NULL) in umem_error()
1098 cp = NULL; in umem_error()
1101 if (cp != cparg) in umem_error()
1105 (uintptr_t)sp->slab_base) % cp->cache_chunksize; in umem_error()
1108 if (cp->cache_flags & UMF_BUFTAG) in umem_error()
1109 btp = UMEM_BUFTAG(cp, buf); in umem_error()
1110 if (cp->cache_flags & UMF_HASH) { in umem_error()
1111 (void) mutex_lock(&cp->cache_lock); in umem_error()
1112 for (bcp = *UMEM_HASH(cp, buf); bcp; bcp = bcp->bc_next) in umem_error()
1115 (void) mutex_unlock(&cp->cache_lock); in umem_error()
1118 if (umem_findslab(cp->cache_bufctl_cache, bcp) == in umem_error()
1131 umem_abort_info.ump_realcache = cp; in umem_error()
1141 off = verify_pattern(UMEM_FREE_PATTERN, buf, cp->cache_verify); in umem_error()
1175 umem_printf("buffer was allocated from %s,\n", cp->cache_name); in umem_error()
1195 if (bcp != NULL && (cp->cache_flags & UMF_AUDIT) && in umem_error()
1209 (void *)sp, cp->cache_name); in umem_error()
1228 umem_alloc_retry(umem_cache_t *cp, int umflag) in umem_alloc_retry() argument
1230 if (cp == &umem_null_cache) { in umem_alloc_retry()
1371 #define UMEM_AUDIT(lp, cp, bcp) \ argument
1377 (cp != NULL) && (cp->cache_flags & UMF_CHECKSIGNAL)); \
1383 umem_log_event(umem_log_header_t *lp, umem_cache_t *cp, in umem_log_event() argument
1392 bcp->bc_cache = cp; in umem_log_event()
1393 UMEM_AUDIT(lp, cp, bcp); in umem_log_event()
1400 umem_slab_create(umem_cache_t *cp, int umflag) in umem_slab_create() argument
1402 size_t slabsize = cp->cache_slabsize; in umem_slab_create()
1403 size_t chunksize = cp->cache_chunksize; in umem_slab_create()
1404 int cache_flags = cp->cache_flags; in umem_slab_create()
1409 vmem_t *vmp = cp->cache_arena; in umem_slab_create()
1411 color = cp->cache_color + cp->cache_align; in umem_slab_create()
1412 if (color > cp->cache_maxcolor) in umem_slab_create()
1413 color = cp->cache_mincolor; in umem_slab_create()
1414 cp->cache_color = color; in umem_slab_create()
1423 if (!(cp->cache_cflags & UMC_NOTOUCH) && in umem_slab_create()
1424 (cp->cache_flags & UMF_DEADBEEF)) in umem_slab_create()
1432 sp = UMEM_SLAB(cp, slab); in umem_slab_create()
1436 sp->slab_cache = cp; in umem_slab_create()
1445 bcp = _umem_cache_alloc(cp->cache_bufctl_cache, umflag); in umem_slab_create()
1452 bcap->bc_cache = cp; in umem_slab_create()
1457 bcp = UMEM_BUFCTL(cp, buf); in umem_slab_create()
1460 umem_buftag_t *btp = UMEM_BUFTAG(cp, buf); in umem_slab_create()
1466 cp->cache_verify); in umem_slab_create()
1474 umem_log_event(umem_slab_log, cp, sp, slab); in umem_slab_create()
1482 _umem_cache_free(cp->cache_bufctl_cache, bcp); in umem_slab_create()
1492 umem_log_event(umem_failure_log, cp, NULL, NULL); in umem_slab_create()
1493 atomic_add_64(&cp->cache_alloc_fail, 1); in umem_slab_create()
1502 umem_slab_destroy(umem_cache_t *cp, umem_slab_t *sp) in umem_slab_destroy() argument
1504 vmem_t *vmp = cp->cache_arena; in umem_slab_destroy()
1507 if (cp->cache_flags & UMF_HASH) { in umem_slab_destroy()
1511 _umem_cache_free(cp->cache_bufctl_cache, bcp); in umem_slab_destroy()
1515 vmem_free(vmp, slab, cp->cache_slabsize); in umem_slab_destroy()
1522 umem_slab_alloc(umem_cache_t *cp, int umflag) in umem_slab_alloc() argument
1528 (void) mutex_lock(&cp->cache_lock); in umem_slab_alloc()
1529 cp->cache_slab_alloc++; in umem_slab_alloc()
1530 sp = cp->cache_freelist; in umem_slab_alloc()
1531 ASSERT(sp->slab_cache == cp); in umem_slab_alloc()
1536 (void) mutex_unlock(&cp->cache_lock); in umem_slab_alloc()
1537 if (cp == &umem_null_cache) in umem_slab_alloc()
1539 if ((sp = umem_slab_create(cp, umflag)) == NULL) in umem_slab_alloc()
1541 (void) mutex_lock(&cp->cache_lock); in umem_slab_alloc()
1542 cp->cache_slab_create++; in umem_slab_alloc()
1543 if ((cp->cache_buftotal += sp->slab_chunks) > cp->cache_bufmax) in umem_slab_alloc()
1544 cp->cache_bufmax = cp->cache_buftotal; in umem_slab_alloc()
1545 sp->slab_next = cp->cache_freelist; in umem_slab_alloc()
1546 sp->slab_prev = cp->cache_freelist->slab_prev; in umem_slab_alloc()
1549 cp->cache_freelist = sp; in umem_slab_alloc()
1561 cp->cache_freelist = sp->slab_next; in umem_slab_alloc()
1565 if (cp->cache_flags & UMF_HASH) { in umem_slab_alloc()
1570 hash_bucket = UMEM_HASH(cp, buf); in umem_slab_alloc()
1573 if ((cp->cache_flags & (UMF_AUDIT | UMF_BUFTAG)) == UMF_AUDIT) { in umem_slab_alloc()
1574 UMEM_AUDIT(umem_transaction_log, cp, bcp); in umem_slab_alloc()
1577 buf = UMEM_BUF(cp, bcp); in umem_slab_alloc()
1582 (void) mutex_unlock(&cp->cache_lock); in umem_slab_alloc()
1591 umem_slab_free(umem_cache_t *cp, void *buf) in umem_slab_free() argument
1598 (void) mutex_lock(&cp->cache_lock); in umem_slab_free()
1599 cp->cache_slab_free++; in umem_slab_free()
1601 if (cp->cache_flags & UMF_HASH) { in umem_slab_free()
1605 prev_bcpp = UMEM_HASH(cp, buf); in umem_slab_free()
1612 cp->cache_lookup_depth++; in umem_slab_free()
1616 bcp = UMEM_BUFCTL(cp, buf); in umem_slab_free()
1617 sp = UMEM_SLAB(cp, buf); in umem_slab_free()
1620 if (bcp == NULL || sp->slab_cache != cp || !UMEM_SLAB_MEMBER(sp, buf)) { in umem_slab_free()
1621 (void) mutex_unlock(&cp->cache_lock); in umem_slab_free()
1622 umem_error(UMERR_BADADDR, cp, buf); in umem_slab_free()
1626 if ((cp->cache_flags & (UMF_AUDIT | UMF_BUFTAG)) == UMF_AUDIT) { in umem_slab_free()
1627 if (cp->cache_flags & UMF_CONTENTS) in umem_slab_free()
1630 cp->cache_contents); in umem_slab_free()
1631 UMEM_AUDIT(umem_transaction_log, cp, bcp); in umem_slab_free()
1639 ASSERT(cp->cache_freelist != sp); in umem_slab_free()
1642 sp->slab_next = cp->cache_freelist; in umem_slab_free()
1643 sp->slab_prev = cp->cache_freelist->slab_prev; in umem_slab_free()
1646 cp->cache_freelist = sp; in umem_slab_free()
1660 if (sp == cp->cache_freelist) in umem_slab_free()
1661 cp->cache_freelist = sp->slab_next; in umem_slab_free()
1662 cp->cache_slab_destroy++; in umem_slab_free()
1663 cp->cache_buftotal -= sp->slab_chunks; in umem_slab_free()
1664 (void) mutex_unlock(&cp->cache_lock); in umem_slab_free()
1665 umem_slab_destroy(cp, sp); in umem_slab_free()
1668 (void) mutex_unlock(&cp->cache_lock); in umem_slab_free()
1672 umem_cache_alloc_debug(umem_cache_t *cp, void *buf, int umflag) in umem_cache_alloc_debug() argument
1674 umem_buftag_t *btp = UMEM_BUFTAG(cp, buf); in umem_cache_alloc_debug()
1680 umem_error(UMERR_BADBUFTAG, cp, buf); in umem_cache_alloc_debug()
1686 if ((cp->cache_flags & UMF_HASH) && bcp->bc_addr != buf) { in umem_cache_alloc_debug()
1687 umem_error(UMERR_BADBUFCTL, cp, buf); in umem_cache_alloc_debug()
1693 if (cp->cache_flags & UMF_DEADBEEF) { in umem_cache_alloc_debug()
1695 UMEM_UNINITIALIZED_PATTERN, buf, cp->cache_verify)) { in umem_cache_alloc_debug()
1696 umem_error(UMERR_MODIFIED, cp, buf); in umem_cache_alloc_debug()
1701 if ((mtbf = umem_mtbf | cp->cache_mtbf) != 0 && in umem_cache_alloc_debug()
1704 umem_log_event(umem_failure_log, cp, NULL, NULL); in umem_cache_alloc_debug()
1714 if (mtbf || (cp->cache_constructor != NULL && in umem_cache_alloc_debug()
1715 cp->cache_constructor(buf, cp->cache_private, flags_nfatal) != 0)) { in umem_cache_alloc_debug()
1716 atomic_add_64(&cp->cache_alloc_fail, 1); in umem_cache_alloc_debug()
1718 copy_pattern(UMEM_FREE_PATTERN, buf, cp->cache_verify); in umem_cache_alloc_debug()
1719 umem_slab_free(cp, buf); in umem_cache_alloc_debug()
1723 if (cp->cache_flags & UMF_AUDIT) { in umem_cache_alloc_debug()
1724 UMEM_AUDIT(umem_transaction_log, cp, bcp); in umem_cache_alloc_debug()
1731 umem_cache_free_debug(umem_cache_t *cp, void *buf) in umem_cache_free_debug() argument
1733 umem_buftag_t *btp = UMEM_BUFTAG(cp, buf); in umem_cache_free_debug()
1739 umem_error(UMERR_DUPFREE, cp, buf); in umem_cache_free_debug()
1742 sp = umem_findslab(cp, buf); in umem_cache_free_debug()
1743 if (sp == NULL || sp->slab_cache != cp) in umem_cache_free_debug()
1744 umem_error(UMERR_BADADDR, cp, buf); in umem_cache_free_debug()
1746 umem_error(UMERR_REDZONE, cp, buf); in umem_cache_free_debug()
1752 if ((cp->cache_flags & UMF_HASH) && bcp->bc_addr != buf) { in umem_cache_free_debug()
1753 umem_error(UMERR_BADBUFCTL, cp, buf); in umem_cache_free_debug()
1758 umem_error(UMERR_REDZONE, cp, buf); in umem_cache_free_debug()
1762 if (cp->cache_flags & UMF_AUDIT) { in umem_cache_free_debug()
1763 if (cp->cache_flags & UMF_CONTENTS) in umem_cache_free_debug()
1765 buf, cp->cache_contents); in umem_cache_free_debug()
1766 UMEM_AUDIT(umem_transaction_log, cp, bcp); in umem_cache_free_debug()
1769 if (cp->cache_destructor != NULL) in umem_cache_free_debug()
1770 cp->cache_destructor(buf, cp->cache_private); in umem_cache_free_debug()
1772 if (cp->cache_flags & UMF_DEADBEEF) in umem_cache_free_debug()
1773 copy_pattern(UMEM_FREE_PATTERN, buf, cp->cache_verify); in umem_cache_free_debug()
1782 umem_magazine_destroy(umem_cache_t *cp, umem_magazine_t *mp, int nrounds) in umem_magazine_destroy() argument
1786 ASSERT(cp->cache_next == NULL || IN_UPDATE()); in umem_magazine_destroy()
1791 if ((cp->cache_flags & UMF_DEADBEEF) && in umem_magazine_destroy()
1793 cp->cache_verify) != NULL) { in umem_magazine_destroy()
1794 umem_error(UMERR_MODIFIED, cp, buf); in umem_magazine_destroy()
1798 if (!(cp->cache_flags & UMF_BUFTAG) && in umem_magazine_destroy()
1799 cp->cache_destructor != NULL) in umem_magazine_destroy()
1800 cp->cache_destructor(buf, cp->cache_private); in umem_magazine_destroy()
1802 umem_slab_free(cp, buf); in umem_magazine_destroy()
1804 ASSERT(UMEM_MAGAZINE_VALID(cp, mp)); in umem_magazine_destroy()
1805 _umem_cache_free(cp->cache_magtype->mt_cache, mp); in umem_magazine_destroy()
1812 umem_depot_alloc(umem_cache_t *cp, umem_maglist_t *mlp) in umem_depot_alloc() argument
1822 if (mutex_trylock(&cp->cache_depot_lock) != 0) { in umem_depot_alloc()
1823 (void) mutex_lock(&cp->cache_depot_lock); in umem_depot_alloc()
1824 cp->cache_depot_contention++; in umem_depot_alloc()
1828 ASSERT(UMEM_MAGAZINE_VALID(cp, mp)); in umem_depot_alloc()
1835 (void) mutex_unlock(&cp->cache_depot_lock); in umem_depot_alloc()
1844 umem_depot_free(umem_cache_t *cp, umem_maglist_t *mlp, umem_magazine_t *mp) in umem_depot_free() argument
1846 (void) mutex_lock(&cp->cache_depot_lock); in umem_depot_free()
1847 ASSERT(UMEM_MAGAZINE_VALID(cp, mp)); in umem_depot_free()
1851 (void) mutex_unlock(&cp->cache_depot_lock); in umem_depot_free()
1858 umem_depot_ws_update(umem_cache_t *cp) in umem_depot_ws_update() argument
1860 (void) mutex_lock(&cp->cache_depot_lock); in umem_depot_ws_update()
1861 cp->cache_full.ml_reaplimit = cp->cache_full.ml_min; in umem_depot_ws_update()
1862 cp->cache_full.ml_min = cp->cache_full.ml_total; in umem_depot_ws_update()
1863 cp->cache_empty.ml_reaplimit = cp->cache_empty.ml_min; in umem_depot_ws_update()
1864 cp->cache_empty.ml_min = cp->cache_empty.ml_total; in umem_depot_ws_update()
1865 (void) mutex_unlock(&cp->cache_depot_lock); in umem_depot_ws_update()
1872 umem_depot_ws_reap(umem_cache_t *cp) in umem_depot_ws_reap() argument
1877 ASSERT(cp->cache_next == NULL || IN_REAP()); in umem_depot_ws_reap()
1879 reap = MIN(cp->cache_full.ml_reaplimit, cp->cache_full.ml_min); in umem_depot_ws_reap()
1880 while (reap-- && (mp = umem_depot_alloc(cp, &cp->cache_full)) != NULL) in umem_depot_ws_reap()
1881 umem_magazine_destroy(cp, mp, cp->cache_magtype->mt_magsize); in umem_depot_ws_reap()
1883 reap = MIN(cp->cache_empty.ml_reaplimit, cp->cache_empty.ml_min); in umem_depot_ws_reap()
1884 while (reap-- && (mp = umem_depot_alloc(cp, &cp->cache_empty)) != NULL) in umem_depot_ws_reap()
1885 umem_magazine_destroy(cp, mp, 0); in umem_depot_ws_reap()
1906 _umem_cache_alloc(umem_cache_t *cp, int umflag) in _umem_cache_alloc() argument
1914 ccp = UMEM_CPU_CACHE(cp, CPU(cp->cache_cpu_mask)); in _umem_cache_alloc()
1926 umem_cache_alloc_debug(cp, buf, umflag) == -1) { in _umem_cache_alloc()
1927 if (umem_alloc_retry(cp, umflag)) { in _umem_cache_alloc()
1954 fmp = umem_depot_alloc(cp, &cp->cache_full); in _umem_cache_alloc()
1957 umem_depot_free(cp, &cp->cache_empty, in _umem_cache_alloc()
1975 buf = umem_slab_alloc(cp, umflag); in _umem_cache_alloc()
1978 if (cp == &umem_null_cache) in _umem_cache_alloc()
1980 if (umem_alloc_retry(cp, umflag)) { in _umem_cache_alloc()
1987 if (cp->cache_flags & UMF_BUFTAG) { in _umem_cache_alloc()
1991 if (umem_cache_alloc_debug(cp, buf, umflag) == -1) { in _umem_cache_alloc()
1992 if (umem_alloc_retry(cp, umflag)) { in _umem_cache_alloc()
2005 if (cp->cache_constructor != NULL && in _umem_cache_alloc()
2006 cp->cache_constructor(buf, cp->cache_private, flags_nfatal) != 0) { in _umem_cache_alloc()
2007 atomic_add_64(&cp->cache_alloc_fail, 1); in _umem_cache_alloc()
2008 umem_slab_free(cp, buf); in _umem_cache_alloc()
2010 if (umem_alloc_retry(cp, umflag)) { in _umem_cache_alloc()
2024 _umem_cache_free(umem_cache_t *cp, void *buf) in _umem_cache_free() argument
2026 umem_cpu_cache_t *ccp = UMEM_CPU_CACHE(cp, CPU(cp->cache_cpu_mask)); in _umem_cache_free()
2031 if (umem_cache_free_debug(cp, buf) == -1) in _umem_cache_free()
2065 emp = umem_depot_alloc(cp, &cp->cache_empty); in _umem_cache_free()
2068 umem_depot_free(cp, &cp->cache_full, in _umem_cache_free()
2080 mtp = cp->cache_magtype; in _umem_cache_free()
2103 umem_depot_free(cp, &cp->cache_empty, emp); in _umem_cache_free()
2121 if (!(cp->cache_flags & UMF_BUFTAG) && cp->cache_destructor != NULL) in _umem_cache_free()
2122 cp->cache_destructor(buf, cp->cache_private); in _umem_cache_free()
2124 umem_slab_free(cp, buf); in _umem_cache_free()
2136 umem_cache_t *cp = umem_alloc_table[index]; in _umem_zalloc() local
2137 buf = _umem_cache_alloc(cp, umflag); in _umem_zalloc()
2139 if (cp->cache_flags & UMF_BUFTAG) { in _umem_zalloc()
2140 umem_buftag_t *btp = UMEM_BUFTAG(cp, buf); in _umem_zalloc()
2145 } else if (umem_alloc_retry(cp, umflag)) in _umem_zalloc()
2163 umem_cache_t *cp = umem_alloc_table[index]; in _umem_alloc() local
2164 buf = _umem_cache_alloc(cp, umflag); in _umem_alloc()
2165 if ((cp->cache_flags & UMF_BUFTAG) && buf != NULL) { in _umem_alloc()
2166 umem_buftag_t *btp = UMEM_BUFTAG(cp, buf); in _umem_alloc()
2170 if (buf == NULL && umem_alloc_retry(cp, umflag)) in _umem_alloc()
2228 umem_cache_t *cp = umem_alloc_table[index]; in _umem_free() local
2229 if (cp->cache_flags & UMF_BUFTAG) { in _umem_free()
2230 umem_buftag_t *btp = UMEM_BUFTAG(cp, buf); in _umem_free()
2234 umem_error(UMERR_DUPFREE, cp, buf); in _umem_free()
2239 umem_error(UMERR_BADSIZE, cp, buf); in _umem_free()
2241 umem_error(UMERR_REDZONE, cp, buf); in _umem_free()
2246 umem_error(UMERR_REDZONE, cp, buf); in _umem_free()
2251 _umem_cache_free(cp, buf); in _umem_free()
2298 umem_cache_reap(umem_cache_t *cp) in umem_cache_reap() argument
2307 if (cp->cache_reclaim != NULL) in umem_cache_reap()
2308 cp->cache_reclaim(cp->cache_private); in umem_cache_reap()
2310 umem_depot_ws_reap(cp); in umem_cache_reap()
2319 umem_cache_magazine_purge(umem_cache_t *cp) in umem_cache_magazine_purge() argument
2325 ASSERT(cp->cache_next == NULL || IN_UPDATE()); in umem_cache_magazine_purge()
2328 ccp = &cp->cache_cpu[cpu_seqid]; in umem_cache_magazine_purge()
2343 umem_magazine_destroy(cp, mp, rounds); in umem_cache_magazine_purge()
2345 umem_magazine_destroy(cp, pmp, prounds); in umem_cache_magazine_purge()
2353 umem_depot_ws_update(cp); in umem_cache_magazine_purge()
2354 umem_depot_ws_update(cp); in umem_cache_magazine_purge()
2356 umem_depot_ws_reap(cp); in umem_cache_magazine_purge()
2363 umem_cache_magazine_enable(umem_cache_t *cp) in umem_cache_magazine_enable() argument
2367 if (cp->cache_flags & UMF_NOMAGAZINE) in umem_cache_magazine_enable()
2371 umem_cpu_cache_t *ccp = &cp->cache_cpu[cpu_seqid]; in umem_cache_magazine_enable()
2373 ccp->cc_magsize = cp->cache_magtype->mt_magsize; in umem_cache_magazine_enable()
2392 umem_cache_magazine_resize(umem_cache_t *cp) in umem_cache_magazine_resize() argument
2394 umem_magtype_t *mtp = cp->cache_magtype; in umem_cache_magazine_resize()
2398 if (cp->cache_chunksize < mtp->mt_maxbuf) { in umem_cache_magazine_resize()
2399 umem_cache_magazine_purge(cp); in umem_cache_magazine_resize()
2400 (void) mutex_lock(&cp->cache_depot_lock); in umem_cache_magazine_resize()
2401 cp->cache_magtype = ++mtp; in umem_cache_magazine_resize()
2402 cp->cache_depot_contention_prev = in umem_cache_magazine_resize()
2403 cp->cache_depot_contention + INT_MAX; in umem_cache_magazine_resize()
2404 (void) mutex_unlock(&cp->cache_depot_lock); in umem_cache_magazine_resize()
2405 umem_cache_magazine_enable(cp); in umem_cache_magazine_resize()
2414 umem_hash_rescale(umem_cache_t *cp) in umem_hash_rescale() argument
2422 1 << (highbit(3 * cp->cache_buftotal + 4) - 2)); in umem_hash_rescale()
2423 old_size = cp->cache_hash_mask + 1; in umem_hash_rescale()
2434 (void) mutex_lock(&cp->cache_lock); in umem_hash_rescale()
2436 old_size = cp->cache_hash_mask + 1; in umem_hash_rescale()
2437 old_table = cp->cache_hash_table; in umem_hash_rescale()
2439 cp->cache_hash_mask = new_size - 1; in umem_hash_rescale()
2440 cp->cache_hash_table = new_table; in umem_hash_rescale()
2441 cp->cache_rescale++; in umem_hash_rescale()
2448 umem_bufctl_t **hash_bucket = UMEM_HASH(cp, addr); in umem_hash_rescale()
2455 (void) mutex_unlock(&cp->cache_lock); in umem_hash_rescale()
2465 umem_cache_update(umem_cache_t *cp) in umem_cache_update() argument
2475 (void) mutex_lock(&cp->cache_lock); in umem_cache_update()
2477 if ((cp->cache_flags & UMF_HASH) && in umem_cache_update()
2478 (cp->cache_buftotal > (cp->cache_hash_mask << 1) || in umem_cache_update()
2479 (cp->cache_buftotal < (cp->cache_hash_mask >> 1) && in umem_cache_update()
2480 cp->cache_hash_mask > UMEM_HASH_INITIAL))) in umem_cache_update()
2483 (void) mutex_unlock(&cp->cache_lock); in umem_cache_update()
2488 umem_depot_ws_update(cp); in umem_cache_update()
2494 (void) mutex_lock(&cp->cache_depot_lock); in umem_cache_update()
2496 if (cp->cache_chunksize < cp->cache_magtype->mt_maxbuf && in umem_cache_update()
2497 (int)(cp->cache_depot_contention - in umem_cache_update()
2498 cp->cache_depot_contention_prev) > umem_depot_contention) in umem_cache_update()
2501 cp->cache_depot_contention_prev = cp->cache_depot_contention; in umem_cache_update()
2503 (void) mutex_unlock(&cp->cache_depot_lock); in umem_cache_update()
2506 umem_add_update(cp, update_flags); in umem_cache_update()
2521 umem_cache_t *cp = umem_null_cache.cache_unext; in umem_process_updates() local
2523 cp->cache_uprev->cache_unext = cp->cache_unext; in umem_process_updates()
2524 cp->cache_unext->cache_uprev = cp->cache_uprev; in umem_process_updates()
2525 cp->cache_uprev = cp->cache_unext = NULL; in umem_process_updates()
2527 ASSERT(!(cp->cache_uflags & UMU_ACTIVE)); in umem_process_updates()
2529 while (cp->cache_uflags) { in umem_process_updates()
2530 int uflags = (cp->cache_uflags |= UMU_ACTIVE); in umem_process_updates()
2539 umem_hash_rescale(cp); in umem_process_updates()
2542 umem_cache_magazine_resize(cp); in umem_process_updates()
2545 umem_cache_reap(cp); in umem_process_updates()
2552 if (cp->cache_uflags & UMU_NOTIFY) { in umem_process_updates()
2556 cp->cache_uflags &= ~uflags; in umem_process_updates()
2657 umem_cache_t *cp, *cnext, *cprev; in umem_cache_create() local
2715 cp = vmem_xalloc(umem_cache_arena, csize, UMEM_CPU_CACHE_SIZE, phase, in umem_cache_create()
2718 if (cp == NULL) { in umem_cache_create()
2723 bzero(cp, csize); in umem_cache_create()
2729 cp->cache_flags = umem_flags | (cflags & UMF_DEBUG); in umem_cache_create()
2735 if (cp->cache_flags & UMF_LITE) { in umem_cache_create()
2739 cp->cache_flags |= UMF_BUFTAG; in umem_cache_create()
2740 cp->cache_flags &= ~(UMF_AUDIT | UMF_FIREWALL); in umem_cache_create()
2742 cp->cache_flags &= ~UMF_DEBUG; in umem_cache_create()
2746 if ((cflags & UMC_QCACHE) && (cp->cache_flags & UMF_AUDIT)) in umem_cache_create()
2747 cp->cache_flags |= UMF_NOMAGAZINE; in umem_cache_create()
2750 cp->cache_flags &= ~UMF_DEBUG; in umem_cache_create()
2753 cp->cache_flags &= ~UMF_TOUCH; in umem_cache_create()
2756 cp->cache_flags &= ~(UMF_AUDIT | UMF_FIREWALL); in umem_cache_create()
2759 cp->cache_flags |= UMF_NOMAGAZINE; in umem_cache_create()
2761 if ((cp->cache_flags & UMF_AUDIT) && !(cflags & UMC_NOTOUCH)) in umem_cache_create()
2762 cp->cache_flags |= UMF_REDZONE; in umem_cache_create()
2764 if ((cp->cache_flags & UMF_BUFTAG) && bufsize >= umem_minfirewall && in umem_cache_create()
2765 !(cp->cache_flags & UMF_LITE) && !(cflags & UMC_NOHASH)) in umem_cache_create()
2766 cp->cache_flags |= UMF_FIREWALL; in umem_cache_create()
2769 cp->cache_flags &= ~UMF_FIREWALL; in umem_cache_create()
2771 if (cp->cache_flags & UMF_FIREWALL) { in umem_cache_create()
2772 cp->cache_flags &= ~UMF_BUFTAG; in umem_cache_create()
2773 cp->cache_flags |= UMF_NOMAGAZINE; in umem_cache_create()
2781 (void) strncpy(cp->cache_name, name, sizeof (cp->cache_name) - 1); in umem_cache_create()
2782 cp->cache_bufsize = bufsize; in umem_cache_create()
2783 cp->cache_align = align; in umem_cache_create()
2784 cp->cache_constructor = constructor; in umem_cache_create()
2785 cp->cache_destructor = destructor; in umem_cache_create()
2786 cp->cache_reclaim = reclaim; in umem_cache_create()
2787 cp->cache_private = private; in umem_cache_create()
2788 cp->cache_arena = vmp; in umem_cache_create()
2789 cp->cache_cflags = cflags; in umem_cache_create()
2790 cp->cache_cpu_mask = umem_cpu_mask; in umem_cache_create()
2799 cp->cache_bufctl = chunksize - UMEM_ALIGN; in umem_cache_create()
2802 if (cp->cache_flags & UMF_BUFTAG) { in umem_cache_create()
2803 cp->cache_bufctl = chunksize; in umem_cache_create()
2804 cp->cache_buftag = chunksize; in umem_cache_create()
2808 if (cp->cache_flags & UMF_DEADBEEF) { in umem_cache_create()
2809 cp->cache_verify = MIN(cp->cache_buftag, umem_maxverify); in umem_cache_create()
2810 if (cp->cache_flags & UMF_LITE) in umem_cache_create()
2811 cp->cache_verify = MIN(cp->cache_verify, UMEM_ALIGN); in umem_cache_create()
2814 cp->cache_contents = MIN(cp->cache_bufctl, umem_content_maxsave); in umem_cache_create()
2816 cp->cache_chunksize = chunksize = P2ROUNDUP(chunksize, align); in umem_cache_create()
2827 cp->cache_slabsize = P2ROUNDUP(chunksize, vmp->vm_quantum); in umem_cache_create()
2828 cp->cache_mincolor = cp->cache_slabsize - chunksize; in umem_cache_create()
2829 cp->cache_maxcolor = cp->cache_mincolor; in umem_cache_create()
2830 cp->cache_flags |= UMF_HASH; in umem_cache_create()
2831 ASSERT(!(cp->cache_flags & UMF_BUFTAG)); in umem_cache_create()
2833 !(cp->cache_flags & UMF_AUDIT) && in umem_cache_create()
2835 cp->cache_slabsize = vmp->vm_quantum; in umem_cache_create()
2836 cp->cache_mincolor = 0; in umem_cache_create()
2837 cp->cache_maxcolor = in umem_cache_create()
2838 (cp->cache_slabsize - sizeof (umem_slab_t)) % chunksize; in umem_cache_create()
2840 if (chunksize + sizeof (umem_slab_t) > cp->cache_slabsize) { in umem_cache_create()
2844 ASSERT(!(cp->cache_flags & UMF_AUDIT)); in umem_cache_create()
2873 cp->cache_slabsize = bestfit; in umem_cache_create()
2874 cp->cache_mincolor = 0; in umem_cache_create()
2875 cp->cache_maxcolor = bestfit % chunksize; in umem_cache_create()
2876 cp->cache_flags |= UMF_HASH; in umem_cache_create()
2879 if (cp->cache_flags & UMF_HASH) { in umem_cache_create()
2881 cp->cache_bufctl_cache = (cp->cache_flags & UMF_AUDIT) ? in umem_cache_create()
2885 if (cp->cache_maxcolor >= vmp->vm_quantum) in umem_cache_create()
2886 cp->cache_maxcolor = vmp->vm_quantum - 1; in umem_cache_create()
2888 cp->cache_color = cp->cache_mincolor; in umem_cache_create()
2893 (void) mutex_init(&cp->cache_lock, USYNC_THREAD, NULL); in umem_cache_create()
2895 cp->cache_freelist = &cp->cache_nullslab; in umem_cache_create()
2896 cp->cache_nullslab.slab_cache = cp; in umem_cache_create()
2897 cp->cache_nullslab.slab_refcnt = -1; in umem_cache_create()
2898 cp->cache_nullslab.slab_next = &cp->cache_nullslab; in umem_cache_create()
2899 cp->cache_nullslab.slab_prev = &cp->cache_nullslab; in umem_cache_create()
2901 if (cp->cache_flags & UMF_HASH) { in umem_cache_create()
2902 cp->cache_hash_table = vmem_alloc(umem_hash_arena, in umem_cache_create()
2904 if (cp->cache_hash_table == NULL) { in umem_cache_create()
2908 bzero(cp->cache_hash_table, in umem_cache_create()
2910 cp->cache_hash_mask = UMEM_HASH_INITIAL - 1; in umem_cache_create()
2911 cp->cache_hash_shift = highbit((ulong_t)chunksize) - 1; in umem_cache_create()
2917 (void) mutex_init(&cp->cache_depot_lock, USYNC_THREAD, NULL); in umem_cache_create()
2922 cp->cache_magtype = mtp; in umem_cache_create()
2928 umem_cpu_cache_t *ccp = &cp->cache_cpu[cpu_seqid]; in umem_cache_create()
2930 ccp->cc_flags = cp->cache_flags; in umem_cache_create()
2940 cp->cache_next = cnext = &umem_null_cache; in umem_cache_create()
2941 cp->cache_prev = cprev = umem_null_cache.cache_prev; in umem_cache_create()
2942 cnext->cache_prev = cp; in umem_cache_create()
2943 cprev->cache_next = cp; in umem_cache_create()
2947 umem_cache_magazine_enable(cp); in umem_cache_create()
2949 return (cp); in umem_cache_create()
2952 (void) mutex_destroy(&cp->cache_lock); in umem_cache_create()
2954 vmem_xfree(umem_cache_arena, cp, csize); in umem_cache_create()
2959 umem_cache_destroy(umem_cache_t *cp) in umem_cache_destroy() argument
2969 cp->cache_prev->cache_next = cp->cache_next; in umem_cache_destroy()
2970 cp->cache_next->cache_prev = cp->cache_prev; in umem_cache_destroy()
2971 cp->cache_prev = cp->cache_next = NULL; in umem_cache_destroy()
2974 umem_remove_updates(cp); in umem_cache_destroy()
2976 umem_cache_magazine_purge(cp); in umem_cache_destroy()
2978 (void) mutex_lock(&cp->cache_lock); in umem_cache_destroy()
2979 if (cp->cache_buftotal != 0) in umem_cache_destroy()
2981 cp->cache_name, (void *)cp); in umem_cache_destroy()
2982 cp->cache_reclaim = NULL; in umem_cache_destroy()
2988 cp->cache_constructor = (umem_constructor_t *)1; in umem_cache_destroy()
2989 cp->cache_destructor = (umem_destructor_t *)2; in umem_cache_destroy()
2990 (void) mutex_unlock(&cp->cache_lock); in umem_cache_destroy()
2992 if (cp->cache_hash_table != NULL) in umem_cache_destroy()
2993 vmem_free(umem_hash_arena, cp->cache_hash_table, in umem_cache_destroy()
2994 (cp->cache_hash_mask + 1) * sizeof (void *)); in umem_cache_destroy()
2997 (void) mutex_destroy(&cp->cache_cpu[cpu_seqid].cc_lock); in umem_cache_destroy()
2999 (void) mutex_destroy(&cp->cache_depot_lock); in umem_cache_destroy()
3000 (void) mutex_destroy(&cp->cache_lock); in umem_cache_destroy()
3002 vmem_free(umem_cache_arena, cp, UMEM_CACHE_SIZE(umem_max_ncpus)); in umem_cache_destroy()
3098 umem_cache_t *cp; in umem_cache_tmem_cleanup() local
3101 cp = umem_alloc_table[(size - 1) >> UMEM_ALIGN_SHIFT]; in umem_cache_tmem_cleanup()
3102 _umem_cache_free(cp, buf); in umem_cache_tmem_cleanup()
3110 umem_cache_t *cp; in umem_cache_init() local
3212 cp = umem_cache_create(name, cache_size, align, in umem_cache_init()
3214 if (cp == NULL) in umem_cache_init()
3217 umem_alloc_caches[i] = cp; in umem_cache_init()
3245 cp = umem_alloc_caches[i]; in umem_cache_init()
3248 umem_alloc_table[(size - 1) >> UMEM_ALIGN_SHIFT] = cp; in umem_cache_init()