1fa9e4066Sahrens /*
2fa9e4066Sahrens * CDDL HEADER START
3fa9e4066Sahrens *
4fa9e4066Sahrens * The contents of this file are subject to the terms of the
591ebeef5Sahrens * Common Development and Distribution License (the "License").
691ebeef5Sahrens * You may not use this file except in compliance with the License.
7fa9e4066Sahrens *
8fa9e4066Sahrens * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9fa9e4066Sahrens * or http://www.opensolaris.org/os/licensing.
10fa9e4066Sahrens * See the License for the specific language governing permissions
11fa9e4066Sahrens * and limitations under the License.
12fa9e4066Sahrens *
13fa9e4066Sahrens * When distributing Covered Code, include this CDDL HEADER in each
14fa9e4066Sahrens * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15fa9e4066Sahrens * If applicable, add the following below this CDDL HEADER, with the
16fa9e4066Sahrens * fields enclosed by brackets "[]" replaced with your own identifying
17fa9e4066Sahrens * information: Portions Copyright [yyyy] [name of copyright owner]
18fa9e4066Sahrens *
19fa9e4066Sahrens * CDDL HEADER END
20fa9e4066Sahrens */
21fa9e4066Sahrens /*
223f9d6ad7SLin Ling * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
230f7643c7SGeorge Wilson * Copyright (c) 2012, 2015 by Delphix. All rights reserved.
24fa9e4066Sahrens */
25fa9e4066Sahrens
26fa9e4066Sahrens #include <sys/zfs_context.h>
27fa9e4066Sahrens #include <sys/refcount.h>
28fa9e4066Sahrens
29744947dcSTom Erickson #ifdef ZFS_DEBUG
30fa9e4066Sahrens
31fa9e4066Sahrens #ifdef _KERNEL
32fa9e4066Sahrens int reference_tracking_enable = FALSE; /* runs out of memory too easily */
33fa9e4066Sahrens #else
34fa9e4066Sahrens int reference_tracking_enable = TRUE;
35fa9e4066Sahrens #endif
363b2aab18SMatthew Ahrens int reference_history = 3; /* tunable */
37fa9e4066Sahrens
38fa9e4066Sahrens static kmem_cache_t *reference_cache;
39fa9e4066Sahrens static kmem_cache_t *reference_history_cache;
40fa9e4066Sahrens
41fa9e4066Sahrens void
zfs_refcount_init(void)42e914ace2STim Schumacher zfs_refcount_init(void)
43fa9e4066Sahrens {
44fa9e4066Sahrens reference_cache = kmem_cache_create("reference_cache",
45fa9e4066Sahrens sizeof (reference_t), 0, NULL, NULL, NULL, NULL, NULL, 0);
46fa9e4066Sahrens
47fa9e4066Sahrens reference_history_cache = kmem_cache_create("reference_history_cache",
48fa9e4066Sahrens sizeof (uint64_t), 0, NULL, NULL, NULL, NULL, NULL, 0);
49fa9e4066Sahrens }
50fa9e4066Sahrens
51fa9e4066Sahrens void
zfs_refcount_fini(void)52e914ace2STim Schumacher zfs_refcount_fini(void)
53fa9e4066Sahrens {
54fa9e4066Sahrens kmem_cache_destroy(reference_cache);
55fa9e4066Sahrens kmem_cache_destroy(reference_history_cache);
56fa9e4066Sahrens }
57fa9e4066Sahrens
58*9a8c5287SAlexander Motin static int
zfs_refcount_compare(const void * x1,const void * x2)59*9a8c5287SAlexander Motin zfs_refcount_compare(const void *x1, const void *x2)
60*9a8c5287SAlexander Motin {
61*9a8c5287SAlexander Motin const reference_t *r1 = (const reference_t *)x1;
62*9a8c5287SAlexander Motin const reference_t *r2 = (const reference_t *)x2;
63*9a8c5287SAlexander Motin
64*9a8c5287SAlexander Motin int cmp1 = TREE_CMP(r1->ref_holder, r2->ref_holder);
65*9a8c5287SAlexander Motin int cmp2 = TREE_CMP(r1->ref_number, r2->ref_number);
66*9a8c5287SAlexander Motin int cmp = cmp1 ? cmp1 : cmp2;
67*9a8c5287SAlexander Motin return ((cmp || r1->ref_search) ? cmp : TREE_PCMP(r1, r2));
68*9a8c5287SAlexander Motin }
69*9a8c5287SAlexander Motin
70fa9e4066Sahrens void
zfs_refcount_create(zfs_refcount_t * rc)71e914ace2STim Schumacher zfs_refcount_create(zfs_refcount_t *rc)
72fa9e4066Sahrens {
7391ebeef5Sahrens mutex_init(&rc->rc_mtx, NULL, MUTEX_DEFAULT, NULL);
74*9a8c5287SAlexander Motin avl_create(&rc->rc_tree, zfs_refcount_compare, sizeof (reference_t),
75*9a8c5287SAlexander Motin offsetof(reference_t, ref_link.a));
76fa9e4066Sahrens list_create(&rc->rc_removed, sizeof (reference_t),
77*9a8c5287SAlexander Motin offsetof(reference_t, ref_link.l));
7891ebeef5Sahrens rc->rc_count = 0;
7991ebeef5Sahrens rc->rc_removed_count = 0;
803b2aab18SMatthew Ahrens rc->rc_tracked = reference_tracking_enable;
813b2aab18SMatthew Ahrens }
823b2aab18SMatthew Ahrens
830f7643c7SGeorge Wilson void
zfs_refcount_create_tracked(zfs_refcount_t * rc)84e914ace2STim Schumacher zfs_refcount_create_tracked(zfs_refcount_t *rc)
850f7643c7SGeorge Wilson {
86e914ace2STim Schumacher zfs_refcount_create(rc);
870f7643c7SGeorge Wilson rc->rc_tracked = B_TRUE;
880f7643c7SGeorge Wilson }
890f7643c7SGeorge Wilson
903b2aab18SMatthew Ahrens void
zfs_refcount_create_untracked(zfs_refcount_t * rc)91e914ace2STim Schumacher zfs_refcount_create_untracked(zfs_refcount_t *rc)
923b2aab18SMatthew Ahrens {
93e914ace2STim Schumacher zfs_refcount_create(rc);
943b2aab18SMatthew Ahrens rc->rc_tracked = B_FALSE;
95fa9e4066Sahrens }
96fa9e4066Sahrens
97fa9e4066Sahrens void
zfs_refcount_destroy_many(zfs_refcount_t * rc,uint64_t number)98e914ace2STim Schumacher zfs_refcount_destroy_many(zfs_refcount_t *rc, uint64_t number)
99fa9e4066Sahrens {
100fa9e4066Sahrens reference_t *ref;
101*9a8c5287SAlexander Motin void *cookie = NULL;
102fa9e4066Sahrens
103*9a8c5287SAlexander Motin ASSERT3U(rc->rc_count, ==, number);
104*9a8c5287SAlexander Motin while ((ref = avl_destroy_nodes(&rc->rc_tree, &cookie)) != NULL)
105fa9e4066Sahrens kmem_cache_free(reference_cache, ref);
106*9a8c5287SAlexander Motin avl_destroy(&rc->rc_tree);
107fa9e4066Sahrens
108*9a8c5287SAlexander Motin while ((ref = list_remove_head(&rc->rc_removed))) {
109fa9e4066Sahrens kmem_cache_free(reference_history_cache, ref->ref_removed);
110fa9e4066Sahrens kmem_cache_free(reference_cache, ref);
111fa9e4066Sahrens }
112fa9e4066Sahrens list_destroy(&rc->rc_removed);
113fa9e4066Sahrens mutex_destroy(&rc->rc_mtx);
114fa9e4066Sahrens }
115fa9e4066Sahrens
116fa9e4066Sahrens void
zfs_refcount_destroy(zfs_refcount_t * rc)117e914ace2STim Schumacher zfs_refcount_destroy(zfs_refcount_t *rc)
118fa9e4066Sahrens {
119e914ace2STim Schumacher zfs_refcount_destroy_many(rc, 0);
120fa9e4066Sahrens }
121fa9e4066Sahrens
122fa9e4066Sahrens int
zfs_refcount_is_zero(zfs_refcount_t * rc)123e914ace2STim Schumacher zfs_refcount_is_zero(zfs_refcount_t *rc)
124fa9e4066Sahrens {
125*9a8c5287SAlexander Motin return (zfs_refcount_count(rc) == 0);
126fa9e4066Sahrens }
127fa9e4066Sahrens
128fa9e4066Sahrens int64_t
zfs_refcount_count(zfs_refcount_t * rc)129e914ace2STim Schumacher zfs_refcount_count(zfs_refcount_t *rc)
130fa9e4066Sahrens {
131*9a8c5287SAlexander Motin return (atomic_load_64(&rc->rc_count));
132fa9e4066Sahrens }
133fa9e4066Sahrens
134fa9e4066Sahrens int64_t
zfs_refcount_add_many(zfs_refcount_t * rc,uint64_t number,const void * holder)135ecd18decSAlexander Motin zfs_refcount_add_many(zfs_refcount_t *rc, uint64_t number, const void *holder)
136fa9e4066Sahrens {
137*9a8c5287SAlexander Motin reference_t *ref;
138fa9e4066Sahrens int64_t count;
139fa9e4066Sahrens
140*9a8c5287SAlexander Motin if (likely(!rc->rc_tracked)) {
141*9a8c5287SAlexander Motin count = atomic_add_64_nv(&(rc)->rc_count, number);
142*9a8c5287SAlexander Motin ASSERT3U(count, >=, number);
143*9a8c5287SAlexander Motin return (count);
144fa9e4066Sahrens }
145*9a8c5287SAlexander Motin
146*9a8c5287SAlexander Motin ref = kmem_cache_alloc(reference_cache, KM_SLEEP);
147*9a8c5287SAlexander Motin ref->ref_holder = holder;
148*9a8c5287SAlexander Motin ref->ref_number = number;
149*9a8c5287SAlexander Motin ref->ref_search = B_FALSE;
150fa9e4066Sahrens mutex_enter(&rc->rc_mtx);
151*9a8c5287SAlexander Motin avl_add(&rc->rc_tree, ref);
152fa9e4066Sahrens rc->rc_count += number;
153fa9e4066Sahrens count = rc->rc_count;
154fa9e4066Sahrens mutex_exit(&rc->rc_mtx);
155fa9e4066Sahrens
156fa9e4066Sahrens return (count);
157fa9e4066Sahrens }
158fa9e4066Sahrens
159fa9e4066Sahrens int64_t
zfs_refcount_add(zfs_refcount_t * rc,const void * holder)160ecd18decSAlexander Motin zfs_refcount_add(zfs_refcount_t *rc, const void *holder)
161fa9e4066Sahrens {
162e914ace2STim Schumacher return (zfs_refcount_add_many(rc, 1, holder));
163fa9e4066Sahrens }
164fa9e4066Sahrens
165ecd18decSAlexander Motin void
zfs_refcount_add_few(zfs_refcount_t * rc,uint64_t number,const void * holder)166ecd18decSAlexander Motin zfs_refcount_add_few(zfs_refcount_t *rc, uint64_t number, const void *holder)
167ecd18decSAlexander Motin {
168*9a8c5287SAlexander Motin if (likely(!rc->rc_tracked)) {
169ecd18decSAlexander Motin (void) zfs_refcount_add_many(rc, number, holder);
170ecd18decSAlexander Motin } else {
171ecd18decSAlexander Motin for (; number > 0; number--)
172ecd18decSAlexander Motin (void) zfs_refcount_add(rc, holder);
173ecd18decSAlexander Motin }
174ecd18decSAlexander Motin }
175ecd18decSAlexander Motin
176fa9e4066Sahrens int64_t
zfs_refcount_remove_many(zfs_refcount_t * rc,uint64_t number,const void * holder)177ecd18decSAlexander Motin zfs_refcount_remove_many(zfs_refcount_t *rc, uint64_t number,
178ecd18decSAlexander Motin const void *holder)
179fa9e4066Sahrens {
180*9a8c5287SAlexander Motin reference_t *ref, s;
181fa9e4066Sahrens int64_t count;
182fa9e4066Sahrens
183*9a8c5287SAlexander Motin if (likely(!rc->rc_tracked)) {
184*9a8c5287SAlexander Motin count = atomic_add_64_nv(&(rc)->rc_count, -number);
185*9a8c5287SAlexander Motin ASSERT3S(count, >=, 0);
186fa9e4066Sahrens return (count);
187fa9e4066Sahrens }
188fa9e4066Sahrens
189*9a8c5287SAlexander Motin s.ref_holder = holder;
190*9a8c5287SAlexander Motin s.ref_number = number;
191*9a8c5287SAlexander Motin s.ref_search = B_TRUE;
192*9a8c5287SAlexander Motin mutex_enter(&rc->rc_mtx);
193*9a8c5287SAlexander Motin ASSERT3U(rc->rc_count, >=, number);
194*9a8c5287SAlexander Motin ref = avl_find(&rc->rc_tree, &s, NULL);
195*9a8c5287SAlexander Motin if (unlikely(ref == NULL)) {
196*9a8c5287SAlexander Motin panic("No such hold %p with count %" PRIu64 " on refcount %llx",
197*9a8c5287SAlexander Motin holder, number, (u_longlong_t)(uintptr_t)rc);
198*9a8c5287SAlexander Motin }
199*9a8c5287SAlexander Motin avl_remove(&rc->rc_tree, ref);
200*9a8c5287SAlexander Motin if (reference_history > 0) {
201*9a8c5287SAlexander Motin ref->ref_removed = kmem_cache_alloc(reference_history_cache,
202*9a8c5287SAlexander Motin KM_SLEEP);
203*9a8c5287SAlexander Motin list_insert_head(&rc->rc_removed, ref);
204*9a8c5287SAlexander Motin if (rc->rc_removed_count >= reference_history) {
205*9a8c5287SAlexander Motin ref = list_remove_tail(&rc->rc_removed);
206*9a8c5287SAlexander Motin kmem_cache_free(reference_history_cache,
207*9a8c5287SAlexander Motin ref->ref_removed);
208*9a8c5287SAlexander Motin kmem_cache_free(reference_cache, ref);
209*9a8c5287SAlexander Motin } else {
210*9a8c5287SAlexander Motin rc->rc_removed_count++;
211fa9e4066Sahrens }
212*9a8c5287SAlexander Motin } else {
213*9a8c5287SAlexander Motin kmem_cache_free(reference_cache, ref);
214fa9e4066Sahrens }
215*9a8c5287SAlexander Motin rc->rc_count -= number;
216*9a8c5287SAlexander Motin count = rc->rc_count;
217*9a8c5287SAlexander Motin mutex_exit(&rc->rc_mtx);
218*9a8c5287SAlexander Motin return (count);
219fa9e4066Sahrens }
220fa9e4066Sahrens
221fa9e4066Sahrens int64_t
zfs_refcount_remove(zfs_refcount_t * rc,const void * holder)222ecd18decSAlexander Motin zfs_refcount_remove(zfs_refcount_t *rc, const void *holder)
223fa9e4066Sahrens {
224e914ace2STim Schumacher return (zfs_refcount_remove_many(rc, 1, holder));
225fa9e4066Sahrens }
226fa9e4066Sahrens
227ecd18decSAlexander Motin void
zfs_refcount_remove_few(zfs_refcount_t * rc,uint64_t number,const void * holder)228ecd18decSAlexander Motin zfs_refcount_remove_few(zfs_refcount_t *rc, uint64_t number, const void *holder)
229ecd18decSAlexander Motin {
230*9a8c5287SAlexander Motin if (likely(!rc->rc_tracked)) {
231ecd18decSAlexander Motin (void) zfs_refcount_remove_many(rc, number, holder);
232ecd18decSAlexander Motin } else {
233ecd18decSAlexander Motin for (; number > 0; number--)
234ecd18decSAlexander Motin (void) zfs_refcount_remove(rc, holder);
235ecd18decSAlexander Motin }
236ecd18decSAlexander Motin }
237ecd18decSAlexander Motin
238744947dcSTom Erickson void
zfs_refcount_transfer(zfs_refcount_t * dst,zfs_refcount_t * src)239e914ace2STim Schumacher zfs_refcount_transfer(zfs_refcount_t *dst, zfs_refcount_t *src)
240744947dcSTom Erickson {
241*9a8c5287SAlexander Motin avl_tree_t tree;
242*9a8c5287SAlexander Motin list_t removed;
243*9a8c5287SAlexander Motin reference_t *ref;
244*9a8c5287SAlexander Motin void *cookie = NULL;
245*9a8c5287SAlexander Motin uint64_t count;
246*9a8c5287SAlexander Motin uint_t removed_count;
247744947dcSTom Erickson
248*9a8c5287SAlexander Motin avl_create(&tree, zfs_refcount_compare, sizeof (reference_t),
249*9a8c5287SAlexander Motin offsetof(reference_t, ref_link.a));
250744947dcSTom Erickson list_create(&removed, sizeof (reference_t),
251*9a8c5287SAlexander Motin offsetof(reference_t, ref_link.l));
252744947dcSTom Erickson
253744947dcSTom Erickson mutex_enter(&src->rc_mtx);
254744947dcSTom Erickson count = src->rc_count;
255744947dcSTom Erickson removed_count = src->rc_removed_count;
256744947dcSTom Erickson src->rc_count = 0;
257744947dcSTom Erickson src->rc_removed_count = 0;
258*9a8c5287SAlexander Motin avl_swap(&tree, &src->rc_tree);
259744947dcSTom Erickson list_move_tail(&removed, &src->rc_removed);
260744947dcSTom Erickson mutex_exit(&src->rc_mtx);
261744947dcSTom Erickson
262744947dcSTom Erickson mutex_enter(&dst->rc_mtx);
263744947dcSTom Erickson dst->rc_count += count;
264744947dcSTom Erickson dst->rc_removed_count += removed_count;
265*9a8c5287SAlexander Motin if (avl_is_empty(&dst->rc_tree))
266*9a8c5287SAlexander Motin avl_swap(&dst->rc_tree, &tree);
267*9a8c5287SAlexander Motin else while ((ref = avl_destroy_nodes(&tree, &cookie)) != NULL)
268*9a8c5287SAlexander Motin avl_add(&dst->rc_tree, ref);
269744947dcSTom Erickson list_move_tail(&dst->rc_removed, &removed);
270744947dcSTom Erickson mutex_exit(&dst->rc_mtx);
271744947dcSTom Erickson
272*9a8c5287SAlexander Motin avl_destroy(&tree);
273744947dcSTom Erickson list_destroy(&removed);
274744947dcSTom Erickson }
275744947dcSTom Erickson
276dcbf3bd6SGeorge Wilson void
zfs_refcount_transfer_ownership_many(zfs_refcount_t * rc,uint64_t number,const void * current_holder,const void * new_holder)277eb633035STom Caputi zfs_refcount_transfer_ownership_many(zfs_refcount_t *rc, uint64_t number,
278*9a8c5287SAlexander Motin const void *current_holder, const void *new_holder)
279dcbf3bd6SGeorge Wilson {
280*9a8c5287SAlexander Motin reference_t *ref, s;
281dcbf3bd6SGeorge Wilson
282*9a8c5287SAlexander Motin if (likely(!rc->rc_tracked))
283dcbf3bd6SGeorge Wilson return;
284dcbf3bd6SGeorge Wilson
285*9a8c5287SAlexander Motin s.ref_holder = current_holder;
286*9a8c5287SAlexander Motin s.ref_number = number;
287*9a8c5287SAlexander Motin s.ref_search = B_TRUE;
288*9a8c5287SAlexander Motin mutex_enter(&rc->rc_mtx);
289*9a8c5287SAlexander Motin ref = avl_find(&rc->rc_tree, &s, NULL);
290*9a8c5287SAlexander Motin ASSERT3P(ref, !=, NULL);
291*9a8c5287SAlexander Motin ref->ref_holder = new_holder;
292*9a8c5287SAlexander Motin avl_update(&rc->rc_tree, ref);
293dcbf3bd6SGeorge Wilson mutex_exit(&rc->rc_mtx);
294dcbf3bd6SGeorge Wilson }
2950f7643c7SGeorge Wilson
296eb633035STom Caputi void
zfs_refcount_transfer_ownership(zfs_refcount_t * rc,const void * current_holder,const void * new_holder)297*9a8c5287SAlexander Motin zfs_refcount_transfer_ownership(zfs_refcount_t *rc, const void *current_holder,
298*9a8c5287SAlexander Motin const void *new_holder)
299eb633035STom Caputi {
300eb633035STom Caputi zfs_refcount_transfer_ownership_many(rc, 1, current_holder,
301eb633035STom Caputi new_holder);
302eb633035STom Caputi }
303eb633035STom Caputi
3040f7643c7SGeorge Wilson /*
3050f7643c7SGeorge Wilson * If tracking is enabled, return true if a reference exists that matches
3060f7643c7SGeorge Wilson * the "holder" tag. If tracking is disabled, then return true if a reference
3070f7643c7SGeorge Wilson * might be held.
3080f7643c7SGeorge Wilson */
3090f7643c7SGeorge Wilson boolean_t
zfs_refcount_held(zfs_refcount_t * rc,const void * holder)310*9a8c5287SAlexander Motin zfs_refcount_held(zfs_refcount_t *rc, const void *holder)
3110f7643c7SGeorge Wilson {
312*9a8c5287SAlexander Motin reference_t *ref, s;
313*9a8c5287SAlexander Motin avl_index_t idx;
314*9a8c5287SAlexander Motin boolean_t res;
3150f7643c7SGeorge Wilson
316*9a8c5287SAlexander Motin if (likely(!rc->rc_tracked))
317*9a8c5287SAlexander Motin return (zfs_refcount_count(rc) > 0);
3180f7643c7SGeorge Wilson
319*9a8c5287SAlexander Motin s.ref_holder = holder;
320*9a8c5287SAlexander Motin s.ref_number = 0;
321*9a8c5287SAlexander Motin s.ref_search = B_TRUE;
322*9a8c5287SAlexander Motin mutex_enter(&rc->rc_mtx);
323*9a8c5287SAlexander Motin ref = avl_find(&rc->rc_tree, &s, &idx);
324*9a8c5287SAlexander Motin if (likely(ref == NULL))
325*9a8c5287SAlexander Motin ref = avl_nearest(&rc->rc_tree, idx, AVL_AFTER);
326*9a8c5287SAlexander Motin res = ref && ref->ref_holder == holder;
3270f7643c7SGeorge Wilson mutex_exit(&rc->rc_mtx);
328*9a8c5287SAlexander Motin return (res);
3290f7643c7SGeorge Wilson }
3300f7643c7SGeorge Wilson
3310f7643c7SGeorge Wilson /*
3320f7643c7SGeorge Wilson * If tracking is enabled, return true if a reference does not exist that
3330f7643c7SGeorge Wilson * matches the "holder" tag. If tracking is disabled, always return true
3340f7643c7SGeorge Wilson * since the reference might not be held.
3350f7643c7SGeorge Wilson */
3360f7643c7SGeorge Wilson boolean_t
zfs_refcount_not_held(zfs_refcount_t * rc,const void * holder)337*9a8c5287SAlexander Motin zfs_refcount_not_held(zfs_refcount_t *rc, const void *holder)
3380f7643c7SGeorge Wilson {
339*9a8c5287SAlexander Motin reference_t *ref, s;
340*9a8c5287SAlexander Motin avl_index_t idx;
341*9a8c5287SAlexander Motin boolean_t res;
3420f7643c7SGeorge Wilson
343*9a8c5287SAlexander Motin if (likely(!rc->rc_tracked))
3440f7643c7SGeorge Wilson return (B_TRUE);
3450f7643c7SGeorge Wilson
346*9a8c5287SAlexander Motin mutex_enter(&rc->rc_mtx);
347*9a8c5287SAlexander Motin s.ref_holder = holder;
348*9a8c5287SAlexander Motin s.ref_number = 0;
349*9a8c5287SAlexander Motin s.ref_search = B_TRUE;
350*9a8c5287SAlexander Motin ref = avl_find(&rc->rc_tree, &s, &idx);
351*9a8c5287SAlexander Motin if (likely(ref == NULL))
352*9a8c5287SAlexander Motin ref = avl_nearest(&rc->rc_tree, idx, AVL_AFTER);
353*9a8c5287SAlexander Motin res = ref == NULL || ref->ref_holder != holder;
3540f7643c7SGeorge Wilson mutex_exit(&rc->rc_mtx);
355*9a8c5287SAlexander Motin return (res);
3560f7643c7SGeorge Wilson }
357744947dcSTom Erickson #endif /* ZFS_DEBUG */
358