1fa9e406ahrens/*
2fa9e406ahrens * CDDL HEADER START
3fa9e406ahrens *
4fa9e406ahrens * The contents of this file are subject to the terms of the
591ebeefahrens * Common Development and Distribution License (the "License").
691ebeefahrens * You may not use this file except in compliance with the License.
7fa9e406ahrens *
8fa9e406ahrens * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9fa9e406ahrens * or http://www.opensolaris.org/os/licensing.
10fa9e406ahrens * See the License for the specific language governing permissions
11fa9e406ahrens * and limitations under the License.
12fa9e406ahrens *
13fa9e406ahrens * When distributing Covered Code, include this CDDL HEADER in each
14fa9e406ahrens * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15fa9e406ahrens * If applicable, add the following below this CDDL HEADER, with the
16fa9e406ahrens * fields enclosed by brackets "[]" replaced with your own identifying
17fa9e406ahrens * information: Portions Copyright [yyyy] [name of copyright owner]
18fa9e406ahrens *
19fa9e406ahrens * CDDL HEADER END
20fa9e406ahrens */
21fa9e406ahrens/*
223f9d6adLin Ling * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
230f7643cGeorge Wilson * Copyright (c) 2012, 2015 by Delphix. All rights reserved.
24fa9e406ahrens */
25fa9e406ahrens
26fa9e406ahrens#include <sys/zfs_context.h>
27fa9e406ahrens#include <sys/refcount.h>
28fa9e406ahrens
29744947dTom Erickson#ifdef	ZFS_DEBUG
30fa9e406ahrens
31fa9e406ahrens#ifdef _KERNEL
32fa9e406ahrensint reference_tracking_enable = FALSE; /* runs out of memory too easily */
33fa9e406ahrens#else
34fa9e406ahrensint reference_tracking_enable = TRUE;
35fa9e406ahrens#endif
363b2aab1Matthew Ahrensint reference_history = 3; /* tunable */
37fa9e406ahrens
38fa9e406ahrensstatic kmem_cache_t *reference_cache;
39fa9e406ahrensstatic kmem_cache_t *reference_history_cache;
40fa9e406ahrens
41fa9e406ahrensvoid
42e914aceTim Schumacherzfs_refcount_init(void)
43fa9e406ahrens{
44fa9e406ahrens	reference_cache = kmem_cache_create("reference_cache",
45fa9e406ahrens	    sizeof (reference_t), 0, NULL, NULL, NULL, NULL, NULL, 0);
46fa9e406ahrens
47fa9e406ahrens	reference_history_cache = kmem_cache_create("reference_history_cache",
48fa9e406ahrens	    sizeof (uint64_t), 0, NULL, NULL, NULL, NULL, NULL, 0);
49fa9e406ahrens}
50fa9e406ahrens
51fa9e406ahrensvoid
52e914aceTim Schumacherzfs_refcount_fini(void)
53fa9e406ahrens{
54fa9e406ahrens	kmem_cache_destroy(reference_cache);
55fa9e406ahrens	kmem_cache_destroy(reference_history_cache);
56fa9e406ahrens}
57fa9e406ahrens
58fa9e406ahrensvoid
59e914aceTim Schumacherzfs_refcount_create(zfs_refcount_t *rc)
60fa9e406ahrens{
6191ebeefahrens	mutex_init(&rc->rc_mtx, NULL, MUTEX_DEFAULT, NULL);
62fa9e406ahrens	list_create(&rc->rc_list, sizeof (reference_t),
63fa9e406ahrens	    offsetof(reference_t, ref_link));
64fa9e406ahrens	list_create(&rc->rc_removed, sizeof (reference_t),
65fa9e406ahrens	    offsetof(reference_t, ref_link));
6691ebeefahrens	rc->rc_count = 0;
6791ebeefahrens	rc->rc_removed_count = 0;
683b2aab1Matthew Ahrens	rc->rc_tracked = reference_tracking_enable;
693b2aab1Matthew Ahrens}
703b2aab1Matthew Ahrens
713b2aab1Matthew Ahrensvoid
72e914aceTim Schumacherzfs_refcount_create_tracked(zfs_refcount_t *rc)
730f7643cGeorge Wilson{
74e914aceTim Schumacher	zfs_refcount_create(rc);
750f7643cGeorge Wilson	rc->rc_tracked = B_TRUE;
760f7643cGeorge Wilson}
770f7643cGeorge Wilson
780f7643cGeorge Wilsonvoid
79e914aceTim Schumacherzfs_refcount_create_untracked(zfs_refcount_t *rc)
803b2aab1Matthew Ahrens{
81e914aceTim Schumacher	zfs_refcount_create(rc);
823b2aab1Matthew Ahrens	rc->rc_tracked = B_FALSE;
83fa9e406ahrens}
84fa9e406ahrens
85fa9e406ahrensvoid
86e914aceTim Schumacherzfs_refcount_destroy_many(zfs_refcount_t *rc, uint64_t number)
87fa9e406ahrens{
88fa9e406ahrens	reference_t *ref;
89fa9e406ahrens
90fa9e406ahrens	ASSERT(rc->rc_count == number);
91fa9e406ahrens	while (ref = list_head(&rc->rc_list)) {
92fa9e406ahrens		list_remove(&rc->rc_list, ref);
93fa9e406ahrens		kmem_cache_free(reference_cache, ref);
94fa9e406ahrens	}
95fa9e406ahrens	list_destroy(&rc->rc_list);
96fa9e406ahrens
97fa9e406ahrens	while (ref = list_head(&rc->rc_removed)) {
98fa9e406ahrens		list_remove(&rc->rc_removed, ref);
99fa9e406ahrens		kmem_cache_free(reference_history_cache, ref->ref_removed);
100fa9e406ahrens		kmem_cache_free(reference_cache, ref);
101fa9e406ahrens	}
102fa9e406ahrens	list_destroy(&rc->rc_removed);
103fa9e406ahrens	mutex_destroy(&rc->rc_mtx);
104fa9e406ahrens}
105fa9e406ahrens
106fa9e406ahrensvoid
107e914aceTim Schumacherzfs_refcount_destroy(zfs_refcount_t *rc)
108fa9e406ahrens{
109e914aceTim Schumacher	zfs_refcount_destroy_many(rc, 0);
110fa9e406ahrens}
111fa9e406ahrens
112fa9e406ahrensint
113e914aceTim Schumacherzfs_refcount_is_zero(zfs_refcount_t *rc)
114fa9e406ahrens{
115fa9e406ahrens	return (rc->rc_count == 0);
116fa9e406ahrens}
117fa9e406ahrens
118fa9e406ahrensint64_t
119e914aceTim Schumacherzfs_refcount_count(zfs_refcount_t *rc)
120fa9e406ahrens{
121fa9e406ahrens	return (rc->rc_count);
122fa9e406ahrens}
123fa9e406ahrens
124fa9e406ahrensint64_t
125e914aceTim Schumacherzfs_refcount_add_many(zfs_refcount_t *rc, uint64_t number, void *holder)
126fa9e406ahrens{
127d5285caGeorge Wilson	reference_t *ref = NULL;
128fa9e406ahrens	int64_t count;
129fa9e406ahrens
1303b2aab1Matthew Ahrens	if (rc->rc_tracked) {
131fa9e406ahrens		ref = kmem_cache_alloc(reference_cache, KM_SLEEP);
132fa9e406ahrens		ref->ref_holder = holder;
133fa9e406ahrens		ref->ref_number = number;
134fa9e406ahrens	}
135fa9e406ahrens	mutex_enter(&rc->rc_mtx);
136fa9e406ahrens	ASSERT(rc->rc_count >= 0);
1373b2aab1Matthew Ahrens	if (rc->rc_tracked)
138fa9e406ahrens		list_insert_head(&rc->rc_list, ref);
139fa9e406ahrens	rc->rc_count += number;
140fa9e406ahrens	count = rc->rc_count;
141fa9e406ahrens	mutex_exit(&rc->rc_mtx);
142fa9e406ahrens
143fa9e406ahrens	return (count);
144fa9e406ahrens}
145fa9e406ahrens
146fa9e406ahrensint64_t
147e914aceTim Schumacherzfs_refcount_add(zfs_refcount_t *rc, void *holder)
148fa9e406ahrens{
149e914aceTim Schumacher	return (zfs_refcount_add_many(rc, 1, holder));
150fa9e406ahrens}
151fa9e406ahrens
152fa9e406ahrensint64_t
153e914aceTim Schumacherzfs_refcount_remove_many(zfs_refcount_t *rc, uint64_t number, void *holder)
154fa9e406ahrens{
155fa9e406ahrens	reference_t *ref;
156fa9e406ahrens	int64_t count;
157fa9e406ahrens
158fa9e406ahrens	mutex_enter(&rc->rc_mtx);
159fa9e406ahrens	ASSERT(rc->rc_count >= number);
160fa9e406ahrens
1613b2aab1Matthew Ahrens	if (!rc->rc_tracked) {
162fa9e406ahrens		rc->rc_count -= number;
163fa9e406ahrens		count = rc->rc_count;
164fa9e406ahrens		mutex_exit(&rc->rc_mtx);
165fa9e406ahrens		return (count);
166fa9e406ahrens	}
167fa9e406ahrens
168fa9e406ahrens	for (ref = list_head(&rc->rc_list); ref;
169fa9e406ahrens	    ref = list_next(&rc->rc_list, ref)) {
170fa9e406ahrens		if (ref->ref_holder == holder && ref->ref_number == number) {
171fa9e406ahrens			list_remove(&rc->rc_list, ref);
172fa9e406ahrens			if (reference_history > 0) {
173fa9e406ahrens				ref->ref_removed =
174fa9e406ahrens				    kmem_cache_alloc(reference_history_cache,
175fa9e406ahrens				    KM_SLEEP);
176fa9e406ahrens				list_insert_head(&rc->rc_removed, ref);
177fa9e406ahrens				rc->rc_removed_count++;
1783b2aab1Matthew Ahrens				if (rc->rc_removed_count > reference_history) {
179fa9e406ahrens					ref = list_tail(&rc->rc_removed);
180fa9e406ahrens					list_remove(&rc->rc_removed, ref);
181fa9e406ahrens					kmem_cache_free(reference_history_cache,
182fa9e406ahrens					    ref->ref_removed);
183fa9e406ahrens					kmem_cache_free(reference_cache, ref);
184fa9e406ahrens					rc->rc_removed_count--;
185fa9e406ahrens				}
186fa9e406ahrens			} else {
187fa9e406ahrens				kmem_cache_free(reference_cache, ref);
188fa9e406ahrens			}
189fa9e406ahrens			rc->rc_count -= number;
190fa9e406ahrens			count = rc->rc_count;
191fa9e406ahrens			mutex_exit(&rc->rc_mtx);
192fa9e406ahrens			return (count);
193fa9e406ahrens		}
194fa9e406ahrens	}
195fa9e406ahrens	panic("No such hold %p on refcount %llx", holder,
196fa9e406ahrens	    (u_longlong_t)(uintptr_t)rc);
197fa9e406ahrens	return (-1);
198fa9e406ahrens}
199fa9e406ahrens
200fa9e406ahrensint64_t
201e914aceTim Schumacherzfs_refcount_remove(zfs_refcount_t *rc, void *holder)
202fa9e406ahrens{
203e914aceTim Schumacher	return (zfs_refcount_remove_many(rc, 1, holder));
204fa9e406ahrens}
205fa9e406ahrens
206744947dTom Ericksonvoid
207e914aceTim Schumacherzfs_refcount_transfer(zfs_refcount_t *dst, zfs_refcount_t *src)
208744947dTom Erickson{
209744947dTom Erickson	int64_t count, removed_count;
210744947dTom Erickson	list_t list, removed;
211744947dTom Erickson
212744947dTom Erickson	list_create(&list, sizeof (reference_t),
213744947dTom Erickson	    offsetof(reference_t, ref_link));
214744947dTom Erickson	list_create(&removed, sizeof (reference_t),
215744947dTom Erickson	    offsetof(reference_t, ref_link));
216744947dTom Erickson
217744947dTom Erickson	mutex_enter(&src->rc_mtx);
218744947dTom Erickson	count = src->rc_count;
219744947dTom Erickson	removed_count = src->rc_removed_count;
220744947dTom Erickson	src->rc_count = 0;
221744947dTom Erickson	src->rc_removed_count = 0;
222744947dTom Erickson	list_move_tail(&list, &src->rc_list);
223744947dTom Erickson	list_move_tail(&removed, &src->rc_removed);
224744947dTom Erickson	mutex_exit(&src->rc_mtx);
225744947dTom Erickson
226744947dTom Erickson	mutex_enter(&dst->rc_mtx);
227744947dTom Erickson	dst->rc_count += count;
228744947dTom Erickson	dst->rc_removed_count += removed_count;
229744947dTom Erickson	list_move_tail(&dst->rc_list, &list);
230744947dTom Erickson	list_move_tail(&dst->rc_removed, &removed);
231744947dTom Erickson	mutex_exit(&dst->rc_mtx);
232744947dTom Erickson
233744947dTom Erickson	list_destroy(&list);
234744947dTom Erickson	list_destroy(&removed);
235744947dTom Erickson}
236744947dTom Erickson
237eb63303Tom Caputi/* ARGSUSED */
238dcbf3bdGeorge Wilsonvoid
239eb63303Tom Caputizfs_refcount_transfer_ownership_many(zfs_refcount_t *rc, uint64_t number,
240eb63303Tom Caputi    void *current_holder, void *new_holder)
241dcbf3bdGeorge Wilson{
242dcbf3bdGeorge Wilson	reference_t *ref;
243dcbf3bdGeorge Wilson	boolean_t found = B_FALSE;
244dcbf3bdGeorge Wilson
245dcbf3bdGeorge Wilson	mutex_enter(&rc->rc_mtx);
246dcbf3bdGeorge Wilson	if (!rc->rc_tracked) {
247dcbf3bdGeorge Wilson		mutex_exit(&rc->rc_mtx);
248dcbf3bdGeorge Wilson		return;
249dcbf3bdGeorge Wilson	}
250dcbf3bdGeorge Wilson
251dcbf3bdGeorge Wilson	for (ref = list_head(&rc->rc_list); ref;
252dcbf3bdGeorge Wilson	    ref = list_next(&rc->rc_list, ref)) {
253eb63303Tom Caputi		if (ref->ref_holder == current_holder &&
254eb63303Tom Caputi		    ref->ref_number == number) {
255dcbf3bdGeorge Wilson			ref->ref_holder = new_holder;
256dcbf3bdGeorge Wilson			found = B_TRUE;
257dcbf3bdGeorge Wilson			break;
258dcbf3bdGeorge Wilson		}
259dcbf3bdGeorge Wilson	}
260dcbf3bdGeorge Wilson	ASSERT(found);
261dcbf3bdGeorge Wilson	mutex_exit(&rc->rc_mtx);
262dcbf3bdGeorge Wilson}
2630f7643cGeorge Wilson
264eb63303Tom Caputivoid
265eb63303Tom Caputizfs_refcount_transfer_ownership(zfs_refcount_t *rc, void *current_holder,
266eb63303Tom Caputi    void *new_holder)
267eb63303Tom Caputi{
268eb63303Tom Caputi	zfs_refcount_transfer_ownership_many(rc, 1, current_holder,
269eb63303Tom Caputi	    new_holder);
270eb63303Tom Caputi}
271eb63303Tom Caputi
2720f7643cGeorge Wilson/*
2730f7643cGeorge Wilson * If tracking is enabled, return true if a reference exists that matches
2740f7643cGeorge Wilson * the "holder" tag. If tracking is disabled, then return true if a reference
2750f7643cGeorge Wilson * might be held.
2760f7643cGeorge Wilson */
2770f7643cGeorge Wilsonboolean_t
278e914aceTim Schumacherzfs_refcount_held(zfs_refcount_t *rc, void *holder)
2790f7643cGeorge Wilson{
2800f7643cGeorge Wilson	reference_t *ref;
2810f7643cGeorge Wilson
2820f7643cGeorge Wilson	mutex_enter(&rc->rc_mtx);
2830f7643cGeorge Wilson
2840f7643cGeorge Wilson	if (!rc->rc_tracked) {
2850f7643cGeorge Wilson		mutex_exit(&rc->rc_mtx);
2860f7643cGeorge Wilson		return (rc->rc_count > 0);
2870f7643cGeorge Wilson	}
2880f7643cGeorge Wilson
2890f7643cGeorge Wilson	for (ref = list_head(&rc->rc_list); ref;
2900f7643cGeorge Wilson	    ref = list_next(&rc->rc_list, ref)) {
2910f7643cGeorge Wilson		if (ref->ref_holder == holder) {
2920f7643cGeorge Wilson			mutex_exit(&rc->rc_mtx);
2930f7643cGeorge Wilson			return (B_TRUE);
2940f7643cGeorge Wilson		}
2950f7643cGeorge Wilson	}
2960f7643cGeorge Wilson	mutex_exit(&rc->rc_mtx);
2970f7643cGeorge Wilson	return (B_FALSE);
2980f7643cGeorge Wilson}
2990f7643cGeorge Wilson
3000f7643cGeorge Wilson/*
3010f7643cGeorge Wilson * If tracking is enabled, return true if a reference does not exist that
3020f7643cGeorge Wilson * matches the "holder" tag. If tracking is disabled, always return true
3030f7643cGeorge Wilson * since the reference might not be held.
3040f7643cGeorge Wilson */
3050f7643cGeorge Wilsonboolean_t
306e914aceTim Schumacherzfs_refcount_not_held(zfs_refcount_t *rc, void *holder)
3070f7643cGeorge Wilson{
3080f7643cGeorge Wilson	reference_t *ref;
3090f7643cGeorge Wilson
3100f7643cGeorge Wilson	mutex_enter(&rc->rc_mtx);
3110f7643cGeorge Wilson
3120f7643cGeorge Wilson	if (!rc->rc_tracked) {
3130f7643cGeorge Wilson		mutex_exit(&rc->rc_mtx);
3140f7643cGeorge Wilson		return (B_TRUE);
3150f7643cGeorge Wilson	}
3160f7643cGeorge Wilson
3170f7643cGeorge Wilson	for (ref = list_head(&rc->rc_list); ref;
3180f7643cGeorge Wilson	    ref = list_next(&rc->rc_list, ref)) {
3190f7643cGeorge Wilson		if (ref->ref_holder == holder) {
3200f7643cGeorge Wilson			mutex_exit(&rc->rc_mtx);
3210f7643cGeorge Wilson			return (B_FALSE);
3220f7643cGeorge Wilson		}
3230f7643cGeorge Wilson	}
3240f7643cGeorge Wilson	mutex_exit(&rc->rc_mtx);
3250f7643cGeorge Wilson	return (B_TRUE);
3260f7643cGeorge Wilson}
327744947dTom Erickson#endif	/* ZFS_DEBUG */
328