xref: /illumos-gate/usr/src/uts/common/fs/zfs/multilist.c (revision 94c2d0eb)
1244781f1SPrakash Surya /*
2244781f1SPrakash Surya  * CDDL HEADER START
3244781f1SPrakash Surya  *
4244781f1SPrakash Surya  * This file and its contents are supplied under the terms of the
5244781f1SPrakash Surya  * Common Development and Distribution License ("CDDL"), version 1.0.
6244781f1SPrakash Surya  * You may only use this file in accordance with the terms of version
7244781f1SPrakash Surya  * 1.0 of the CDDL.
8244781f1SPrakash Surya  *
9244781f1SPrakash Surya  * A full copy of the text of the CDDL should have accompanied this
10244781f1SPrakash Surya  * source.  A copy of the CDDL is also available via the Internet at
11244781f1SPrakash Surya  * http://www.illumos.org/license/CDDL.
12244781f1SPrakash Surya  *
13244781f1SPrakash Surya  * CDDL HEADER END
14244781f1SPrakash Surya  */
15244781f1SPrakash Surya /*
1610fbdecbSMatthew Ahrens  * Copyright (c) 2013, 2017 by Delphix. All rights reserved.
17244781f1SPrakash Surya  */
18244781f1SPrakash Surya 
19244781f1SPrakash Surya #include <sys/zfs_context.h>
20244781f1SPrakash Surya #include <sys/multilist.h>
21244781f1SPrakash Surya 
22244781f1SPrakash Surya /* needed for spa_get_random() */
23244781f1SPrakash Surya #include <sys/spa.h>
24244781f1SPrakash Surya 
2510fbdecbSMatthew Ahrens /*
2610fbdecbSMatthew Ahrens  * This overrides the number of sublists in each multilist_t, which defaults
2710fbdecbSMatthew Ahrens  * to the number of CPUs in the system (see multilist_create()).
2810fbdecbSMatthew Ahrens  */
2910fbdecbSMatthew Ahrens int zfs_multilist_num_sublists = 0;
3010fbdecbSMatthew Ahrens 
31244781f1SPrakash Surya /*
32244781f1SPrakash Surya  * Given the object contained on the list, return a pointer to the
33244781f1SPrakash Surya  * object's multilist_node_t structure it contains.
34244781f1SPrakash Surya  */
35244781f1SPrakash Surya static multilist_node_t *
multilist_d2l(multilist_t * ml,void * obj)36244781f1SPrakash Surya multilist_d2l(multilist_t *ml, void *obj)
37244781f1SPrakash Surya {
38244781f1SPrakash Surya 	return ((multilist_node_t *)((char *)obj + ml->ml_offset));
39244781f1SPrakash Surya }
40244781f1SPrakash Surya 
41244781f1SPrakash Surya /*
42244781f1SPrakash Surya  * Initialize a new mutlilist using the parameters specified.
43244781f1SPrakash Surya  *
44244781f1SPrakash Surya  *  - 'size' denotes the size of the structure containing the
45244781f1SPrakash Surya  *     multilist_node_t.
46244781f1SPrakash Surya  *  - 'offset' denotes the byte offset of the mutlilist_node_t within
47244781f1SPrakash Surya  *     the structure that contains it.
48244781f1SPrakash Surya  *  - 'num' specifies the number of internal sublists to create.
49244781f1SPrakash Surya  *  - 'index_func' is used to determine which sublist to insert into
50244781f1SPrakash Surya  *     when the multilist_insert() function is called; as well as which
51244781f1SPrakash Surya  *     sublist to remove from when multilist_remove() is called. The
52244781f1SPrakash Surya  *     requirements this function must meet, are the following:
53244781f1SPrakash Surya  *
54244781f1SPrakash Surya  *      - It must always return the same value when called on the same
55244781f1SPrakash Surya  *        object (to ensure the object is removed from the list it was
56244781f1SPrakash Surya  *        inserted into).
57244781f1SPrakash Surya  *
58244781f1SPrakash Surya  *      - It must return a value in the range [0, number of sublists).
59244781f1SPrakash Surya  *        The multilist_get_num_sublists() function may be used to
60244781f1SPrakash Surya  *        determine the number of sublists in the multilist.
61244781f1SPrakash Surya  *
62244781f1SPrakash Surya  *     Also, in order to reduce internal contention between the sublists
63244781f1SPrakash Surya  *     during insertion and removal, this function should choose evenly
64244781f1SPrakash Surya  *     between all available sublists when inserting. This isn't a hard
65244781f1SPrakash Surya  *     requirement, but a general rule of thumb in order to garner the
66244781f1SPrakash Surya  *     best multi-threaded performance out of the data structure.
67244781f1SPrakash Surya  */
68*94c2d0ebSMatthew Ahrens static multilist_t *
multilist_create_impl(size_t size,size_t offset,unsigned int num,multilist_sublist_index_func_t * index_func)69*94c2d0ebSMatthew Ahrens multilist_create_impl(size_t size, size_t offset,
7010fbdecbSMatthew Ahrens     unsigned int num, multilist_sublist_index_func_t *index_func)
71244781f1SPrakash Surya {
72244781f1SPrakash Surya 	ASSERT3U(size, >, 0);
73244781f1SPrakash Surya 	ASSERT3U(size, >=, offset + sizeof (multilist_node_t));
74244781f1SPrakash Surya 	ASSERT3U(num, >, 0);
75244781f1SPrakash Surya 	ASSERT3P(index_func, !=, NULL);
76244781f1SPrakash Surya 
77*94c2d0ebSMatthew Ahrens 	multilist_t *ml = kmem_alloc(sizeof (*ml), KM_SLEEP);
78244781f1SPrakash Surya 	ml->ml_offset = offset;
79244781f1SPrakash Surya 	ml->ml_num_sublists = num;
80244781f1SPrakash Surya 	ml->ml_index_func = index_func;
81244781f1SPrakash Surya 
82244781f1SPrakash Surya 	ml->ml_sublists = kmem_zalloc(sizeof (multilist_sublist_t) *
83244781f1SPrakash Surya 	    ml->ml_num_sublists, KM_SLEEP);
84244781f1SPrakash Surya 
85244781f1SPrakash Surya 	ASSERT3P(ml->ml_sublists, !=, NULL);
86244781f1SPrakash Surya 
87244781f1SPrakash Surya 	for (int i = 0; i < ml->ml_num_sublists; i++) {
88244781f1SPrakash Surya 		multilist_sublist_t *mls = &ml->ml_sublists[i];
89244781f1SPrakash Surya 		mutex_init(&mls->mls_lock, NULL, MUTEX_DEFAULT, NULL);
90244781f1SPrakash Surya 		list_create(&mls->mls_list, size, offset);
91244781f1SPrakash Surya 	}
92*94c2d0ebSMatthew Ahrens 	return (ml);
93244781f1SPrakash Surya }
94244781f1SPrakash Surya 
9510fbdecbSMatthew Ahrens /*
96*94c2d0ebSMatthew Ahrens  * Allocate a new multilist, using the default number of sublists
9710fbdecbSMatthew Ahrens  * (the number of CPUs, or at least 4, or the tunable
9810fbdecbSMatthew Ahrens  * zfs_multilist_num_sublists).
9910fbdecbSMatthew Ahrens  */
100*94c2d0ebSMatthew Ahrens multilist_t *
multilist_create(size_t size,size_t offset,multilist_sublist_index_func_t * index_func)101*94c2d0ebSMatthew Ahrens multilist_create(size_t size, size_t offset,
10210fbdecbSMatthew Ahrens     multilist_sublist_index_func_t *index_func)
10310fbdecbSMatthew Ahrens {
10410fbdecbSMatthew Ahrens 	int num_sublists;
10510fbdecbSMatthew Ahrens 
10610fbdecbSMatthew Ahrens 	if (zfs_multilist_num_sublists > 0) {
10710fbdecbSMatthew Ahrens 		num_sublists = zfs_multilist_num_sublists;
10810fbdecbSMatthew Ahrens 	} else {
10910fbdecbSMatthew Ahrens 		num_sublists = MAX(boot_ncpus, 4);
11010fbdecbSMatthew Ahrens 	}
11110fbdecbSMatthew Ahrens 
112*94c2d0ebSMatthew Ahrens 	return (multilist_create_impl(size, offset, num_sublists, index_func));
11310fbdecbSMatthew Ahrens }
11410fbdecbSMatthew Ahrens 
115244781f1SPrakash Surya /*
116244781f1SPrakash Surya  * Destroy the given multilist object, and free up any memory it holds.
117244781f1SPrakash Surya  */
118244781f1SPrakash Surya void
multilist_destroy(multilist_t * ml)119244781f1SPrakash Surya multilist_destroy(multilist_t *ml)
120244781f1SPrakash Surya {
121244781f1SPrakash Surya 	ASSERT(multilist_is_empty(ml));
122244781f1SPrakash Surya 
123244781f1SPrakash Surya 	for (int i = 0; i < ml->ml_num_sublists; i++) {
124244781f1SPrakash Surya 		multilist_sublist_t *mls = &ml->ml_sublists[i];
125244781f1SPrakash Surya 
126244781f1SPrakash Surya 		ASSERT(list_is_empty(&mls->mls_list));
127244781f1SPrakash Surya 
128244781f1SPrakash Surya 		list_destroy(&mls->mls_list);
129244781f1SPrakash Surya 		mutex_destroy(&mls->mls_lock);
130244781f1SPrakash Surya 	}
131244781f1SPrakash Surya 
132244781f1SPrakash Surya 	ASSERT3P(ml->ml_sublists, !=, NULL);
133244781f1SPrakash Surya 	kmem_free(ml->ml_sublists,
134244781f1SPrakash Surya 	    sizeof (multilist_sublist_t) * ml->ml_num_sublists);
135244781f1SPrakash Surya 
136244781f1SPrakash Surya 	ml->ml_num_sublists = 0;
137244781f1SPrakash Surya 	ml->ml_offset = 0;
138*94c2d0ebSMatthew Ahrens 	kmem_free(ml, sizeof (multilist_t));
139244781f1SPrakash Surya }
140244781f1SPrakash Surya 
141244781f1SPrakash Surya /*
142244781f1SPrakash Surya  * Insert the given object into the multilist.
143244781f1SPrakash Surya  *
144244781f1SPrakash Surya  * This function will insert the object specified into the sublist
145244781f1SPrakash Surya  * determined using the function given at multilist creation time.
146244781f1SPrakash Surya  *
147244781f1SPrakash Surya  * The sublist locks are automatically acquired if not already held, to
148244781f1SPrakash Surya  * ensure consistency when inserting and removing from multiple threads.
149244781f1SPrakash Surya  */
150244781f1SPrakash Surya void
multilist_insert(multilist_t * ml,void * obj)151244781f1SPrakash Surya multilist_insert(multilist_t *ml, void *obj)
152244781f1SPrakash Surya {
153244781f1SPrakash Surya 	unsigned int sublist_idx = ml->ml_index_func(ml, obj);
154244781f1SPrakash Surya 	multilist_sublist_t *mls;
155244781f1SPrakash Surya 	boolean_t need_lock;
156244781f1SPrakash Surya 
157244781f1SPrakash Surya 	DTRACE_PROBE3(multilist__insert, multilist_t *, ml,
158244781f1SPrakash Surya 	    unsigned int, sublist_idx, void *, obj);
159244781f1SPrakash Surya 
160244781f1SPrakash Surya 	ASSERT3U(sublist_idx, <, ml->ml_num_sublists);
161244781f1SPrakash Surya 
162244781f1SPrakash Surya 	mls = &ml->ml_sublists[sublist_idx];
163244781f1SPrakash Surya 
164244781f1SPrakash Surya 	/*
165244781f1SPrakash Surya 	 * Note: Callers may already hold the sublist lock by calling
166244781f1SPrakash Surya 	 * multilist_sublist_lock().  Here we rely on MUTEX_HELD()
167244781f1SPrakash Surya 	 * returning TRUE if and only if the current thread holds the
168244781f1SPrakash Surya 	 * lock.  While it's a little ugly to make the lock recursive in
169244781f1SPrakash Surya 	 * this way, it works and allows the calling code to be much
170244781f1SPrakash Surya 	 * simpler -- otherwise it would have to pass around a flag
171244781f1SPrakash Surya 	 * indicating that it already has the lock.
172244781f1SPrakash Surya 	 */
173244781f1SPrakash Surya 	need_lock = !MUTEX_HELD(&mls->mls_lock);
174244781f1SPrakash Surya 
175244781f1SPrakash Surya 	if (need_lock)
176244781f1SPrakash Surya 		mutex_enter(&mls->mls_lock);
177244781f1SPrakash Surya 
178244781f1SPrakash Surya 	ASSERT(!multilist_link_active(multilist_d2l(ml, obj)));
179244781f1SPrakash Surya 
180244781f1SPrakash Surya 	multilist_sublist_insert_head(mls, obj);
181244781f1SPrakash Surya 
182244781f1SPrakash Surya 	if (need_lock)
183244781f1SPrakash Surya 		mutex_exit(&mls->mls_lock);
184244781f1SPrakash Surya }
185244781f1SPrakash Surya 
186244781f1SPrakash Surya /*
187244781f1SPrakash Surya  * Remove the given object from the multilist.
188244781f1SPrakash Surya  *
189244781f1SPrakash Surya  * This function will remove the object specified from the sublist
190244781f1SPrakash Surya  * determined using the function given at multilist creation time.
191244781f1SPrakash Surya  *
192244781f1SPrakash Surya  * The necessary sublist locks are automatically acquired, to ensure
193244781f1SPrakash Surya  * consistency when inserting and removing from multiple threads.
194244781f1SPrakash Surya  */
195244781f1SPrakash Surya void
multilist_remove(multilist_t * ml,void * obj)196244781f1SPrakash Surya multilist_remove(multilist_t *ml, void *obj)
197244781f1SPrakash Surya {
198244781f1SPrakash Surya 	unsigned int sublist_idx = ml->ml_index_func(ml, obj);
199244781f1SPrakash Surya 	multilist_sublist_t *mls;
200244781f1SPrakash Surya 	boolean_t need_lock;
201244781f1SPrakash Surya 
202244781f1SPrakash Surya 	DTRACE_PROBE3(multilist__remove, multilist_t *, ml,
203244781f1SPrakash Surya 	    unsigned int, sublist_idx, void *, obj);
204244781f1SPrakash Surya 
205244781f1SPrakash Surya 	ASSERT3U(sublist_idx, <, ml->ml_num_sublists);
206244781f1SPrakash Surya 
207244781f1SPrakash Surya 	mls = &ml->ml_sublists[sublist_idx];
208244781f1SPrakash Surya 	/* See comment in multilist_insert(). */
209244781f1SPrakash Surya 	need_lock = !MUTEX_HELD(&mls->mls_lock);
210244781f1SPrakash Surya 
211244781f1SPrakash Surya 	if (need_lock)
212244781f1SPrakash Surya 		mutex_enter(&mls->mls_lock);
213244781f1SPrakash Surya 
214244781f1SPrakash Surya 	ASSERT(multilist_link_active(multilist_d2l(ml, obj)));
215244781f1SPrakash Surya 
216244781f1SPrakash Surya 	multilist_sublist_remove(mls, obj);
217244781f1SPrakash Surya 
218244781f1SPrakash Surya 	if (need_lock)
219244781f1SPrakash Surya 		mutex_exit(&mls->mls_lock);
220244781f1SPrakash Surya }
221244781f1SPrakash Surya 
222244781f1SPrakash Surya /*
223244781f1SPrakash Surya  * Check to see if this multilist object is empty.
224244781f1SPrakash Surya  *
225244781f1SPrakash Surya  * This will return TRUE if it finds all of the sublists of this
226244781f1SPrakash Surya  * multilist to be empty, and FALSE otherwise. Each sublist lock will be
227244781f1SPrakash Surya  * automatically acquired as necessary.
228244781f1SPrakash Surya  *
229244781f1SPrakash Surya  * If concurrent insertions and removals are occurring, the semantics
230244781f1SPrakash Surya  * of this function become a little fuzzy. Instead of locking all
231244781f1SPrakash Surya  * sublists for the entire call time of the function, each sublist is
232244781f1SPrakash Surya  * only locked as it is individually checked for emptiness. Thus, it's
233244781f1SPrakash Surya  * possible for this function to return TRUE with non-empty sublists at
234244781f1SPrakash Surya  * the time the function returns. This would be due to another thread
235244781f1SPrakash Surya  * inserting into a given sublist, after that specific sublist was check
236244781f1SPrakash Surya  * and deemed empty, but before all sublists have been checked.
237244781f1SPrakash Surya  */
238244781f1SPrakash Surya int
multilist_is_empty(multilist_t * ml)239244781f1SPrakash Surya multilist_is_empty(multilist_t *ml)
240244781f1SPrakash Surya {
241244781f1SPrakash Surya 	for (int i = 0; i < ml->ml_num_sublists; i++) {
242244781f1SPrakash Surya 		multilist_sublist_t *mls = &ml->ml_sublists[i];
243244781f1SPrakash Surya 		/* See comment in multilist_insert(). */
244244781f1SPrakash Surya 		boolean_t need_lock = !MUTEX_HELD(&mls->mls_lock);
245244781f1SPrakash Surya 
246244781f1SPrakash Surya 		if (need_lock)
247244781f1SPrakash Surya 			mutex_enter(&mls->mls_lock);
248244781f1SPrakash Surya 
249244781f1SPrakash Surya 		if (!list_is_empty(&mls->mls_list)) {
250244781f1SPrakash Surya 			if (need_lock)
251244781f1SPrakash Surya 				mutex_exit(&mls->mls_lock);
252244781f1SPrakash Surya 
253244781f1SPrakash Surya 			return (FALSE);
254244781f1SPrakash Surya 		}
255244781f1SPrakash Surya 
256244781f1SPrakash Surya 		if (need_lock)
257244781f1SPrakash Surya 			mutex_exit(&mls->mls_lock);
258244781f1SPrakash Surya 	}
259244781f1SPrakash Surya 
260244781f1SPrakash Surya 	return (TRUE);
261244781f1SPrakash Surya }
262244781f1SPrakash Surya 
263244781f1SPrakash Surya /* Return the number of sublists composing this multilist */
264244781f1SPrakash Surya unsigned int
multilist_get_num_sublists(multilist_t * ml)265244781f1SPrakash Surya multilist_get_num_sublists(multilist_t *ml)
266244781f1SPrakash Surya {
267244781f1SPrakash Surya 	return (ml->ml_num_sublists);
268244781f1SPrakash Surya }
269244781f1SPrakash Surya 
270244781f1SPrakash Surya /* Return a randomly selected, valid sublist index for this multilist */
271244781f1SPrakash Surya unsigned int
multilist_get_random_index(multilist_t * ml)272244781f1SPrakash Surya multilist_get_random_index(multilist_t *ml)
273244781f1SPrakash Surya {
274244781f1SPrakash Surya 	return (spa_get_random(ml->ml_num_sublists));
275244781f1SPrakash Surya }
276244781f1SPrakash Surya 
277244781f1SPrakash Surya /* Lock and return the sublist specified at the given index */
278244781f1SPrakash Surya multilist_sublist_t *
multilist_sublist_lock(multilist_t * ml,unsigned int sublist_idx)279244781f1SPrakash Surya multilist_sublist_lock(multilist_t *ml, unsigned int sublist_idx)
280244781f1SPrakash Surya {
281244781f1SPrakash Surya 	multilist_sublist_t *mls;
282244781f1SPrakash Surya 
283244781f1SPrakash Surya 	ASSERT3U(sublist_idx, <, ml->ml_num_sublists);
284244781f1SPrakash Surya 	mls = &ml->ml_sublists[sublist_idx];
285244781f1SPrakash Surya 	mutex_enter(&mls->mls_lock);
286244781f1SPrakash Surya 
287244781f1SPrakash Surya 	return (mls);
288244781f1SPrakash Surya }
289244781f1SPrakash Surya 
290*94c2d0ebSMatthew Ahrens /* Lock and return the sublist that would be used to store the specified obj */
291*94c2d0ebSMatthew Ahrens multilist_sublist_t *
multilist_sublist_lock_obj(multilist_t * ml,void * obj)292*94c2d0ebSMatthew Ahrens multilist_sublist_lock_obj(multilist_t *ml, void *obj)
293*94c2d0ebSMatthew Ahrens {
294*94c2d0ebSMatthew Ahrens 	return (multilist_sublist_lock(ml, ml->ml_index_func(ml, obj)));
295*94c2d0ebSMatthew Ahrens }
296*94c2d0ebSMatthew Ahrens 
297244781f1SPrakash Surya void
multilist_sublist_unlock(multilist_sublist_t * mls)298244781f1SPrakash Surya multilist_sublist_unlock(multilist_sublist_t *mls)
299244781f1SPrakash Surya {
300244781f1SPrakash Surya 	mutex_exit(&mls->mls_lock);
301244781f1SPrakash Surya }
302244781f1SPrakash Surya 
303244781f1SPrakash Surya /*
304244781f1SPrakash Surya  * We're allowing any object to be inserted into this specific sublist,
305244781f1SPrakash Surya  * but this can lead to trouble if multilist_remove() is called to
306244781f1SPrakash Surya  * remove this object. Specifically, if calling ml_index_func on this
307244781f1SPrakash Surya  * object returns an index for sublist different than what is passed as
308244781f1SPrakash Surya  * a parameter here, any call to multilist_remove() with this newly
309244781f1SPrakash Surya  * inserted object is undefined! (the call to multilist_remove() will
310244781f1SPrakash Surya  * remove the object from a list that it isn't contained in)
311244781f1SPrakash Surya  */
312244781f1SPrakash Surya void
multilist_sublist_insert_head(multilist_sublist_t * mls,void * obj)313244781f1SPrakash Surya multilist_sublist_insert_head(multilist_sublist_t *mls, void *obj)
314244781f1SPrakash Surya {
315244781f1SPrakash Surya 	ASSERT(MUTEX_HELD(&mls->mls_lock));
316244781f1SPrakash Surya 	list_insert_head(&mls->mls_list, obj);
317244781f1SPrakash Surya }
318244781f1SPrakash Surya 
319244781f1SPrakash Surya /* please see comment above multilist_sublist_insert_head */
320244781f1SPrakash Surya void
multilist_sublist_insert_tail(multilist_sublist_t * mls,void * obj)321244781f1SPrakash Surya multilist_sublist_insert_tail(multilist_sublist_t *mls, void *obj)
322244781f1SPrakash Surya {
323244781f1SPrakash Surya 	ASSERT(MUTEX_HELD(&mls->mls_lock));
324244781f1SPrakash Surya 	list_insert_tail(&mls->mls_list, obj);
325244781f1SPrakash Surya }
326244781f1SPrakash Surya 
327244781f1SPrakash Surya /*
328244781f1SPrakash Surya  * Move the object one element forward in the list.
329244781f1SPrakash Surya  *
330244781f1SPrakash Surya  * This function will move the given object forward in the list (towards
331244781f1SPrakash Surya  * the head) by one object. So, in essence, it will swap its position in
332244781f1SPrakash Surya  * the list with its "prev" pointer. If the given object is already at the
333244781f1SPrakash Surya  * head of the list, it cannot be moved forward any more than it already
334244781f1SPrakash Surya  * is, so no action is taken.
335244781f1SPrakash Surya  *
336244781f1SPrakash Surya  * NOTE: This function **must not** remove any object from the list other
337244781f1SPrakash Surya  *       than the object given as the parameter. This is relied upon in
338244781f1SPrakash Surya  *       arc_evict_state_impl().
339244781f1SPrakash Surya  */
340244781f1SPrakash Surya void
multilist_sublist_move_forward(multilist_sublist_t * mls,void * obj)341244781f1SPrakash Surya multilist_sublist_move_forward(multilist_sublist_t *mls, void *obj)
342244781f1SPrakash Surya {
343244781f1SPrakash Surya 	void *prev = list_prev(&mls->mls_list, obj);
344244781f1SPrakash Surya 
345244781f1SPrakash Surya 	ASSERT(MUTEX_HELD(&mls->mls_lock));
346244781f1SPrakash Surya 	ASSERT(!list_is_empty(&mls->mls_list));
347244781f1SPrakash Surya 
348244781f1SPrakash Surya 	/* 'obj' must be at the head of the list, nothing to do */
349244781f1SPrakash Surya 	if (prev == NULL)
350244781f1SPrakash Surya 		return;
351244781f1SPrakash Surya 
352244781f1SPrakash Surya 	list_remove(&mls->mls_list, obj);
353244781f1SPrakash Surya 	list_insert_before(&mls->mls_list, prev, obj);
354244781f1SPrakash Surya }
355244781f1SPrakash Surya 
356244781f1SPrakash Surya void
multilist_sublist_remove(multilist_sublist_t * mls,void * obj)357244781f1SPrakash Surya multilist_sublist_remove(multilist_sublist_t *mls, void *obj)
358244781f1SPrakash Surya {
359244781f1SPrakash Surya 	ASSERT(MUTEX_HELD(&mls->mls_lock));
360244781f1SPrakash Surya 	list_remove(&mls->mls_list, obj);
361244781f1SPrakash Surya }
362244781f1SPrakash Surya 
363244781f1SPrakash Surya void *
multilist_sublist_head(multilist_sublist_t * mls)364244781f1SPrakash Surya multilist_sublist_head(multilist_sublist_t *mls)
365244781f1SPrakash Surya {
366244781f1SPrakash Surya 	ASSERT(MUTEX_HELD(&mls->mls_lock));
367244781f1SPrakash Surya 	return (list_head(&mls->mls_list));
368244781f1SPrakash Surya }
369244781f1SPrakash Surya 
370244781f1SPrakash Surya void *
multilist_sublist_tail(multilist_sublist_t * mls)371244781f1SPrakash Surya multilist_sublist_tail(multilist_sublist_t *mls)
372244781f1SPrakash Surya {
373244781f1SPrakash Surya 	ASSERT(MUTEX_HELD(&mls->mls_lock));
374244781f1SPrakash Surya 	return (list_tail(&mls->mls_list));
375244781f1SPrakash Surya }
376244781f1SPrakash Surya 
377244781f1SPrakash Surya void *
multilist_sublist_next(multilist_sublist_t * mls,void * obj)378244781f1SPrakash Surya multilist_sublist_next(multilist_sublist_t *mls, void *obj)
379244781f1SPrakash Surya {
380244781f1SPrakash Surya 	ASSERT(MUTEX_HELD(&mls->mls_lock));
381244781f1SPrakash Surya 	return (list_next(&mls->mls_list, obj));
382244781f1SPrakash Surya }
383244781f1SPrakash Surya 
384244781f1SPrakash Surya void *
multilist_sublist_prev(multilist_sublist_t * mls,void * obj)385244781f1SPrakash Surya multilist_sublist_prev(multilist_sublist_t *mls, void *obj)
386244781f1SPrakash Surya {
387244781f1SPrakash Surya 	ASSERT(MUTEX_HELD(&mls->mls_lock));
388244781f1SPrakash Surya 	return (list_prev(&mls->mls_list, obj));
389244781f1SPrakash Surya }
390244781f1SPrakash Surya 
391244781f1SPrakash Surya void
multilist_link_init(multilist_node_t * link)392244781f1SPrakash Surya multilist_link_init(multilist_node_t *link)
393244781f1SPrakash Surya {
394244781f1SPrakash Surya 	list_link_init(link);
395244781f1SPrakash Surya }
396244781f1SPrakash Surya 
397244781f1SPrakash Surya int
multilist_link_active(multilist_node_t * link)398244781f1SPrakash Surya multilist_link_active(multilist_node_t *link)
399244781f1SPrakash Surya {
400244781f1SPrakash Surya 	return (list_link_active(link));
401244781f1SPrakash Surya }
402