17c478bdstevel@tonic-gate/*
27c478bdstevel@tonic-gate * CDDL HEADER START
37c478bdstevel@tonic-gate *
47c478bdstevel@tonic-gate * The contents of this file are subject to the terms of the
52e0c549Jonathan Adams * Common Development and Distribution License (the "License").
62e0c549Jonathan Adams * You may not use this file except in compliance with the License.
77c478bdstevel@tonic-gate *
87c478bdstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bdstevel@tonic-gate * or http://www.opensolaris.org/os/licensing.
107c478bdstevel@tonic-gate * See the License for the specific language governing permissions
117c478bdstevel@tonic-gate * and limitations under the License.
127c478bdstevel@tonic-gate *
137c478bdstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each
147c478bdstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bdstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the
167c478bdstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying
177c478bdstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bdstevel@tonic-gate *
197c478bdstevel@tonic-gate * CDDL HEADER END
207c478bdstevel@tonic-gate */
217c478bdstevel@tonic-gate/*
2256f3320Jonathan Adams * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
237c478bdstevel@tonic-gate * Use is subject to license terms.
247c478bdstevel@tonic-gate */
257c478bdstevel@tonic-gate
267c478bdstevel@tonic-gate/*
27e548823Gordon Ross * Copyright 2015 Nexenta Systems, Inc.  All rights reserved.
28216d772Prakash Surya * Copyright (c) 2017 by Delphix. All rights reserved.
294c99eccTim Kordas * Copyright 2018, Joyent, Inc.
305aeb947Garrett D'Amore */
315aeb947Garrett D'Amore
325aeb947Garrett D'Amore/*
337c478bdstevel@tonic-gate * Kernel task queues: general-purpose asynchronous task scheduling.
347c478bdstevel@tonic-gate *
357c478bdstevel@tonic-gate * A common problem in kernel programming is the need to schedule tasks
367c478bdstevel@tonic-gate * to be performed later, by another thread. There are several reasons
377c478bdstevel@tonic-gate * you may want or need to do this:
387c478bdstevel@tonic-gate *
397c478bdstevel@tonic-gate * (1) The task isn't time-critical, but your current code path is.
407c478bdstevel@tonic-gate *
417c478bdstevel@tonic-gate * (2) The task may require grabbing locks that you already hold.
427c478bdstevel@tonic-gate *
437c478bdstevel@tonic-gate * (3) The task may need to block (e.g. to wait for memory), but you
447c478bdstevel@tonic-gate *     cannot block in your current context.
457c478bdstevel@tonic-gate *
467c478bdstevel@tonic-gate * (4) Your code path can't complete because of some condition, but you can't
477c478bdstevel@tonic-gate *     sleep or fail, so you queue the task for later execution when condition
487c478bdstevel@tonic-gate *     disappears.
497c478bdstevel@tonic-gate *
507c478bdstevel@tonic-gate * (5) You just want a simple way to launch multiple tasks in parallel.
517c478bdstevel@tonic-gate *
527c478bdstevel@tonic-gate * Task queues provide such a facility. In its simplest form (used when
537c478bdstevel@tonic-gate * performance is not a critical consideration) a task queue consists of a
547c478bdstevel@tonic-gate * single list of tasks, together with one or more threads to service the
557c478bdstevel@tonic-gate * list. There are some cases when this simple queue is not sufficient:
567c478bdstevel@tonic-gate *
577c478bdstevel@tonic-gate * (1) The task queues are very hot and there is a need to avoid data and lock
587c478bdstevel@tonic-gate *	contention over global resources.
597c478bdstevel@tonic-gate *
607c478bdstevel@tonic-gate * (2) Some tasks may depend on other tasks to complete, so they can't be put in
617c478bdstevel@tonic-gate *	the same list managed by the same thread.
627c478bdstevel@tonic-gate *
637c478bdstevel@tonic-gate * (3) Some tasks may block for a long time, and this should not block other
6435a5a35Jonathan Adams *	tasks in the queue.
657c478bdstevel@tonic-gate *
667c478bdstevel@tonic-gate * To provide useful service in such cases we define a "dynamic task queue"
677c478bdstevel@tonic-gate * which has an individual thread for each of the tasks. These threads are
687c478bdstevel@tonic-gate * dynamically created as they are needed and destroyed when they are not in
697c478bdstevel@tonic-gate * use. The API for managing task pools is the same as for managing task queues
707c478bdstevel@tonic-gate * with the exception of a taskq creation flag TASKQ_DYNAMIC which tells that
717c478bdstevel@tonic-gate * dynamic task pool behavior is desired.
727c478bdstevel@tonic-gate *
737c478bdstevel@tonic-gate * Dynamic task queues may also place tasks in the normal queue (called "backing
747c478bdstevel@tonic-gate * queue") when task pool runs out of resources. Users of task queues may
757c478bdstevel@tonic-gate * disallow such queued scheduling by specifying TQ_NOQUEUE in the dispatch
767c478bdstevel@tonic-gate * flags.
777c478bdstevel@tonic-gate *
787c478bdstevel@tonic-gate * The backing task queue is also used for scheduling internal tasks needed for
797c478bdstevel@tonic-gate * dynamic task queue maintenance.
807c478bdstevel@tonic-gate *
812e0c549Jonathan Adams * INTERFACES ==================================================================
827c478bdstevel@tonic-gate *
83a676a7cSergio Aguayo * taskq_t *taskq_create(name, nthreads, pri, minalloc, maxalloc, flags);
847c478bdstevel@tonic-gate *
857c478bdstevel@tonic-gate *	Create a taskq with specified properties.
867c478bdstevel@tonic-gate *	Possible 'flags':
877c478bdstevel@tonic-gate *
887c478bdstevel@tonic-gate *	  TASKQ_DYNAMIC: Create task pool for task management. If this flag is
892e0c549Jonathan Adams *		specified, 'nthreads' specifies the maximum number of threads in
907c478bdstevel@tonic-gate *		the task queue. Task execution order for dynamic task queues is
917c478bdstevel@tonic-gate *		not predictable.
927c478bdstevel@tonic-gate *
937c478bdstevel@tonic-gate *		If this flag is not specified (default case) a
942e0c549Jonathan Adams *		single-list task queue is created with 'nthreads' threads
952e0c549Jonathan Adams *		servicing it. Entries in this queue are managed by
962e0c549Jonathan Adams *		taskq_ent_alloc() and taskq_ent_free() which try to keep the
972e0c549Jonathan Adams *		task population between 'minalloc' and 'maxalloc', but the
987c478bdstevel@tonic-gate *		latter limit is only advisory for TQ_SLEEP dispatches and the
997c478bdstevel@tonic-gate *		former limit is only advisory for TQ_NOALLOC dispatches. If
1007c478bdstevel@tonic-gate *		TASKQ_PREPOPULATE is set in 'flags', the taskq will be
1017c478bdstevel@tonic-gate *		prepopulated with 'minalloc' task structures.
1027c478bdstevel@tonic-gate *
1037c478bdstevel@tonic-gate *		Since non-DYNAMIC taskqs are queues, tasks are guaranteed to be
1047c478bdstevel@tonic-gate *		executed in the order they are scheduled if nthreads == 1.
1057c478bdstevel@tonic-gate *		If nthreads > 1, task execution order is not predictable.
1067c478bdstevel@tonic-gate *
1077c478bdstevel@tonic-gate *	  TASKQ_PREPOPULATE: Prepopulate task queue with threads.
1087c478bdstevel@tonic-gate *		Also prepopulate the task queue with 'minalloc' task structures.
1097c478bdstevel@tonic-gate *
1102e0c549Jonathan Adams *	  TASKQ_THREADS_CPU_PCT: This flag specifies that 'nthreads' should be
1112e0c549Jonathan Adams *		interpreted as a percentage of the # of online CPUs on the
1122e0c549Jonathan Adams *		system.  The taskq subsystem will automatically adjust the
1132e0c549Jonathan Adams *		number of threads in the taskq in response to CPU online
1142e0c549Jonathan Adams *		and offline events, to keep the ratio.  nthreads must be in
1152e0c549Jonathan Adams *		the range [0,100].
1162e0c549Jonathan Adams *
1172e0c549Jonathan Adams *		The calculation used is:
1182e0c549Jonathan Adams *
1192e0c549Jonathan Adams *			MAX((ncpus_online * percentage)/100, 1)
1202e0c549Jonathan Adams *
1212e0c549Jonathan Adams *		This flag is not supported for DYNAMIC task queues.
1222e0c549Jonathan Adams *		This flag is not compatible with TASKQ_CPR_SAFE.
1232e0c549Jonathan Adams *
1247c478bdstevel@tonic-gate *	  TASKQ_CPR_SAFE: This flag specifies that users of the task queue will
1252e0c549Jonathan Adams *		use their own protocol for handling CPR issues. This flag is not
1262e0c549Jonathan Adams *		supported for DYNAMIC task queues.  This flag is not compatible
1272e0c549Jonathan Adams *		with TASKQ_THREADS_CPU_PCT.
1287c478bdstevel@tonic-gate *
1297c478bdstevel@tonic-gate *	The 'pri' field specifies the default priority for the threads that
1307c478bdstevel@tonic-gate *	service all scheduled tasks.
1317c478bdstevel@tonic-gate *
13235a5a35Jonathan Adams * taskq_t *taskq_create_instance(name, instance, nthreads, pri, minalloc,
133a676a7cSergio Aguayo *    maxalloc, flags);
13435a5a35Jonathan Adams *
13535a5a35Jonathan Adams *	Like taskq_create(), but takes an instance number (or -1 to indicate
13635a5a35Jonathan Adams *	no instance).
13735a5a35Jonathan Adams *
138a676a7cSergio Aguayo * taskq_t *taskq_create_proc(name, nthreads, pri, minalloc, maxalloc, proc,
13935a5a35Jonathan Adams *    flags);
14035a5a35Jonathan Adams *
14135a5a35Jonathan Adams *	Like taskq_create(), but creates the taskq threads in the specified
14235a5a35Jonathan Adams *	system process.  If proc != &p0, this must be called from a thread
14335a5a35Jonathan Adams *	in that process.
14435a5a35Jonathan Adams *
145a676a7cSergio Aguayo * taskq_t *taskq_create_sysdc(name, nthreads, minalloc, maxalloc, proc,
14635a5a35Jonathan Adams *    dc, flags);
14735a5a35Jonathan Adams *
14835a5a35Jonathan Adams *	Like taskq_create_proc(), but the taskq threads will use the
14935a5a35Jonathan Adams *	System Duty Cycle (SDC) scheduling class with a duty cycle of dc.
15035a5a35Jonathan Adams *
1517c478bdstevel@tonic-gate * void taskq_destroy(tap):
1527c478bdstevel@tonic-gate *
1537c478bdstevel@tonic-gate *	Waits for any scheduled tasks to complete, then destroys the taskq.
1547c478bdstevel@tonic-gate *	Caller should guarantee that no new tasks are scheduled in the closing
1557c478bdstevel@tonic-gate *	taskq.
1567c478bdstevel@tonic-gate *
1577c478bdstevel@tonic-gate * taskqid_t taskq_dispatch(tq, func, arg, flags):
1587c478bdstevel@tonic-gate *
1597c478bdstevel@tonic-gate *	Dispatches the task "func(arg)" to taskq. The 'flags' indicates whether
1607c478bdstevel@tonic-gate *	the caller is willing to block for memory.  The function returns an
1617c478bdstevel@tonic-gate *	opaque value which is zero iff dispatch fails.  If flags is TQ_NOSLEEP
1627c478bdstevel@tonic-gate *	or TQ_NOALLOC and the task can't be dispatched, taskq_dispatch() fails
163fc8ae2eToomas Soome *	and returns TASKQID_INVALID.
1647c478bdstevel@tonic-gate *
1657c478bdstevel@tonic-gate *	ASSUMES: func != NULL.
1667c478bdstevel@tonic-gate *
1677c478bdstevel@tonic-gate *	Possible flags:
1687c478bdstevel@tonic-gate *	  TQ_NOSLEEP: Do not wait for resources; may fail.
1697c478bdstevel@tonic-gate *
1707c478bdstevel@tonic-gate *	  TQ_NOALLOC: Do not allocate memory; may fail.  May only be used with
1717c478bdstevel@tonic-gate *		non-dynamic task queues.
1727c478bdstevel@tonic-gate *
1737c478bdstevel@tonic-gate *	  TQ_NOQUEUE: Do not enqueue a task if it can't dispatch it due to
1747c478bdstevel@tonic-gate *		lack of available resources and fail. If this flag is not
17535a5a35Jonathan Adams *		set, and the task pool is exhausted, the task may be scheduled
1767c478bdstevel@tonic-gate *		in the backing queue. This flag may ONLY be used with dynamic
1777c478bdstevel@tonic-gate *		task queues.
1787c478bdstevel@tonic-gate *
1797c478bdstevel@tonic-gate *		NOTE: This flag should always be used when a task queue is used
1807c478bdstevel@tonic-gate *		for tasks that may depend on each other for completion.
1817c478bdstevel@tonic-gate *		Enqueueing dependent tasks may create deadlocks.
1827c478bdstevel@tonic-gate *
1837c478bdstevel@tonic-gate *	  TQ_SLEEP:   May block waiting for resources. May still fail for
18435a5a35Jonathan Adams *		dynamic task queues if TQ_NOQUEUE is also specified, otherwise
1857c478bdstevel@tonic-gate *		always succeed.
1867c478bdstevel@tonic-gate *
18735a5a35Jonathan Adams *	  TQ_FRONT:   Puts the new task at the front of the queue.  Be careful.
18835a5a35Jonathan Adams *
1897c478bdstevel@tonic-gate *	NOTE: Dynamic task queues are much more likely to fail in
1907c478bdstevel@tonic-gate *		taskq_dispatch() (especially if TQ_NOQUEUE was specified), so it
1917c478bdstevel@tonic-gate *		is important to have backup strategies handling such failures.
1927c478bdstevel@tonic-gate *
1935aeb947Garrett D'Amore * void taskq_dispatch_ent(tq, func, arg, flags, tqent)
1945aeb947Garrett D'Amore *
1955aeb947Garrett D'Amore *	This is a light-weight form of taskq_dispatch(), that uses a
1965aeb947Garrett D'Amore *	preallocated taskq_ent_t structure for scheduling.  As a
1975aeb947Garrett D'Amore *	result, it does not perform allocations and cannot ever fail.
1985aeb947Garrett D'Amore *	Note especially that it cannot be used with TASKQ_DYNAMIC
1995aeb947Garrett D'Amore *	taskqs.  The memory for the tqent must not be modified or used
2005aeb947Garrett D'Amore *	until the function (func) is called.  (However, func itself
2015aeb947Garrett D'Amore *	may safely modify or free this memory, once it is called.)
2025aeb947Garrett D'Amore *	Note that the taskq framework will NOT free this memory.
2035aeb947Garrett D'Amore *
2044c99eccTim Kordas * boolean_t taskq_empty(tq)
2054c99eccTim Kordas *
2064c99eccTim Kordas *	Queries if there are tasks pending on the queue.
2074c99eccTim Kordas *
2087c478bdstevel@tonic-gate * void taskq_wait(tq):
2097c478bdstevel@tonic-gate *
2107c478bdstevel@tonic-gate *	Waits for all previously scheduled tasks to complete.
2117c478bdstevel@tonic-gate *
2127c478bdstevel@tonic-gate *	NOTE: It does not stop any new task dispatches.
2137c478bdstevel@tonic-gate *	      Do NOT call taskq_wait() from a task: it will cause deadlock.
2147c478bdstevel@tonic-gate *
2157c478bdstevel@tonic-gate * void taskq_suspend(tq)
2167c478bdstevel@tonic-gate *
2177c478bdstevel@tonic-gate *	Suspend all task execution. Tasks already scheduled for a dynamic task
2187c478bdstevel@tonic-gate *	queue will still be executed, but all new scheduled tasks will be
2197c478bdstevel@tonic-gate *	suspended until taskq_resume() is called.
2207c478bdstevel@tonic-gate *
2217c478bdstevel@tonic-gate * int  taskq_suspended(tq)
2227c478bdstevel@tonic-gate *
2237c478bdstevel@tonic-gate *	Returns 1 if taskq is suspended and 0 otherwise. It is intended to
2247c478bdstevel@tonic-gate *	ASSERT that the task queue is suspended.
2257c478bdstevel@tonic-gate *
2267c478bdstevel@tonic-gate * void taskq_resume(tq)
2277c478bdstevel@tonic-gate *
2287c478bdstevel@tonic-gate *	Resume task queue execution.
2297c478bdstevel@tonic-gate *
2307c478bdstevel@tonic-gate * int  taskq_member(tq, thread)
2317c478bdstevel@tonic-gate *
2327c478bdstevel@tonic-gate *	Returns 1 if 'thread' belongs to taskq 'tq' and 0 otherwise. The
2337c478bdstevel@tonic-gate *	intended use is to ASSERT that a given function is called in taskq
2347c478bdstevel@tonic-gate *	context only.
2357c478bdstevel@tonic-gate *
2367c478bdstevel@tonic-gate * system_taskq
2377c478bdstevel@tonic-gate *
2387c478bdstevel@tonic-gate *	Global system-wide dynamic task queue for common uses. It may be used by
2397c478bdstevel@tonic-gate *	any subsystem that needs to schedule tasks and does not need to manage
2407c478bdstevel@tonic-gate *	its own task queues. It is initialized quite early during system boot.
2417c478bdstevel@tonic-gate *
2422e0c549Jonathan Adams * IMPLEMENTATION ==============================================================
2437c478bdstevel@tonic-gate *
2447c478bdstevel@tonic-gate * This is schematic representation of the task queue structures.
2457c478bdstevel@tonic-gate *
2467c478bdstevel@tonic-gate *   taskq:
2477c478bdstevel@tonic-gate *   +-------------+
248e0ad97eJonathan Adams *   | tq_lock     | +---< taskq_ent_free()
2497c478bdstevel@tonic-gate *   +-------------+ |
2507c478bdstevel@tonic-gate *   |...          | | tqent:                  tqent:
2517c478bdstevel@tonic-gate *   +-------------+ | +------------+          +------------+
2527c478bdstevel@tonic-gate *   | tq_freelist |-->| tqent_next |--> ... ->| tqent_next |
2537c478bdstevel@tonic-gate *   +-------------+   +------------+          +------------+
2547c478bdstevel@tonic-gate *   |...          |   | ...        |          | ...        |
2557c478bdstevel@tonic-gate *   +-------------+   +------------+          +------------+
2567c478bdstevel@tonic-gate *   | tq_task     |    |
2577c478bdstevel@tonic-gate *   |             |    +-------------->taskq_ent_alloc()
2587c478bdstevel@tonic-gate * +--------------------------------------------------------------------------+
2597c478bdstevel@tonic-gate * | |                     |            tqent                   tqent         |
2607c478bdstevel@tonic-gate * | +---------------------+     +--> +------------+     +--> +------------+  |
2617c478bdstevel@tonic-gate * | | ...		   |     |    | func, arg  |     |    | func, arg  |  |
2627c478bdstevel@tonic-gate * +>+---------------------+ <---|-+  +------------+ <---|-+  +------------+  |
2637c478bdstevel@tonic-gate *   | tq_taskq.tqent_next | ----+ |  | tqent_next | --->+ |  | tqent_next |--+
2647c478bdstevel@tonic-gate *   +---------------------+	   |  +------------+     ^ |  +------------+
2657c478bdstevel@tonic-gate * +-| tq_task.tqent_prev  |	   +--| tqent_prev |     | +--| tqent_prev |  ^
2667c478bdstevel@tonic-gate * | +---------------------+	      +------------+     |    +------------+  |
2677c478bdstevel@tonic-gate * | |...		   |	      | ...        |     |    | ...        |  |
2687c478bdstevel@tonic-gate * | +---------------------+	      +------------+     |    +------------+  |
2697c478bdstevel@tonic-gate * |                                      ^              |                    |
2707c478bdstevel@tonic-gate * |                                      |              |                    |
2717c478bdstevel@tonic-gate * +--------------------------------------+--------------+       TQ_APPEND() -+
2727c478bdstevel@tonic-gate *   |             |                      |
2737c478bdstevel@tonic-gate *   |...          |   taskq_thread()-----+
2747c478bdstevel@tonic-gate *   +-------------+
2757c478bdstevel@tonic-gate *   | tq_buckets  |--+-------> [ NULL ] (for regular task queues)
2767c478bdstevel@tonic-gate *   +-------------+  |
2777c478bdstevel@tonic-gate *                    |   DYNAMIC TASK QUEUES:
2787c478bdstevel@tonic-gate *                    |
27935a5a35Jonathan Adams *                    +-> taskq_bucket[nCPU]		taskq_bucket_dispatch()
2807c478bdstevel@tonic-gate *                        +-------------------+                    ^
2817c478bdstevel@tonic-gate *                   +--->| tqbucket_lock     |                    |
2827c478bdstevel@tonic-gate *                   |    +-------------------+   +--------+      +--------+
2837c478bdstevel@tonic-gate *                   |    | tqbucket_freelist |-->| tqent  |-->...| tqent  | ^
2847c478bdstevel@tonic-gate *                   |    +-------------------+<--+--------+<--...+--------+ |
2857c478bdstevel@tonic-gate *                   |    | ...               |   | thread |      | thread | |
2867c478bdstevel@tonic-gate *                   |    +-------------------+   +--------+      +--------+ |
2877c478bdstevel@tonic-gate *                   |    +-------------------+                              |
2887c478bdstevel@tonic-gate * taskq_dispatch()--+--->| tqbucket_lock     |             TQ_APPEND()------+
2897c478bdstevel@tonic-gate *      TQ_HASH()    |    +-------------------+   +--------+      +--------+
2907c478bdstevel@tonic-gate *                   |    | tqbucket_freelist |-->| tqent  |-->...| tqent  |
2917c478bdstevel@tonic-gate *                   |    +-------------------+<--+--------+<--...+--------+
2927c478bdstevel@tonic-gate *                   |    | ...               |   | thread |      | thread |
2937c478bdstevel@tonic-gate *                   |    +-------------------+   +--------+      +--------+
29435a5a35Jonathan Adams *		     +--->	...
2957c478bdstevel@tonic-gate *
2967c478bdstevel@tonic-gate *
2977c478bdstevel@tonic-gate * Task queues use tq_task field to link new entry in the queue. The queue is a
2987c478bdstevel@tonic-gate * circular doubly-linked list. Entries are put in the end of the list with
2997c478bdstevel@tonic-gate * TQ_APPEND() and processed from the front of the list by taskq_thread() in
3007c478bdstevel@tonic-gate * FIFO order. Task queue entries are cached in the free list managed by
3017c478bdstevel@tonic-gate * taskq_ent_alloc() and taskq_ent_free() functions.
3027c478bdstevel@tonic-gate *
3037c478bdstevel@tonic-gate *	All threads used by task queues mark t_taskq field of the thread to
3047c478bdstevel@tonic-gate *	point to the task queue.
3057c478bdstevel@tonic-gate *
3062e0c549Jonathan Adams * Taskq Thread Management -----------------------------------------------------
3072e0c549Jonathan Adams *
3082e0c549Jonathan Adams * Taskq's non-dynamic threads are managed with several variables and flags:
3092e0c549Jonathan Adams *
3102e0c549Jonathan Adams *	* tq_nthreads	- The number of threads in taskq_thread() for the
3112e0c549Jonathan Adams *			  taskq.
3122e0c549Jonathan Adams *
3132e0c549Jonathan Adams *	* tq_active	- The number of threads not waiting on a CV in
3142e0c549Jonathan Adams *			  taskq_thread(); includes newly created threads
3152e0c549Jonathan Adams *			  not yet counted in tq_nthreads.
3162e0c549Jonathan Adams *
3172e0c549Jonathan Adams *	* tq_nthreads_target
3182e0c549Jonathan Adams *			- The number of threads desired for the taskq.
3192e0c549Jonathan Adams *
3202e0c549Jonathan Adams *	* tq_flags & TASKQ_CHANGING
3212e0c549Jonathan Adams *			- Indicates that tq_nthreads != tq_nthreads_target.
3222e0c549Jonathan Adams *
3232e0c549Jonathan Adams *	* tq_flags & TASKQ_THREAD_CREATED
3242e0c549Jonathan Adams *			- Indicates that a thread is being created in the taskq.
3252e0c549Jonathan Adams *
3262e0c549Jonathan Adams * During creation, tq_nthreads and tq_active are set to 0, and
3272e0c549Jonathan Adams * tq_nthreads_target is set to the number of threads desired.  The
32835a5a35Jonathan Adams * TASKQ_CHANGING flag is set, and taskq_thread_create() is called to
32935a5a35Jonathan Adams * create the first thread. taskq_thread_create() increments tq_active,
3302e0c549Jonathan Adams * sets TASKQ_THREAD_CREATED, and creates the new thread.
3312e0c549Jonathan Adams *
3322e0c549Jonathan Adams * Each thread starts in taskq_thread(), clears the TASKQ_THREAD_CREATED
3332e0c549Jonathan Adams * flag, and increments tq_nthreads.  It stores the new value of
3342e0c549Jonathan Adams * tq_nthreads as its "thread_id", and stores its thread pointer in the
3352e0c549Jonathan Adams * tq_threadlist at the (thread_id - 1).  We keep the thread_id space
3362e0c549Jonathan Adams * densely packed by requiring that only the largest thread_id can exit during
3372e0c549Jonathan Adams * normal adjustment.   The exception is during the destruction of the
3382e0c549Jonathan Adams * taskq; once tq_nthreads_target is set to zero, no new threads will be created
3392e0c549Jonathan Adams * for the taskq queue, so every thread can exit without any ordering being
3402e0c549Jonathan Adams * necessary.
3412e0c549Jonathan Adams *
3422e0c549Jonathan Adams * Threads will only process work if their thread id is <= tq_nthreads_target.
3432e0c549Jonathan Adams *
3442e0c549Jonathan Adams * When TASKQ_CHANGING is set, threads will check the current thread target
3452e0c549Jonathan Adams * whenever they wake up, and do whatever they can to apply its effects.
3462e0c549Jonathan Adams *
3472e0c549Jonathan Adams * TASKQ_THREAD_CPU_PCT --------------------------------------------------------
3482e0c549Jonathan Adams *
3492e0c549Jonathan Adams * When a taskq is created with TASKQ_THREAD_CPU_PCT, we store their requested
3502e0c549Jonathan Adams * percentage in tq_threads_ncpus_pct, start them off with the correct thread
3512e0c549Jonathan Adams * target, and add them to the taskq_cpupct_list for later adjustment.
3522e0c549Jonathan Adams *
3532e0c549Jonathan Adams * We register taskq_cpu_setup() to be called whenever a CPU changes state.  It
3542e0c549Jonathan Adams * walks the list of TASKQ_THREAD_CPU_PCT taskqs, adjusts their nthread_target
3552e0c549Jonathan Adams * if need be, and wakes up all of the threads to process the change.
3562e0c549Jonathan Adams *
3572e0c549Jonathan Adams * Dynamic Task Queues Implementation ------------------------------------------
3587c478bdstevel@tonic-gate *
3597c478bdstevel@tonic-gate * For a dynamic task queues there is a 1-to-1 mapping between a thread and
3607c478bdstevel@tonic-gate * taskq_ent_structure. Each entry is serviced by its own thread and each thread
3617c478bdstevel@tonic-gate * is controlled by a single entry.
3627c478bdstevel@tonic-gate *
3637c478bdstevel@tonic-gate * Entries are distributed over a set of buckets. To avoid using modulo
3647c478bdstevel@tonic-gate * arithmetics the number of buckets is 2^n and is determined as the nearest
3657c478bdstevel@tonic-gate * power of two roundown of the number of CPUs in the system. Tunable
3667c478bdstevel@tonic-gate * variable 'taskq_maxbuckets' limits the maximum number of buckets. Each entry
3677c478bdstevel@tonic-gate * is attached to a bucket for its lifetime and can't migrate to other buckets.
3687c478bdstevel@tonic-gate *
3697c478bdstevel@tonic-gate * Entries that have scheduled tasks are not placed in any list. The dispatch
3707c478bdstevel@tonic-gate * function sets their "func" and "arg" fields and signals the corresponding
3717c478bdstevel@tonic-gate * thread to execute the task. Once the thread executes the task it clears the
3727c478bdstevel@tonic-gate * "func" field and places an entry on the bucket cache of free entries pointed
3737c478bdstevel@tonic-gate * by "tqbucket_freelist" field. ALL entries on the free list should have "func"
3747c478bdstevel@tonic-gate * field equal to NULL. The free list is a circular doubly-linked list identical
3757c478bdstevel@tonic-gate * in structure to the tq_task list above, but entries are taken from it in LIFO
3767c478bdstevel@tonic-gate * order - the last freed entry is the first to be allocated. The
3777c478bdstevel@tonic-gate * taskq_bucket_dispatch() function gets the most recently used entry from the
3787c478bdstevel@tonic-gate * free list, sets its "func" and "arg" fields and signals a worker thread.
3797c478bdstevel@tonic-gate *
3807c478bdstevel@tonic-gate * After executing each task a per-entry thread taskq_d_thread() places its
3817c478bdstevel@tonic-gate * entry on the bucket free list and goes to a timed sleep. If it wakes up
3827c478bdstevel@tonic-gate * without getting new task it removes the entry from the free list and destroys
3837c478bdstevel@tonic-gate * itself. The thread sleep time is controlled by a tunable variable
3847c478bdstevel@tonic-gate * `taskq_thread_timeout'.
3857c478bdstevel@tonic-gate *
3862e0c549Jonathan Adams * There are various statistics kept in the bucket which allows for later
3877c478bdstevel@tonic-gate * analysis of taskq usage patterns. Also, a global copy of taskq creation and
3887c478bdstevel@tonic-gate * death statistics is kept in the global taskq data structure. Since thread
3897c478bdstevel@tonic-gate * creation and death happen rarely, updating such global data does not present
3907c478bdstevel@tonic-gate * a performance problem.
3917c478bdstevel@tonic-gate *
3927c478bdstevel@tonic-gate * NOTE: Threads are not bound to any CPU and there is absolutely no association
3937c478bdstevel@tonic-gate *       between the bucket and actual thread CPU, so buckets are used only to
3947c478bdstevel@tonic-gate *	 split resources and reduce resource contention. Having threads attached
3957c478bdstevel@tonic-gate *	 to the CPU denoted by a bucket may reduce number of times the job
3967c478bdstevel@tonic-gate *	 switches between CPUs.
3977c478bdstevel@tonic-gate *
3987c478bdstevel@tonic-gate *	 Current algorithm creates a thread whenever a bucket has no free
3997c478bdstevel@tonic-gate *	 entries. It would be nice to know how many threads are in the running
4007c478bdstevel@tonic-gate *	 state and don't create threads if all CPUs are busy with existing
4017c478bdstevel@tonic-gate *	 tasks, but it is unclear how such strategy can be implemented.
4027c478bdstevel@tonic-gate *
4037c478bdstevel@tonic-gate *	 Currently buckets are created statically as an array attached to task
4047c478bdstevel@tonic-gate *	 queue. On some system with nCPUs < max_ncpus it may waste system
4057c478bdstevel@tonic-gate *	 memory. One solution may be allocation of buckets when they are first
4067c478bdstevel@tonic-gate *	 touched, but it is not clear how useful it is.
4077c478bdstevel@tonic-gate *
4082e0c549Jonathan Adams * SUSPEND/RESUME implementation -----------------------------------------------
4097c478bdstevel@tonic-gate *
4107c478bdstevel@tonic-gate *	Before executing a task taskq_thread() (executing non-dynamic task
4117c478bdstevel@tonic-gate *	queues) obtains taskq's thread lock as a reader. The taskq_suspend()
4127c478bdstevel@tonic-gate *	function gets the same lock as a writer blocking all non-dynamic task
4137c478bdstevel@tonic-gate *	execution. The taskq_resume() function releases the lock allowing
4147c478bdstevel@tonic-gate *	taskq_thread to continue execution.
4157c478bdstevel@tonic-gate *
4167c478bdstevel@tonic-gate *	For dynamic task queues, each bucket is marked as TQBUCKET_SUSPEND by
4177c478bdstevel@tonic-gate *	taskq_suspend() function. After that taskq_bucket_dispatch() always
4187c478bdstevel@tonic-gate *	fails, so that taskq_dispatch() will either enqueue tasks for a
4197c478bdstevel@tonic-gate *	suspended backing queue or fail if TQ_NOQUEUE is specified in dispatch
4207c478bdstevel@tonic-gate *	flags.
4217c478bdstevel@tonic-gate *
4227c478bdstevel@tonic-gate *	NOTE: taskq_suspend() does not immediately block any tasks already
4237c478bdstevel@tonic-gate *	      scheduled for dynamic task queues. It only suspends new tasks
4247c478bdstevel@tonic-gate *	      scheduled after taskq_suspend() was called.
4257c478bdstevel@tonic-gate *
4267c478bdstevel@tonic-gate *	taskq_member() function works by comparing a thread t_taskq pointer with
4277c478bdstevel@tonic-gate *	the passed thread pointer.
4287c478bdstevel@tonic-gate *
4292e0c549Jonathan Adams * LOCKS and LOCK Hierarchy ----------------------------------------------------
4307c478bdstevel@tonic-gate *
4312e0c549Jonathan Adams *   There are three locks used in task queues:
4327c478bdstevel@tonic-gate *
4332e0c549Jonathan Adams *   1) The taskq_t's tq_lock, protecting global task queue state.
4347c478bdstevel@tonic-gate *
4357c478bdstevel@tonic-gate *   2) Each per-CPU bucket has a lock for bucket management.
4367c478bdstevel@tonic-gate *
4372e0c549Jonathan Adams *   3) The global taskq_cpupct_lock, which protects the list of
4382e0c549Jonathan Adams *      TASKQ_THREADS_CPU_PCT taskqs.
4392e0c549Jonathan Adams *
4402e0c549Jonathan Adams *   If both (1) and (2) are needed, tq_lock should be taken *after* the bucket
4417c478bdstevel@tonic-gate *   lock.
4427c478bdstevel@tonic-gate *
4432e0c549Jonathan Adams *   If both (1) and (3) are needed, tq_lock should be taken *after*
4442e0c549Jonathan Adams *   taskq_cpupct_lock.
4452e0c549Jonathan Adams *
4462e0c549Jonathan Adams * DEBUG FACILITIES ------------------------------------------------------------
4477c478bdstevel@tonic-gate *
4487c478bdstevel@tonic-gate * For DEBUG kernels it is possible to induce random failures to
4497c478bdstevel@tonic-gate * taskq_dispatch() function when it is given TQ_NOSLEEP argument. The value of
4507c478bdstevel@tonic-gate * taskq_dmtbf and taskq_smtbf tunables control the mean time between induced
4517c478bdstevel@tonic-gate * failures for dynamic and static task queues respectively.
4527c478bdstevel@tonic-gate *
4537c478bdstevel@tonic-gate * Setting TASKQ_STATISTIC to 0 will disable per-bucket statistics.
4547c478bdstevel@tonic-gate *
4552e0c549Jonathan Adams * TUNABLES --------------------------------------------------------------------
4567c478bdstevel@tonic-gate *
4577c478bdstevel@tonic-gate *	system_taskq_size	- Size of the global system_taskq.
4587c478bdstevel@tonic-gate *				  This value is multiplied by nCPUs to determine
4597c478bdstevel@tonic-gate *				  actual size.
4607c478bdstevel@tonic-gate *				  Default value: 64
4617c478bdstevel@tonic-gate *
4622e0c549Jonathan Adams *	taskq_minimum_nthreads_max
4632e0c549Jonathan Adams *				- Minimum size of the thread list for a taskq.
4642e0c549Jonathan Adams *				  Useful for testing different thread pool
4652e0c549Jonathan Adams *				  sizes by overwriting tq_nthreads_target.
4662e0c549Jonathan Adams *
4677c478bdstevel@tonic-gate *	taskq_thread_timeout	- Maximum idle time for taskq_d_thread()
4687c478bdstevel@tonic-gate *				  Default value: 5 minutes
4697c478bdstevel@tonic-gate *
4707c478bdstevel@tonic-gate *	taskq_maxbuckets	- Maximum number of buckets in any task queue
4717c478bdstevel@tonic-gate *				  Default value: 128
4727c478bdstevel@tonic-gate *
4737c478bdstevel@tonic-gate *	taskq_search_depth	- Maximum # of buckets searched for a free entry
4747c478bdstevel@tonic-gate *				  Default value: 4
4757c478bdstevel@tonic-gate *
4767c478bdstevel@tonic-gate *	taskq_dmtbf		- Mean time between induced dispatch failures
4777c478bdstevel@tonic-gate *				  for dynamic task queues.
4787c478bdstevel@tonic-gate *				  Default value: UINT_MAX (no induced failures)
4797c478bdstevel@tonic-gate *
4807c478bdstevel@tonic-gate *	taskq_smtbf		- Mean time between induced dispatch failures
4817c478bdstevel@tonic-gate *				  for static task queues.
4827c478bdstevel@tonic-gate *				  Default value: UINT_MAX (no induced failures)
4837c478bdstevel@tonic-gate *
4842e0c549Jonathan Adams * CONDITIONAL compilation -----------------------------------------------------
4857c478bdstevel@tonic-gate *
4867c478bdstevel@tonic-gate *    TASKQ_STATISTIC	- If set will enable bucket statistic (default).
4877c478bdstevel@tonic-gate *
4887c478bdstevel@tonic-gate */
4897c478bdstevel@tonic-gate
4907c478bdstevel@tonic-gate#include <sys/taskq_impl.h>
4917c478bdstevel@tonic-gate#include <sys/thread.h>
4927c478bdstevel@tonic-gate#include <sys/proc.h>
4937c478bdstevel@tonic-gate#include <sys/kmem.h>
4947c478bdstevel@tonic-gate#include <sys/vmem.h>
4957c478bdstevel@tonic-gate#include <sys/callb.h>
49635a5a35Jonathan Adams#include <sys/class.h>
4977c478bdstevel@tonic-gate#include <sys/systm.h>
4987c478bdstevel@tonic-gate#include <sys/cmn_err.h>
4997c478bdstevel@tonic-gate#include <sys/debug.h>
5007c478bdstevel@tonic-gate#include <sys/vmsystm.h>	/* For throttlefree */
5017c478bdstevel@tonic-gate#include <sys/sysmacros.h>
5027c478bdstevel@tonic-gate#include <sys/cpuvar.h>
50335a5a35Jonathan Adams#include <sys/cpupart.h>
5047c478bdstevel@tonic-gate#include <sys/sdt.h>
50535a5a35Jonathan Adams#include <sys/sysdc.h>
5062e0c549Jonathan Adams#include <sys/note.h>
5077c478bdstevel@tonic-gate
5087c478bdstevel@tonic-gatestatic kmem_cache_t *taskq_ent_cache, *taskq_cache;
5097c478bdstevel@tonic-gate
5107c478bdstevel@tonic-gate/*
5112e0c549Jonathan Adams * Pseudo instance numbers for taskqs without explicitly provided instance.
5127c478bdstevel@tonic-gate */
5137c478bdstevel@tonic-gatestatic vmem_t *taskq_id_arena;
5147c478bdstevel@tonic-gate
5157c478bdstevel@tonic-gate/* Global system task queue for common use */
5167c478bdstevel@tonic-gatetaskq_t	*system_taskq;
5177c478bdstevel@tonic-gate
5187c478bdstevel@tonic-gate/*
5192e0c549Jonathan Adams * Maximum number of entries in global system taskq is
52035a5a35Jonathan Adams *	system_taskq_size * max_ncpus
5217c478bdstevel@tonic-gate */
5227c478bdstevel@tonic-gate#define	SYSTEM_TASKQ_SIZE 64
5237c478bdstevel@tonic-gateint system_taskq_size = SYSTEM_TASKQ_SIZE;
5247c478bdstevel@tonic-gate
5257c478bdstevel@tonic-gate/*
5262e0c549Jonathan Adams * Minimum size for tq_nthreads_max; useful for those who want to play around
5272e0c549Jonathan Adams * with increasing a taskq's tq_nthreads_target.
5282e0c549Jonathan Adams */
5292e0c549Jonathan Adamsint taskq_minimum_nthreads_max = 1;
5302e0c549Jonathan Adams
53135a5a35Jonathan Adams/*
53235a5a35Jonathan Adams * We want to ensure that when taskq_create() returns, there is at least
53335a5a35Jonathan Adams * one thread ready to handle requests.  To guarantee this, we have to wait
53435a5a35Jonathan Adams * for the second thread, since the first one cannot process requests until
53535a5a35Jonathan Adams * the second thread has been created.
53635a5a35Jonathan Adams */
53735a5a35Jonathan Adams#define	TASKQ_CREATE_ACTIVE_THREADS	2
53835a5a35Jonathan Adams
5392e0c549Jonathan Adams/* Maximum percentage allowed for TASKQ_THREADS_CPU_PCT */
5402e0c549Jonathan Adams#define	TASKQ_CPUPCT_MAX_PERCENT	1000
5412e0c549Jonathan Adamsint taskq_cpupct_max_percent = TASKQ_CPUPCT_MAX_PERCENT;
5422e0c549Jonathan Adams
5432e0c549Jonathan Adams/*
5447c478bdstevel@tonic-gate * Dynamic task queue threads that don't get any work within
5457c478bdstevel@tonic-gate * taskq_thread_timeout destroy themselves
5467c478bdstevel@tonic-gate */
5477c478bdstevel@tonic-gate#define	TASKQ_THREAD_TIMEOUT (60 * 5)
5487c478bdstevel@tonic-gateint taskq_thread_timeout = TASKQ_THREAD_TIMEOUT;
5497c478bdstevel@tonic-gate
5507c478bdstevel@tonic-gate#define	TASKQ_MAXBUCKETS 128
5517c478bdstevel@tonic-gateint taskq_maxbuckets = TASKQ_MAXBUCKETS;
5527c478bdstevel@tonic-gate
5537c478bdstevel@tonic-gate/*
5547c478bdstevel@tonic-gate * When a bucket has no available entries another buckets are tried.
5557c478bdstevel@tonic-gate * taskq_search_depth parameter limits the amount of buckets that we search
5567c478bdstevel@tonic-gate * before failing. This is mostly useful in systems with many CPUs where we may
5577c478bdstevel@tonic-gate * spend too much time scanning busy buckets.
5587c478bdstevel@tonic-gate */
5597c478bdstevel@tonic-gate#define	TASKQ_SEARCH_DEPTH 4
5607c478bdstevel@tonic-gateint taskq_search_depth = TASKQ_SEARCH_DEPTH;
5617c478bdstevel@tonic-gate
5627c478bdstevel@tonic-gate/*
5637c478bdstevel@tonic-gate * Hashing function: mix various bits of x. May be pretty much anything.
5647c478bdstevel@tonic-gate */
5657c478bdstevel@tonic-gate#define	TQ_HASH(x) ((x) ^ ((x) >> 11) ^ ((x) >> 17) ^ ((x) ^ 27))
5667c478bdstevel@tonic-gate
5677c478bdstevel@tonic-gate/*
5687c478bdstevel@tonic-gate * We do not create any new threads when the system is low on memory and start
5697c478bdstevel@tonic-gate * throttling memory allocations. The following macro tries to estimate such
5707c478bdstevel@tonic-gate * condition.
5717c478bdstevel@tonic-gate */
5727c478bdstevel@tonic-gate#define	ENOUGH_MEMORY() (freemem > throttlefree)
5737c478bdstevel@tonic-gate
5747c478bdstevel@tonic-gate/*
5757c478bdstevel@tonic-gate * Static functions.
5767c478bdstevel@tonic-gate */
5777c478bdstevel@tonic-gatestatic taskq_t	*taskq_create_common(const char *, int, int, pri_t, int,
57835a5a35Jonathan Adams    int, proc_t *, uint_t, uint_t);
5797c478bdstevel@tonic-gatestatic void taskq_thread(void *);
5807c478bdstevel@tonic-gatestatic void taskq_d_thread(taskq_ent_t *);
5817c478bdstevel@tonic-gatestatic void taskq_bucket_extend(void *);
5827c478bdstevel@tonic-gatestatic int  taskq_constructor(void *, void *, int);
5837c478bdstevel@tonic-gatestatic void taskq_destructor(void *, void *);
5847c478bdstevel@tonic-gatestatic int  taskq_ent_constructor(void *, void *, int);
5857c478bdstevel@tonic-gatestatic void taskq_ent_destructor(void *, void *);
5867c478bdstevel@tonic-gatestatic taskq_ent_t *taskq_ent_alloc(taskq_t *, int);
5877c478bdstevel@tonic-gatestatic void taskq_ent_free(taskq_t *, taskq_ent_t *);
5886410974Chris Hornestatic int taskq_ent_exists(taskq_t *, task_func_t, void *);
5897c478bdstevel@tonic-gatestatic taskq_ent_t *taskq_bucket_dispatch(taskq_bucket_t *, task_func_t,
5907c478bdstevel@tonic-gate    void *);
5917c478bdstevel@tonic-gate
5927c478bdstevel@tonic-gate/*
5937c478bdstevel@tonic-gate * Task queues kstats.
5947c478bdstevel@tonic-gate */
5957c478bdstevel@tonic-gatestruct taskq_kstat {
59635a5a35Jonathan Adams	kstat_named_t	tq_pid;
5977c478bdstevel@tonic-gate	kstat_named_t	tq_tasks;
5987c478bdstevel@tonic-gate	kstat_named_t	tq_executed;
5997c478bdstevel@tonic-gate	kstat_named_t	tq_maxtasks;
6007c478bdstevel@tonic-gate	kstat_named_t	tq_totaltime;
6017c478bdstevel@tonic-gate	kstat_named_t	tq_nalloc;
6027c478bdstevel@tonic-gate	kstat_named_t	tq_nactive;
6037c478bdstevel@tonic-gate	kstat_named_t	tq_pri;
6047c478bdstevel@tonic-gate	kstat_named_t	tq_nthreads;
605216d772Prakash Surya	kstat_named_t	tq_nomem;
6067c478bdstevel@tonic-gate} taskq_kstat = {
60735a5a35Jonathan Adams	{ "pid",		KSTAT_DATA_UINT64 },
6087c478bdstevel@tonic-gate	{ "tasks",		KSTAT_DATA_UINT64 },
6097c478bdstevel@tonic-gate	{ "executed",		KSTAT_DATA_UINT64 },
6107c478bdstevel@tonic-gate	{ "maxtasks",		KSTAT_DATA_UINT64 },
6117c478bdstevel@tonic-gate	{ "totaltime",		KSTAT_DATA_UINT64 },
6127c478bdstevel@tonic-gate	{ "nalloc",		KSTAT_DATA_UINT64 },
6137c5f01cRobert Mustacchi	{ "nactive",		KSTAT_DATA_UINT64 },
6147c478bdstevel@tonic-gate	{ "priority",		KSTAT_DATA_UINT64 },
6157c478bdstevel@tonic-gate	{ "threads",		KSTAT_DATA_UINT64 },
616216d772Prakash Surya	{ "nomem",		KSTAT_DATA_UINT64 },
6177c478bdstevel@tonic-gate};
6187c478bdstevel@tonic-gate
6197c478bdstevel@tonic-gatestruct taskq_d_kstat {
6207c478bdstevel@tonic-gate	kstat_named_t	tqd_pri;
6217c478bdstevel@tonic-gate	kstat_named_t	tqd_btasks;
6227c478bdstevel@tonic-gate	kstat_named_t	tqd_bexecuted;
6237c478bdstevel@tonic-gate	kstat_named_t	tqd_bmaxtasks;
6247c478bdstevel@tonic-gate	kstat_named_t	tqd_bnalloc;
6257c478bdstevel@tonic-gate	kstat_named_t	tqd_bnactive;
6267c478bdstevel@tonic-gate	kstat_named_t	tqd_btotaltime;
6277c478bdstevel@tonic-gate	kstat_named_t	tqd_hits;
6287c478bdstevel@tonic-gate	kstat_named_t	tqd_misses;
6297c478bdstevel@tonic-gate	kstat_named_t	tqd_overflows;
6307c478bdstevel@tonic-gate	kstat_named_t	tqd_tcreates;
6317c478bdstevel@tonic-gate	kstat_named_t	tqd_tdeaths;
6327c478bdstevel@tonic-gate	kstat_named_t	tqd_maxthreads;
6337c478bdstevel@tonic-gate	kstat_named_t	tqd_nomem;
6347c478bdstevel@tonic-gate	kstat_named_t	tqd_disptcreates;
6357c478bdstevel@tonic-gate	kstat_named_t	tqd_totaltime;
6367c478bdstevel@tonic-gate	kstat_named_t	tqd_nalloc;
6377c478bdstevel@tonic-gate	kstat_named_t	tqd_nfree;
6387c478bdstevel@tonic-gate} taskq_d_kstat = {
6397c478bdstevel@tonic-gate	{ "priority",		KSTAT_DATA_UINT64 },
6407c478bdstevel@tonic-gate	{ "btasks",		KSTAT_DATA_UINT64 },
6417c478bdstevel@tonic-gate	{ "bexecuted",		KSTAT_DATA_UINT64 },
6427c478bdstevel@tonic-gate	{ "bmaxtasks",		KSTAT_DATA_UINT64 },
6437c478bdstevel@tonic-gate	{ "bnalloc",		KSTAT_DATA_UINT64 },
6447c478bdstevel@tonic-gate	{ "bnactive",		KSTAT_DATA_UINT64 },
6457c478bdstevel@tonic-gate	{ "btotaltime",		KSTAT_DATA_UINT64 },
6467c478bdstevel@tonic-gate	{ "hits",		KSTAT_DATA_UINT64 },
6477c478bdstevel@tonic-gate	{ "misses",		KSTAT_DATA_UINT64 },
6487c478bdstevel@tonic-gate	{ "overflows",		KSTAT_DATA_UINT64 },
6497c478bdstevel@tonic-gate	{ "tcreates",		KSTAT_DATA_UINT64 },
6507c478bdstevel@tonic-gate	{ "tdeaths",		KSTAT_DATA_UINT64 },
6517c478bdstevel@tonic-gate	{ "maxthreads",		KSTAT_DATA_UINT64 },
6527c478bdstevel@tonic-gate	{ "nomem",		KSTAT_DATA_UINT64 },
6537c478bdstevel@tonic-gate	{ "disptcreates",	KSTAT_DATA_UINT64 },
6547c478bdstevel@tonic-gate	{ "totaltime",		KSTAT_DATA_UINT64 },
6557c478bdstevel@tonic-gate	{ "nalloc",		KSTAT_DATA_UINT64 },
6567c478bdstevel@tonic-gate	{ "nfree",		KSTAT_DATA_UINT64 },
6577c478bdstevel@tonic-gate};
6587c478bdstevel@tonic-gate
6597c478bdstevel@tonic-gatestatic kmutex_t taskq_kstat_lock;
6607c478bdstevel@tonic-gatestatic kmutex_t taskq_d_kstat_lock;
6617c478bdstevel@tonic-gatestatic int taskq_kstat_update(kstat_t *, int);
6627c478bdstevel@tonic-gatestatic int taskq_d_kstat_update(kstat_t *, int);
6637c478bdstevel@tonic-gate
6642e0c549Jonathan Adams/*
66535a5a35Jonathan Adams * List of all TASKQ_THREADS_CPU_PCT taskqs.
6662e0c549Jonathan Adams */
66735a5a35Jonathan Adamsstatic list_t taskq_cpupct_list;	/* protected by cpu_lock */
6687c478bdstevel@tonic-gate
6697c478bdstevel@tonic-gate/*
6707c478bdstevel@tonic-gate * Collect per-bucket statistic when TASKQ_STATISTIC is defined.
6717c478bdstevel@tonic-gate */
6727c478bdstevel@tonic-gate#define	TASKQ_STATISTIC 1
6737c478bdstevel@tonic-gate
6747c478bdstevel@tonic-gate#if TASKQ_STATISTIC
6757c478bdstevel@tonic-gate#define	TQ_STAT(b, x)	b->tqbucket_stat.x++
6767c478bdstevel@tonic-gate#else
6777c478bdstevel@tonic-gate#define	TQ_STAT(b, x)
6787c478bdstevel@tonic-gate#endif
6797c478bdstevel@tonic-gate
6807c478bdstevel@tonic-gate/*
6817c478bdstevel@tonic-gate * Random fault injection.
6827c478bdstevel@tonic-gate */
6837c478bdstevel@tonic-gateuint_t taskq_random;
6847c478bdstevel@tonic-gateuint_t taskq_dmtbf = UINT_MAX;    /* mean time between injected failures */
6857c478bdstevel@tonic-gateuint_t taskq_smtbf = UINT_MAX;    /* mean time between injected failures */
6867c478bdstevel@tonic-gate
6877c478bdstevel@tonic-gate/*
6887c478bdstevel@tonic-gate * TQ_NOSLEEP dispatches on dynamic task queues are always allowed to fail.
6897c478bdstevel@tonic-gate *
6907c478bdstevel@tonic-gate * TQ_NOSLEEP dispatches on static task queues can't arbitrarily fail because
6917c478bdstevel@tonic-gate * they could prepopulate the cache and make sure that they do not use more
6927c478bdstevel@tonic-gate * then minalloc entries.  So, fault injection in this case insures that
6937c478bdstevel@tonic-gate * either TASKQ_PREPOPULATE is not set or there are more entries allocated
6947c478bdstevel@tonic-gate * than is specified by minalloc.  TQ_NOALLOC dispatches are always allowed
6957c478bdstevel@tonic-gate * to fail, but for simplicity we treat them identically to TQ_NOSLEEP
6967c478bdstevel@tonic-gate * dispatches.
6977c478bdstevel@tonic-gate */
6987c478bdstevel@tonic-gate#ifdef DEBUG
6997c478bdstevel@tonic-gate#define	TASKQ_D_RANDOM_DISPATCH_FAILURE(tq, flag)		\
7007c478bdstevel@tonic-gate	taskq_random = (taskq_random * 2416 + 374441) % 1771875;\
7017c478bdstevel@tonic-gate	if ((flag & TQ_NOSLEEP) &&				\
7027c478bdstevel@tonic-gate	    taskq_random < 1771875 / taskq_dmtbf) {		\
703fc8ae2eToomas Soome		return (TASKQID_INVALID);			\
7047c478bdstevel@tonic-gate	}
7057c478bdstevel@tonic-gate
7067c478bdstevel@tonic-gate#define	TASKQ_S_RANDOM_DISPATCH_FAILURE(tq, flag)		\
7077c478bdstevel@tonic-gate	taskq_random = (taskq_random * 2416 + 374441) % 1771875;\
7087c478bdstevel@tonic-gate	if ((flag & (TQ_NOSLEEP | TQ_NOALLOC)) &&		\
7097c478bdstevel@tonic-gate	    (!(tq->tq_flags & TASKQ_PREPOPULATE) ||		\
7107c478bdstevel@tonic-gate	    (tq->tq_nalloc > tq->tq_minalloc)) &&		\
7117c478bdstevel@tonic-gate	    (taskq_random < (1771875 / taskq_smtbf))) {		\
7127c478bdstevel@tonic-gate		mutex_exit(&tq->tq_lock);			\
713fc8ae2eToomas Soome		return (TASKQID_INVALID);			\
7147c478bdstevel@tonic-gate	}
7157c478bdstevel@tonic-gate#else
7167c478bdstevel@tonic-gate#define	TASKQ_S_RANDOM_DISPATCH_FAILURE(tq, flag)
7177c478bdstevel@tonic-gate#define	TASKQ_D_RANDOM_DISPATCH_FAILURE(tq, flag)
7187c478bdstevel@tonic-gate#endif
7197c478bdstevel@tonic-gate
7207c478bdstevel@tonic-gate#define	IS_EMPTY(l) (((l).tqent_prev == (l).tqent_next) &&	\
7217c478bdstevel@tonic-gate	((l).tqent_prev == &(l)))
7227c478bdstevel@tonic-gate
7237c478bdstevel@tonic-gate/*
7247c478bdstevel@tonic-gate * Append `tqe' in the end of the doubly-linked list denoted by l.
7257c478bdstevel@tonic-gate */
7267c478bdstevel@tonic-gate#define	TQ_APPEND(l, tqe) {					\
7277c478bdstevel@tonic-gate	tqe->tqent_next = &l;					\
7287c478bdstevel@tonic-gate	tqe->tqent_prev = l.tqent_prev;				\
7297c478bdstevel@tonic-gate	tqe->tqent_next->tqent_prev = tqe;			\
7307c478bdstevel@tonic-gate	tqe->tqent_prev->tqent_next = tqe;			\
7317c478bdstevel@tonic-gate}
73235a5a35Jonathan Adams/*
73335a5a35Jonathan Adams * Prepend 'tqe' to the beginning of l
73435a5a35Jonathan Adams */
73535a5a35Jonathan Adams#define	TQ_PREPEND(l, tqe) {					\
73635a5a35Jonathan Adams	tqe->tqent_next = l.tqent_next;				\
73735a5a35Jonathan Adams	tqe->tqent_prev = &l;					\
73835a5a35Jonathan Adams	tqe->tqent_next->tqent_prev = tqe;			\
73935a5a35Jonathan Adams	tqe->tqent_prev->tqent_next = tqe;			\
74035a5a35Jonathan Adams}
7417c478bdstevel@tonic-gate
7427c478bdstevel@tonic-gate/*
7437c478bdstevel@tonic-gate * Schedule a task specified by func and arg into the task queue entry tqe.
7447c478bdstevel@tonic-gate */
74535a5a35Jonathan Adams#define	TQ_DO_ENQUEUE(tq, tqe, func, arg, front) {			\
74635a5a35Jonathan Adams	ASSERT(MUTEX_HELD(&tq->tq_lock));				\
74735a5a35Jonathan Adams	_NOTE(CONSTCOND)						\
74835a5a35Jonathan Adams	if (front) {							\
74935a5a35Jonathan Adams		TQ_PREPEND(tq->tq_task, tqe);				\
75035a5a35Jonathan Adams	} else {							\
75135a5a35Jonathan Adams		TQ_APPEND(tq->tq_task, tqe);				\
75235a5a35Jonathan Adams	}								\
75335a5a35Jonathan Adams	tqe->tqent_func = (func);					\
75435a5a35Jonathan Adams	tqe->tqent_arg = (arg);						\
75535a5a35Jonathan Adams	tq->tq_tasks++;							\
75635a5a35Jonathan Adams	if (tq->tq_tasks - tq->tq_executed > tq->tq_maxtasks)		\
7577c478bdstevel@tonic-gate		tq->tq_maxtasks = tq->tq_tasks - tq->tq_executed;	\
75835a5a35Jonathan Adams	cv_signal(&tq->tq_dispatch_cv);					\
7597c478bdstevel@tonic-gate	DTRACE_PROBE2(taskq__enqueue, taskq_t *, tq, taskq_ent_t *, tqe); \
7607c478bdstevel@tonic-gate}
7617c478bdstevel@tonic-gate
76235a5a35Jonathan Adams#define	TQ_ENQUEUE(tq, tqe, func, arg)					\
76335a5a35Jonathan Adams	TQ_DO_ENQUEUE(tq, tqe, func, arg, 0)
76435a5a35Jonathan Adams
76535a5a35Jonathan Adams#define	TQ_ENQUEUE_FRONT(tq, tqe, func, arg)				\
76635a5a35Jonathan Adams	TQ_DO_ENQUEUE(tq, tqe, func, arg, 1)
76735a5a35Jonathan Adams
7687c478bdstevel@tonic-gate/*
7697c478bdstevel@tonic-gate * Do-nothing task which may be used to prepopulate thread caches.
7707c478bdstevel@tonic-gate */
7717c478bdstevel@tonic-gate/*ARGSUSED*/
7727c478bdstevel@tonic-gatevoid
7737c478bdstevel@tonic-gatenulltask(void *unused)
7747c478bdstevel@tonic-gate{
7757c478bdstevel@tonic-gate}
7767c478bdstevel@tonic-gate
7777c478bdstevel@tonic-gate/*ARGSUSED*/
7787c478bdstevel@tonic-gatestatic int
7797c478bdstevel@tonic-gatetaskq_constructor(void *buf, void *cdrarg, int kmflags)
7807c478bdstevel@tonic-gate{
7817c478bdstevel@tonic-gate	taskq_t *tq = buf;
7827c478bdstevel@tonic-gate
7837c478bdstevel@tonic-gate	bzero(tq, sizeof (taskq_t));
7847c478bdstevel@tonic-gate
7857c478bdstevel@tonic-gate	mutex_init(&tq->tq_lock, NULL, MUTEX_DEFAULT, NULL);
7867c478bdstevel@tonic-gate	rw_init(&tq->tq_threadlock, NULL, RW_DEFAULT, NULL);
7877c478bdstevel@tonic-gate	cv_init(&tq->tq_dispatch_cv, NULL, CV_DEFAULT, NULL);
7882e0c549Jonathan Adams	cv_init(&tq->tq_exit_cv, NULL, CV_DEFAULT, NULL);
7897c478bdstevel@tonic-gate	cv_init(&tq->tq_wait_cv, NULL, CV_DEFAULT, NULL);
7906410974Chris Horne	cv_init(&tq->tq_maxalloc_cv, NULL, CV_DEFAULT, NULL);
7917c478bdstevel@tonic-gate
7927c478bdstevel@tonic-gate	tq->tq_task.tqent_next = &tq->tq_task;
7937c478bdstevel@tonic-gate	tq->tq_task.tqent_prev = &tq->tq_task;
7947c478bdstevel@tonic-gate
7957c478bdstevel@tonic-gate	return (0);
7967c478bdstevel@tonic-gate}
7977c478bdstevel@tonic-gate
7987c478bdstevel@tonic-gate/*ARGSUSED*/
7997c478bdstevel@tonic-gatestatic void
8007c478bdstevel@tonic-gatetaskq_destructor(void *buf, void *cdrarg)
8017c478bdstevel@tonic-gate{
8027c478bdstevel@tonic-gate	taskq_t *tq = buf;
8037c478bdstevel@tonic-gate
8042e0c549Jonathan Adams	ASSERT(tq->tq_nthreads == 0);
8052e0c549Jonathan Adams	ASSERT(tq->tq_buckets == NULL);
8062e0c549Jonathan Adams	ASSERT(tq->tq_tcreates == 0);
8072e0c549Jonathan Adams	ASSERT(tq->tq_tdeaths == 0);
8082e0c549Jonathan Adams
8097c478bdstevel@tonic-gate	mutex_destroy(&tq->tq_lock);
8107c478bdstevel@tonic-gate	rw_destroy(&tq->tq_threadlock);
8117c478bdstevel@tonic-gate	cv_destroy(&tq->tq_dispatch_cv);
8122e0c549Jonathan Adams	cv_destroy(&tq->tq_exit_cv);
8137c478bdstevel@tonic-gate	cv_destroy(&tq->tq_wait_cv);
8146410974Chris Horne	cv_destroy(&tq->tq_maxalloc_cv);
8157c478bdstevel@tonic-gate}
8167c478bdstevel@tonic-gate
8177c478bdstevel@tonic-gate/*ARGSUSED*/
8187c478bdstevel@tonic-gatestatic int
8197c478bdstevel@tonic-gatetaskq_ent_constructor(void *buf, void *cdrarg, int kmflags)
8207c478bdstevel@tonic-gate{
8217c478bdstevel@tonic-gate	taskq_ent_t *tqe = buf;
8227c478bdstevel@tonic-gate
8237c478bdstevel@tonic-gate	tqe->tqent_thread = NULL;
8247c478bdstevel@tonic-gate	cv_init(&tqe->tqent_cv, NULL, CV_DEFAULT, NULL);
8257c478bdstevel@tonic-gate
8267c478bdstevel@tonic-gate	return (0);
8277c478bdstevel@tonic-gate}
8287c478bdstevel@tonic-gate
8297c478bdstevel@tonic-gate/*ARGSUSED*/
8307c478bdstevel@tonic-gatestatic void
8317c478bdstevel@tonic-gatetaskq_ent_destructor(void *buf, void *cdrarg)
8327c478bdstevel@tonic-gate{
8337c478bdstevel@tonic-gate	taskq_ent_t *tqe = buf;
8347c478bdstevel@tonic-gate
8357c478bdstevel@tonic-gate	ASSERT(tqe->tqent_thread == NULL);
8367c478bdstevel@tonic-gate	cv_destroy(&tqe->tqent_cv);
8377c478bdstevel@tonic-gate}
8387c478bdstevel@tonic-gate
8397c478bdstevel@tonic-gatevoid
8407c478bdstevel@tonic-gatetaskq_init(void)
8417c478bdstevel@tonic-gate{
8427c478bdstevel@tonic-gate	taskq_ent_cache = kmem_cache_create("taskq_ent_cache",
8437c478bdstevel@tonic-gate	    sizeof (taskq_ent_t), 0, taskq_ent_constructor,
8447c478bdstevel@tonic-gate	    taskq_ent_destructor, NULL, NULL, NULL, 0);
8457c478bdstevel@tonic-gate	taskq_cache = kmem_cache_create("taskq_cache", sizeof (taskq_t),
8467c478bdstevel@tonic-gate	    0, taskq_constructor, taskq_destructor, NULL, NULL, NULL, 0);
8477c478bdstevel@tonic-gate	taskq_id_arena = vmem_create("taskq_id_arena",
8487c478bdstevel@tonic-gate	    (void *)1, INT32_MAX, 1, NULL, NULL, NULL, 0,
8497c478bdstevel@tonic-gate	    VM_SLEEP | VMC_IDENTIFIER);
8502e0c549Jonathan Adams
85135a5a35Jonathan Adams	list_create(&taskq_cpupct_list, sizeof (taskq_t),
85235a5a35Jonathan Adams	    offsetof(taskq_t, tq_cpupct_link));
85335a5a35Jonathan Adams}
85435a5a35Jonathan Adams
85535a5a35Jonathan Adamsstatic void
85635a5a35Jonathan Adamstaskq_update_nthreads(taskq_t *tq, uint_t ncpus)
85735a5a35Jonathan Adams{
85835a5a35Jonathan Adams	uint_t newtarget = TASKQ_THREADS_PCT(ncpus, tq->tq_threads_ncpus_pct);
85935a5a35Jonathan Adams
86035a5a35Jonathan Adams	ASSERT(MUTEX_HELD(&cpu_lock));
86135a5a35Jonathan Adams	ASSERT(MUTEX_HELD(&tq->tq_lock));
86235a5a35Jonathan Adams
86335a5a35Jonathan Adams	/* We must be going from non-zero to non-zero; no exiting. */
86435a5a35Jonathan Adams	ASSERT3U(tq->tq_nthreads_target, !=, 0);
86535a5a35Jonathan Adams	ASSERT3U(newtarget, !=, 0);
86635a5a35Jonathan Adams
86735a5a35Jonathan Adams	ASSERT3U(newtarget, <=, tq->tq_nthreads_max);
86835a5a35Jonathan Adams	if (newtarget != tq->tq_nthreads_target) {
86935a5a35Jonathan Adams		tq->tq_flags |= TASKQ_CHANGING;
87035a5a35Jonathan Adams		tq->tq_nthreads_target = newtarget;
87135a5a35Jonathan Adams		cv_broadcast(&tq->tq_dispatch_cv);
87235a5a35Jonathan Adams		cv_broadcast(&tq->tq_exit_cv);
87335a5a35Jonathan Adams	}
87435a5a35Jonathan Adams}
87535a5a35Jonathan Adams
87635a5a35Jonathan Adams/* called during task queue creation */
87735a5a35Jonathan Adamsstatic void
87835a5a35Jonathan Adamstaskq_cpupct_install(taskq_t *tq, cpupart_t *cpup)
87935a5a35Jonathan Adams{
88035a5a35Jonathan Adams	ASSERT(tq->tq_flags & TASKQ_THREADS_CPU_PCT);
88135a5a35Jonathan Adams
88235a5a35Jonathan Adams	mutex_enter(&cpu_lock);
88335a5a35Jonathan Adams	mutex_enter(&tq->tq_lock);
88435a5a35Jonathan Adams	tq->tq_cpupart = cpup->cp_id;
88535a5a35Jonathan Adams	taskq_update_nthreads(tq, cpup->cp_ncpus);
88635a5a35Jonathan Adams	mutex_exit(&tq->tq_lock);
88735a5a35Jonathan Adams
88835a5a35Jonathan Adams	list_insert_tail(&taskq_cpupct_list, tq);
88935a5a35Jonathan Adams	mutex_exit(&cpu_lock);
89035a5a35Jonathan Adams}
89135a5a35Jonathan Adams
89235a5a35Jonathan Adamsstatic void
89335a5a35Jonathan Adamstaskq_cpupct_remove(taskq_t *tq)
89435a5a35Jonathan Adams{
89535a5a35Jonathan Adams	ASSERT(tq->tq_flags & TASKQ_THREADS_CPU_PCT);
89635a5a35Jonathan Adams
89735a5a35Jonathan Adams	mutex_enter(&cpu_lock);
89835a5a35Jonathan Adams	list_remove(&taskq_cpupct_list, tq);
89935a5a35Jonathan Adams	mutex_exit(&cpu_lock);
9002e0c549Jonathan Adams}
9012e0c549Jonathan Adams
9022e0c549Jonathan Adams/*ARGSUSED*/
9032e0c549Jonathan Adamsstatic int
9042e0c549Jonathan Adamstaskq_cpu_setup(cpu_setup_t what, int id, void *arg)
9052e0c549Jonathan Adams{
90635a5a35Jonathan Adams	taskq_t *tq;
90735a5a35Jonathan Adams	cpupart_t *cp = cpu[id]->cpu_part;
90835a5a35Jonathan Adams	uint_t ncpus = cp->cp_ncpus;
90935a5a35Jonathan Adams
91035a5a35Jonathan Adams	ASSERT(MUTEX_HELD(&cpu_lock));
91135a5a35Jonathan Adams	ASSERT(ncpus > 0);
91235a5a35Jonathan Adams
91335a5a35Jonathan Adams	switch (what) {
91435a5a35Jonathan Adams	case CPU_OFF:
91535a5a35Jonathan Adams	case CPU_CPUPART_OUT:
91635a5a35Jonathan Adams		/* offlines are called *before* the cpu is offlined. */
91735a5a35Jonathan Adams		if (ncpus > 1)
91835a5a35Jonathan Adams			ncpus--;
91935a5a35Jonathan Adams		break;
92035a5a35Jonathan Adams
92135a5a35Jonathan Adams	case CPU_ON:
92235a5a35Jonathan Adams	case CPU_CPUPART_IN:
92335a5a35Jonathan Adams		break;
92435a5a35Jonathan Adams
92535a5a35Jonathan Adams	default:
92635a5a35Jonathan Adams		return (0);		/* doesn't affect cpu count */
9272e0c549Jonathan Adams	}
9282e0c549Jonathan Adams
92935a5a35Jonathan Adams	for (tq = list_head(&taskq_cpupct_list); tq != NULL;
93035a5a35Jonathan Adams	    tq = list_next(&taskq_cpupct_list, tq)) {
9312e0c549Jonathan Adams
9322e0c549Jonathan Adams		mutex_enter(&tq->tq_lock);
93335a5a35Jonathan Adams		/*
93435a5a35Jonathan Adams		 * If the taskq is part of the cpuset which is changing,
93535a5a35Jonathan Adams		 * update its nthreads_target.
93635a5a35Jonathan Adams		 */
93735a5a35Jonathan Adams		if (tq->tq_cpupart == cp->cp_id) {
93835a5a35Jonathan Adams			taskq_update_nthreads(tq, ncpus);
9392e0c549Jonathan Adams		}
9402e0c549Jonathan Adams		mutex_exit(&tq->tq_lock);
9412e0c549Jonathan Adams	}
9422e0c549Jonathan Adams	return (0);
9432e0c549Jonathan Adams}
9442e0c549Jonathan Adams
9452e0c549Jonathan Adamsvoid
9462e0c549Jonathan Adamstaskq_mp_init(void)
9472e0c549Jonathan Adams{
9482e0c549Jonathan Adams	mutex_enter(&cpu_lock);
9492e0c549Jonathan Adams	register_cpu_setup_func(taskq_cpu_setup, NULL);
95035a5a35Jonathan Adams	/*
95135a5a35Jonathan Adams	 * Make sure we're up to date.  At this point in boot, there is only
95235a5a35Jonathan Adams	 * one processor set, so we only have to update the current CPU.
95335a5a35Jonathan Adams	 */
95435a5a35Jonathan Adams	(void) taskq_cpu_setup(CPU_ON, CPU->cpu_id, NULL);
9552e0c549Jonathan Adams	mutex_exit(&cpu_lock);
9567c478bdstevel@tonic-gate}
9577c478bdstevel@tonic-gate
9587c478bdstevel@tonic-gate/*
9597c478bdstevel@tonic-gate * Create global system dynamic task queue.
9607c478bdstevel@tonic-gate */
9617c478bdstevel@tonic-gatevoid
9627c478bdstevel@tonic-gatesystem_taskq_init(void)
9637c478bdstevel@tonic-gate{
9647c478bdstevel@tonic-gate	system_taskq = taskq_create_common("system_taskq", 0,
96535a5a35Jonathan Adams	    system_taskq_size * max_ncpus, minclsyspri, 4, 512, &p0, 0,
9667c478bdstevel@tonic-gate	    TASKQ_DYNAMIC | TASKQ_PREPOPULATE);
9677c478bdstevel@tonic-gate}
9687c478bdstevel@tonic-gate
9697c478bdstevel@tonic-gate/*
9707c478bdstevel@tonic-gate * taskq_ent_alloc()
9717c478bdstevel@tonic-gate *
9727c478bdstevel@tonic-gate * Allocates a new taskq_ent_t structure either from the free list or from the
9737c478bdstevel@tonic-gate * cache. Returns NULL if it can't be allocated.
9747c478bdstevel@tonic-gate *
9757c478bdstevel@tonic-gate * Assumes: tq->tq_lock is held.
9767c478bdstevel@tonic-gate */
9777c478bdstevel@tonic-gatestatic taskq_ent_t *
9787c478bdstevel@tonic-gatetaskq_ent_alloc(taskq_t *tq, int flags)
9797c478bdstevel@tonic-gate{
9807c478bdstevel@tonic-gate	int kmflags = (flags & TQ_NOSLEEP) ? KM_NOSLEEP : KM_SLEEP;
9817c478bdstevel@tonic-gate	taskq_ent_t *tqe;
9826410974Chris Horne	clock_t wait_time;
9836410974Chris Horne	clock_t	wait_rv;
9847c478bdstevel@tonic-gate
9857c478bdstevel@tonic-gate	ASSERT(MUTEX_HELD(&tq->tq_lock));
9867c478bdstevel@tonic-gate
9877c478bdstevel@tonic-gate	/*
9887c478bdstevel@tonic-gate	 * TQ_NOALLOC allocations are allowed to use the freelist, even if
9897c478bdstevel@tonic-gate	 * we are below tq_minalloc.
9907c478bdstevel@tonic-gate	 */
9916410974Chris Horneagain:	if ((tqe = tq->tq_freelist) != NULL &&
9927c478bdstevel@tonic-gate	    ((flags & TQ_NOALLOC) || tq->tq_nalloc >= tq->tq_minalloc)) {
9937c478bdstevel@tonic-gate		tq->tq_freelist = tqe->tqent_next;
9947c478bdstevel@tonic-gate	} else {
9957c478bdstevel@tonic-gate		if (flags & TQ_NOALLOC)
9967c478bdstevel@tonic-gate			return (NULL);
9977c478bdstevel@tonic-gate
9987c478bdstevel@tonic-gate		if (tq->tq_nalloc >= tq->tq_maxalloc) {
9996410974Chris Horne			if (kmflags & KM_NOSLEEP)
10007c478bdstevel@tonic-gate				return (NULL);
10016410974Chris Horne
10027c478bdstevel@tonic-gate			/*
10037c478bdstevel@tonic-gate			 * We don't want to exceed tq_maxalloc, but we can't
10047c478bdstevel@tonic-gate			 * wait for other tasks to complete (and thus free up
10057c478bdstevel@tonic-gate			 * task structures) without risking deadlock with
10067c478bdstevel@tonic-gate			 * the caller.  So, we just delay for one second
10076410974Chris Horne			 * to throttle the allocation rate. If we have tasks
10086410974Chris Horne			 * complete before one second timeout expires then
10096410974Chris Horne			 * taskq_ent_free will signal us and we will
10106410974Chris Horne			 * immediately retry the allocation (reap free).
10117c478bdstevel@tonic-gate			 */
10126410974Chris Horne			wait_time = ddi_get_lbolt() + hz;
10136410974Chris Horne			while (tq->tq_freelist == NULL) {
10146410974Chris Horne				tq->tq_maxalloc_wait++;
10156410974Chris Horne				wait_rv = cv_timedwait(&tq->tq_maxalloc_cv,
10166410974Chris Horne				    &tq->tq_lock, wait_time);
10176410974Chris Horne				tq->tq_maxalloc_wait--;
10186410974Chris Horne				if (wait_rv == -1)
10196410974Chris Horne					break;
10206410974Chris Horne			}
10216410974Chris Horne			if (tq->tq_freelist)
10226410974Chris Horne				goto again;		/* reap freelist */
10236410974Chris Horne
10247c478bdstevel@tonic-gate		}
10256410974Chris Horne		mutex_exit(&tq->tq_lock);
10266410974Chris Horne
10277c478bdstevel@tonic-gate		tqe = kmem_cache_alloc(taskq_ent_cache, kmflags);
10286410974Chris Horne
10297c478bdstevel@tonic-gate		mutex_enter(&tq->tq_lock);
10307c478bdstevel@tonic-gate		if (tqe != NULL)
10317c478bdstevel@tonic-gate			tq->tq_nalloc++;
10327c478bdstevel@tonic-gate	}
10337c478bdstevel@tonic-gate	return (tqe);
10347c478bdstevel@tonic-gate}
10357c478bdstevel@tonic-gate
10367c478bdstevel@tonic-gate/*
10377c478bdstevel@tonic-gate * taskq_ent_free()
10387c478bdstevel@tonic-gate *
10397c478bdstevel@tonic-gate * Free taskq_ent_t structure by either putting it on the free list or freeing
10407c478bdstevel@tonic-gate * it to the cache.
10417c478bdstevel@tonic-gate *
10427c478bdstevel@tonic-gate * Assumes: tq->tq_lock is held.
10437c478bdstevel@tonic-gate */
10447c478bdstevel@tonic-gatestatic void
10457c478bdstevel@tonic-gatetaskq_ent_free(taskq_t *tq, taskq_ent_t *tqe)
10467c478bdstevel@tonic-gate{
10477c478bdstevel@tonic-gate	ASSERT(MUTEX_HELD(&tq->tq_lock));
10487c478bdstevel@tonic-gate
10497c478bdstevel@tonic-gate	if (tq->tq_nalloc <= tq->tq_minalloc) {
10507c478bdstevel@tonic-gate		tqe->tqent_next = tq->tq_freelist;
10517c478bdstevel@tonic-gate		tq->tq_freelist = tqe;
10527c478bdstevel@tonic-gate	} else {
10537c478bdstevel@tonic-gate		tq->tq_nalloc--;
10547c478bdstevel@tonic-gate		mutex_exit(&tq->tq_lock);
10557c478bdstevel@tonic-gate		kmem_cache_free(taskq_ent_cache, tqe);
10567c478bdstevel@tonic-gate		mutex_enter(&tq->tq_lock);
10577c478bdstevel@tonic-gate	}
10586410974Chris Horne
10596410974Chris Horne	if (tq->tq_maxalloc_wait)
10606410974Chris Horne		cv_signal(&tq->tq_maxalloc_cv);
10616410974Chris Horne}
10626410974Chris Horne
10636410974Chris Horne/*
10646410974Chris Horne * taskq_ent_exists()
10656410974Chris Horne *
10666410974Chris Horne * Return 1 if taskq already has entry for calling 'func(arg)'.
10676410974Chris Horne *
10686410974Chris Horne * Assumes: tq->tq_lock is held.
10696410974Chris Horne */
10706410974Chris Hornestatic int
10716410974Chris Hornetaskq_ent_exists(taskq_t *tq, task_func_t func, void *arg)
10726410974Chris Horne{
10736410974Chris Horne	taskq_ent_t	*tqe;
10746410974Chris Horne
10756410974Chris Horne	ASSERT(MUTEX_HELD(&tq->tq_lock));
10766410974Chris Horne
10776410974Chris Horne	for (tqe = tq->tq_task.tqent_next; tqe != &tq->tq_task;
10786410974Chris Horne	    tqe = tqe->tqent_next)
10796410974Chris Horne		if ((tqe->tqent_func == func) && (tqe->tqent_arg == arg))
10806410974Chris Horne			return (1);
10816410974Chris Horne	return (0);
10827c478bdstevel@tonic-gate}
10837c478bdstevel@tonic-gate
10847c478bdstevel@tonic-gate/*
10857c478bdstevel@tonic-gate * Dispatch a task "func(arg)" to a free entry of bucket b.
10867c478bdstevel@tonic-gate *
10877c478bdstevel@tonic-gate * Assumes: no bucket locks is held.
10887c478bdstevel@tonic-gate *
10897c478bdstevel@tonic-gate * Returns: a pointer to an entry if dispatch was successful.
10907c478bdstevel@tonic-gate *	    NULL if there are no free entries or if the bucket is suspended.
10917c478bdstevel@tonic-gate */
10927c478bdstevel@tonic-gatestatic taskq_ent_t *
10937c478bdstevel@tonic-gatetaskq_bucket_dispatch(taskq_bucket_t *b, task_func_t func, void *arg)
10947c478bdstevel@tonic-gate{
10957c478bdstevel@tonic-gate	taskq_ent_t *tqe;
10967c478bdstevel@tonic-gate
10977c478bdstevel@tonic-gate	ASSERT(MUTEX_NOT_HELD(&b->tqbucket_lock));
10987c478bdstevel@tonic-gate	ASSERT(func != NULL);
10997c478bdstevel@tonic-gate
11007c478bdstevel@tonic-gate	mutex_enter(&b->tqbucket_lock);
11017c478bdstevel@tonic-gate
11027c478bdstevel@tonic-gate	ASSERT(b->tqbucket_nfree != 0 || IS_EMPTY(b->tqbucket_freelist));
11037c478bdstevel@tonic-gate	ASSERT(b->tqbucket_nfree == 0 || !IS_EMPTY(b->tqbucket_freelist));
11047c478bdstevel@tonic-gate
11057c478bdstevel@tonic-gate	/*
11067c478bdstevel@tonic-gate	 * Get en entry from the freelist if there is one.
11077c478bdstevel@tonic-gate	 * Schedule task into the entry.
11087c478bdstevel@tonic-gate	 */
11097c478bdstevel@tonic-gate	if ((b->tqbucket_nfree != 0) &&
11107c478bdstevel@tonic-gate	    !(b->tqbucket_flags & TQBUCKET_SUSPEND)) {
11117c478bdstevel@tonic-gate		tqe = b->tqbucket_freelist.tqent_prev;
11127c478bdstevel@tonic-gate
11137c478bdstevel@tonic-gate		ASSERT(tqe != &b->tqbucket_freelist);
11147c478bdstevel@tonic-gate		ASSERT(tqe->tqent_thread != NULL);
11157c478bdstevel@tonic-gate
11167c478bdstevel@tonic-gate		tqe->tqent_prev->tqent_next = tqe->tqent_next;
11177c478bdstevel@tonic-gate		tqe->tqent_next->tqent_prev = tqe->tqent_prev;
11187c478bdstevel@tonic-gate		b->tqbucket_nalloc++;
11197c478bdstevel@tonic-gate		b->tqbucket_nfree--;
11207c478bdstevel@tonic-gate		tqe->tqent_func = func;
11217c478bdstevel@tonic-gate		tqe->tqent_arg = arg;
11227c478bdstevel@tonic-gate		TQ_STAT(b, tqs_hits);
11237c478bdstevel@tonic-gate		cv_signal(&tqe->tqent_cv);
11247c478bdstevel@tonic-gate		DTRACE_PROBE2(taskq__d__enqueue, taskq_bucket_t *, b,
11257c478bdstevel@tonic-gate		    taskq_ent_t *, tqe);
11267c478bdstevel@tonic-gate	} else {
11277c478bdstevel@tonic-gate		tqe = NULL;
11287c478bdstevel@tonic-gate		TQ_STAT(b, tqs_misses);
11297c478bdstevel@tonic-gate	}
11307c478bdstevel@tonic-gate	mutex_exit(&b->tqbucket_lock);
11317c478bdstevel@tonic-gate	return (tqe);
11327c478bdstevel@tonic-gate}
11337c478bdstevel@tonic-gate
11347c478bdstevel@tonic-gate/*
11357c478bdstevel@tonic-gate * Dispatch a task.
11367c478bdstevel@tonic-gate *
11377c478bdstevel@tonic-gate * Assumes: func != NULL
11387c478bdstevel@tonic-gate *
11397c478bdstevel@tonic-gate * Returns: NULL if dispatch failed.
11407c478bdstevel@tonic-gate *	    non-NULL if task dispatched successfully.
11417c478bdstevel@tonic-gate *	    Actual return value is the pointer to taskq entry that was used to
11427c478bdstevel@tonic-gate *	    dispatch a task. This is useful for debugging.
11437c478bdstevel@tonic-gate */
11447c478bdstevel@tonic-gatetaskqid_t
11457c478bdstevel@tonic-gatetaskq_dispatch(taskq_t *tq, task_func_t func, void *arg, uint_t flags)
11467c478bdstevel@tonic-gate{
11477c478bdstevel@tonic-gate	taskq_bucket_t *bucket = NULL;	/* Which bucket needs extension */
11487c478bdstevel@tonic-gate	taskq_ent_t *tqe = NULL;
11497c478bdstevel@tonic-gate	taskq_ent_t *tqe1;
11507c478bdstevel@tonic-gate	uint_t bsize;
11517c478bdstevel@tonic-gate
11527c478bdstevel@tonic-gate	ASSERT(tq != NULL);
11537c478bdstevel@tonic-gate	ASSERT(func != NULL);
11547c478bdstevel@tonic-gate
11557c478bdstevel@tonic-gate	if (!(tq->tq_flags & TASKQ_DYNAMIC)) {
11567c478bdstevel@tonic-gate		/*
11577c478bdstevel@tonic-gate		 * TQ_NOQUEUE flag can't be used with non-dynamic task queues.
11587c478bdstevel@tonic-gate		 */
11595aeb947Garrett D'Amore		ASSERT(!(flags & TQ_NOQUEUE));
11607c478bdstevel@tonic-gate		/*
11617c478bdstevel@tonic-gate		 * Enqueue the task to the underlying queue.
11627c478bdstevel@tonic-gate		 */
11637c478bdstevel@tonic-gate		mutex_enter(&tq->tq_lock);
11647c478bdstevel@tonic-gate
11657c478bdstevel@tonic-gate		TASKQ_S_RANDOM_DISPATCH_FAILURE(tq, flags);
11667c478bdstevel@tonic-gate
11677c478bdstevel@tonic-gate		if ((tqe = taskq_ent_alloc(tq, flags)) == NULL) {
1168216d772Prakash Surya			tq->tq_nomem++;
11697c478bdstevel@tonic-gate			mutex_exit(&tq->tq_lock);
11707e12cebToomas Soome			return ((taskqid_t)tqe);
11717c478bdstevel@tonic-gate		}
11725aeb947Garrett D'Amore		/* Make sure we start without any flags */
11735aeb947Garrett D'Amore		tqe->tqent_un.tqent_flags = 0;
11745aeb947Garrett D'Amore
117535a5a35Jonathan Adams		if (flags & TQ_FRONT) {
117635a5a35Jonathan Adams			TQ_ENQUEUE_FRONT(tq, tqe, func, arg);
117735a5a35Jonathan Adams		} else {
117835a5a35Jonathan Adams			TQ_ENQUEUE(tq, tqe, func, arg);
117935a5a35Jonathan Adams		}
11807c478bdstevel@tonic-gate		mutex_exit(&tq->tq_lock);
11817c478bdstevel@tonic-gate		return ((taskqid_t)tqe);
11827c478bdstevel@tonic-gate	}
11837c478bdstevel@tonic-gate
11847c478bdstevel@tonic-gate	/*
11857c478bdstevel@tonic-gate	 * Dynamic taskq dispatching.
11867c478bdstevel@tonic-gate	 */
118735a5a35Jonathan Adams	ASSERT(!(flags & (TQ_NOALLOC | TQ_FRONT)));
11887c478bdstevel@tonic-gate	TASKQ_D_RANDOM_DISPATCH_FAILURE(tq, flags);
11897c478bdstevel@tonic-gate
11907c478bdstevel@tonic-gate	bsize = tq->tq_nbuckets;
11917c478bdstevel@tonic-gate
11927c478bdstevel@tonic-gate	if (bsize == 1) {
11937c478bdstevel@tonic-gate		/*
11947c478bdstevel@tonic-gate		 * In a single-CPU case there is only one bucket, so get
11957c478bdstevel@tonic-gate		 * entry directly from there.
11967c478bdstevel@tonic-gate		 */
11977c478bdstevel@tonic-gate		if ((tqe = taskq_bucket_dispatch(tq->tq_buckets, func, arg))
11987c478bdstevel@tonic-gate		    != NULL)
11997c478bdstevel@tonic-gate			return ((taskqid_t)tqe);	/* Fastpath */
12007c478bdstevel@tonic-gate		bucket = tq->tq_buckets;
12017c478bdstevel@tonic-gate	} else {
12027c478bdstevel@tonic-gate		int loopcount;
12037c478bdstevel@tonic-gate		taskq_bucket_t *b;
12047c478bdstevel@tonic-gate		uintptr_t h = ((uintptr_t)CPU + (uintptr_t)arg) >> 3;
12057c478bdstevel@tonic-gate
12067c478bdstevel@tonic-gate		h = TQ_HASH(h);
12077c478bdstevel@tonic-gate
12087c478bdstevel@tonic-gate		/*
12097c478bdstevel@tonic-gate		 * The 'bucket' points to the original bucket that we hit. If we
12107c478bdstevel@tonic-gate		 * can't allocate from it, we search other buckets, but only
12117c478bdstevel@tonic-gate		 * extend this one.
12127c478bdstevel@tonic-gate		 */
12137c478bdstevel@tonic-gate		b = &tq->tq_buckets[h & (bsize - 1)];
12147c478bdstevel@tonic-gate		ASSERT(b->tqbucket_taskq == tq);	/* Sanity check */
12157c478bdstevel@tonic-gate
12167c478bdstevel@tonic-gate		/*
12177c478bdstevel@tonic-gate		 * Do a quick check before grabbing the lock. If the bucket does
12187c478bdstevel@tonic-gate		 * not have free entries now, chances are very small that it
12197c478bdstevel@tonic-gate		 * will after we take the lock, so we just skip it.
12207c478bdstevel@tonic-gate		 */
12217c478bdstevel@tonic-gate		if (b->tqbucket_nfree != 0) {
12227c478bdstevel@tonic-gate			if ((tqe = taskq_bucket_dispatch(b, func, arg)) != NULL)
12237c478bdstevel@tonic-gate				return ((taskqid_t)tqe);	/* Fastpath */
12247c478bdstevel@tonic-gate		} else {
12257c478bdstevel@tonic-gate			TQ_STAT(b, tqs_misses);
12267c478bdstevel@tonic-gate		}
12277c478bdstevel@tonic-gate
12287c478bdstevel@tonic-gate		bucket = b;
12297c478bdstevel@tonic-gate		loopcount = MIN(taskq_search_depth, bsize);
12307c478bdstevel@tonic-gate		/*
12317c478bdstevel@tonic-gate		 * If bucket dispatch failed, search loopcount number of buckets
12327c478bdstevel@tonic-gate		 * before we give up and fail.
12337c478bdstevel@tonic-gate		 */
12347c478bdstevel@tonic-gate		do {
12357c478bdstevel@tonic-gate			b = &tq->tq_buckets[++h & (bsize - 1)];
12367c478bdstevel@tonic-gate			ASSERT(b->tqbucket_taskq == tq);  /* Sanity check */
12377c478bdstevel@tonic-gate			loopcount--;
12387c478bdstevel@tonic-gate
12397c478bdstevel@tonic-gate			if (b->tqbucket_nfree != 0) {
12407c478bdstevel@tonic-gate				tqe = taskq_bucket_dispatch(b, func, arg);
12417c478bdstevel@tonic-gate			} else {
12427c478bdstevel@tonic-gate				TQ_STAT(b, tqs_misses);
12437c478bdstevel@tonic-gate			}
12447c478bdstevel@tonic-gate		} while ((tqe == NULL) && (loopcount > 0));
12457c478bdstevel@tonic-gate	}
12467c478bdstevel@tonic-gate
12477c478bdstevel@tonic-gate	/*
12487c478bdstevel@tonic-gate	 * At this point we either scheduled a task and (tqe != NULL) or failed
12497c478bdstevel@tonic-gate	 * (tqe == NULL). Try to recover from fails.
12507c478bdstevel@tonic-gate	 */
12517c478bdstevel@tonic-gate
12527c478bdstevel@tonic-gate	/*
12537c478bdstevel@tonic-gate	 * For KM_SLEEP dispatches, try to extend the bucket and retry dispatch.
12547c478bdstevel@tonic-gate	 */
12557c478bdstevel@tonic-gate	if ((tqe == NULL) && !(flags & TQ_NOSLEEP)) {
12567c478bdstevel@tonic-gate		/*
12577c478bdstevel@tonic-gate		 * taskq_bucket_extend() may fail to do anything, but this is
12587c478bdstevel@tonic-gate		 * fine - we deal with it later. If the bucket was successfully
12597c478bdstevel@tonic-gate		 * extended, there is a good chance that taskq_bucket_dispatch()
12607c478bdstevel@tonic-gate		 * will get this new entry, unless someone is racing with us and
12617c478bdstevel@tonic-gate		 * stealing the new entry from under our nose.
12627c478bdstevel@tonic-gate		 * taskq_bucket_extend() may sleep.
12637c478bdstevel@tonic-gate		 */
12647c478bdstevel@tonic-gate		taskq_bucket_extend(bucket);
12657c478bdstevel@tonic-gate		TQ_STAT(bucket, tqs_disptcreates);
12667c478bdstevel@tonic-gate		if ((tqe = taskq_bucket_dispatch(bucket, func, arg)) != NULL)
12677c478bdstevel@tonic-gate			return ((taskqid_t)tqe);
12687c478bdstevel@tonic-gate	}
12697c478bdstevel@tonic-gate
12707c478bdstevel@tonic-gate	ASSERT(bucket != NULL);
12716410974Chris Horne
12727c478bdstevel@tonic-gate	/*
12736410974Chris Horne	 * Since there are not enough free entries in the bucket, add a
12746410974Chris Horne	 * taskq entry to extend it in the background using backing queue
12756410974Chris Horne	 * (unless we already have a taskq entry to perform that extension).
12767c478bdstevel@tonic-gate	 */
12777c478bdstevel@tonic-gate	mutex_enter(&tq->tq_lock);
12786410974Chris Horne	if (!taskq_ent_exists(tq, taskq_bucket_extend, bucket)) {
12796410974Chris Horne		if ((tqe1 = taskq_ent_alloc(tq, TQ_NOSLEEP)) != NULL) {
12806410974Chris Horne			TQ_ENQUEUE_FRONT(tq, tqe1, taskq_bucket_extend, bucket);
12816410974Chris Horne		} else {
1282216d772Prakash Surya			tq->tq_nomem++;
12836410974Chris Horne		}
12847c478bdstevel@tonic-gate	}
12857c478bdstevel@tonic-gate
12867c478bdstevel@tonic-gate	/*
12877c478bdstevel@tonic-gate	 * Dispatch failed and we can't find an entry to schedule a task.
12887c478bdstevel@tonic-gate	 * Revert to the backing queue unless TQ_NOQUEUE was asked.
12897c478bdstevel@tonic-gate	 */
12907c478bdstevel@tonic-gate	if ((tqe == NULL) && !(flags & TQ_NOQUEUE)) {
12917c478bdstevel@tonic-gate		if ((tqe = taskq_ent_alloc(tq, flags)) != NULL) {
12927c478bdstevel@tonic-gate			TQ_ENQUEUE(tq, tqe, func, arg);
12937c478bdstevel@tonic-gate		} else {
1294216d772Prakash Surya			tq->tq_nomem++;
12957c478bdstevel@tonic-gate		}
12967c478bdstevel@tonic-gate	}
12977c478bdstevel@tonic-gate	mutex_exit(&tq->tq_lock);
12987c478bdstevel@tonic-gate
12997c478bdstevel@tonic-gate	return ((taskqid_t)tqe);
13007c478bdstevel@tonic-gate}
13017c478bdstevel@tonic-gate
13025aeb947Garrett D'Amorevoid
13035aeb947Garrett D'Amoretaskq_dispatch_ent(taskq_t *tq, task_func_t func, void *arg, uint_t flags,
13045aeb947Garrett D'Amore    taskq_ent_t *tqe)
13055aeb947Garrett D'Amore{
13065aeb947Garrett D'Amore	ASSERT(func != NULL);
13075aeb947Garrett D'Amore	ASSERT(!(tq->tq_flags & TASKQ_DYNAMIC));
13085aeb947Garrett D'Amore
13095aeb947Garrett D'Amore	/*
13105aeb947Garrett D'Amore	 * Mark it as a prealloc'd task.  This is important
13115aeb947Garrett D'Amore	 * to ensure that we don't free it later.
13125aeb947Garrett D'Amore	 */
13135aeb947Garrett D'Amore	tqe->tqent_un.tqent_flags |= TQENT_FLAG_PREALLOC;
13145aeb947Garrett D'Amore	/*
13155aeb947Garrett D'Amore	 * Enqueue the task to the underlying queue.
13165aeb947Garrett D'Amore	 */
13175aeb947Garrett D'Amore	mutex_enter(&tq->tq_lock);
13185aeb947Garrett D'Amore
13195aeb947Garrett D'Amore	if (flags & TQ_FRONT) {
13205aeb947Garrett D'Amore		TQ_ENQUEUE_FRONT(tq, tqe, func, arg);
13215aeb947Garrett D'Amore	} else {
13225aeb947Garrett D'Amore		TQ_ENQUEUE(tq, tqe, func, arg);
13235aeb947Garrett D'Amore	}
13245aeb947Garrett D'Amore	mutex_exit(&tq->tq_lock);
13255aeb947Garrett D'Amore}
13265aeb947Garrett D'Amore
13277c478bdstevel@tonic-gate/*
13284c99eccTim Kordas * Allow our caller to ask if there are tasks pending on the queue.
13294c99eccTim Kordas */
13304c99eccTim Kordasboolean_t
13314c99eccTim Kordastaskq_empty(taskq_t *tq)
13324c99eccTim Kordas{
13334c99eccTim Kordas	boolean_t rv;
13344c99eccTim Kordas
13354c99eccTim Kordas	ASSERT3P(tq, !=, curthread->t_taskq);
13364c99eccTim Kordas	mutex_enter(&tq->tq_lock);
13374c99eccTim Kordas	rv = (tq->tq_task.tqent_next == &tq->tq_task) && (tq->tq_active == 0);
13384c99eccTim Kordas	mutex_exit(&tq->tq_lock);
13394c99eccTim Kordas
13404c99eccTim Kordas	return (rv);
13414c99eccTim Kordas}
13424c99eccTim Kordas
13434c99eccTim Kordas/*
13447c478bdstevel@tonic-gate * Wait for all pending tasks to complete.
13457c478bdstevel@tonic-gate * Calling taskq_wait from a task will cause deadlock.
13467c478bdstevel@tonic-gate */
13477c478bdstevel@tonic-gatevoid
13487c478bdstevel@tonic-gatetaskq_wait(taskq_t *tq)
13497c478bdstevel@tonic-gate{
13507c478bdstevel@tonic-gate	ASSERT(tq != curthread->t_taskq);
13517c478bdstevel@tonic-gate
13527c478bdstevel@tonic-gate	mutex_enter(&tq->tq_lock);
13537c478bdstevel@tonic-gate	while (tq->tq_task.tqent_next != &tq->tq_task || tq->tq_active != 0)
13547c478bdstevel@tonic-gate		cv_wait(&tq->tq_wait_cv, &tq->tq_lock);
13557c478bdstevel@tonic-gate	mutex_exit(&tq->tq_lock);
13567c478bdstevel@tonic-gate
13577c478bdstevel@tonic-gate	if (tq->tq_flags & TASKQ_DYNAMIC) {
13587c478bdstevel@tonic-gate		taskq_bucket_t *b = tq->tq_buckets;
13597c478bdstevel@tonic-gate		int bid = 0;
13607c478bdstevel@tonic-gate		for (; (b != NULL) && (bid < tq->tq_nbuckets); b++, bid++) {
13617c478bdstevel@tonic-gate			mutex_enter(&b->tqbucket_lock);
13627c478bdstevel@tonic-gate			while (b->tqbucket_nalloc > 0)
13637c478bdstevel@tonic-gate				cv_wait(&b->tqbucket_cv, &b->tqbucket_lock);
13647c478bdstevel@tonic-gate			mutex_exit(&b->tqbucket_lock);
13657c478bdstevel@tonic-gate		}
13667c478bdstevel@tonic-gate	}
13677c478bdstevel@tonic-gate}
13687c478bdstevel@tonic-gate
1369a3874b8Toomas Soomevoid
1370a3874b8Toomas Soometaskq_wait_id(taskq_t *tq, taskqid_t id __unused)
1371a3874b8Toomas Soome{
1372a3874b8Toomas Soome	taskq_wait(tq);
1373a3874b8Toomas Soome}
1374a3874b8Toomas Soome
13757c478bdstevel@tonic-gate/*
13767c478bdstevel@tonic-gate * Suspend execution of tasks.
13777c478bdstevel@tonic-gate *
13787c478bdstevel@tonic-gate * Tasks in the queue part will be suspended immediately upon return from this
13797c478bdstevel@tonic-gate * function. Pending tasks in the dynamic part will continue to execute, but all
13807c478bdstevel@tonic-gate * new tasks will  be suspended.
13817c478bdstevel@tonic-gate */
13827c478bdstevel@tonic-gatevoid
13837c478bdstevel@tonic-gatetaskq_suspend(taskq_t *tq)
13847c478bdstevel@tonic-gate{
13857c478bdstevel@tonic-gate	rw_enter(&tq->tq_threadlock, RW_WRITER);
13867c478bdstevel@tonic-gate
13877c478bdstevel@tonic-gate	if (tq->tq_flags & TASKQ_DYNAMIC) {
13887c478bdstevel@tonic-gate		taskq_bucket_t *b = tq->tq_buckets;
13897c478bdstevel@tonic-gate		int bid = 0;
13907c478bdstevel@tonic-gate		for (; (b != NULL) && (bid < tq->tq_nbuckets); b++, bid++) {
13917c478bdstevel@tonic-gate			mutex_enter(&b->tqbucket_lock);
13927c478bdstevel@tonic-gate			b->tqbucket_flags |= TQBUCKET_SUSPEND;
13937c478bdstevel@tonic-gate			mutex_exit(&b->tqbucket_lock);
13947c478bdstevel@tonic-gate		}
13957c478bdstevel@tonic-gate	}
13967c478bdstevel@tonic-gate	/*
13977c478bdstevel@tonic-gate	 * Mark task queue as being suspended. Needed for taskq_suspended().
13987c478bdstevel@tonic-gate	 */
13997c478bdstevel@tonic-gate	mutex_enter(&tq->tq_lock);
14007c478bdstevel@tonic-gate	ASSERT(!(tq->tq_flags & TASKQ_SUSPENDED));
14017c478bdstevel@tonic-gate	tq->tq_flags |= TASKQ_SUSPENDED;
14027c478bdstevel@tonic-gate	mutex_exit(&tq->tq_lock);
14037c478bdstevel@tonic-gate}
14047c478bdstevel@tonic-gate
14057c478bdstevel@tonic-gate/*
14067c478bdstevel@tonic-gate * returns: 1 if tq is suspended, 0 otherwise.
14077c478bdstevel@tonic-gate */
14087c478bdstevel@tonic-gateint
14097c478bdstevel@tonic-gatetaskq_suspended(taskq_t *tq)
14107c478bdstevel@tonic-gate{
14117c478bdstevel@tonic-gate	return ((tq->tq_flags & TASKQ_SUSPENDED) != 0);
14127c478bdstevel@tonic-gate}
14137c478bdstevel@tonic-gate
14147c478bdstevel@tonic-gate/*
14157c478bdstevel@tonic-gate * Resume taskq execution.
14167c478bdstevel@tonic-gate */
14177c478bdstevel@tonic-gatevoid
14187c478bdstevel@tonic-gatetaskq_resume(taskq_t *tq)
14197c478bdstevel@tonic-gate{
14207c478bdstevel@tonic-gate	ASSERT(RW_WRITE_HELD(&tq->tq_threadlock));
14217c478bdstevel@tonic-gate
14227c478bdstevel@tonic-gate	if (tq->tq_flags & TASKQ_DYNAMIC) {
14237c478bdstevel@tonic-gate		taskq_bucket_t *b = tq->tq_buckets;
14247c478bdstevel@tonic-gate		int bid = 0;
14257c478bdstevel@tonic-gate		for (; (b != NULL) && (bid < tq->tq_nbuckets); b++, bid++) {
14267c478bdstevel@tonic-gate			mutex_enter(&b->tqbucket_lock);
14277c478bdstevel@tonic-gate			b->tqbucket_flags &= ~TQBUCKET_SUSPEND;
14287c478bdstevel@tonic-gate			mutex_exit(&b->tqbucket_lock);
14297c478bdstevel@tonic-gate		}
14307c478bdstevel@tonic-gate	}
14317c478bdstevel@tonic-gate	mutex_enter(&tq->tq_lock);
14327c478bdstevel@tonic-gate	ASSERT(tq->tq_flags & TASKQ_SUSPENDED);
14337c478bdstevel@tonic-gate	tq->tq_flags &= ~TASKQ_SUSPENDED;
14347c478bdstevel@tonic-gate	mutex_exit(&tq->tq_lock);
14357c478bdstevel@tonic-gate
14367c478bdstevel@tonic-gate	rw_exit(&tq->tq_threadlock);
14377c478bdstevel@tonic-gate}
14387c478bdstevel@tonic-gate
14397c478bdstevel@tonic-gateint
14407c478bdstevel@tonic-gatetaskq_member(taskq_t *tq, kthread_t *thread)
14417c478bdstevel@tonic-gate{
14427c478bdstevel@tonic-gate	return (thread->t_taskq == tq);
14437c478bdstevel@tonic-gate}
14447c478bdstevel@tonic-gate
144535a5a35Jonathan Adams/*
144635a5a35Jonathan Adams * Creates a thread in the taskq.  We only allow one outstanding create at
144735a5a35Jonathan Adams * a time.  We drop and reacquire the tq_lock in order to avoid blocking other
144835a5a35Jonathan Adams * taskq activity while thread_create() or lwp_kernel_create() run.
144935a5a35Jonathan Adams *
145035a5a35Jonathan Adams * The first time we're called, we do some additional setup, and do not
145135a5a35Jonathan Adams * return until there are enough threads to start servicing requests.
145235a5a35Jonathan Adams */
14532e0c549Jonathan Adamsstatic void
14542e0c549Jonathan Adamstaskq_thread_create(taskq_t *tq)
14552e0c549Jonathan Adams{
145635a5a35Jonathan Adams	kthread_t	*t;
145735a5a35Jonathan Adams	const boolean_t	first = (tq->tq_nthreads == 0);
14582e0c549Jonathan Adams
14592e0c549Jonathan Adams	ASSERT(MUTEX_HELD(&tq->tq_lock));
146035a5a35Jonathan Adams	ASSERT(tq->tq_flags & TASKQ_CHANGING);
146135a5a35Jonathan Adams	ASSERT(tq->tq_nthreads < tq->tq_nthreads_target);
14622e0c549Jonathan Adams	ASSERT(!(tq->tq_flags & TASKQ_THREAD_CREATED));
14632e0c549Jonathan Adams
146435a5a35Jonathan Adams
14652e0c549Jonathan Adams	tq->tq_flags |= TASKQ_THREAD_CREATED;
14662e0c549Jonathan Adams	tq->tq_active++;
146735a5a35Jonathan Adams	mutex_exit(&tq->tq_lock);
146835a5a35Jonathan Adams
1469e548823Gordon Ross	/*
1470e548823Gordon Ross	 * With TASKQ_DUTY_CYCLE the new thread must have an LWP
1471e548823Gordon Ross	 * as explained in ../disp/sysdc.c (for the msacct data).
1472e548823Gordon Ross	 * Otherwise simple kthreads are preferred.
1473e548823Gordon Ross	 */
1474e548823Gordon Ross	if ((tq->tq_flags & TASKQ_DUTY_CYCLE) != 0) {
1475e548823Gordon Ross		/* Enforced in taskq_create_common */
1476e548823Gordon Ross		ASSERT3P(tq->tq_proc, !=, &p0);
147735a5a35Jonathan Adams		t = lwp_kernel_create(tq->tq_proc, taskq_thread, tq, TS_RUN,
147835a5a35Jonathan Adams		    tq->tq_pri);
147935a5a35Jonathan Adams	} else {
1480e548823Gordon Ross		t = thread_create(NULL, 0, taskq_thread, tq, 0, tq->tq_proc,
1481e548823Gordon Ross		    TS_RUN, tq->tq_pri);
148235a5a35Jonathan Adams	}
148335a5a35Jonathan Adams
148435a5a35Jonathan Adams	if (!first) {
148535a5a35Jonathan Adams		mutex_enter(&tq->tq_lock);
148635a5a35Jonathan Adams		return;
148735a5a35Jonathan Adams	}
148835a5a35Jonathan Adams
148935a5a35Jonathan Adams	/*
149035a5a35Jonathan Adams	 * We know the thread cannot go away, since tq cannot be
149135a5a35Jonathan Adams	 * destroyed until creation has completed.  We can therefore
149235a5a35Jonathan Adams	 * safely dereference t.
149335a5a35Jonathan Adams	 */
149435a5a35Jonathan Adams	if (tq->tq_flags & TASKQ_THREADS_CPU_PCT) {
149535a5a35Jonathan Adams		taskq_cpupct_install(tq, t->t_cpupart);
149635a5a35Jonathan Adams	}
149735a5a35Jonathan Adams	mutex_enter(&tq->tq_lock);
149835a5a35Jonathan Adams
149935a5a35Jonathan Adams	/* Wait until we can service requests. */
150035a5a35Jonathan Adams	while (tq->tq_nthreads != tq->tq_nthreads_target &&
150135a5a35Jonathan Adams	    tq->tq_nthreads < TASKQ_CREATE_ACTIVE_THREADS) {
150235a5a35Jonathan Adams		cv_wait(&tq->tq_wait_cv, &tq->tq_lock);
150335a5a35Jonathan Adams	}
15042e0c549Jonathan Adams}
15052e0c549Jonathan Adams
1506e0ad97eJonathan Adams/*
1507e0ad97eJonathan Adams * Common "sleep taskq thread" function, which handles CPR stuff, as well
1508e0ad97eJonathan Adams * as giving a nice common point for debuggers to find inactive threads.
1509e0ad97eJonathan Adams */
1510e0ad97eJonathan Adamsstatic clock_t
1511e0ad97eJonathan Adamstaskq_thread_wait(taskq_t *tq, kmutex_t *mx, kcondvar_t *cv,
1512e0ad97eJonathan Adams    callb_cpr_t *cprinfo, clock_t timeout)
15132e0c549Jonathan Adams{
1514e0ad97eJonathan Adams	clock_t ret = 0;
1515e0ad97eJonathan Adams
1516e0ad97eJonathan Adams	if (!(tq->tq_flags & TASKQ_CPR_SAFE)) {
15172e0c549Jonathan Adams		CALLB_CPR_SAFE_BEGIN(cprinfo);
15182e0c549Jonathan Adams	}
1519e0ad97eJonathan Adams	if (timeout < 0)
1520e0ad97eJonathan Adams		cv_wait(cv, mx);
1521e0ad97eJonathan Adams	else
1522d3d5073Rafael Vanoni		ret = cv_reltimedwait(cv, mx, timeout, TR_CLOCK_TICK);
1523e0ad97eJonathan Adams
1524e0ad97eJonathan Adams	if (!(tq->tq_flags & TASKQ_CPR_SAFE)) {
1525e0ad97eJonathan Adams		CALLB_CPR_SAFE_END(cprinfo, mx);
1526e0ad97eJonathan Adams	}
1527e0ad97eJonathan Adams
1528e0ad97eJonathan Adams	return (ret);
15292e0c549Jonathan Adams}
15302e0c549Jonathan Adams
15317c478bdstevel@tonic-gate/*
15327c478bdstevel@tonic-gate * Worker thread for processing task queue.
15337c478bdstevel@tonic-gate */
15347c478bdstevel@tonic-gatestatic void
15357c478bdstevel@tonic-gatetaskq_thread(void *arg)
15367c478bdstevel@tonic-gate{
15372e0c549Jonathan Adams	int thread_id;
15382e0c549Jonathan Adams
15397c478bdstevel@tonic-gate	taskq_t *tq = arg;
15407c478bdstevel@tonic-gate	taskq_ent_t *tqe;
15417c478bdstevel@tonic-gate	callb_cpr_t cprinfo;
15427c478bdstevel@tonic-gate	hrtime_t start, end;
15435aeb947Garrett D'Amore	boolean_t freeit;
15447c478bdstevel@tonic-gate
154535a5a35Jonathan Adams	curthread->t_taskq = tq;	/* mark ourselves for taskq_member() */
154635a5a35Jonathan Adams
154735a5a35Jonathan Adams	if (curproc != &p0 && (tq->tq_flags & TASKQ_DUTY_CYCLE)) {
154835a5a35Jonathan Adams		sysdc_thread_enter(curthread, tq->tq_DC,
154935a5a35Jonathan Adams		    (tq->tq_flags & TASKQ_DC_BATCH) ? SYSDC_THREAD_BATCH : 0);
155035a5a35Jonathan Adams	}
155135a5a35Jonathan Adams
15527c478bdstevel@tonic-gate	if (tq->tq_flags & TASKQ_CPR_SAFE) {
15537c478bdstevel@tonic-gate		CALLB_CPR_INIT_SAFE(curthread, tq->tq_name);
15547c478bdstevel@tonic-gate	} else {
15557c478bdstevel@tonic-gate		CALLB_CPR_INIT(&cprinfo, &tq->tq_lock, callb_generic_cpr,
15567c478bdstevel@tonic-gate		    tq->tq_name);
15577c478bdstevel@tonic-gate	}
15587c478bdstevel@tonic-gate	mutex_enter(&tq->tq_lock);
15592e0c549Jonathan Adams	thread_id = ++tq->tq_nthreads;
15602e0c549Jonathan Adams	ASSERT(tq->tq_flags & TASKQ_THREAD_CREATED);
156135a5a35Jonathan Adams	ASSERT(tq->tq_flags & TASKQ_CHANGING);
15622e0c549Jonathan Adams	tq->tq_flags &= ~TASKQ_THREAD_CREATED;
15632e0c549Jonathan Adams
15642e0c549Jonathan Adams	VERIFY3S(thread_id, <=, tq->tq_nthreads_max);
15652e0c549Jonathan Adams
15662e0c549Jonathan Adams	if (tq->tq_nthreads_max == 1)
15672e0c549Jonathan Adams		tq->tq_thread = curthread;
15682e0c549Jonathan Adams	else
15692e0c549Jonathan Adams		tq->tq_threadlist[thread_id - 1] = curthread;
15702e0c549Jonathan Adams
157135a5a35Jonathan Adams	/* Allow taskq_create_common()'s taskq_thread_create() to return. */
157235a5a35Jonathan Adams	if (tq->tq_nthreads == TASKQ_CREATE_ACTIVE_THREADS)
157335a5a35Jonathan Adams		cv_broadcast(&tq->tq_wait_cv);
157435a5a35Jonathan Adams
15752e0c549Jonathan Adams	for (;;) {
15762e0c549Jonathan Adams		if (tq->tq_flags & TASKQ_CHANGING) {
157735a5a35Jonathan Adams			/* See if we're no longer needed */
15782e0c549Jonathan Adams			if (thread_id > tq->tq_nthreads_target) {
15792e0c549Jonathan Adams				/*
15802e0c549Jonathan Adams				 * To preserve the one-to-one mapping between
15812e0c549Jonathan Adams				 * thread_id and thread, we must exit from
15822e0c549Jonathan Adams				 * highest thread ID to least.
15832e0c549Jonathan Adams				 *
15842e0c549Jonathan Adams				 * However, if everyone is exiting, the order
15852e0c549Jonathan Adams				 * doesn't matter, so just exit immediately.
15862e0c549Jonathan Adams				 * (this is safe, since you must wait for
15872e0c549Jonathan Adams				 * nthreads to reach 0 after setting
15882e0c549Jonathan Adams				 * tq_nthreads_target to 0)
15892e0c549Jonathan Adams				 */
15902e0c549Jonathan Adams				if (thread_id == tq->tq_nthreads ||
15912e0c549Jonathan Adams				    tq->tq_nthreads_target == 0)
15922e0c549Jonathan Adams					break;
15932e0c549Jonathan Adams
15942e0c549Jonathan Adams				/* Wait for higher thread_ids to exit */
1595e0ad97eJonathan Adams				(void) taskq_thread_wait(tq, &tq->tq_lock,
1596e0ad97eJonathan Adams				    &tq->tq_exit_cv, &cprinfo, -1);
15972e0c549Jonathan Adams				continue;
15982e0c549Jonathan Adams			}
159935a5a35Jonathan Adams
160035a5a35Jonathan Adams			/*
160135a5a35Jonathan Adams			 * If no thread is starting taskq_thread(), we can
160235a5a35Jonathan Adams			 * do some bookkeeping.
160335a5a35Jonathan Adams			 */
160435a5a35Jonathan Adams			if (!(tq->tq_flags & TASKQ_THREAD_CREATED)) {
160535a5a35Jonathan Adams				/* Check if we've reached our target */
160635a5a35Jonathan Adams				if (tq->tq_nthreads == tq->tq_nthreads_target) {
160735a5a35Jonathan Adams					tq->tq_flags &= ~TASKQ_CHANGING;
160835a5a35Jonathan Adams					cv_broadcast(&tq->tq_wait_cv);
160935a5a35Jonathan Adams				}
161035a5a35Jonathan Adams				/* Check if we need to create a thread */
161135a5a35Jonathan Adams				if (tq->tq_nthreads < tq->tq_nthreads_target) {
161235a5a35Jonathan Adams					taskq_thread_create(tq);
161335a5a35Jonathan Adams					continue; /* tq_lock was dropped */
161435a5a35Jonathan Adams				}
161535a5a35Jonathan Adams			}
16162e0c549Jonathan Adams		}
16177c478bdstevel@tonic-gate		if ((tqe = tq->tq_task.tqent_next) == &tq->tq_task) {
16187c478bdstevel@tonic-gate			if (--tq->tq_active == 0)
16197c478bdstevel@tonic-gate				cv_broadcast(&tq->tq_wait_cv);
1620e0ad97eJonathan Adams			(void) taskq_thread_wait(tq, &tq->tq_lock,
1621e0ad97eJonathan Adams			    &tq->tq_dispatch_cv, &cprinfo, -1);
16227c478bdstevel@tonic-gate			tq->tq_active++;
16237c478bdstevel@tonic-gate			continue;
16247c478bdstevel@tonic-gate		}
162535a5a35Jonathan Adams
16267c478bdstevel@tonic-gate		tqe->tqent_prev->tqent_next = tqe->tqent_next;
16277c478bdstevel@tonic-gate		tqe->tqent_next->tqent_prev = tqe->tqent_prev;
16287c478bdstevel@tonic-gate		mutex_exit(&tq->tq_lock);
16297c478bdstevel@tonic-gate
16305aeb947Garrett D'Amore		/*
16315aeb947Garrett D'Amore		 * For prealloc'd tasks, we don't free anything.  We
16325aeb947Garrett D'Amore		 * have to check this now, because once we call the
16335aeb947Garrett D'Amore		 * function for a prealloc'd taskq, we can't touch the
16345aeb947Garrett D'Amore		 * tqent any longer (calling the function returns the
16355aeb947Garrett D'Amore		 * ownershp of the tqent back to caller of
16365aeb947Garrett D'Amore		 * taskq_dispatch.)
16375aeb947Garrett D'Amore		 */
16385aeb947Garrett D'Amore		if ((!(tq->tq_flags & TASKQ_DYNAMIC)) &&
16395aeb947Garrett D'Amore		    (tqe->tqent_un.tqent_flags & TQENT_FLAG_PREALLOC)) {
16405aeb947Garrett D'Amore			/* clear pointers to assist assertion checks */
16415aeb947Garrett D'Amore			tqe->tqent_next = tqe->tqent_prev = NULL;
16425aeb947Garrett D'Amore			freeit = B_FALSE;
16435aeb947Garrett D'Amore		} else {
16445aeb947Garrett D'Amore			freeit = B_TRUE;
16455aeb947Garrett D'Amore		}
16465aeb947Garrett D'Amore
16477c478bdstevel@tonic-gate		rw_enter(&tq->tq_threadlock, RW_READER);
16487c478bdstevel@tonic-gate		start = gethrtime();
16497c478bdstevel@tonic-gate		DTRACE_PROBE2(taskq__exec__start, taskq_t *, tq,
16507c478bdstevel@tonic-gate		    taskq_ent_t *, tqe);
16517c478bdstevel@tonic-gate		tqe->tqent_func(tqe->tqent_arg);
16527c478bdstevel@tonic-gate		DTRACE_PROBE2(taskq__exec__end, taskq_t *, tq,
16537c478bdstevel@tonic-gate		    taskq_ent_t *, tqe);
16547c478bdstevel@tonic-gate		end = gethrtime();
16557c478bdstevel@tonic-gate		rw_exit(&tq->tq_threadlock);
16567c478bdstevel@tonic-gate
16577c478bdstevel@tonic-gate		mutex_enter(&tq->tq_lock);
16587c478bdstevel@tonic-gate		tq->tq_totaltime += end - start;
16597c478bdstevel@tonic-gate		tq->tq_executed++;
16607c478bdstevel@tonic-gate
16615aeb947Garrett D'Amore		if (freeit)
16625aeb947Garrett D'Amore			taskq_ent_free(tq, tqe);
16637c478bdstevel@tonic-gate	}
16642e0c549Jonathan Adams
16652e0c549Jonathan Adams	if (tq->tq_nthreads_max == 1)
16662e0c549Jonathan Adams		tq->tq_thread = NULL;
16672e0c549Jonathan Adams	else
16682e0c549Jonathan Adams		tq->tq_threadlist[thread_id - 1] = NULL;
16692e0c549Jonathan Adams
167035a5a35Jonathan Adams	/* We're exiting, and therefore no longer active */
167135a5a35Jonathan Adams	ASSERT(tq->tq_active > 0);
167235a5a35Jonathan Adams	tq->tq_active--;
167335a5a35Jonathan Adams
16742e0c549Jonathan Adams	ASSERT(tq->tq_nthreads > 0);
167535a5a35Jonathan Adams	tq->tq_nthreads--;
16762e0c549Jonathan Adams
167735a5a35Jonathan Adams	/* Wake up anyone waiting for us to exit */
16782e0c549Jonathan Adams	cv_broadcast(&tq->tq_exit_cv);
167935a5a35Jonathan Adams	if (tq->tq_nthreads == tq->tq_nthreads_target) {
168035a5a35Jonathan Adams		if (!(tq