xref: /illumos-gate/usr/src/uts/common/os/taskq.c (revision 7c5f01c17875fc96e2b6119c01a189afd6081e87)
17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate  * CDDL HEADER START
37c478bd9Sstevel@tonic-gate  *
47c478bd9Sstevel@tonic-gate  * The contents of this file are subject to the terms of the
52e0c549eSJonathan Adams  * Common Development and Distribution License (the "License").
62e0c549eSJonathan Adams  * You may not use this file except in compliance with the License.
77c478bd9Sstevel@tonic-gate  *
87c478bd9Sstevel@tonic-gate  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bd9Sstevel@tonic-gate  * or http://www.opensolaris.org/os/licensing.
107c478bd9Sstevel@tonic-gate  * See the License for the specific language governing permissions
117c478bd9Sstevel@tonic-gate  * and limitations under the License.
127c478bd9Sstevel@tonic-gate  *
137c478bd9Sstevel@tonic-gate  * When distributing Covered Code, include this CDDL HEADER in each
147c478bd9Sstevel@tonic-gate  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bd9Sstevel@tonic-gate  * If applicable, add the following below this CDDL HEADER, with the
167c478bd9Sstevel@tonic-gate  * fields enclosed by brackets "[]" replaced with your own identifying
177c478bd9Sstevel@tonic-gate  * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bd9Sstevel@tonic-gate  *
197c478bd9Sstevel@tonic-gate  * CDDL HEADER END
207c478bd9Sstevel@tonic-gate  */
217c478bd9Sstevel@tonic-gate /*
2256f33205SJonathan Adams  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
237c478bd9Sstevel@tonic-gate  * Use is subject to license terms.
247c478bd9Sstevel@tonic-gate  */
257c478bd9Sstevel@tonic-gate 
265aeb9474SGarrett D'Amore /*
27e5488233SGordon Ross  * Copyright 2015 Nexenta Systems, Inc.  All rights reserved.
285aeb9474SGarrett D'Amore  */
295aeb9474SGarrett D'Amore 
307c478bd9Sstevel@tonic-gate /*
317c478bd9Sstevel@tonic-gate  * Kernel task queues: general-purpose asynchronous task scheduling.
327c478bd9Sstevel@tonic-gate  *
337c478bd9Sstevel@tonic-gate  * A common problem in kernel programming is the need to schedule tasks
347c478bd9Sstevel@tonic-gate  * to be performed later, by another thread. There are several reasons
357c478bd9Sstevel@tonic-gate  * you may want or need to do this:
367c478bd9Sstevel@tonic-gate  *
377c478bd9Sstevel@tonic-gate  * (1) The task isn't time-critical, but your current code path is.
387c478bd9Sstevel@tonic-gate  *
397c478bd9Sstevel@tonic-gate  * (2) The task may require grabbing locks that you already hold.
407c478bd9Sstevel@tonic-gate  *
417c478bd9Sstevel@tonic-gate  * (3) The task may need to block (e.g. to wait for memory), but you
427c478bd9Sstevel@tonic-gate  *     cannot block in your current context.
437c478bd9Sstevel@tonic-gate  *
447c478bd9Sstevel@tonic-gate  * (4) Your code path can't complete because of some condition, but you can't
457c478bd9Sstevel@tonic-gate  *     sleep or fail, so you queue the task for later execution when condition
467c478bd9Sstevel@tonic-gate  *     disappears.
477c478bd9Sstevel@tonic-gate  *
487c478bd9Sstevel@tonic-gate  * (5) You just want a simple way to launch multiple tasks in parallel.
497c478bd9Sstevel@tonic-gate  *
507c478bd9Sstevel@tonic-gate  * Task queues provide such a facility. In its simplest form (used when
517c478bd9Sstevel@tonic-gate  * performance is not a critical consideration) a task queue consists of a
527c478bd9Sstevel@tonic-gate  * single list of tasks, together with one or more threads to service the
537c478bd9Sstevel@tonic-gate  * list. There are some cases when this simple queue is not sufficient:
547c478bd9Sstevel@tonic-gate  *
557c478bd9Sstevel@tonic-gate  * (1) The task queues are very hot and there is a need to avoid data and lock
567c478bd9Sstevel@tonic-gate  *	contention over global resources.
577c478bd9Sstevel@tonic-gate  *
587c478bd9Sstevel@tonic-gate  * (2) Some tasks may depend on other tasks to complete, so they can't be put in
597c478bd9Sstevel@tonic-gate  *	the same list managed by the same thread.
607c478bd9Sstevel@tonic-gate  *
617c478bd9Sstevel@tonic-gate  * (3) Some tasks may block for a long time, and this should not block other
6235a5a358SJonathan Adams  *	tasks in the queue.
637c478bd9Sstevel@tonic-gate  *
647c478bd9Sstevel@tonic-gate  * To provide useful service in such cases we define a "dynamic task queue"
657c478bd9Sstevel@tonic-gate  * which has an individual thread for each of the tasks. These threads are
667c478bd9Sstevel@tonic-gate  * dynamically created as they are needed and destroyed when they are not in
677c478bd9Sstevel@tonic-gate  * use. The API for managing task pools is the same as for managing task queues
687c478bd9Sstevel@tonic-gate  * with the exception of a taskq creation flag TASKQ_DYNAMIC which tells that
697c478bd9Sstevel@tonic-gate  * dynamic task pool behavior is desired.
707c478bd9Sstevel@tonic-gate  *
717c478bd9Sstevel@tonic-gate  * Dynamic task queues may also place tasks in the normal queue (called "backing
727c478bd9Sstevel@tonic-gate  * queue") when task pool runs out of resources. Users of task queues may
737c478bd9Sstevel@tonic-gate  * disallow such queued scheduling by specifying TQ_NOQUEUE in the dispatch
747c478bd9Sstevel@tonic-gate  * flags.
757c478bd9Sstevel@tonic-gate  *
767c478bd9Sstevel@tonic-gate  * The backing task queue is also used for scheduling internal tasks needed for
777c478bd9Sstevel@tonic-gate  * dynamic task queue maintenance.
787c478bd9Sstevel@tonic-gate  *
792e0c549eSJonathan Adams  * INTERFACES ==================================================================
807c478bd9Sstevel@tonic-gate  *
81a676a7c9SSergio Aguayo  * taskq_t *taskq_create(name, nthreads, pri, minalloc, maxalloc, flags);
827c478bd9Sstevel@tonic-gate  *
837c478bd9Sstevel@tonic-gate  *	Create a taskq with specified properties.
847c478bd9Sstevel@tonic-gate  *	Possible 'flags':
857c478bd9Sstevel@tonic-gate  *
867c478bd9Sstevel@tonic-gate  *	  TASKQ_DYNAMIC: Create task pool for task management. If this flag is
872e0c549eSJonathan Adams  *		specified, 'nthreads' specifies the maximum number of threads in
887c478bd9Sstevel@tonic-gate  *		the task queue. Task execution order for dynamic task queues is
897c478bd9Sstevel@tonic-gate  *		not predictable.
907c478bd9Sstevel@tonic-gate  *
917c478bd9Sstevel@tonic-gate  *		If this flag is not specified (default case) a
922e0c549eSJonathan Adams  *		single-list task queue is created with 'nthreads' threads
932e0c549eSJonathan Adams  *		servicing it. Entries in this queue are managed by
942e0c549eSJonathan Adams  *		taskq_ent_alloc() and taskq_ent_free() which try to keep the
952e0c549eSJonathan Adams  *		task population between 'minalloc' and 'maxalloc', but the
967c478bd9Sstevel@tonic-gate  *		latter limit is only advisory for TQ_SLEEP dispatches and the
977c478bd9Sstevel@tonic-gate  *		former limit is only advisory for TQ_NOALLOC dispatches. If
987c478bd9Sstevel@tonic-gate  *		TASKQ_PREPOPULATE is set in 'flags', the taskq will be
997c478bd9Sstevel@tonic-gate  *		prepopulated with 'minalloc' task structures.
1007c478bd9Sstevel@tonic-gate  *
1017c478bd9Sstevel@tonic-gate  *		Since non-DYNAMIC taskqs are queues, tasks are guaranteed to be
1027c478bd9Sstevel@tonic-gate  *		executed in the order they are scheduled if nthreads == 1.
1037c478bd9Sstevel@tonic-gate  *		If nthreads > 1, task execution order is not predictable.
1047c478bd9Sstevel@tonic-gate  *
1057c478bd9Sstevel@tonic-gate  *	  TASKQ_PREPOPULATE: Prepopulate task queue with threads.
1067c478bd9Sstevel@tonic-gate  *		Also prepopulate the task queue with 'minalloc' task structures.
1077c478bd9Sstevel@tonic-gate  *
1082e0c549eSJonathan Adams  *	  TASKQ_THREADS_CPU_PCT: This flag specifies that 'nthreads' should be
1092e0c549eSJonathan Adams  *		interpreted as a percentage of the # of online CPUs on the
1102e0c549eSJonathan Adams  *		system.  The taskq subsystem will automatically adjust the
1112e0c549eSJonathan Adams  *		number of threads in the taskq in response to CPU online
1122e0c549eSJonathan Adams  *		and offline events, to keep the ratio.  nthreads must be in
1132e0c549eSJonathan Adams  *		the range [0,100].
1142e0c549eSJonathan Adams  *
1152e0c549eSJonathan Adams  *		The calculation used is:
1162e0c549eSJonathan Adams  *
1172e0c549eSJonathan Adams  *			MAX((ncpus_online * percentage)/100, 1)
1182e0c549eSJonathan Adams  *
1192e0c549eSJonathan Adams  *		This flag is not supported for DYNAMIC task queues.
1202e0c549eSJonathan Adams  *		This flag is not compatible with TASKQ_CPR_SAFE.
1212e0c549eSJonathan Adams  *
1227c478bd9Sstevel@tonic-gate  *	  TASKQ_CPR_SAFE: This flag specifies that users of the task queue will
1232e0c549eSJonathan Adams  *		use their own protocol for handling CPR issues. This flag is not
1242e0c549eSJonathan Adams  *		supported for DYNAMIC task queues.  This flag is not compatible
1252e0c549eSJonathan Adams  *		with TASKQ_THREADS_CPU_PCT.
1267c478bd9Sstevel@tonic-gate  *
1277c478bd9Sstevel@tonic-gate  *	The 'pri' field specifies the default priority for the threads that
1287c478bd9Sstevel@tonic-gate  *	service all scheduled tasks.
1297c478bd9Sstevel@tonic-gate  *
13035a5a358SJonathan Adams  * taskq_t *taskq_create_instance(name, instance, nthreads, pri, minalloc,
131a676a7c9SSergio Aguayo  *    maxalloc, flags);
13235a5a358SJonathan Adams  *
13335a5a358SJonathan Adams  *	Like taskq_create(), but takes an instance number (or -1 to indicate
13435a5a358SJonathan Adams  *	no instance).
13535a5a358SJonathan Adams  *
136a676a7c9SSergio Aguayo  * taskq_t *taskq_create_proc(name, nthreads, pri, minalloc, maxalloc, proc,
13735a5a358SJonathan Adams  *    flags);
13835a5a358SJonathan Adams  *
13935a5a358SJonathan Adams  *	Like taskq_create(), but creates the taskq threads in the specified
14035a5a358SJonathan Adams  *	system process.  If proc != &p0, this must be called from a thread
14135a5a358SJonathan Adams  *	in that process.
14235a5a358SJonathan Adams  *
143a676a7c9SSergio Aguayo  * taskq_t *taskq_create_sysdc(name, nthreads, minalloc, maxalloc, proc,
14435a5a358SJonathan Adams  *    dc, flags);
14535a5a358SJonathan Adams  *
14635a5a358SJonathan Adams  *	Like taskq_create_proc(), but the taskq threads will use the
14735a5a358SJonathan Adams  *	System Duty Cycle (SDC) scheduling class with a duty cycle of dc.
14835a5a358SJonathan Adams  *
1497c478bd9Sstevel@tonic-gate  * void taskq_destroy(tap):
1507c478bd9Sstevel@tonic-gate  *
1517c478bd9Sstevel@tonic-gate  *	Waits for any scheduled tasks to complete, then destroys the taskq.
1527c478bd9Sstevel@tonic-gate  *	Caller should guarantee that no new tasks are scheduled in the closing
1537c478bd9Sstevel@tonic-gate  *	taskq.
1547c478bd9Sstevel@tonic-gate  *
1557c478bd9Sstevel@tonic-gate  * taskqid_t taskq_dispatch(tq, func, arg, flags):
1567c478bd9Sstevel@tonic-gate  *
1577c478bd9Sstevel@tonic-gate  *	Dispatches the task "func(arg)" to taskq. The 'flags' indicates whether
1587c478bd9Sstevel@tonic-gate  *	the caller is willing to block for memory.  The function returns an
1597c478bd9Sstevel@tonic-gate  *	opaque value which is zero iff dispatch fails.  If flags is TQ_NOSLEEP
1607c478bd9Sstevel@tonic-gate  *	or TQ_NOALLOC and the task can't be dispatched, taskq_dispatch() fails
1617c478bd9Sstevel@tonic-gate  *	and returns (taskqid_t)0.
1627c478bd9Sstevel@tonic-gate  *
1637c478bd9Sstevel@tonic-gate  *	ASSUMES: func != NULL.
1647c478bd9Sstevel@tonic-gate  *
1657c478bd9Sstevel@tonic-gate  *	Possible flags:
1667c478bd9Sstevel@tonic-gate  *	  TQ_NOSLEEP: Do not wait for resources; may fail.
1677c478bd9Sstevel@tonic-gate  *
1687c478bd9Sstevel@tonic-gate  *	  TQ_NOALLOC: Do not allocate memory; may fail.  May only be used with
1697c478bd9Sstevel@tonic-gate  *		non-dynamic task queues.
1707c478bd9Sstevel@tonic-gate  *
1717c478bd9Sstevel@tonic-gate  *	  TQ_NOQUEUE: Do not enqueue a task if it can't dispatch it due to
1727c478bd9Sstevel@tonic-gate  *		lack of available resources and fail. If this flag is not
17335a5a358SJonathan Adams  *		set, and the task pool is exhausted, the task may be scheduled
1747c478bd9Sstevel@tonic-gate  *		in the backing queue. This flag may ONLY be used with dynamic
1757c478bd9Sstevel@tonic-gate  *		task queues.
1767c478bd9Sstevel@tonic-gate  *
1777c478bd9Sstevel@tonic-gate  *		NOTE: This flag should always be used when a task queue is used
1787c478bd9Sstevel@tonic-gate  *		for tasks that may depend on each other for completion.
1797c478bd9Sstevel@tonic-gate  *		Enqueueing dependent tasks may create deadlocks.
1807c478bd9Sstevel@tonic-gate  *
1817c478bd9Sstevel@tonic-gate  *	  TQ_SLEEP:   May block waiting for resources. May still fail for
18235a5a358SJonathan Adams  *		dynamic task queues if TQ_NOQUEUE is also specified, otherwise
1837c478bd9Sstevel@tonic-gate  *		always succeed.
1847c478bd9Sstevel@tonic-gate  *
18535a5a358SJonathan Adams  *	  TQ_FRONT:   Puts the new task at the front of the queue.  Be careful.
18635a5a358SJonathan Adams  *
1877c478bd9Sstevel@tonic-gate  *	NOTE: Dynamic task queues are much more likely to fail in
1887c478bd9Sstevel@tonic-gate  *		taskq_dispatch() (especially if TQ_NOQUEUE was specified), so it
1897c478bd9Sstevel@tonic-gate  *		is important to have backup strategies handling such failures.
1907c478bd9Sstevel@tonic-gate  *
1915aeb9474SGarrett D'Amore  * void taskq_dispatch_ent(tq, func, arg, flags, tqent)
1925aeb9474SGarrett D'Amore  *
1935aeb9474SGarrett D'Amore  *	This is a light-weight form of taskq_dispatch(), that uses a
1945aeb9474SGarrett D'Amore  *	preallocated taskq_ent_t structure for scheduling.  As a
1955aeb9474SGarrett D'Amore  *	result, it does not perform allocations and cannot ever fail.
1965aeb9474SGarrett D'Amore  *	Note especially that it cannot be used with TASKQ_DYNAMIC
1975aeb9474SGarrett D'Amore  *	taskqs.  The memory for the tqent must not be modified or used
1985aeb9474SGarrett D'Amore  *	until the function (func) is called.  (However, func itself
1995aeb9474SGarrett D'Amore  *	may safely modify or free this memory, once it is called.)
2005aeb9474SGarrett D'Amore  *	Note that the taskq framework will NOT free this memory.
2015aeb9474SGarrett D'Amore  *
2027c478bd9Sstevel@tonic-gate  * void taskq_wait(tq):
2037c478bd9Sstevel@tonic-gate  *
2047c478bd9Sstevel@tonic-gate  *	Waits for all previously scheduled tasks to complete.
2057c478bd9Sstevel@tonic-gate  *
2067c478bd9Sstevel@tonic-gate  *	NOTE: It does not stop any new task dispatches.
2077c478bd9Sstevel@tonic-gate  *	      Do NOT call taskq_wait() from a task: it will cause deadlock.
2087c478bd9Sstevel@tonic-gate  *
2097c478bd9Sstevel@tonic-gate  * void taskq_suspend(tq)
2107c478bd9Sstevel@tonic-gate  *
2117c478bd9Sstevel@tonic-gate  *	Suspend all task execution. Tasks already scheduled for a dynamic task
2127c478bd9Sstevel@tonic-gate  *	queue will still be executed, but all new scheduled tasks will be
2137c478bd9Sstevel@tonic-gate  *	suspended until taskq_resume() is called.
2147c478bd9Sstevel@tonic-gate  *
2157c478bd9Sstevel@tonic-gate  * int  taskq_suspended(tq)
2167c478bd9Sstevel@tonic-gate  *
2177c478bd9Sstevel@tonic-gate  *	Returns 1 if taskq is suspended and 0 otherwise. It is intended to
2187c478bd9Sstevel@tonic-gate  *	ASSERT that the task queue is suspended.
2197c478bd9Sstevel@tonic-gate  *
2207c478bd9Sstevel@tonic-gate  * void taskq_resume(tq)
2217c478bd9Sstevel@tonic-gate  *
2227c478bd9Sstevel@tonic-gate  *	Resume task queue execution.
2237c478bd9Sstevel@tonic-gate  *
2247c478bd9Sstevel@tonic-gate  * int  taskq_member(tq, thread)
2257c478bd9Sstevel@tonic-gate  *
2267c478bd9Sstevel@tonic-gate  *	Returns 1 if 'thread' belongs to taskq 'tq' and 0 otherwise. The
2277c478bd9Sstevel@tonic-gate  *	intended use is to ASSERT that a given function is called in taskq
2287c478bd9Sstevel@tonic-gate  *	context only.
2297c478bd9Sstevel@tonic-gate  *
2307c478bd9Sstevel@tonic-gate  * system_taskq
2317c478bd9Sstevel@tonic-gate  *
2327c478bd9Sstevel@tonic-gate  *	Global system-wide dynamic task queue for common uses. It may be used by
2337c478bd9Sstevel@tonic-gate  *	any subsystem that needs to schedule tasks and does not need to manage
2347c478bd9Sstevel@tonic-gate  *	its own task queues. It is initialized quite early during system boot.
2357c478bd9Sstevel@tonic-gate  *
2362e0c549eSJonathan Adams  * IMPLEMENTATION ==============================================================
2377c478bd9Sstevel@tonic-gate  *
2387c478bd9Sstevel@tonic-gate  * This is schematic representation of the task queue structures.
2397c478bd9Sstevel@tonic-gate  *
2407c478bd9Sstevel@tonic-gate  *   taskq:
2417c478bd9Sstevel@tonic-gate  *   +-------------+
242e0ad97e3SJonathan Adams  *   | tq_lock     | +---< taskq_ent_free()
2437c478bd9Sstevel@tonic-gate  *   +-------------+ |
2447c478bd9Sstevel@tonic-gate  *   |...          | | tqent:                  tqent:
2457c478bd9Sstevel@tonic-gate  *   +-------------+ | +------------+          +------------+
2467c478bd9Sstevel@tonic-gate  *   | tq_freelist |-->| tqent_next |--> ... ->| tqent_next |
2477c478bd9Sstevel@tonic-gate  *   +-------------+   +------------+          +------------+
2487c478bd9Sstevel@tonic-gate  *   |...          |   | ...        |          | ...        |
2497c478bd9Sstevel@tonic-gate  *   +-------------+   +------------+          +------------+
2507c478bd9Sstevel@tonic-gate  *   | tq_task     |    |
2517c478bd9Sstevel@tonic-gate  *   |             |    +-------------->taskq_ent_alloc()
2527c478bd9Sstevel@tonic-gate  * +--------------------------------------------------------------------------+
2537c478bd9Sstevel@tonic-gate  * | |                     |            tqent                   tqent         |
2547c478bd9Sstevel@tonic-gate  * | +---------------------+     +--> +------------+     +--> +------------+  |
2557c478bd9Sstevel@tonic-gate  * | | ...		   |     |    | func, arg  |     |    | func, arg  |  |
2567c478bd9Sstevel@tonic-gate  * +>+---------------------+ <---|-+  +------------+ <---|-+  +------------+  |
2577c478bd9Sstevel@tonic-gate  *   | tq_taskq.tqent_next | ----+ |  | tqent_next | --->+ |  | tqent_next |--+
2587c478bd9Sstevel@tonic-gate  *   +---------------------+	   |  +------------+     ^ |  +------------+
2597c478bd9Sstevel@tonic-gate  * +-| tq_task.tqent_prev  |	   +--| tqent_prev |     | +--| tqent_prev |  ^
2607c478bd9Sstevel@tonic-gate  * | +---------------------+	      +------------+     |    +------------+  |
2617c478bd9Sstevel@tonic-gate  * | |...		   |	      | ...        |     |    | ...        |  |
2627c478bd9Sstevel@tonic-gate  * | +---------------------+	      +------------+     |    +------------+  |
2637c478bd9Sstevel@tonic-gate  * |                                      ^              |                    |
2647c478bd9Sstevel@tonic-gate  * |                                      |              |                    |
2657c478bd9Sstevel@tonic-gate  * +--------------------------------------+--------------+       TQ_APPEND() -+
2667c478bd9Sstevel@tonic-gate  *   |             |                      |
2677c478bd9Sstevel@tonic-gate  *   |...          |   taskq_thread()-----+
2687c478bd9Sstevel@tonic-gate  *   +-------------+
2697c478bd9Sstevel@tonic-gate  *   | tq_buckets  |--+-------> [ NULL ] (for regular task queues)
2707c478bd9Sstevel@tonic-gate  *   +-------------+  |
2717c478bd9Sstevel@tonic-gate  *                    |   DYNAMIC TASK QUEUES:
2727c478bd9Sstevel@tonic-gate  *                    |
27335a5a358SJonathan Adams  *                    +-> taskq_bucket[nCPU]		taskq_bucket_dispatch()
2747c478bd9Sstevel@tonic-gate  *                        +-------------------+                    ^
2757c478bd9Sstevel@tonic-gate  *                   +--->| tqbucket_lock     |                    |
2767c478bd9Sstevel@tonic-gate  *                   |    +-------------------+   +--------+      +--------+
2777c478bd9Sstevel@tonic-gate  *                   |    | tqbucket_freelist |-->| tqent  |-->...| tqent  | ^
2787c478bd9Sstevel@tonic-gate  *                   |    +-------------------+<--+--------+<--...+--------+ |
2797c478bd9Sstevel@tonic-gate  *                   |    | ...               |   | thread |      | thread | |
2807c478bd9Sstevel@tonic-gate  *                   |    +-------------------+   +--------+      +--------+ |
2817c478bd9Sstevel@tonic-gate  *                   |    +-------------------+                              |
2827c478bd9Sstevel@tonic-gate  * taskq_dispatch()--+--->| tqbucket_lock     |             TQ_APPEND()------+
2837c478bd9Sstevel@tonic-gate  *      TQ_HASH()    |    +-------------------+   +--------+      +--------+
2847c478bd9Sstevel@tonic-gate  *                   |    | tqbucket_freelist |-->| tqent  |-->...| tqent  |
2857c478bd9Sstevel@tonic-gate  *                   |    +-------------------+<--+--------+<--...+--------+
2867c478bd9Sstevel@tonic-gate  *                   |    | ...               |   | thread |      | thread |
2877c478bd9Sstevel@tonic-gate  *                   |    +-------------------+   +--------+      +--------+
28835a5a358SJonathan Adams  *		     +--->	...
2897c478bd9Sstevel@tonic-gate  *
2907c478bd9Sstevel@tonic-gate  *
2917c478bd9Sstevel@tonic-gate  * Task queues use tq_task field to link new entry in the queue. The queue is a
2927c478bd9Sstevel@tonic-gate  * circular doubly-linked list. Entries are put in the end of the list with
2937c478bd9Sstevel@tonic-gate  * TQ_APPEND() and processed from the front of the list by taskq_thread() in
2947c478bd9Sstevel@tonic-gate  * FIFO order. Task queue entries are cached in the free list managed by
2957c478bd9Sstevel@tonic-gate  * taskq_ent_alloc() and taskq_ent_free() functions.
2967c478bd9Sstevel@tonic-gate  *
2977c478bd9Sstevel@tonic-gate  *	All threads used by task queues mark t_taskq field of the thread to
2987c478bd9Sstevel@tonic-gate  *	point to the task queue.
2997c478bd9Sstevel@tonic-gate  *
3002e0c549eSJonathan Adams  * Taskq Thread Management -----------------------------------------------------
3012e0c549eSJonathan Adams  *
3022e0c549eSJonathan Adams  * Taskq's non-dynamic threads are managed with several variables and flags:
3032e0c549eSJonathan Adams  *
3042e0c549eSJonathan Adams  *	* tq_nthreads	- The number of threads in taskq_thread() for the
3052e0c549eSJonathan Adams  *			  taskq.
3062e0c549eSJonathan Adams  *
3072e0c549eSJonathan Adams  *	* tq_active	- The number of threads not waiting on a CV in
3082e0c549eSJonathan Adams  *			  taskq_thread(); includes newly created threads
3092e0c549eSJonathan Adams  *			  not yet counted in tq_nthreads.
3102e0c549eSJonathan Adams  *
3112e0c549eSJonathan Adams  *	* tq_nthreads_target
3122e0c549eSJonathan Adams  *			- The number of threads desired for the taskq.
3132e0c549eSJonathan Adams  *
3142e0c549eSJonathan Adams  *	* tq_flags & TASKQ_CHANGING
3152e0c549eSJonathan Adams  *			- Indicates that tq_nthreads != tq_nthreads_target.
3162e0c549eSJonathan Adams  *
3172e0c549eSJonathan Adams  *	* tq_flags & TASKQ_THREAD_CREATED
3182e0c549eSJonathan Adams  *			- Indicates that a thread is being created in the taskq.
3192e0c549eSJonathan Adams  *
3202e0c549eSJonathan Adams  * During creation, tq_nthreads and tq_active are set to 0, and
3212e0c549eSJonathan Adams  * tq_nthreads_target is set to the number of threads desired.  The
32235a5a358SJonathan Adams  * TASKQ_CHANGING flag is set, and taskq_thread_create() is called to
32335a5a358SJonathan Adams  * create the first thread. taskq_thread_create() increments tq_active,
3242e0c549eSJonathan Adams  * sets TASKQ_THREAD_CREATED, and creates the new thread.
3252e0c549eSJonathan Adams  *
3262e0c549eSJonathan Adams  * Each thread starts in taskq_thread(), clears the TASKQ_THREAD_CREATED
3272e0c549eSJonathan Adams  * flag, and increments tq_nthreads.  It stores the new value of
3282e0c549eSJonathan Adams  * tq_nthreads as its "thread_id", and stores its thread pointer in the
3292e0c549eSJonathan Adams  * tq_threadlist at the (thread_id - 1).  We keep the thread_id space
3302e0c549eSJonathan Adams  * densely packed by requiring that only the largest thread_id can exit during
3312e0c549eSJonathan Adams  * normal adjustment.   The exception is during the destruction of the
3322e0c549eSJonathan Adams  * taskq; once tq_nthreads_target is set to zero, no new threads will be created
3332e0c549eSJonathan Adams  * for the taskq queue, so every thread can exit without any ordering being
3342e0c549eSJonathan Adams  * necessary.
3352e0c549eSJonathan Adams  *
3362e0c549eSJonathan Adams  * Threads will only process work if their thread id is <= tq_nthreads_target.
3372e0c549eSJonathan Adams  *
3382e0c549eSJonathan Adams  * When TASKQ_CHANGING is set, threads will check the current thread target
3392e0c549eSJonathan Adams  * whenever they wake up, and do whatever they can to apply its effects.
3402e0c549eSJonathan Adams  *
3412e0c549eSJonathan Adams  * TASKQ_THREAD_CPU_PCT --------------------------------------------------------
3422e0c549eSJonathan Adams  *
3432e0c549eSJonathan Adams  * When a taskq is created with TASKQ_THREAD_CPU_PCT, we store their requested
3442e0c549eSJonathan Adams  * percentage in tq_threads_ncpus_pct, start them off with the correct thread
3452e0c549eSJonathan Adams  * target, and add them to the taskq_cpupct_list for later adjustment.
3462e0c549eSJonathan Adams  *
3472e0c549eSJonathan Adams  * We register taskq_cpu_setup() to be called whenever a CPU changes state.  It
3482e0c549eSJonathan Adams  * walks the list of TASKQ_THREAD_CPU_PCT taskqs, adjusts their nthread_target
3492e0c549eSJonathan Adams  * if need be, and wakes up all of the threads to process the change.
3502e0c549eSJonathan Adams  *
3512e0c549eSJonathan Adams  * Dynamic Task Queues Implementation ------------------------------------------
3527c478bd9Sstevel@tonic-gate  *
3537c478bd9Sstevel@tonic-gate  * For a dynamic task queues there is a 1-to-1 mapping between a thread and
3547c478bd9Sstevel@tonic-gate  * taskq_ent_structure. Each entry is serviced by its own thread and each thread
3557c478bd9Sstevel@tonic-gate  * is controlled by a single entry.
3567c478bd9Sstevel@tonic-gate  *
3577c478bd9Sstevel@tonic-gate  * Entries are distributed over a set of buckets. To avoid using modulo
3587c478bd9Sstevel@tonic-gate  * arithmetics the number of buckets is 2^n and is determined as the nearest
3597c478bd9Sstevel@tonic-gate  * power of two roundown of the number of CPUs in the system. Tunable
3607c478bd9Sstevel@tonic-gate  * variable 'taskq_maxbuckets' limits the maximum number of buckets. Each entry
3617c478bd9Sstevel@tonic-gate  * is attached to a bucket for its lifetime and can't migrate to other buckets.
3627c478bd9Sstevel@tonic-gate  *
3637c478bd9Sstevel@tonic-gate  * Entries that have scheduled tasks are not placed in any list. The dispatch
3647c478bd9Sstevel@tonic-gate  * function sets their "func" and "arg" fields and signals the corresponding
3657c478bd9Sstevel@tonic-gate  * thread to execute the task. Once the thread executes the task it clears the
3667c478bd9Sstevel@tonic-gate  * "func" field and places an entry on the bucket cache of free entries pointed
3677c478bd9Sstevel@tonic-gate  * by "tqbucket_freelist" field. ALL entries on the free list should have "func"
3687c478bd9Sstevel@tonic-gate  * field equal to NULL. The free list is a circular doubly-linked list identical
3697c478bd9Sstevel@tonic-gate  * in structure to the tq_task list above, but entries are taken from it in LIFO
3707c478bd9Sstevel@tonic-gate  * order - the last freed entry is the first to be allocated. The
3717c478bd9Sstevel@tonic-gate  * taskq_bucket_dispatch() function gets the most recently used entry from the
3727c478bd9Sstevel@tonic-gate  * free list, sets its "func" and "arg" fields and signals a worker thread.
3737c478bd9Sstevel@tonic-gate  *
3747c478bd9Sstevel@tonic-gate  * After executing each task a per-entry thread taskq_d_thread() places its
3757c478bd9Sstevel@tonic-gate  * entry on the bucket free list and goes to a timed sleep. If it wakes up
3767c478bd9Sstevel@tonic-gate  * without getting new task it removes the entry from the free list and destroys
3777c478bd9Sstevel@tonic-gate  * itself. The thread sleep time is controlled by a tunable variable
3787c478bd9Sstevel@tonic-gate  * `taskq_thread_timeout'.
3797c478bd9Sstevel@tonic-gate  *
3802e0c549eSJonathan Adams  * There are various statistics kept in the bucket which allows for later
3817c478bd9Sstevel@tonic-gate  * analysis of taskq usage patterns. Also, a global copy of taskq creation and
3827c478bd9Sstevel@tonic-gate  * death statistics is kept in the global taskq data structure. Since thread
3837c478bd9Sstevel@tonic-gate  * creation and death happen rarely, updating such global data does not present
3847c478bd9Sstevel@tonic-gate  * a performance problem.
3857c478bd9Sstevel@tonic-gate  *
3867c478bd9Sstevel@tonic-gate  * NOTE: Threads are not bound to any CPU and there is absolutely no association
3877c478bd9Sstevel@tonic-gate  *       between the bucket and actual thread CPU, so buckets are used only to
3887c478bd9Sstevel@tonic-gate  *	 split resources and reduce resource contention. Having threads attached
3897c478bd9Sstevel@tonic-gate  *	 to the CPU denoted by a bucket may reduce number of times the job
3907c478bd9Sstevel@tonic-gate  *	 switches between CPUs.
3917c478bd9Sstevel@tonic-gate  *
3927c478bd9Sstevel@tonic-gate  *	 Current algorithm creates a thread whenever a bucket has no free
3937c478bd9Sstevel@tonic-gate  *	 entries. It would be nice to know how many threads are in the running
3947c478bd9Sstevel@tonic-gate  *	 state and don't create threads if all CPUs are busy with existing
3957c478bd9Sstevel@tonic-gate  *	 tasks, but it is unclear how such strategy can be implemented.
3967c478bd9Sstevel@tonic-gate  *
3977c478bd9Sstevel@tonic-gate  *	 Currently buckets are created statically as an array attached to task
3987c478bd9Sstevel@tonic-gate  *	 queue. On some system with nCPUs < max_ncpus it may waste system
3997c478bd9Sstevel@tonic-gate  *	 memory. One solution may be allocation of buckets when they are first
4007c478bd9Sstevel@tonic-gate  *	 touched, but it is not clear how useful it is.
4017c478bd9Sstevel@tonic-gate  *
4022e0c549eSJonathan Adams  * SUSPEND/RESUME implementation -----------------------------------------------
4037c478bd9Sstevel@tonic-gate  *
4047c478bd9Sstevel@tonic-gate  *	Before executing a task taskq_thread() (executing non-dynamic task
4057c478bd9Sstevel@tonic-gate  *	queues) obtains taskq's thread lock as a reader. The taskq_suspend()
4067c478bd9Sstevel@tonic-gate  *	function gets the same lock as a writer blocking all non-dynamic task
4077c478bd9Sstevel@tonic-gate  *	execution. The taskq_resume() function releases the lock allowing
4087c478bd9Sstevel@tonic-gate  *	taskq_thread to continue execution.
4097c478bd9Sstevel@tonic-gate  *
4107c478bd9Sstevel@tonic-gate  *	For dynamic task queues, each bucket is marked as TQBUCKET_SUSPEND by
4117c478bd9Sstevel@tonic-gate  *	taskq_suspend() function. After that taskq_bucket_dispatch() always
4127c478bd9Sstevel@tonic-gate  *	fails, so that taskq_dispatch() will either enqueue tasks for a
4137c478bd9Sstevel@tonic-gate  *	suspended backing queue or fail if TQ_NOQUEUE is specified in dispatch
4147c478bd9Sstevel@tonic-gate  *	flags.
4157c478bd9Sstevel@tonic-gate  *
4167c478bd9Sstevel@tonic-gate  *	NOTE: taskq_suspend() does not immediately block any tasks already
4177c478bd9Sstevel@tonic-gate  *	      scheduled for dynamic task queues. It only suspends new tasks
4187c478bd9Sstevel@tonic-gate  *	      scheduled after taskq_suspend() was called.
4197c478bd9Sstevel@tonic-gate  *
4207c478bd9Sstevel@tonic-gate  *	taskq_member() function works by comparing a thread t_taskq pointer with
4217c478bd9Sstevel@tonic-gate  *	the passed thread pointer.
4227c478bd9Sstevel@tonic-gate  *
4232e0c549eSJonathan Adams  * LOCKS and LOCK Hierarchy ----------------------------------------------------
4247c478bd9Sstevel@tonic-gate  *
4252e0c549eSJonathan Adams  *   There are three locks used in task queues:
4267c478bd9Sstevel@tonic-gate  *
4272e0c549eSJonathan Adams  *   1) The taskq_t's tq_lock, protecting global task queue state.
4287c478bd9Sstevel@tonic-gate  *
4297c478bd9Sstevel@tonic-gate  *   2) Each per-CPU bucket has a lock for bucket management.
4307c478bd9Sstevel@tonic-gate  *
4312e0c549eSJonathan Adams  *   3) The global taskq_cpupct_lock, which protects the list of
4322e0c549eSJonathan Adams  *      TASKQ_THREADS_CPU_PCT taskqs.
4332e0c549eSJonathan Adams  *
4342e0c549eSJonathan Adams  *   If both (1) and (2) are needed, tq_lock should be taken *after* the bucket
4357c478bd9Sstevel@tonic-gate  *   lock.
4367c478bd9Sstevel@tonic-gate  *
4372e0c549eSJonathan Adams  *   If both (1) and (3) are needed, tq_lock should be taken *after*
4382e0c549eSJonathan Adams  *   taskq_cpupct_lock.
4392e0c549eSJonathan Adams  *
4402e0c549eSJonathan Adams  * DEBUG FACILITIES ------------------------------------------------------------
4417c478bd9Sstevel@tonic-gate  *
4427c478bd9Sstevel@tonic-gate  * For DEBUG kernels it is possible to induce random failures to
4437c478bd9Sstevel@tonic-gate  * taskq_dispatch() function when it is given TQ_NOSLEEP argument. The value of
4447c478bd9Sstevel@tonic-gate  * taskq_dmtbf and taskq_smtbf tunables control the mean time between induced
4457c478bd9Sstevel@tonic-gate  * failures for dynamic and static task queues respectively.
4467c478bd9Sstevel@tonic-gate  *
4477c478bd9Sstevel@tonic-gate  * Setting TASKQ_STATISTIC to 0 will disable per-bucket statistics.
4487c478bd9Sstevel@tonic-gate  *
4492e0c549eSJonathan Adams  * TUNABLES --------------------------------------------------------------------
4507c478bd9Sstevel@tonic-gate  *
4517c478bd9Sstevel@tonic-gate  *	system_taskq_size	- Size of the global system_taskq.
4527c478bd9Sstevel@tonic-gate  *				  This value is multiplied by nCPUs to determine
4537c478bd9Sstevel@tonic-gate  *				  actual size.
4547c478bd9Sstevel@tonic-gate  *				  Default value: 64
4557c478bd9Sstevel@tonic-gate  *
4562e0c549eSJonathan Adams  *	taskq_minimum_nthreads_max
4572e0c549eSJonathan Adams  *				- Minimum size of the thread list for a taskq.
4582e0c549eSJonathan Adams  *				  Useful for testing different thread pool
4592e0c549eSJonathan Adams  *				  sizes by overwriting tq_nthreads_target.
4602e0c549eSJonathan Adams  *
4617c478bd9Sstevel@tonic-gate  *	taskq_thread_timeout	- Maximum idle time for taskq_d_thread()
4627c478bd9Sstevel@tonic-gate  *				  Default value: 5 minutes
4637c478bd9Sstevel@tonic-gate  *
4647c478bd9Sstevel@tonic-gate  *	taskq_maxbuckets	- Maximum number of buckets in any task queue
4657c478bd9Sstevel@tonic-gate  *				  Default value: 128
4667c478bd9Sstevel@tonic-gate  *
4677c478bd9Sstevel@tonic-gate  *	taskq_search_depth	- Maximum # of buckets searched for a free entry
4687c478bd9Sstevel@tonic-gate  *				  Default value: 4
4697c478bd9Sstevel@tonic-gate  *
4707c478bd9Sstevel@tonic-gate  *	taskq_dmtbf		- Mean time between induced dispatch failures
4717c478bd9Sstevel@tonic-gate  *				  for dynamic task queues.
4727c478bd9Sstevel@tonic-gate  *				  Default value: UINT_MAX (no induced failures)
4737c478bd9Sstevel@tonic-gate  *
4747c478bd9Sstevel@tonic-gate  *	taskq_smtbf		- Mean time between induced dispatch failures
4757c478bd9Sstevel@tonic-gate  *				  for static task queues.
4767c478bd9Sstevel@tonic-gate  *				  Default value: UINT_MAX (no induced failures)
4777c478bd9Sstevel@tonic-gate  *
4782e0c549eSJonathan Adams  * CONDITIONAL compilation -----------------------------------------------------
4797c478bd9Sstevel@tonic-gate  *
4807c478bd9Sstevel@tonic-gate  *    TASKQ_STATISTIC	- If set will enable bucket statistic (default).
4817c478bd9Sstevel@tonic-gate  *
4827c478bd9Sstevel@tonic-gate  */
4837c478bd9Sstevel@tonic-gate 
4847c478bd9Sstevel@tonic-gate #include <sys/taskq_impl.h>
4857c478bd9Sstevel@tonic-gate #include <sys/thread.h>
4867c478bd9Sstevel@tonic-gate #include <sys/proc.h>
4877c478bd9Sstevel@tonic-gate #include <sys/kmem.h>
4887c478bd9Sstevel@tonic-gate #include <sys/vmem.h>
4897c478bd9Sstevel@tonic-gate #include <sys/callb.h>
49035a5a358SJonathan Adams #include <sys/class.h>
4917c478bd9Sstevel@tonic-gate #include <sys/systm.h>
4927c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h>
4937c478bd9Sstevel@tonic-gate #include <sys/debug.h>
4947c478bd9Sstevel@tonic-gate #include <sys/vmsystm.h>	/* For throttlefree */
4957c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h>
4967c478bd9Sstevel@tonic-gate #include <sys/cpuvar.h>
49735a5a358SJonathan Adams #include <sys/cpupart.h>
4987c478bd9Sstevel@tonic-gate #include <sys/sdt.h>
49935a5a358SJonathan Adams #include <sys/sysdc.h>
5002e0c549eSJonathan Adams #include <sys/note.h>
5017c478bd9Sstevel@tonic-gate 
5027c478bd9Sstevel@tonic-gate static kmem_cache_t *taskq_ent_cache, *taskq_cache;
5037c478bd9Sstevel@tonic-gate 
5047c478bd9Sstevel@tonic-gate /*
5052e0c549eSJonathan Adams  * Pseudo instance numbers for taskqs without explicitly provided instance.
5067c478bd9Sstevel@tonic-gate  */
5077c478bd9Sstevel@tonic-gate static vmem_t *taskq_id_arena;
5087c478bd9Sstevel@tonic-gate 
5097c478bd9Sstevel@tonic-gate /* Global system task queue for common use */
5107c478bd9Sstevel@tonic-gate taskq_t	*system_taskq;
5117c478bd9Sstevel@tonic-gate 
5127c478bd9Sstevel@tonic-gate /*
5132e0c549eSJonathan Adams  * Maximum number of entries in global system taskq is
51435a5a358SJonathan Adams  *	system_taskq_size * max_ncpus
5157c478bd9Sstevel@tonic-gate  */
5167c478bd9Sstevel@tonic-gate #define	SYSTEM_TASKQ_SIZE 64
5177c478bd9Sstevel@tonic-gate int system_taskq_size = SYSTEM_TASKQ_SIZE;
5187c478bd9Sstevel@tonic-gate 
5192e0c549eSJonathan Adams /*
5202e0c549eSJonathan Adams  * Minimum size for tq_nthreads_max; useful for those who want to play around
5212e0c549eSJonathan Adams  * with increasing a taskq's tq_nthreads_target.
5222e0c549eSJonathan Adams  */
5232e0c549eSJonathan Adams int taskq_minimum_nthreads_max = 1;
5242e0c549eSJonathan Adams 
52535a5a358SJonathan Adams /*
52635a5a358SJonathan Adams  * We want to ensure that when taskq_create() returns, there is at least
52735a5a358SJonathan Adams  * one thread ready to handle requests.  To guarantee this, we have to wait
52835a5a358SJonathan Adams  * for the second thread, since the first one cannot process requests until
52935a5a358SJonathan Adams  * the second thread has been created.
53035a5a358SJonathan Adams  */
53135a5a358SJonathan Adams #define	TASKQ_CREATE_ACTIVE_THREADS	2
53235a5a358SJonathan Adams 
5332e0c549eSJonathan Adams /* Maximum percentage allowed for TASKQ_THREADS_CPU_PCT */
5342e0c549eSJonathan Adams #define	TASKQ_CPUPCT_MAX_PERCENT	1000
5352e0c549eSJonathan Adams int taskq_cpupct_max_percent = TASKQ_CPUPCT_MAX_PERCENT;
5362e0c549eSJonathan Adams 
5377c478bd9Sstevel@tonic-gate /*
5387c478bd9Sstevel@tonic-gate  * Dynamic task queue threads that don't get any work within
5397c478bd9Sstevel@tonic-gate  * taskq_thread_timeout destroy themselves
5407c478bd9Sstevel@tonic-gate  */
5417c478bd9Sstevel@tonic-gate #define	TASKQ_THREAD_TIMEOUT (60 * 5)
5427c478bd9Sstevel@tonic-gate int taskq_thread_timeout = TASKQ_THREAD_TIMEOUT;
5437c478bd9Sstevel@tonic-gate 
5447c478bd9Sstevel@tonic-gate #define	TASKQ_MAXBUCKETS 128
5457c478bd9Sstevel@tonic-gate int taskq_maxbuckets = TASKQ_MAXBUCKETS;
5467c478bd9Sstevel@tonic-gate 
5477c478bd9Sstevel@tonic-gate /*
5487c478bd9Sstevel@tonic-gate  * When a bucket has no available entries another buckets are tried.
5497c478bd9Sstevel@tonic-gate  * taskq_search_depth parameter limits the amount of buckets that we search
5507c478bd9Sstevel@tonic-gate  * before failing. This is mostly useful in systems with many CPUs where we may
5517c478bd9Sstevel@tonic-gate  * spend too much time scanning busy buckets.
5527c478bd9Sstevel@tonic-gate  */
5537c478bd9Sstevel@tonic-gate #define	TASKQ_SEARCH_DEPTH 4
5547c478bd9Sstevel@tonic-gate int taskq_search_depth = TASKQ_SEARCH_DEPTH;
5557c478bd9Sstevel@tonic-gate 
5567c478bd9Sstevel@tonic-gate /*
5577c478bd9Sstevel@tonic-gate  * Hashing function: mix various bits of x. May be pretty much anything.
5587c478bd9Sstevel@tonic-gate  */
5597c478bd9Sstevel@tonic-gate #define	TQ_HASH(x) ((x) ^ ((x) >> 11) ^ ((x) >> 17) ^ ((x) ^ 27))
5607c478bd9Sstevel@tonic-gate 
5617c478bd9Sstevel@tonic-gate /*
5627c478bd9Sstevel@tonic-gate  * We do not create any new threads when the system is low on memory and start
5637c478bd9Sstevel@tonic-gate  * throttling memory allocations. The following macro tries to estimate such
5647c478bd9Sstevel@tonic-gate  * condition.
5657c478bd9Sstevel@tonic-gate  */
5667c478bd9Sstevel@tonic-gate #define	ENOUGH_MEMORY() (freemem > throttlefree)
5677c478bd9Sstevel@tonic-gate 
5687c478bd9Sstevel@tonic-gate /*
5697c478bd9Sstevel@tonic-gate  * Static functions.
5707c478bd9Sstevel@tonic-gate  */
5717c478bd9Sstevel@tonic-gate static taskq_t	*taskq_create_common(const char *, int, int, pri_t, int,
57235a5a358SJonathan Adams     int, proc_t *, uint_t, uint_t);
5737c478bd9Sstevel@tonic-gate static void taskq_thread(void *);
5747c478bd9Sstevel@tonic-gate static void taskq_d_thread(taskq_ent_t *);
5757c478bd9Sstevel@tonic-gate static void taskq_bucket_extend(void *);
5767c478bd9Sstevel@tonic-gate static int  taskq_constructor(void *, void *, int);
5777c478bd9Sstevel@tonic-gate static void taskq_destructor(void *, void *);
5787c478bd9Sstevel@tonic-gate static int  taskq_ent_constructor(void *, void *, int);
5797c478bd9Sstevel@tonic-gate static void taskq_ent_destructor(void *, void *);
5807c478bd9Sstevel@tonic-gate static taskq_ent_t *taskq_ent_alloc(taskq_t *, int);
5817c478bd9Sstevel@tonic-gate static void taskq_ent_free(taskq_t *, taskq_ent_t *);
58264109744SChris Horne static int taskq_ent_exists(taskq_t *, task_func_t, void *);
5837c478bd9Sstevel@tonic-gate static taskq_ent_t *taskq_bucket_dispatch(taskq_bucket_t *, task_func_t,
5847c478bd9Sstevel@tonic-gate     void *);
5857c478bd9Sstevel@tonic-gate 
5867c478bd9Sstevel@tonic-gate /*
5877c478bd9Sstevel@tonic-gate  * Task queues kstats.
5887c478bd9Sstevel@tonic-gate  */
5897c478bd9Sstevel@tonic-gate struct taskq_kstat {
59035a5a358SJonathan Adams 	kstat_named_t	tq_pid;
5917c478bd9Sstevel@tonic-gate 	kstat_named_t	tq_tasks;
5927c478bd9Sstevel@tonic-gate 	kstat_named_t	tq_executed;
5937c478bd9Sstevel@tonic-gate 	kstat_named_t	tq_maxtasks;
5947c478bd9Sstevel@tonic-gate 	kstat_named_t	tq_totaltime;
5957c478bd9Sstevel@tonic-gate 	kstat_named_t	tq_nalloc;
5967c478bd9Sstevel@tonic-gate 	kstat_named_t	tq_nactive;
5977c478bd9Sstevel@tonic-gate 	kstat_named_t	tq_pri;
5987c478bd9Sstevel@tonic-gate 	kstat_named_t	tq_nthreads;
5997c478bd9Sstevel@tonic-gate } taskq_kstat = {
60035a5a358SJonathan Adams 	{ "pid",		KSTAT_DATA_UINT64 },
6017c478bd9Sstevel@tonic-gate 	{ "tasks",		KSTAT_DATA_UINT64 },
6027c478bd9Sstevel@tonic-gate 	{ "executed",		KSTAT_DATA_UINT64 },
6037c478bd9Sstevel@tonic-gate 	{ "maxtasks",		KSTAT_DATA_UINT64 },
6047c478bd9Sstevel@tonic-gate 	{ "totaltime",		KSTAT_DATA_UINT64 },
6057c478bd9Sstevel@tonic-gate 	{ "nalloc",		KSTAT_DATA_UINT64 },
606*7c5f01c1SRobert Mustacchi 	{ "nactive",		KSTAT_DATA_UINT64 },
6077c478bd9Sstevel@tonic-gate 	{ "priority",		KSTAT_DATA_UINT64 },
6087c478bd9Sstevel@tonic-gate 	{ "threads",		KSTAT_DATA_UINT64 },
6097c478bd9Sstevel@tonic-gate };
6107c478bd9Sstevel@tonic-gate 
6117c478bd9Sstevel@tonic-gate struct taskq_d_kstat {
6127c478bd9Sstevel@tonic-gate 	kstat_named_t	tqd_pri;
6137c478bd9Sstevel@tonic-gate 	kstat_named_t	tqd_btasks;
6147c478bd9Sstevel@tonic-gate 	kstat_named_t	tqd_bexecuted;
6157c478bd9Sstevel@tonic-gate 	kstat_named_t	tqd_bmaxtasks;
6167c478bd9Sstevel@tonic-gate 	kstat_named_t	tqd_bnalloc;
6177c478bd9Sstevel@tonic-gate 	kstat_named_t	tqd_bnactive;
6187c478bd9Sstevel@tonic-gate 	kstat_named_t	tqd_btotaltime;
6197c478bd9Sstevel@tonic-gate 	kstat_named_t	tqd_hits;
6207c478bd9Sstevel@tonic-gate 	kstat_named_t	tqd_misses;
6217c478bd9Sstevel@tonic-gate 	kstat_named_t	tqd_overflows;
6227c478bd9Sstevel@tonic-gate 	kstat_named_t	tqd_tcreates;
6237c478bd9Sstevel@tonic-gate 	kstat_named_t	tqd_tdeaths;
6247c478bd9Sstevel@tonic-gate 	kstat_named_t	tqd_maxthreads;
6257c478bd9Sstevel@tonic-gate 	kstat_named_t	tqd_nomem;
6267c478bd9Sstevel@tonic-gate 	kstat_named_t	tqd_disptcreates;
6277c478bd9Sstevel@tonic-gate 	kstat_named_t	tqd_totaltime;
6287c478bd9Sstevel@tonic-gate 	kstat_named_t	tqd_nalloc;
6297c478bd9Sstevel@tonic-gate 	kstat_named_t	tqd_nfree;
6307c478bd9Sstevel@tonic-gate } taskq_d_kstat = {
6317c478bd9Sstevel@tonic-gate 	{ "priority",		KSTAT_DATA_UINT64 },
6327c478bd9Sstevel@tonic-gate 	{ "btasks",		KSTAT_DATA_UINT64 },
6337c478bd9Sstevel@tonic-gate 	{ "bexecuted",		KSTAT_DATA_UINT64 },
6347c478bd9Sstevel@tonic-gate 	{ "bmaxtasks",		KSTAT_DATA_UINT64 },
6357c478bd9Sstevel@tonic-gate 	{ "bnalloc",		KSTAT_DATA_UINT64 },
6367c478bd9Sstevel@tonic-gate 	{ "bnactive",		KSTAT_DATA_UINT64 },
6377c478bd9Sstevel@tonic-gate 	{ "btotaltime",		KSTAT_DATA_UINT64 },
6387c478bd9Sstevel@tonic-gate 	{ "hits",		KSTAT_DATA_UINT64 },
6397c478bd9Sstevel@tonic-gate 	{ "misses",		KSTAT_DATA_UINT64 },
6407c478bd9Sstevel@tonic-gate 	{ "overflows",		KSTAT_DATA_UINT64 },
6417c478bd9Sstevel@tonic-gate 	{ "tcreates",		KSTAT_DATA_UINT64 },
6427c478bd9Sstevel@tonic-gate 	{ "tdeaths",		KSTAT_DATA_UINT64 },
6437c478bd9Sstevel@tonic-gate 	{ "maxthreads",		KSTAT_DATA_UINT64 },
6447c478bd9Sstevel@tonic-gate 	{ "nomem",		KSTAT_DATA_UINT64 },
6457c478bd9Sstevel@tonic-gate 	{ "disptcreates",	KSTAT_DATA_UINT64 },
6467c478bd9Sstevel@tonic-gate 	{ "totaltime",		KSTAT_DATA_UINT64 },
6477c478bd9Sstevel@tonic-gate 	{ "nalloc",		KSTAT_DATA_UINT64 },
6487c478bd9Sstevel@tonic-gate 	{ "nfree",		KSTAT_DATA_UINT64 },
6497c478bd9Sstevel@tonic-gate };
6507c478bd9Sstevel@tonic-gate 
6517c478bd9Sstevel@tonic-gate static kmutex_t taskq_kstat_lock;
6527c478bd9Sstevel@tonic-gate static kmutex_t taskq_d_kstat_lock;
6537c478bd9Sstevel@tonic-gate static int taskq_kstat_update(kstat_t *, int);
6547c478bd9Sstevel@tonic-gate static int taskq_d_kstat_update(kstat_t *, int);
6557c478bd9Sstevel@tonic-gate 
6562e0c549eSJonathan Adams /*
65735a5a358SJonathan Adams  * List of all TASKQ_THREADS_CPU_PCT taskqs.
6582e0c549eSJonathan Adams  */
65935a5a358SJonathan Adams static list_t taskq_cpupct_list;	/* protected by cpu_lock */
6607c478bd9Sstevel@tonic-gate 
6617c478bd9Sstevel@tonic-gate /*
6627c478bd9Sstevel@tonic-gate  * Collect per-bucket statistic when TASKQ_STATISTIC is defined.
6637c478bd9Sstevel@tonic-gate  */
6647c478bd9Sstevel@tonic-gate #define	TASKQ_STATISTIC 1
6657c478bd9Sstevel@tonic-gate 
6667c478bd9Sstevel@tonic-gate #if TASKQ_STATISTIC
6677c478bd9Sstevel@tonic-gate #define	TQ_STAT(b, x)	b->tqbucket_stat.x++
6687c478bd9Sstevel@tonic-gate #else
6697c478bd9Sstevel@tonic-gate #define	TQ_STAT(b, x)
6707c478bd9Sstevel@tonic-gate #endif
6717c478bd9Sstevel@tonic-gate 
6727c478bd9Sstevel@tonic-gate /*
6737c478bd9Sstevel@tonic-gate  * Random fault injection.
6747c478bd9Sstevel@tonic-gate  */
6757c478bd9Sstevel@tonic-gate uint_t taskq_random;
6767c478bd9Sstevel@tonic-gate uint_t taskq_dmtbf = UINT_MAX;    /* mean time between injected failures */
6777c478bd9Sstevel@tonic-gate uint_t taskq_smtbf = UINT_MAX;    /* mean time between injected failures */
6787c478bd9Sstevel@tonic-gate 
6797c478bd9Sstevel@tonic-gate /*
6807c478bd9Sstevel@tonic-gate  * TQ_NOSLEEP dispatches on dynamic task queues are always allowed to fail.
6817c478bd9Sstevel@tonic-gate  *
6827c478bd9Sstevel@tonic-gate  * TQ_NOSLEEP dispatches on static task queues can't arbitrarily fail because
6837c478bd9Sstevel@tonic-gate  * they could prepopulate the cache and make sure that they do not use more
6847c478bd9Sstevel@tonic-gate  * then minalloc entries.  So, fault injection in this case insures that
6857c478bd9Sstevel@tonic-gate  * either TASKQ_PREPOPULATE is not set or there are more entries allocated
6867c478bd9Sstevel@tonic-gate  * than is specified by minalloc.  TQ_NOALLOC dispatches are always allowed
6877c478bd9Sstevel@tonic-gate  * to fail, but for simplicity we treat them identically to TQ_NOSLEEP
6887c478bd9Sstevel@tonic-gate  * dispatches.
6897c478bd9Sstevel@tonic-gate  */
6907c478bd9Sstevel@tonic-gate #ifdef DEBUG
6917c478bd9Sstevel@tonic-gate #define	TASKQ_D_RANDOM_DISPATCH_FAILURE(tq, flag)		\
6927c478bd9Sstevel@tonic-gate 	taskq_random = (taskq_random * 2416 + 374441) % 1771875;\
6937c478bd9Sstevel@tonic-gate 	if ((flag & TQ_NOSLEEP) &&				\
6947c478bd9Sstevel@tonic-gate 	    taskq_random < 1771875 / taskq_dmtbf) {		\
6957c478bd9Sstevel@tonic-gate 		return (NULL);					\
6967c478bd9Sstevel@tonic-gate 	}
6977c478bd9Sstevel@tonic-gate 
6987c478bd9Sstevel@tonic-gate #define	TASKQ_S_RANDOM_DISPATCH_FAILURE(tq, flag)		\
6997c478bd9Sstevel@tonic-gate 	taskq_random = (taskq_random * 2416 + 374441) % 1771875;\
7007c478bd9Sstevel@tonic-gate 	if ((flag & (TQ_NOSLEEP | TQ_NOALLOC)) &&		\
7017c478bd9Sstevel@tonic-gate 	    (!(tq->tq_flags & TASKQ_PREPOPULATE) ||		\
7027c478bd9Sstevel@tonic-gate 	    (tq->tq_nalloc > tq->tq_minalloc)) &&		\
7037c478bd9Sstevel@tonic-gate 	    (taskq_random < (1771875 / taskq_smtbf))) {		\
7047c478bd9Sstevel@tonic-gate 		mutex_exit(&tq->tq_lock);			\
7057c478bd9Sstevel@tonic-gate 		return (NULL);					\
7067c478bd9Sstevel@tonic-gate 	}
7077c478bd9Sstevel@tonic-gate #else
7087c478bd9Sstevel@tonic-gate #define	TASKQ_S_RANDOM_DISPATCH_FAILURE(tq, flag)
7097c478bd9Sstevel@tonic-gate #define	TASKQ_D_RANDOM_DISPATCH_FAILURE(tq, flag)
7107c478bd9Sstevel@tonic-gate #endif
7117c478bd9Sstevel@tonic-gate 
7127c478bd9Sstevel@tonic-gate #define	IS_EMPTY(l) (((l).tqent_prev == (l).tqent_next) &&	\
7137c478bd9Sstevel@tonic-gate 	((l).tqent_prev == &(l)))
7147c478bd9Sstevel@tonic-gate 
7157c478bd9Sstevel@tonic-gate /*
7167c478bd9Sstevel@tonic-gate  * Append `tqe' in the end of the doubly-linked list denoted by l.
7177c478bd9Sstevel@tonic-gate  */
7187c478bd9Sstevel@tonic-gate #define	TQ_APPEND(l, tqe) {					\
7197c478bd9Sstevel@tonic-gate 	tqe->tqent_next = &l;					\
7207c478bd9Sstevel@tonic-gate 	tqe->tqent_prev = l.tqent_prev;				\
7217c478bd9Sstevel@tonic-gate 	tqe->tqent_next->tqent_prev = tqe;			\
7227c478bd9Sstevel@tonic-gate 	tqe->tqent_prev->tqent_next = tqe;			\
7237c478bd9Sstevel@tonic-gate }
72435a5a358SJonathan Adams /*
72535a5a358SJonathan Adams  * Prepend 'tqe' to the beginning of l
72635a5a358SJonathan Adams  */
72735a5a358SJonathan Adams #define	TQ_PREPEND(l, tqe) {					\
72835a5a358SJonathan Adams 	tqe->tqent_next = l.tqent_next;				\
72935a5a358SJonathan Adams 	tqe->tqent_prev = &l;					\
73035a5a358SJonathan Adams 	tqe->tqent_next->tqent_prev = tqe;			\
73135a5a358SJonathan Adams 	tqe->tqent_prev->tqent_next = tqe;			\
73235a5a358SJonathan Adams }
7337c478bd9Sstevel@tonic-gate 
7347c478bd9Sstevel@tonic-gate /*
7357c478bd9Sstevel@tonic-gate  * Schedule a task specified by func and arg into the task queue entry tqe.
7367c478bd9Sstevel@tonic-gate  */
73735a5a358SJonathan Adams #define	TQ_DO_ENQUEUE(tq, tqe, func, arg, front) {			\
73835a5a358SJonathan Adams 	ASSERT(MUTEX_HELD(&tq->tq_lock));				\
73935a5a358SJonathan Adams 	_NOTE(CONSTCOND)						\
74035a5a358SJonathan Adams 	if (front) {							\
74135a5a358SJonathan Adams 		TQ_PREPEND(tq->tq_task, tqe);				\
74235a5a358SJonathan Adams 	} else {							\
74335a5a358SJonathan Adams 		TQ_APPEND(tq->tq_task, tqe);				\
74435a5a358SJonathan Adams 	}								\
74535a5a358SJonathan Adams 	tqe->tqent_func = (func);					\
74635a5a358SJonathan Adams 	tqe->tqent_arg = (arg);						\
74735a5a358SJonathan Adams 	tq->tq_tasks++;							\
74835a5a358SJonathan Adams 	if (tq->tq_tasks - tq->tq_executed > tq->tq_maxtasks)		\
7497c478bd9Sstevel@tonic-gate 		tq->tq_maxtasks = tq->tq_tasks - tq->tq_executed;	\
75035a5a358SJonathan Adams 	cv_signal(&tq->tq_dispatch_cv);					\
7517c478bd9Sstevel@tonic-gate 	DTRACE_PROBE2(taskq__enqueue, taskq_t *, tq, taskq_ent_t *, tqe); \
7527c478bd9Sstevel@tonic-gate }
7537c478bd9Sstevel@tonic-gate 
75435a5a358SJonathan Adams #define	TQ_ENQUEUE(tq, tqe, func, arg)					\
75535a5a358SJonathan Adams 	TQ_DO_ENQUEUE(tq, tqe, func, arg, 0)
75635a5a358SJonathan Adams 
75735a5a358SJonathan Adams #define	TQ_ENQUEUE_FRONT(tq, tqe, func, arg)				\
75835a5a358SJonathan Adams 	TQ_DO_ENQUEUE(tq, tqe, func, arg, 1)
75935a5a358SJonathan Adams 
7607c478bd9Sstevel@tonic-gate /*
7617c478bd9Sstevel@tonic-gate  * Do-nothing task which may be used to prepopulate thread caches.
7627c478bd9Sstevel@tonic-gate  */
7637c478bd9Sstevel@tonic-gate /*ARGSUSED*/
7647c478bd9Sstevel@tonic-gate void
7657c478bd9Sstevel@tonic-gate nulltask(void *unused)
7667c478bd9Sstevel@tonic-gate {
7677c478bd9Sstevel@tonic-gate }
7687c478bd9Sstevel@tonic-gate 
7697c478bd9Sstevel@tonic-gate /*ARGSUSED*/
7707c478bd9Sstevel@tonic-gate static int
7717c478bd9Sstevel@tonic-gate taskq_constructor(void *buf, void *cdrarg, int kmflags)
7727c478bd9Sstevel@tonic-gate {
7737c478bd9Sstevel@tonic-gate 	taskq_t *tq = buf;
7747c478bd9Sstevel@tonic-gate 
7757c478bd9Sstevel@tonic-gate 	bzero(tq, sizeof (taskq_t));
7767c478bd9Sstevel@tonic-gate 
7777c478bd9Sstevel@tonic-gate 	mutex_init(&tq->tq_lock, NULL, MUTEX_DEFAULT, NULL);
7787c478bd9Sstevel@tonic-gate 	rw_init(&tq->tq_threadlock, NULL, RW_DEFAULT, NULL);
7797c478bd9Sstevel@tonic-gate 	cv_init(&tq->tq_dispatch_cv, NULL, CV_DEFAULT, NULL);
7802e0c549eSJonathan Adams 	cv_init(&tq->tq_exit_cv, NULL, CV_DEFAULT, NULL);
7817c478bd9Sstevel@tonic-gate 	cv_init(&tq->tq_wait_cv, NULL, CV_DEFAULT, NULL);
78264109744SChris Horne 	cv_init(&tq->tq_maxalloc_cv, NULL, CV_DEFAULT, NULL);
7837c478bd9Sstevel@tonic-gate 
7847c478bd9Sstevel@tonic-gate 	tq->tq_task.tqent_next = &tq->tq_task;
7857c478bd9Sstevel@tonic-gate 	tq->tq_task.tqent_prev = &tq->tq_task;
7867c478bd9Sstevel@tonic-gate 
7877c478bd9Sstevel@tonic-gate 	return (0);
7887c478bd9Sstevel@tonic-gate }
7897c478bd9Sstevel@tonic-gate 
7907c478bd9Sstevel@tonic-gate /*ARGSUSED*/
7917c478bd9Sstevel@tonic-gate static void
7927c478bd9Sstevel@tonic-gate taskq_destructor(void *buf, void *cdrarg)
7937c478bd9Sstevel@tonic-gate {
7947c478bd9Sstevel@tonic-gate 	taskq_t *tq = buf;
7957c478bd9Sstevel@tonic-gate 
7962e0c549eSJonathan Adams 	ASSERT(tq->tq_nthreads == 0);
7972e0c549eSJonathan Adams 	ASSERT(tq->tq_buckets == NULL);
7982e0c549eSJonathan Adams 	ASSERT(tq->tq_tcreates == 0);
7992e0c549eSJonathan Adams 	ASSERT(tq->tq_tdeaths == 0);
8002e0c549eSJonathan Adams 
8017c478bd9Sstevel@tonic-gate 	mutex_destroy(&tq->tq_lock);
8027c478bd9Sstevel@tonic-gate 	rw_destroy(&tq->tq_threadlock);
8037c478bd9Sstevel@tonic-gate 	cv_destroy(&tq->tq_dispatch_cv);
8042e0c549eSJonathan Adams 	cv_destroy(&tq->tq_exit_cv);
8057c478bd9Sstevel@tonic-gate 	cv_destroy(&tq->tq_wait_cv);
80664109744SChris Horne 	cv_destroy(&tq->tq_maxalloc_cv);
8077c478bd9Sstevel@tonic-gate }
8087c478bd9Sstevel@tonic-gate 
8097c478bd9Sstevel@tonic-gate /*ARGSUSED*/
8107c478bd9Sstevel@tonic-gate static int
8117c478bd9Sstevel@tonic-gate taskq_ent_constructor(void *buf, void *cdrarg, int kmflags)
8127c478bd9Sstevel@tonic-gate {
8137c478bd9Sstevel@tonic-gate 	taskq_ent_t *tqe = buf;
8147c478bd9Sstevel@tonic-gate 
8157c478bd9Sstevel@tonic-gate 	tqe->tqent_thread = NULL;
8167c478bd9Sstevel@tonic-gate 	cv_init(&tqe->tqent_cv, NULL, CV_DEFAULT, NULL);
8177c478bd9Sstevel@tonic-gate 
8187c478bd9Sstevel@tonic-gate 	return (0);
8197c478bd9Sstevel@tonic-gate }
8207c478bd9Sstevel@tonic-gate 
8217c478bd9Sstevel@tonic-gate /*ARGSUSED*/
8227c478bd9Sstevel@tonic-gate static void
8237c478bd9Sstevel@tonic-gate taskq_ent_destructor(void *buf, void *cdrarg)
8247c478bd9Sstevel@tonic-gate {
8257c478bd9Sstevel@tonic-gate 	taskq_ent_t *tqe = buf;
8267c478bd9Sstevel@tonic-gate 
8277c478bd9Sstevel@tonic-gate 	ASSERT(tqe->tqent_thread == NULL);
8287c478bd9Sstevel@tonic-gate 	cv_destroy(&tqe->tqent_cv);
8297c478bd9Sstevel@tonic-gate }
8307c478bd9Sstevel@tonic-gate 
8317c478bd9Sstevel@tonic-gate void
8327c478bd9Sstevel@tonic-gate taskq_init(void)
8337c478bd9Sstevel@tonic-gate {
8347c478bd9Sstevel@tonic-gate 	taskq_ent_cache = kmem_cache_create("taskq_ent_cache",
8357c478bd9Sstevel@tonic-gate 	    sizeof (taskq_ent_t), 0, taskq_ent_constructor,
8367c478bd9Sstevel@tonic-gate 	    taskq_ent_destructor, NULL, NULL, NULL, 0);
8377c478bd9Sstevel@tonic-gate 	taskq_cache = kmem_cache_create("taskq_cache", sizeof (taskq_t),
8387c478bd9Sstevel@tonic-gate 	    0, taskq_constructor, taskq_destructor, NULL, NULL, NULL, 0);
8397c478bd9Sstevel@tonic-gate 	taskq_id_arena = vmem_create("taskq_id_arena",
8407c478bd9Sstevel@tonic-gate 	    (void *)1, INT32_MAX, 1, NULL, NULL, NULL, 0,
8417c478bd9Sstevel@tonic-gate 	    VM_SLEEP | VMC_IDENTIFIER);
8422e0c549eSJonathan Adams 
84335a5a358SJonathan Adams 	list_create(&taskq_cpupct_list, sizeof (taskq_t),
84435a5a358SJonathan Adams 	    offsetof(taskq_t, tq_cpupct_link));
84535a5a358SJonathan Adams }
84635a5a358SJonathan Adams 
84735a5a358SJonathan Adams static void
84835a5a358SJonathan Adams taskq_update_nthreads(taskq_t *tq, uint_t ncpus)
84935a5a358SJonathan Adams {
85035a5a358SJonathan Adams 	uint_t newtarget = TASKQ_THREADS_PCT(ncpus, tq->tq_threads_ncpus_pct);
85135a5a358SJonathan Adams 
85235a5a358SJonathan Adams 	ASSERT(MUTEX_HELD(&cpu_lock));
85335a5a358SJonathan Adams 	ASSERT(MUTEX_HELD(&tq->tq_lock));
85435a5a358SJonathan Adams 
85535a5a358SJonathan Adams 	/* We must be going from non-zero to non-zero; no exiting. */
85635a5a358SJonathan Adams 	ASSERT3U(tq->tq_nthreads_target, !=, 0);
85735a5a358SJonathan Adams 	ASSERT3U(newtarget, !=, 0);
85835a5a358SJonathan Adams 
85935a5a358SJonathan Adams 	ASSERT3U(newtarget, <=, tq->tq_nthreads_max);
86035a5a358SJonathan Adams 	if (newtarget != tq->tq_nthreads_target) {
86135a5a358SJonathan Adams 		tq->tq_flags |= TASKQ_CHANGING;
86235a5a358SJonathan Adams 		tq->tq_nthreads_target = newtarget;
86335a5a358SJonathan Adams 		cv_broadcast(&tq->tq_dispatch_cv);
86435a5a358SJonathan Adams 		cv_broadcast(&tq->tq_exit_cv);
86535a5a358SJonathan Adams 	}
86635a5a358SJonathan Adams }
86735a5a358SJonathan Adams 
86835a5a358SJonathan Adams /* called during task queue creation */
86935a5a358SJonathan Adams static void
87035a5a358SJonathan Adams taskq_cpupct_install(taskq_t *tq, cpupart_t *cpup)
87135a5a358SJonathan Adams {
87235a5a358SJonathan Adams 	ASSERT(tq->tq_flags & TASKQ_THREADS_CPU_PCT);
87335a5a358SJonathan Adams 
87435a5a358SJonathan Adams 	mutex_enter(&cpu_lock);
87535a5a358SJonathan Adams 	mutex_enter(&tq->tq_lock);
87635a5a358SJonathan Adams 	tq->tq_cpupart = cpup->cp_id;
87735a5a358SJonathan Adams 	taskq_update_nthreads(tq, cpup->cp_ncpus);
87835a5a358SJonathan Adams 	mutex_exit(&tq->tq_lock);
87935a5a358SJonathan Adams 
88035a5a358SJonathan Adams 	list_insert_tail(&taskq_cpupct_list, tq);
88135a5a358SJonathan Adams 	mutex_exit(&cpu_lock);
88235a5a358SJonathan Adams }
88335a5a358SJonathan Adams 
88435a5a358SJonathan Adams static void
88535a5a358SJonathan Adams taskq_cpupct_remove(taskq_t *tq)
88635a5a358SJonathan Adams {
88735a5a358SJonathan Adams 	ASSERT(tq->tq_flags & TASKQ_THREADS_CPU_PCT);
88835a5a358SJonathan Adams 
88935a5a358SJonathan Adams 	mutex_enter(&cpu_lock);
89035a5a358SJonathan Adams 	list_remove(&taskq_cpupct_list, tq);
89135a5a358SJonathan Adams 	mutex_exit(&cpu_lock);
8922e0c549eSJonathan Adams }
8932e0c549eSJonathan Adams 
8942e0c549eSJonathan Adams /*ARGSUSED*/
8952e0c549eSJonathan Adams static int
8962e0c549eSJonathan Adams taskq_cpu_setup(cpu_setup_t what, int id, void *arg)
8972e0c549eSJonathan Adams {
89835a5a358SJonathan Adams 	taskq_t *tq;
89935a5a358SJonathan Adams 	cpupart_t *cp = cpu[id]->cpu_part;
90035a5a358SJonathan Adams 	uint_t ncpus = cp->cp_ncpus;
90135a5a358SJonathan Adams 
90235a5a358SJonathan Adams 	ASSERT(MUTEX_HELD(&cpu_lock));
90335a5a358SJonathan Adams 	ASSERT(ncpus > 0);
90435a5a358SJonathan Adams 
90535a5a358SJonathan Adams 	switch (what) {
90635a5a358SJonathan Adams 	case CPU_OFF:
90735a5a358SJonathan Adams 	case CPU_CPUPART_OUT:
90835a5a358SJonathan Adams 		/* offlines are called *before* the cpu is offlined. */
90935a5a358SJonathan Adams 		if (ncpus > 1)
91035a5a358SJonathan Adams 			ncpus--;
91135a5a358SJonathan Adams 		break;
91235a5a358SJonathan Adams 
91335a5a358SJonathan Adams 	case CPU_ON:
91435a5a358SJonathan Adams 	case CPU_CPUPART_IN:
91535a5a358SJonathan Adams 		break;
91635a5a358SJonathan Adams 
91735a5a358SJonathan Adams 	default:
91835a5a358SJonathan Adams 		return (0);		/* doesn't affect cpu count */
9192e0c549eSJonathan Adams 	}
9202e0c549eSJonathan Adams 
92135a5a358SJonathan Adams 	for (tq = list_head(&taskq_cpupct_list); tq != NULL;
92235a5a358SJonathan Adams 	    tq = list_next(&taskq_cpupct_list, tq)) {
9232e0c549eSJonathan Adams 
9242e0c549eSJonathan Adams 		mutex_enter(&tq->tq_lock);
92535a5a358SJonathan Adams 		/*
92635a5a358SJonathan Adams 		 * If the taskq is part of the cpuset which is changing,
92735a5a358SJonathan Adams 		 * update its nthreads_target.
92835a5a358SJonathan Adams 		 */
92935a5a358SJonathan Adams 		if (tq->tq_cpupart == cp->cp_id) {
93035a5a358SJonathan Adams 			taskq_update_nthreads(tq, ncpus);
9312e0c549eSJonathan Adams 		}
9322e0c549eSJonathan Adams 		mutex_exit(&tq->tq_lock);
9332e0c549eSJonathan Adams 	}
9342e0c549eSJonathan Adams 	return (0);
9352e0c549eSJonathan Adams }
9362e0c549eSJonathan Adams 
9372e0c549eSJonathan Adams void
9382e0c549eSJonathan Adams taskq_mp_init(void)
9392e0c549eSJonathan Adams {
9402e0c549eSJonathan Adams 	mutex_enter(&cpu_lock);
9412e0c549eSJonathan Adams 	register_cpu_setup_func(taskq_cpu_setup, NULL);
94235a5a358SJonathan Adams 	/*
94335a5a358SJonathan Adams 	 * Make sure we're up to date.  At this point in boot, there is only
94435a5a358SJonathan Adams 	 * one processor set, so we only have to update the current CPU.
94535a5a358SJonathan Adams 	 */
94635a5a358SJonathan Adams 	(void) taskq_cpu_setup(CPU_ON, CPU->cpu_id, NULL);
9472e0c549eSJonathan Adams 	mutex_exit(&cpu_lock);
9487c478bd9Sstevel@tonic-gate }
9497c478bd9Sstevel@tonic-gate 
9507c478bd9Sstevel@tonic-gate /*
9517c478bd9Sstevel@tonic-gate  * Create global system dynamic task queue.
9527c478bd9Sstevel@tonic-gate  */
9537c478bd9Sstevel@tonic-gate void
9547c478bd9Sstevel@tonic-gate system_taskq_init(void)
9557c478bd9Sstevel@tonic-gate {
9567c478bd9Sstevel@tonic-gate 	system_taskq = taskq_create_common("system_taskq", 0,
95735a5a358SJonathan Adams 	    system_taskq_size * max_ncpus, minclsyspri, 4, 512, &p0, 0,
9587c478bd9Sstevel@tonic-gate 	    TASKQ_DYNAMIC | TASKQ_PREPOPULATE);
9597c478bd9Sstevel@tonic-gate }
9607c478bd9Sstevel@tonic-gate 
9617c478bd9Sstevel@tonic-gate /*
9627c478bd9Sstevel@tonic-gate  * taskq_ent_alloc()
9637c478bd9Sstevel@tonic-gate  *
9647c478bd9Sstevel@tonic-gate  * Allocates a new taskq_ent_t structure either from the free list or from the
9657c478bd9Sstevel@tonic-gate  * cache. Returns NULL if it can't be allocated.
9667c478bd9Sstevel@tonic-gate  *
9677c478bd9Sstevel@tonic-gate  * Assumes: tq->tq_lock is held.
9687c478bd9Sstevel@tonic-gate  */
9697c478bd9Sstevel@tonic-gate static taskq_ent_t *
9707c478bd9Sstevel@tonic-gate taskq_ent_alloc(taskq_t *tq, int flags)
9717c478bd9Sstevel@tonic-gate {
9727c478bd9Sstevel@tonic-gate 	int kmflags = (flags & TQ_NOSLEEP) ? KM_NOSLEEP : KM_SLEEP;
9737c478bd9Sstevel@tonic-gate 	taskq_ent_t *tqe;
97464109744SChris Horne 	clock_t wait_time;
97564109744SChris Horne 	clock_t	wait_rv;
9767c478bd9Sstevel@tonic-gate 
9777c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&tq->tq_lock));
9787c478bd9Sstevel@tonic-gate 
9797c478bd9Sstevel@tonic-gate 	/*
9807c478bd9Sstevel@tonic-gate 	 * TQ_NOALLOC allocations are allowed to use the freelist, even if
9817c478bd9Sstevel@tonic-gate 	 * we are below tq_minalloc.
9827c478bd9Sstevel@tonic-gate 	 */
98364109744SChris Horne again:	if ((tqe = tq->tq_freelist) != NULL &&
9847c478bd9Sstevel@tonic-gate 	    ((flags & TQ_NOALLOC) || tq->tq_nalloc >= tq->tq_minalloc)) {
9857c478bd9Sstevel@tonic-gate 		tq->tq_freelist = tqe->tqent_next;
9867c478bd9Sstevel@tonic-gate 	} else {
9877c478bd9Sstevel@tonic-gate 		if (flags & TQ_NOALLOC)
9887c478bd9Sstevel@tonic-gate 			return (NULL);
9897c478bd9Sstevel@tonic-gate 
9907c478bd9Sstevel@tonic-gate 		if (tq->tq_nalloc >= tq->tq_maxalloc) {
99164109744SChris Horne 			if (kmflags & KM_NOSLEEP)
9927c478bd9Sstevel@tonic-gate 				return (NULL);
99364109744SChris Horne 
9947c478bd9Sstevel@tonic-gate 			/*
9957c478bd9Sstevel@tonic-gate 			 * We don't want to exceed tq_maxalloc, but we can't
9967c478bd9Sstevel@tonic-gate 			 * wait for other tasks to complete (and thus free up
9977c478bd9Sstevel@tonic-gate 			 * task structures) without risking deadlock with
9987c478bd9Sstevel@tonic-gate 			 * the caller.  So, we just delay for one second
99964109744SChris Horne 			 * to throttle the allocation rate. If we have tasks
100064109744SChris Horne 			 * complete before one second timeout expires then
100164109744SChris Horne 			 * taskq_ent_free will signal us and we will
100264109744SChris Horne 			 * immediately retry the allocation (reap free).
10037c478bd9Sstevel@tonic-gate 			 */
100464109744SChris Horne 			wait_time = ddi_get_lbolt() + hz;
100564109744SChris Horne 			while (tq->tq_freelist == NULL) {
100664109744SChris Horne 				tq->tq_maxalloc_wait++;
100764109744SChris Horne 				wait_rv = cv_timedwait(&tq->tq_maxalloc_cv,
100864109744SChris Horne 				    &tq->tq_lock, wait_time);
100964109744SChris Horne 				tq->tq_maxalloc_wait--;
101064109744SChris Horne 				if (wait_rv == -1)
101164109744SChris Horne 					break;
101264109744SChris Horne 			}
101364109744SChris Horne 			if (tq->tq_freelist)
101464109744SChris Horne 				goto again;		/* reap freelist */
101564109744SChris Horne 
10167c478bd9Sstevel@tonic-gate 		}
101764109744SChris Horne 		mutex_exit(&tq->tq_lock);
101864109744SChris Horne 
10197c478bd9Sstevel@tonic-gate 		tqe = kmem_cache_alloc(taskq_ent_cache, kmflags);
102064109744SChris Horne 
10217c478bd9Sstevel@tonic-gate 		mutex_enter(&tq->tq_lock);
10227c478bd9Sstevel@tonic-gate 		if (tqe != NULL)
10237c478bd9Sstevel@tonic-gate 			tq->tq_nalloc++;
10247c478bd9Sstevel@tonic-gate 	}
10257c478bd9Sstevel@tonic-gate 	return (tqe);
10267c478bd9Sstevel@tonic-gate }
10277c478bd9Sstevel@tonic-gate 
10287c478bd9Sstevel@tonic-gate /*
10297c478bd9Sstevel@tonic-gate  * taskq_ent_free()
10307c478bd9Sstevel@tonic-gate  *
10317c478bd9Sstevel@tonic-gate  * Free taskq_ent_t structure by either putting it on the free list or freeing
10327c478bd9Sstevel@tonic-gate  * it to the cache.
10337c478bd9Sstevel@tonic-gate  *
10347c478bd9Sstevel@tonic-gate  * Assumes: tq->tq_lock is held.
10357c478bd9Sstevel@tonic-gate  */
10367c478bd9Sstevel@tonic-gate static void
10377c478bd9Sstevel@tonic-gate taskq_ent_free(taskq_t *tq, taskq_ent_t *tqe)
10387c478bd9Sstevel@tonic-gate {
10397c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&tq->tq_lock));
10407c478bd9Sstevel@tonic-gate 
10417c478bd9Sstevel@tonic-gate 	if (tq->tq_nalloc <= tq->tq_minalloc) {
10427c478bd9Sstevel@tonic-gate 		tqe->tqent_next = tq->tq_freelist;
10437c478bd9Sstevel@tonic-gate 		tq->tq_freelist = tqe;
10447c478bd9Sstevel@tonic-gate 	} else {
10457c478bd9Sstevel@tonic-gate 		tq->tq_nalloc--;
10467c478bd9Sstevel@tonic-gate 		mutex_exit(&tq->tq_lock);
10477c478bd9Sstevel@tonic-gate 		kmem_cache_free(taskq_ent_cache, tqe);
10487c478bd9Sstevel@tonic-gate 		mutex_enter(&tq->tq_lock);
10497c478bd9Sstevel@tonic-gate 	}
105064109744SChris Horne 
105164109744SChris Horne 	if (tq->tq_maxalloc_wait)
105264109744SChris Horne 		cv_signal(&tq->tq_maxalloc_cv);
105364109744SChris Horne }
105464109744SChris Horne 
105564109744SChris Horne /*
105664109744SChris Horne  * taskq_ent_exists()
105764109744SChris Horne  *
105864109744SChris Horne  * Return 1 if taskq already has entry for calling 'func(arg)'.
105964109744SChris Horne  *
106064109744SChris Horne  * Assumes: tq->tq_lock is held.
106164109744SChris Horne  */
106264109744SChris Horne static int
106364109744SChris Horne taskq_ent_exists(taskq_t *tq, task_func_t func, void *arg)
106464109744SChris Horne {
106564109744SChris Horne 	taskq_ent_t	*tqe;
106664109744SChris Horne 
106764109744SChris Horne 	ASSERT(MUTEX_HELD(&tq->tq_lock));
106864109744SChris Horne 
106964109744SChris Horne 	for (tqe = tq->tq_task.tqent_next; tqe != &tq->tq_task;
107064109744SChris Horne 	    tqe = tqe->tqent_next)
107164109744SChris Horne 		if ((tqe->tqent_func == func) && (tqe->tqent_arg == arg))
107264109744SChris Horne 			return (1);
107364109744SChris Horne 	return (0);
10747c478bd9Sstevel@tonic-gate }
10757c478bd9Sstevel@tonic-gate 
10767c478bd9Sstevel@tonic-gate /*
10777c478bd9Sstevel@tonic-gate  * Dispatch a task "func(arg)" to a free entry of bucket b.
10787c478bd9Sstevel@tonic-gate  *
10797c478bd9Sstevel@tonic-gate  * Assumes: no bucket locks is held.
10807c478bd9Sstevel@tonic-gate  *
10817c478bd9Sstevel@tonic-gate  * Returns: a pointer to an entry if dispatch was successful.
10827c478bd9Sstevel@tonic-gate  *	    NULL if there are no free entries or if the bucket is suspended.
10837c478bd9Sstevel@tonic-gate  */
10847c478bd9Sstevel@tonic-gate static taskq_ent_t *
10857c478bd9Sstevel@tonic-gate taskq_bucket_dispatch(taskq_bucket_t *b, task_func_t func, void *arg)
10867c478bd9Sstevel@tonic-gate {
10877c478bd9Sstevel@tonic-gate 	taskq_ent_t *tqe;
10887c478bd9Sstevel@tonic-gate 
10897c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_NOT_HELD(&b->tqbucket_lock));
10907c478bd9Sstevel@tonic-gate 	ASSERT(func != NULL);
10917c478bd9Sstevel@tonic-gate 
10927c478bd9Sstevel@tonic-gate 	mutex_enter(&b->tqbucket_lock);
10937c478bd9Sstevel@tonic-gate 
10947c478bd9Sstevel@tonic-gate 	ASSERT(b->tqbucket_nfree != 0 || IS_EMPTY(b->tqbucket_freelist));
10957c478bd9Sstevel@tonic-gate 	ASSERT(b->tqbucket_nfree == 0 || !IS_EMPTY(b->tqbucket_freelist));
10967c478bd9Sstevel@tonic-gate 
10977c478bd9Sstevel@tonic-gate 	/*
10987c478bd9Sstevel@tonic-gate 	 * Get en entry from the freelist if there is one.
10997c478bd9Sstevel@tonic-gate 	 * Schedule task into the entry.
11007c478bd9Sstevel@tonic-gate 	 */
11017c478bd9Sstevel@tonic-gate 	if ((b->tqbucket_nfree != 0) &&
11027c478bd9Sstevel@tonic-gate 	    !(b->tqbucket_flags & TQBUCKET_SUSPEND)) {
11037c478bd9Sstevel@tonic-gate 		tqe = b->tqbucket_freelist.tqent_prev;
11047c478bd9Sstevel@tonic-gate 
11057c478bd9Sstevel@tonic-gate 		ASSERT(tqe != &b->tqbucket_freelist);
11067c478bd9Sstevel@tonic-gate 		ASSERT(tqe->tqent_thread != NULL);
11077c478bd9Sstevel@tonic-gate 
11087c478bd9Sstevel@tonic-gate 		tqe->tqent_prev->tqent_next = tqe->tqent_next;
11097c478bd9Sstevel@tonic-gate 		tqe->tqent_next->tqent_prev = tqe->tqent_prev;
11107c478bd9Sstevel@tonic-gate 		b->tqbucket_nalloc++;
11117c478bd9Sstevel@tonic-gate 		b->tqbucket_nfree--;
11127c478bd9Sstevel@tonic-gate 		tqe->tqent_func = func;
11137c478bd9Sstevel@tonic-gate 		tqe->tqent_arg = arg;
11147c478bd9Sstevel@tonic-gate 		TQ_STAT(b, tqs_hits);
11157c478bd9Sstevel@tonic-gate 		cv_signal(&tqe->tqent_cv);
11167c478bd9Sstevel@tonic-gate 		DTRACE_PROBE2(taskq__d__enqueue, taskq_bucket_t *, b,
11177c478bd9Sstevel@tonic-gate 		    taskq_ent_t *, tqe);
11187c478bd9Sstevel@tonic-gate 	} else {
11197c478bd9Sstevel@tonic-gate 		tqe = NULL;
11207c478bd9Sstevel@tonic-gate 		TQ_STAT(b, tqs_misses);
11217c478bd9Sstevel@tonic-gate 	}
11227c478bd9Sstevel@tonic-gate 	mutex_exit(&b->tqbucket_lock);
11237c478bd9Sstevel@tonic-gate 	return (tqe);
11247c478bd9Sstevel@tonic-gate }
11257c478bd9Sstevel@tonic-gate 
11267c478bd9Sstevel@tonic-gate /*
11277c478bd9Sstevel@tonic-gate  * Dispatch a task.
11287c478bd9Sstevel@tonic-gate  *
11297c478bd9Sstevel@tonic-gate  * Assumes: func != NULL
11307c478bd9Sstevel@tonic-gate  *
11317c478bd9Sstevel@tonic-gate  * Returns: NULL if dispatch failed.
11327c478bd9Sstevel@tonic-gate  *	    non-NULL if task dispatched successfully.
11337c478bd9Sstevel@tonic-gate  *	    Actual return value is the pointer to taskq entry that was used to
11347c478bd9Sstevel@tonic-gate  *	    dispatch a task. This is useful for debugging.
11357c478bd9Sstevel@tonic-gate  */
11367c478bd9Sstevel@tonic-gate taskqid_t
11377c478bd9Sstevel@tonic-gate taskq_dispatch(taskq_t *tq, task_func_t func, void *arg, uint_t flags)
11387c478bd9Sstevel@tonic-gate {
11397c478bd9Sstevel@tonic-gate 	taskq_bucket_t *bucket = NULL;	/* Which bucket needs extension */
11407c478bd9Sstevel@tonic-gate 	taskq_ent_t *tqe = NULL;
11417c478bd9Sstevel@tonic-gate 	taskq_ent_t *tqe1;
11427c478bd9Sstevel@tonic-gate 	uint_t bsize;
11437c478bd9Sstevel@tonic-gate 
11447c478bd9Sstevel@tonic-gate 	ASSERT(tq != NULL);
11457c478bd9Sstevel@tonic-gate 	ASSERT(func != NULL);
11467c478bd9Sstevel@tonic-gate 
11477c478bd9Sstevel@tonic-gate 	if (!(tq->tq_flags & TASKQ_DYNAMIC)) {
11487c478bd9Sstevel@tonic-gate 		/*
11497c478bd9Sstevel@tonic-gate 		 * TQ_NOQUEUE flag can't be used with non-dynamic task queues.
11507c478bd9Sstevel@tonic-gate 		 */
11515aeb9474SGarrett D'Amore 		ASSERT(!(flags & TQ_NOQUEUE));
11527c478bd9Sstevel@tonic-gate 		/*
11537c478bd9Sstevel@tonic-gate 		 * Enqueue the task to the underlying queue.
11547c478bd9Sstevel@tonic-gate 		 */
11557c478bd9Sstevel@tonic-gate 		mutex_enter(&tq->tq_lock);
11567c478bd9Sstevel@tonic-gate 
11577c478bd9Sstevel@tonic-gate 		TASKQ_S_RANDOM_DISPATCH_FAILURE(tq, flags);
11587c478bd9Sstevel@tonic-gate 
11597c478bd9Sstevel@tonic-gate 		if ((tqe = taskq_ent_alloc(tq, flags)) == NULL) {
11607c478bd9Sstevel@tonic-gate 			mutex_exit(&tq->tq_lock);
11617c478bd9Sstevel@tonic-gate 			return (NULL);
11627c478bd9Sstevel@tonic-gate 		}
11635aeb9474SGarrett D'Amore 		/* Make sure we start without any flags */
11645aeb9474SGarrett D'Amore 		tqe->tqent_un.tqent_flags = 0;
11655aeb9474SGarrett D'Amore 
116635a5a358SJonathan Adams 		if (flags & TQ_FRONT) {
116735a5a358SJonathan Adams 			TQ_ENQUEUE_FRONT(tq, tqe, func, arg);
116835a5a358SJonathan Adams 		} else {
116935a5a358SJonathan Adams 			TQ_ENQUEUE(tq, tqe, func, arg);
117035a5a358SJonathan Adams 		}
11717c478bd9Sstevel@tonic-gate 		mutex_exit(&tq->tq_lock);
11727c478bd9Sstevel@tonic-gate 		return ((taskqid_t)tqe);
11737c478bd9Sstevel@tonic-gate 	}
11747c478bd9Sstevel@tonic-gate 
11757c478bd9Sstevel@tonic-gate 	/*
11767c478bd9Sstevel@tonic-gate 	 * Dynamic taskq dispatching.
11777c478bd9Sstevel@tonic-gate 	 */
117835a5a358SJonathan Adams 	ASSERT(!(flags & (TQ_NOALLOC | TQ_FRONT)));
11797c478bd9Sstevel@tonic-gate 	TASKQ_D_RANDOM_DISPATCH_FAILURE(tq, flags);
11807c478bd9Sstevel@tonic-gate 
11817c478bd9Sstevel@tonic-gate 	bsize = tq->tq_nbuckets;
11827c478bd9Sstevel@tonic-gate 
11837c478bd9Sstevel@tonic-gate 	if (bsize == 1) {
11847c478bd9Sstevel@tonic-gate 		/*
11857c478bd9Sstevel@tonic-gate 		 * In a single-CPU case there is only one bucket, so get
11867c478bd9Sstevel@tonic-gate 		 * entry directly from there.
11877c478bd9Sstevel@tonic-gate 		 */
11887c478bd9Sstevel@tonic-gate 		if ((tqe = taskq_bucket_dispatch(tq->tq_buckets, func, arg))
11897c478bd9Sstevel@tonic-gate 		    != NULL)
11907c478bd9Sstevel@tonic-gate 			return ((taskqid_t)tqe);	/* Fastpath */
11917c478bd9Sstevel@tonic-gate 		bucket = tq->tq_buckets;
11927c478bd9Sstevel@tonic-gate 	} else {
11937c478bd9Sstevel@tonic-gate 		int loopcount;
11947c478bd9Sstevel@tonic-gate 		taskq_bucket_t *b;
11957c478bd9Sstevel@tonic-gate 		uintptr_t h = ((uintptr_t)CPU + (uintptr_t)arg) >> 3;
11967c478bd9Sstevel@tonic-gate 
11977c478bd9Sstevel@tonic-gate 		h = TQ_HASH(h);
11987c478bd9Sstevel@tonic-gate 
11997c478bd9Sstevel@tonic-gate 		/*
12007c478bd9Sstevel@tonic-gate 		 * The 'bucket' points to the original bucket that we hit. If we
12017c478bd9Sstevel@tonic-gate 		 * can't allocate from it, we search other buckets, but only
12027c478bd9Sstevel@tonic-gate 		 * extend this one.
12037c478bd9Sstevel@tonic-gate 		 */
12047c478bd9Sstevel@tonic-gate 		b = &tq->tq_buckets[h & (bsize - 1)];
12057c478bd9Sstevel@tonic-gate 		ASSERT(b->tqbucket_taskq == tq);	/* Sanity check */
12067c478bd9Sstevel@tonic-gate 
12077c478bd9Sstevel@tonic-gate 		/*
12087c478bd9Sstevel@tonic-gate 		 * Do a quick check before grabbing the lock. If the bucket does
12097c478bd9Sstevel@tonic-gate 		 * not have free entries now, chances are very small that it
12107c478bd9Sstevel@tonic-gate 		 * will after we take the lock, so we just skip it.
12117c478bd9Sstevel@tonic-gate 		 */
12127c478bd9Sstevel@tonic-gate 		if (b->tqbucket_nfree != 0) {
12137c478bd9Sstevel@tonic-gate 			if ((tqe = taskq_bucket_dispatch(b, func, arg)) != NULL)
12147c478bd9Sstevel@tonic-gate 				return ((taskqid_t)tqe);	/* Fastpath */
12157c478bd9Sstevel@tonic-gate 		} else {
12167c478bd9Sstevel@tonic-gate 			TQ_STAT(b, tqs_misses);
12177c478bd9Sstevel@tonic-gate 		}
12187c478bd9Sstevel@tonic-gate 
12197c478bd9Sstevel@tonic-gate 		bucket = b;
12207c478bd9Sstevel@tonic-gate 		loopcount = MIN(taskq_search_depth, bsize);
12217c478bd9Sstevel@tonic-gate 		/*
12227c478bd9Sstevel@tonic-gate 		 * If bucket dispatch failed, search loopcount number of buckets
12237c478bd9Sstevel@tonic-gate 		 * before we give up and fail.
12247c478bd9Sstevel@tonic-gate 		 */
12257c478bd9Sstevel@tonic-gate 		do {
12267c478bd9Sstevel@tonic-gate 			b = &tq->tq_buckets[++h & (bsize - 1)];
12277c478bd9Sstevel@tonic-gate 			ASSERT(b->tqbucket_taskq == tq);  /* Sanity check */
12287c478bd9Sstevel@tonic-gate 			loopcount--;
12297c478bd9Sstevel@tonic-gate 
12307c478bd9Sstevel@tonic-gate 			if (b->tqbucket_nfree != 0) {
12317c478bd9Sstevel@tonic-gate 				tqe = taskq_bucket_dispatch(b, func, arg);
12327c478bd9Sstevel@tonic-gate 			} else {
12337c478bd9Sstevel@tonic-gate 				TQ_STAT(b, tqs_misses);
12347c478bd9Sstevel@tonic-gate 			}
12357c478bd9Sstevel@tonic-gate 		} while ((tqe == NULL) && (loopcount > 0));
12367c478bd9Sstevel@tonic-gate 	}
12377c478bd9Sstevel@tonic-gate 
12387c478bd9Sstevel@tonic-gate 	/*
12397c478bd9Sstevel@tonic-gate 	 * At this point we either scheduled a task and (tqe != NULL) or failed
12407c478bd9Sstevel@tonic-gate 	 * (tqe == NULL). Try to recover from fails.
12417c478bd9Sstevel@tonic-gate 	 */
12427c478bd9Sstevel@tonic-gate 
12437c478bd9Sstevel@tonic-gate 	/*
12447c478bd9Sstevel@tonic-gate 	 * For KM_SLEEP dispatches, try to extend the bucket and retry dispatch.
12457c478bd9Sstevel@tonic-gate 	 */
12467c478bd9Sstevel@tonic-gate 	if ((tqe == NULL) && !(flags & TQ_NOSLEEP)) {
12477c478bd9Sstevel@tonic-gate 		/*
12487c478bd9Sstevel@tonic-gate 		 * taskq_bucket_extend() may fail to do anything, but this is
12497c478bd9Sstevel@tonic-gate 		 * fine - we deal with it later. If the bucket was successfully
12507c478bd9Sstevel@tonic-gate 		 * extended, there is a good chance that taskq_bucket_dispatch()
12517c478bd9Sstevel@tonic-gate 		 * will get this new entry, unless someone is racing with us and
12527c478bd9Sstevel@tonic-gate 		 * stealing the new entry from under our nose.
12537c478bd9Sstevel@tonic-gate 		 * taskq_bucket_extend() may sleep.
12547c478bd9Sstevel@tonic-gate 		 */
12557c478bd9Sstevel@tonic-gate 		taskq_bucket_extend(bucket);
12567c478bd9Sstevel@tonic-gate 		TQ_STAT(bucket, tqs_disptcreates);
12577c478bd9Sstevel@tonic-gate 		if ((tqe = taskq_bucket_dispatch(bucket, func, arg)) != NULL)
12587c478bd9Sstevel@tonic-gate 			return ((taskqid_t)tqe);
12597c478bd9Sstevel@tonic-gate 	}
12607c478bd9Sstevel@tonic-gate 
12617c478bd9Sstevel@tonic-gate 	ASSERT(bucket != NULL);
126264109744SChris Horne 
12637c478bd9Sstevel@tonic-gate 	/*
126464109744SChris Horne 	 * Since there are not enough free entries in the bucket, add a
126564109744SChris Horne 	 * taskq entry to extend it in the background using backing queue
126664109744SChris Horne 	 * (unless we already have a taskq entry to perform that extension).
12677c478bd9Sstevel@tonic-gate 	 */
12687c478bd9Sstevel@tonic-gate 	mutex_enter(&tq->tq_lock);
126964109744SChris Horne 	if (!taskq_ent_exists(tq, taskq_bucket_extend, bucket)) {
127064109744SChris Horne 		if ((tqe1 = taskq_ent_alloc(tq, TQ_NOSLEEP)) != NULL) {
127164109744SChris Horne 			TQ_ENQUEUE_FRONT(tq, tqe1, taskq_bucket_extend, bucket);
127264109744SChris Horne 		} else {
127364109744SChris Horne 			TQ_STAT(bucket, tqs_nomem);
127464109744SChris Horne 		}
12757c478bd9Sstevel@tonic-gate 	}
12767c478bd9Sstevel@tonic-gate 
12777c478bd9Sstevel@tonic-gate 	/*
12787c478bd9Sstevel@tonic-gate 	 * Dispatch failed and we can't find an entry to schedule a task.
12797c478bd9Sstevel@tonic-gate 	 * Revert to the backing queue unless TQ_NOQUEUE was asked.
12807c478bd9Sstevel@tonic-gate 	 */
12817c478bd9Sstevel@tonic-gate 	if ((tqe == NULL) && !(flags & TQ_NOQUEUE)) {
12827c478bd9Sstevel@tonic-gate 		if ((tqe = taskq_ent_alloc(tq, flags)) != NULL) {
12837c478bd9Sstevel@tonic-gate 			TQ_ENQUEUE(tq, tqe, func, arg);
12847c478bd9Sstevel@tonic-gate 		} else {
12857c478bd9Sstevel@tonic-gate 			TQ_STAT(bucket, tqs_nomem);
12867c478bd9Sstevel@tonic-gate 		}
12877c478bd9Sstevel@tonic-gate 	}
12887c478bd9Sstevel@tonic-gate 	mutex_exit(&tq->tq_lock);
12897c478bd9Sstevel@tonic-gate 
12907c478bd9Sstevel@tonic-gate 	return ((taskqid_t)tqe);
12917c478bd9Sstevel@tonic-gate }
12927c478bd9Sstevel@tonic-gate 
12935aeb9474SGarrett D'Amore void
12945aeb9474SGarrett D'Amore taskq_dispatch_ent(taskq_t *tq, task_func_t func, void *arg, uint_t flags,
12955aeb9474SGarrett D'Amore     taskq_ent_t *tqe)
12965aeb9474SGarrett D'Amore {
12975aeb9474SGarrett D'Amore 	ASSERT(func != NULL);
12985aeb9474SGarrett D'Amore 	ASSERT(!(tq->tq_flags & TASKQ_DYNAMIC));
12995aeb9474SGarrett D'Amore 
13005aeb9474SGarrett D'Amore 	/*
13015aeb9474SGarrett D'Amore 	 * Mark it as a prealloc'd task.  This is important
13025aeb9474SGarrett D'Amore 	 * to ensure that we don't free it later.
13035aeb9474SGarrett D'Amore 	 */
13045aeb9474SGarrett D'Amore 	tqe->tqent_un.tqent_flags |= TQENT_FLAG_PREALLOC;
13055aeb9474SGarrett D'Amore 	/*
13065aeb9474SGarrett D'Amore 	 * Enqueue the task to the underlying queue.
13075aeb9474SGarrett D'Amore 	 */
13085aeb9474SGarrett D'Amore 	mutex_enter(&tq->tq_lock);
13095aeb9474SGarrett D'Amore 
13105aeb9474SGarrett D'Amore 	if (flags & TQ_FRONT) {
13115aeb9474SGarrett D'Amore 		TQ_ENQUEUE_FRONT(tq, tqe, func, arg);
13125aeb9474SGarrett D'Amore 	} else {
13135aeb9474SGarrett D'Amore 		TQ_ENQUEUE(tq, tqe, func, arg);
13145aeb9474SGarrett D'Amore 	}
13155aeb9474SGarrett D'Amore 	mutex_exit(&tq->tq_lock);
13165aeb9474SGarrett D'Amore }
13175aeb9474SGarrett D'Amore 
13187c478bd9Sstevel@tonic-gate /*
13197c478bd9Sstevel@tonic-gate  * Wait for all pending tasks to complete.
13207c478bd9Sstevel@tonic-gate  * Calling taskq_wait from a task will cause deadlock.
13217c478bd9Sstevel@tonic-gate  */
13227c478bd9Sstevel@tonic-gate void
13237c478bd9Sstevel@tonic-gate taskq_wait(taskq_t *tq)
13247c478bd9Sstevel@tonic-gate {
13257c478bd9Sstevel@tonic-gate 	ASSERT(tq != curthread->t_taskq);
13267c478bd9Sstevel@tonic-gate 
13277c478bd9Sstevel@tonic-gate 	mutex_enter(&tq->tq_lock);
13287c478bd9Sstevel@tonic-gate 	while (tq->tq_task.tqent_next != &tq->tq_task || tq->tq_active != 0)
13297c478bd9Sstevel@tonic-gate 		cv_wait(&tq->tq_wait_cv, &tq->tq_lock);
13307c478bd9Sstevel@tonic-gate 	mutex_exit(&tq->tq_lock);
13317c478bd9Sstevel@tonic-gate 
13327c478bd9Sstevel@tonic-gate 	if (tq->tq_flags & TASKQ_DYNAMIC) {
13337c478bd9Sstevel@tonic-gate 		taskq_bucket_t *b = tq->tq_buckets;
13347c478bd9Sstevel@tonic-gate 		int bid = 0;
13357c478bd9Sstevel@tonic-gate 		for (; (b != NULL) && (bid < tq->tq_nbuckets); b++, bid++) {
13367c478bd9Sstevel@tonic-gate 			mutex_enter(&b->tqbucket_lock);
13377c478bd9Sstevel@tonic-gate 			while (b->tqbucket_nalloc > 0)
13387c478bd9Sstevel@tonic-gate 				cv_wait(&b->tqbucket_cv, &b->tqbucket_lock);
13397c478bd9Sstevel@tonic-gate 			mutex_exit(&b->tqbucket_lock);
13407c478bd9Sstevel@tonic-gate 		}
13417c478bd9Sstevel@tonic-gate 	}
13427c478bd9Sstevel@tonic-gate }
13437c478bd9Sstevel@tonic-gate 
13447c478bd9Sstevel@tonic-gate /*
13457c478bd9Sstevel@tonic-gate  * Suspend execution of tasks.
13467c478bd9Sstevel@tonic-gate  *
13477c478bd9Sstevel@tonic-gate  * Tasks in the queue part will be suspended immediately upon return from this
13487c478bd9Sstevel@tonic-gate  * function. Pending tasks in the dynamic part will continue to execute, but all
13497c478bd9Sstevel@tonic-gate  * new tasks will  be suspended.
13507c478bd9Sstevel@tonic-gate  */
13517c478bd9Sstevel@tonic-gate void
13527c478bd9Sstevel@tonic-gate taskq_suspend(taskq_t *tq)
13537c478bd9Sstevel@tonic-gate {
13547c478bd9Sstevel@tonic-gate 	rw_enter(&tq->tq_threadlock, RW_WRITER);
13557c478bd9Sstevel@tonic-gate 
13567c478bd9Sstevel@tonic-gate 	if (tq->tq_flags & TASKQ_DYNAMIC) {
13577c478bd9Sstevel@tonic-gate 		taskq_bucket_t *b = tq->tq_buckets;
13587c478bd9Sstevel@tonic-gate 		int bid = 0;
13597c478bd9Sstevel@tonic-gate 		for (; (b != NULL) && (bid < tq->tq_nbuckets); b++, bid++) {
13607c478bd9Sstevel@tonic-gate 			mutex_enter(&b->tqbucket_lock);
13617c478bd9Sstevel@tonic-gate 			b->tqbucket_flags |= TQBUCKET_SUSPEND;
13627c478bd9Sstevel@tonic-gate 			mutex_exit(&b->tqbucket_lock);
13637c478bd9Sstevel@tonic-gate 		}
13647c478bd9Sstevel@tonic-gate 	}
13657c478bd9Sstevel@tonic-gate 	/*
13667c478bd9Sstevel@tonic-gate 	 * Mark task queue as being suspended. Needed for taskq_suspended().
13677c478bd9Sstevel@tonic-gate 	 */
13687c478bd9Sstevel@tonic-gate 	mutex_enter(&tq->tq_lock);
13697c478bd9Sstevel@tonic-gate 	ASSERT(!(tq->tq_flags & TASKQ_SUSPENDED));
13707c478bd9Sstevel@tonic-gate 	tq->tq_flags |= TASKQ_SUSPENDED;
13717c478bd9Sstevel@tonic-gate 	mutex_exit(&tq->tq_lock);
13727c478bd9Sstevel@tonic-gate }
13737c478bd9Sstevel@tonic-gate 
13747c478bd9Sstevel@tonic-gate /*
13757c478bd9Sstevel@tonic-gate  * returns: 1 if tq is suspended, 0 otherwise.
13767c478bd9Sstevel@tonic-gate  */
13777c478bd9Sstevel@tonic-gate int
13787c478bd9Sstevel@tonic-gate taskq_suspended(taskq_t *tq)
13797c478bd9Sstevel@tonic-gate {
13807c478bd9Sstevel@tonic-gate 	return ((tq->tq_flags & TASKQ_SUSPENDED) != 0);
13817c478bd9Sstevel@tonic-gate }
13827c478bd9Sstevel@tonic-gate 
13837c478bd9Sstevel@tonic-gate /*
13847c478bd9Sstevel@tonic-gate  * Resume taskq execution.
13857c478bd9Sstevel@tonic-gate  */
13867c478bd9Sstevel@tonic-gate void
13877c478bd9Sstevel@tonic-gate taskq_resume(taskq_t *tq)
13887c478bd9Sstevel@tonic-gate {
13897c478bd9Sstevel@tonic-gate 	ASSERT(RW_WRITE_HELD(&tq->tq_threadlock));
13907c478bd9Sstevel@tonic-gate 
13917c478bd9Sstevel@tonic-gate 	if (tq->tq_flags & TASKQ_DYNAMIC) {
13927c478bd9Sstevel@tonic-gate 		taskq_bucket_t *b = tq->tq_buckets;
13937c478bd9Sstevel@tonic-gate 		int bid = 0;
13947c478bd9Sstevel@tonic-gate 		for (; (b != NULL) && (bid < tq->tq_nbuckets); b++, bid++) {
13957c478bd9Sstevel@tonic-gate 			mutex_enter(&b->tqbucket_lock);
13967c478bd9Sstevel@tonic-gate 			b->tqbucket_flags &= ~TQBUCKET_SUSPEND;
13977c478bd9Sstevel@tonic-gate 			mutex_exit(&b->tqbucket_lock);
13987c478bd9Sstevel@tonic-gate 		}
13997c478bd9Sstevel@tonic-gate 	}
14007c478bd9Sstevel@tonic-gate 	mutex_enter(&tq->tq_lock);
14017c478bd9Sstevel@tonic-gate 	ASSERT(tq->tq_flags & TASKQ_SUSPENDED);
14027c478bd9Sstevel@tonic-gate 	tq->tq_flags &= ~TASKQ_SUSPENDED;
14037c478bd9Sstevel@tonic-gate 	mutex_exit(&tq->tq_lock);
14047c478bd9Sstevel@tonic-gate 
14057c478bd9Sstevel@tonic-gate 	rw_exit(&tq->tq_threadlock);
14067c478bd9Sstevel@tonic-gate }
14077c478bd9Sstevel@tonic-gate 
14087c478bd9Sstevel@tonic-gate int
14097c478bd9Sstevel@tonic-gate taskq_member(taskq_t *tq, kthread_t *thread)
14107c478bd9Sstevel@tonic-gate {
14117c478bd9Sstevel@tonic-gate 	return (thread->t_taskq == tq);
14127c478bd9Sstevel@tonic-gate }
14137c478bd9Sstevel@tonic-gate 
141435a5a358SJonathan Adams /*
141535a5a358SJonathan Adams  * Creates a thread in the taskq.  We only allow one outstanding create at
141635a5a358SJonathan Adams  * a time.  We drop and reacquire the tq_lock in order to avoid blocking other
141735a5a358SJonathan Adams  * taskq activity while thread_create() or lwp_kernel_create() run.
141835a5a358SJonathan Adams  *
141935a5a358SJonathan Adams  * The first time we're called, we do some additional setup, and do not
142035a5a358SJonathan Adams  * return until there are enough threads to start servicing requests.
142135a5a358SJonathan Adams  */
14222e0c549eSJonathan Adams static void
14232e0c549eSJonathan Adams taskq_thread_create(taskq_t *tq)
14242e0c549eSJonathan Adams {
142535a5a358SJonathan Adams 	kthread_t	*t;
142635a5a358SJonathan Adams 	const boolean_t	first = (tq->tq_nthreads == 0);
14272e0c549eSJonathan Adams 
14282e0c549eSJonathan Adams 	ASSERT(MUTEX_HELD(&tq->tq_lock));
142935a5a358SJonathan Adams 	ASSERT(tq->tq_flags & TASKQ_CHANGING);
143035a5a358SJonathan Adams 	ASSERT(tq->tq_nthreads < tq->tq_nthreads_target);
14312e0c549eSJonathan Adams 	ASSERT(!(tq->tq_flags & TASKQ_THREAD_CREATED));
14322e0c549eSJonathan Adams 
143335a5a358SJonathan Adams 
14342e0c549eSJonathan Adams 	tq->tq_flags |= TASKQ_THREAD_CREATED;
14352e0c549eSJonathan Adams 	tq->tq_active++;
143635a5a358SJonathan Adams 	mutex_exit(&tq->tq_lock);
143735a5a358SJonathan Adams 
1438e5488233SGordon Ross 	/*
1439e5488233SGordon Ross 	 * With TASKQ_DUTY_CYCLE the new thread must have an LWP
1440e5488233SGordon Ross 	 * as explained in ../disp/sysdc.c (for the msacct data).
1441e5488233SGordon Ross 	 * Otherwise simple kthreads are preferred.
1442e5488233SGordon Ross 	 */
1443e5488233SGordon Ross 	if ((tq->tq_flags & TASKQ_DUTY_CYCLE) != 0) {
1444e5488233SGordon Ross 		/* Enforced in taskq_create_common */
1445e5488233SGordon Ross 		ASSERT3P(tq->tq_proc, !=, &p0);
144635a5a358SJonathan Adams 		t = lwp_kernel_create(tq->tq_proc, taskq_thread, tq, TS_RUN,
144735a5a358SJonathan Adams 		    tq->tq_pri);
144835a5a358SJonathan Adams 	} else {
1449e5488233SGordon Ross 		t = thread_create(NULL, 0, taskq_thread, tq, 0, tq->tq_proc,
1450e5488233SGordon Ross 		    TS_RUN, tq->tq_pri);
145135a5a358SJonathan Adams 	}
145235a5a358SJonathan Adams 
145335a5a358SJonathan Adams 	if (!first) {
145435a5a358SJonathan Adams 		mutex_enter(&tq->tq_lock);
145535a5a358SJonathan Adams 		return;
145635a5a358SJonathan Adams 	}
145735a5a358SJonathan Adams 
145835a5a358SJonathan Adams 	/*
145935a5a358SJonathan Adams 	 * We know the thread cannot go away, since tq cannot be
146035a5a358SJonathan Adams 	 * destroyed until creation has completed.  We can therefore
146135a5a358SJonathan Adams 	 * safely dereference t.
146235a5a358SJonathan Adams 	 */
146335a5a358SJonathan Adams 	if (tq->tq_flags & TASKQ_THREADS_CPU_PCT) {
146435a5a358SJonathan Adams 		taskq_cpupct_install(tq, t->t_cpupart);
146535a5a358SJonathan Adams 	}
146635a5a358SJonathan Adams 	mutex_enter(&tq->tq_lock);
146735a5a358SJonathan Adams 
146835a5a358SJonathan Adams 	/* Wait until we can service requests. */
146935a5a358SJonathan Adams 	while (tq->tq_nthreads != tq->tq_nthreads_target &&
147035a5a358SJonathan Adams 	    tq->tq_nthreads < TASKQ_CREATE_ACTIVE_THREADS) {
147135a5a358SJonathan Adams 		cv_wait(&tq->tq_wait_cv, &tq->tq_lock);
147235a5a358SJonathan Adams 	}
14732e0c549eSJonathan Adams }
14742e0c549eSJonathan Adams 
1475e0ad97e3SJonathan Adams /*
1476e0ad97e3SJonathan Adams  * Common "sleep taskq thread" function, which handles CPR stuff, as well
1477e0ad97e3SJonathan Adams  * as giving a nice common point for debuggers to find inactive threads.
1478e0ad97e3SJonathan Adams  */
1479e0ad97e3SJonathan Adams static clock_t
1480e0ad97e3SJonathan Adams taskq_thread_wait(taskq_t *tq, kmutex_t *mx, kcondvar_t *cv,
1481e0ad97e3SJonathan Adams     callb_cpr_t *cprinfo, clock_t timeout)
14822e0c549eSJonathan Adams {
1483e0ad97e3SJonathan Adams 	clock_t ret = 0;
1484e0ad97e3SJonathan Adams 
1485e0ad97e3SJonathan Adams 	if (!(tq->tq_flags & TASKQ_CPR_SAFE)) {
14862e0c549eSJonathan Adams 		CALLB_CPR_SAFE_BEGIN(cprinfo);
14872e0c549eSJonathan Adams 	}
1488e0ad97e3SJonathan Adams 	if (timeout < 0)
1489e0ad97e3SJonathan Adams 		cv_wait(cv, mx);
1490e0ad97e3SJonathan Adams 	else
1491d3d50737SRafael Vanoni 		ret = cv_reltimedwait(cv, mx, timeout, TR_CLOCK_TICK);
1492e0ad97e3SJonathan Adams 
1493e0ad97e3SJonathan Adams 	if (!(tq->tq_flags & TASKQ_CPR_SAFE)) {
1494e0ad97e3SJonathan Adams 		CALLB_CPR_SAFE_END(cprinfo, mx);
1495e0ad97e3SJonathan Adams 	}
1496e0ad97e3SJonathan Adams 
1497e0ad97e3SJonathan Adams 	return (ret);
14982e0c549eSJonathan Adams }
14992e0c549eSJonathan Adams 
15007c478bd9Sstevel@tonic-gate /*
15017c478bd9Sstevel@tonic-gate  * Worker thread for processing task queue.
15027c478bd9Sstevel@tonic-gate  */
15037c478bd9Sstevel@tonic-gate static void
15047c478bd9Sstevel@tonic-gate taskq_thread(void *arg)
15057c478bd9Sstevel@tonic-gate {
15062e0c549eSJonathan Adams 	int thread_id;
15072e0c549eSJonathan Adams 
15087c478bd9Sstevel@tonic-gate 	taskq_t *tq = arg;
15097c478bd9Sstevel@tonic-gate 	taskq_ent_t *tqe;
15107c478bd9Sstevel@tonic-gate 	callb_cpr_t cprinfo;
15117c478bd9Sstevel@tonic-gate 	hrtime_t start, end;
15125aeb9474SGarrett D'Amore 	boolean_t freeit;
15137c478bd9Sstevel@tonic-gate 
151435a5a358SJonathan Adams 	curthread->t_taskq = tq;	/* mark ourselves for taskq_member() */
151535a5a358SJonathan Adams 
151635a5a358SJonathan Adams 	if (curproc != &p0 && (tq->tq_flags & TASKQ_DUTY_CYCLE)) {
151735a5a358SJonathan Adams 		sysdc_thread_enter(curthread, tq->tq_DC,
151835a5a358SJonathan Adams 		    (tq->tq_flags & TASKQ_DC_BATCH) ? SYSDC_THREAD_BATCH : 0);
151935a5a358SJonathan Adams 	}
152035a5a358SJonathan Adams 
15217c478bd9Sstevel@tonic-gate 	if (tq->tq_flags & TASKQ_CPR_SAFE) {
15227c478bd9Sstevel@tonic-gate 		CALLB_CPR_INIT_SAFE(curthread, tq->tq_name);
15237c478bd9Sstevel@tonic-gate 	} else {
15247c478bd9Sstevel@tonic-gate 		CALLB_CPR_INIT(&cprinfo, &tq->tq_lock, callb_generic_cpr,
15257c478bd9Sstevel@tonic-gate 		    tq->tq_name);
15267c478bd9Sstevel@tonic-gate 	}
15277c478bd9Sstevel@tonic-gate 	mutex_enter(&tq->tq_lock);
15282e0c549eSJonathan Adams 	thread_id = ++tq->tq_nthreads;
15292e0c549eSJonathan Adams 	ASSERT(tq->tq_flags & TASKQ_THREAD_CREATED);
153035a5a358SJonathan Adams 	ASSERT(tq->tq_flags & TASKQ_CHANGING);
15312e0c549eSJonathan Adams 	tq->tq_flags &= ~TASKQ_THREAD_CREATED;
15322e0c549eSJonathan Adams 
15332e0c549eSJonathan Adams 	VERIFY3S(thread_id, <=, tq->tq_nthreads_max);
15342e0c549eSJonathan Adams 
15352e0c549eSJonathan Adams 	if (tq->tq_nthreads_max == 1)
15362e0c549eSJonathan Adams 		tq->tq_thread = curthread;
15372e0c549eSJonathan Adams 	else
15382e0c549eSJonathan Adams 		tq->tq_threadlist[thread_id - 1] = curthread;
15392e0c549eSJonathan Adams 
154035a5a358SJonathan Adams 	/* Allow taskq_create_common()'s taskq_thread_create() to return. */
154135a5a358SJonathan Adams 	if (tq->tq_nthreads == TASKQ_CREATE_ACTIVE_THREADS)
154235a5a358SJonathan Adams 		cv_broadcast(&tq->tq_wait_cv);
154335a5a358SJonathan Adams 
15442e0c549eSJonathan Adams 	for (;;) {
15452e0c549eSJonathan Adams 		if (tq->tq_flags & TASKQ_CHANGING) {
154635a5a358SJonathan Adams 			/* See if we're no longer needed */
15472e0c549eSJonathan Adams 			if (thread_id > tq->tq_nthreads_target) {
15482e0c549eSJonathan Adams 				/*
15492e0c549eSJonathan Adams 				 * To preserve the one-to-one mapping between
15502e0c549eSJonathan Adams 				 * thread_id and thread, we must exit from
15512e0c549eSJonathan Adams 				 * highest thread ID to least.
15522e0c549eSJonathan Adams 				 *
15532e0c549eSJonathan Adams 				 * However, if everyone is exiting, the order
15542e0c549eSJonathan Adams 				 * doesn't matter, so just exit immediately.
15552e0c549eSJonathan Adams 				 * (this is safe, since you must wait for
15562e0c549eSJonathan Adams 				 * nthreads to reach 0 after setting
15572e0c549eSJonathan Adams 				 * tq_nthreads_target to 0)
15582e0c549eSJonathan Adams 				 */
15592e0c549eSJonathan Adams 				if (thread_id == tq->tq_nthreads ||
15602e0c549eSJonathan Adams 				    tq->tq_nthreads_target == 0)
15612e0c549eSJonathan Adams 					break;
15622e0c549eSJonathan Adams 
15632e0c549eSJonathan Adams 				/* Wait for higher thread_ids to exit */
1564e0ad97e3SJonathan Adams 				(void) taskq_thread_wait(tq, &tq->tq_lock,
1565e0ad97e3SJonathan Adams 				    &tq->tq_exit_cv, &cprinfo, -1);
15662e0c549eSJonathan Adams 				continue;
15672e0c549eSJonathan Adams 			}
156835a5a358SJonathan Adams 
156935a5a358SJonathan Adams 			/*
157035a5a358SJonathan Adams 			 * If no thread is starting taskq_thread(), we can
157135a5a358SJonathan Adams 			 * do some bookkeeping.
157235a5a358SJonathan Adams 			 */
157335a5a358SJonathan Adams 			if (!(tq->tq_flags & TASKQ_THREAD_CREATED)) {
157435a5a358SJonathan Adams 				/* Check if we've reached our target */
157535a5a358SJonathan Adams 				if (tq->tq_nthreads == tq->tq_nthreads_target) {
157635a5a358SJonathan Adams 					tq->tq_flags &= ~TASKQ_CHANGING;
157735a5a358SJonathan Adams 					cv_broadcast(&tq->tq_wait_cv);
157835a5a358SJonathan Adams 				}
157935a5a358SJonathan Adams 				/* Check if we need to create a thread */
158035a5a358SJonathan Adams 				if (tq->tq_nthreads < tq->tq_nthreads_target) {
158135a5a358SJonathan Adams 					taskq_thread_create(tq);
158235a5a358SJonathan Adams 					continue; /* tq_lock was dropped */
158335a5a358SJonathan Adams 				}
158435a5a358SJonathan Adams 			}
15852e0c549eSJonathan Adams 		}
15867c478bd9Sstevel@tonic-gate 		if ((tqe = tq->tq_task.tqent_next) == &tq->tq_task) {
15877c478bd9Sstevel@tonic-gate 			if (--tq->tq_active == 0)
15887c478bd9Sstevel@tonic-gate 				cv_broadcast(&tq->tq_wait_cv);
1589e0ad97e3SJonathan Adams 			(void) taskq_thread_wait(tq, &tq->tq_lock,
1590e0ad97e3SJonathan Adams 			    &tq->tq_dispatch_cv, &cprinfo, -1);
15917c478bd9Sstevel@tonic-gate 			tq->tq_active++;
15927c478bd9Sstevel@tonic-gate 			continue;
15937c478bd9Sstevel@tonic-gate 		}
159435a5a358SJonathan Adams 
15957c478bd9Sstevel@tonic-gate 		tqe->tqent_prev->tqent_next = tqe->tqent_next;
15967c478bd9Sstevel@tonic-gate 		tqe->tqent_next->tqent_prev = tqe->tqent_prev;
15977c478bd9Sstevel@tonic-gate 		mutex_exit(&tq->tq_lock);
15987c478bd9Sstevel@tonic-gate 
15995aeb9474SGarrett D'Amore 		/*
16005aeb9474SGarrett D'Amore 		 * For prealloc'd tasks, we don't free anything.  We
16015aeb9474SGarrett D'Amore 		 * have to check this now, because once we call the
16025aeb9474SGarrett D'Amore 		 * function for a prealloc'd taskq, we can't touch the
16035aeb9474SGarrett D'Amore 		 * tqent any longer (calling the function returns the
16045aeb9474SGarrett D'Amore 		 * ownershp of the tqent back to caller of
16055aeb9474SGarrett D'Amore 		 * taskq_dispatch.)
16065aeb9474SGarrett D'Amore 		 */
16075aeb9474SGarrett D'Amore 		if ((!(tq->tq_flags & TASKQ_DYNAMIC)) &&
16085aeb9474SGarrett D'Amore 		    (tqe->tqent_un.tqent_flags & TQENT_FLAG_PREALLOC)) {
16095aeb9474SGarrett D'Amore 			/* clear pointers to assist assertion checks */
16105aeb9474SGarrett D'Amore 			tqe->tqent_next = tqe->tqent_prev = NULL;
16115aeb9474SGarrett D'Amore 			freeit = B_FALSE;
16125aeb9474SGarrett D'Amore 		} else {
16135aeb9474SGarrett D'Amore 			freeit = B_TRUE;
16145aeb9474SGarrett D'Amore 		}
16155aeb9474SGarrett D'Amore 
16167c478bd9Sstevel@tonic-gate 		rw_enter(&tq->tq_threadlock, RW_READER);
16177c478bd9Sstevel@tonic-gate 		start = gethrtime();
16187c478bd9Sstevel@tonic-gate 		DTRACE_PROBE2(taskq__exec__start, taskq_t *, tq,
16197c478bd9Sstevel@tonic-gate 		    taskq_ent_t *, tqe);
16207c478bd9Sstevel@tonic-gate 		tqe->tqent_func(tqe->tqent_arg);
16217c478bd9Sstevel@tonic-gate 		DTRACE_PROBE2(taskq__exec__end, taskq_t *, tq,
16227c478bd9Sstevel@tonic-gate 		    taskq_ent_t *, tqe);
16237c478bd9Sstevel@tonic-gate 		end = gethrtime();
16247c478bd9Sstevel@tonic-gate 		rw_exit(&tq->tq_threadlock);
16257c478bd9Sstevel@tonic-gate 
16267c478bd9Sstevel@tonic-gate 		mutex_enter(&tq->tq_lock);
16277c478bd9Sstevel@tonic-gate 		tq->tq_totaltime += end - start;
16287c478bd9Sstevel@tonic-gate 		tq->tq_executed++;
16297c478bd9Sstevel@tonic-gate 
16305aeb9474SGarrett D'Amore 		if (freeit)
16315aeb9474SGarrett D'Amore 			taskq_ent_free(tq, tqe);
16327c478bd9Sstevel@tonic-gate 	}
16332e0c549eSJonathan Adams 
16342e0c549eSJonathan Adams 	if (tq->tq_nthreads_max == 1)
16352e0c549eSJonathan Adams 		tq->tq_thread = NULL;
16362e0c549eSJonathan Adams 	else
16372e0c549eSJonathan Adams 		tq->tq_threadlist[thread_id - 1] = NULL;
16382e0c549eSJonathan Adams 
163935a5a358SJonathan Adams 	/* We're exiting, and therefore no longer active */
164035a5a358SJonathan Adams 	ASSERT(tq->tq_active > 0);
164135a5a358SJonathan Adams 	tq->tq_active--;
164235a5a358SJonathan Adams 
16432e0c549eSJonathan Adams 	ASSERT(tq->tq_nthreads > 0);
164435a5a358SJonathan Adams 	tq->tq_nthreads--;
16452e0c549eSJonathan Adams 
164635a5a358SJonathan Adams 	/* Wake up anyone waiting for us to exit */
16472e0c549eSJonathan Adams 	cv_broadcast(&tq->tq_exit_cv);
164835a5a358SJonathan Adams 	if (tq->tq_nthreads == tq->tq_nthreads_target) {
164935a5a358SJonathan Adams 		if (!(tq->tq_flags & TASKQ_THREAD_CREATED))
165035a5a358SJonathan Adams 			tq->tq_flags &= ~TASKQ_CHANGING;
16512e0c549eSJonathan Adams 
165235a5a358SJonathan Adams 		cv_broadcast(&tq->tq_wait_cv);
165335a5a358SJonathan Adams 	}
16542e0c549eSJonathan Adams 
16557c478bd9Sstevel@tonic-gate 	ASSERT(!(tq->tq_flags & TASKQ_CPR_SAFE));
165635a5a358SJonathan Adams 	CALLB_CPR_EXIT(&cprinfo);		/* drops tq->tq_lock */
165735a5a358SJonathan Adams 	if (curthread->t_lwp != NULL) {
165835a5a358SJonathan Adams 		mutex_enter(&curproc->p_lock);
165935a5a358SJonathan Adams 		lwp_exit();
166035a5a358SJonathan Adams 	} else {
166135a5a358SJonathan Adams 		thread_exit();
166235a5a358SJonathan Adams 	}
16637c478bd9Sstevel@tonic-gate }
16647c478bd9Sstevel@tonic-gate 
16657c478bd9Sstevel@tonic-gate /*
16667c478bd9Sstevel@tonic-gate  * Worker per-entry thread for dynamic dispatches.
16677c478bd9Sstevel@tonic-gate  */
16687c478bd9Sstevel@tonic-gate static void
16697c478bd9Sstevel@tonic-gate taskq_d_thread(taskq_ent_t *tqe)
16707c478bd9Sstevel@tonic-gate {
16715aeb9474SGarrett D'Amore 	taskq_bucket_t	*bucket = tqe->tqent_un.tqent_bucket;
16727c478bd9Sstevel@tonic-gate 	taskq_t		*tq = bucket->tqbucket_taskq;
16737c478bd9Sstevel@tonic-gate 	kmutex_t	*lock = &bucket->tqbucket_lock;
16747c478bd9Sstevel@tonic-gate 	kcondvar_t	*cv = &tqe->tqent_cv;
16757c478bd9Sstevel@tonic-gate 	callb_cpr_t	cprinfo;
16767c478bd9Sstevel@tonic-gate 	clock_t		w;
16777c478bd9Sstevel@tonic-gate 
16787c478bd9Sstevel@tonic-gate 	CALLB_CPR_INIT(&cprinfo, lock, callb_generic_cpr, tq->tq_name);
16797c478bd9Sstevel@tonic-gate 
16807c478bd9Sstevel@tonic-gate 	mutex_enter(lock);
16817c478bd9Sstevel@tonic-gate 
16827c478bd9Sstevel@tonic-gate 	for (;;) {
16837c478bd9Sstevel@tonic-gate 		/*
16847c478bd9Sstevel@tonic-gate 		 * If a task is scheduled (func != NULL), execute it, otherwise
16857c478bd9Sstevel@tonic-gate 		 * sleep, waiting for a job.
16867c478bd9Sstevel@tonic-gate 		 */
16877c478bd9Sstevel@tonic-gate 		if (tqe->tqent_func != NULL) {
16887c478bd9Sstevel@tonic-gate 			hrtime_t	start;
16897c478bd9Sstevel@tonic-gate 			hrtime_t	end;
16907c478bd9Sstevel@tonic-gate 
16917c478bd9Sstevel@tonic-gate 			ASSERT(bucket->tqbucket_nalloc > 0);
16927c478bd9Sstevel@tonic-gate 
16937c478bd9Sstevel@tonic-gate 			/*
16947c478bd9Sstevel@tonic-gate 			 * It is possible to free the entry right away before
16957c478bd9Sstevel@tonic-gate 			 * actually executing the task so that subsequent
16967c478bd9Sstevel@tonic-gate 			 * dispatches may immediately reuse it. But this,
16977c478bd9Sstevel@tonic-gate 			 * effectively, creates a two-length queue in the entry
16987c478bd9Sstevel@tonic-gate 			 * and may lead to a deadlock if the execution of the
16997c478bd9Sstevel@tonic-gate 			 * current task depends on the execution of the next
17007c478bd9Sstevel@tonic-gate 			 * scheduled task. So, we keep the entry busy until the
17017c478bd9Sstevel@tonic-gate 			 * task is processed.
17027c478bd9Sstevel@tonic-gate 			 */
17037c478bd9Sstevel@tonic-gate 
17047c478bd9Sstevel@tonic-gate 			mutex_exit(lock);
17057c478bd9Sstevel@tonic-gate 			start = gethrtime();
17067c478bd9Sstevel@tonic-gate 			DTRACE_PROBE3(taskq__d__exec__start, taskq_t *, tq,
17077c478bd9Sstevel@tonic-gate 			    taskq_bucket_t *, bucket, taskq_ent_t *, tqe);
17087c478bd9Sstevel@tonic-gate 			tqe->tqent_func(tqe->tqent_arg);
17097c478bd9Sstevel@tonic-gate 			DTRACE_PROBE3(taskq__d__exec__end, taskq_t *, tq,
17107c478bd9Sstevel@tonic-gate 			    taskq_bucket_t *, bucket, taskq_ent_t *, tqe);
17117c478bd9Sstevel@tonic-gate 			end = gethrtime();
17127c478bd9Sstevel@tonic-gate 			mutex_enter(lock);
17137c478bd9Sstevel@tonic-gate 			bucket->tqbucket_totaltime += end - start;
17147c478bd9Sstevel@tonic-gate 
17157c478bd9Sstevel@tonic-gate 			/*
17167c478bd9Sstevel@tonic-gate 			 * Return the entry to the bucket free list.
17177c478bd9Sstevel@tonic-gate 			 */
17187c478bd9Sstevel@tonic-gate 			tqe->tqent_func = NULL;
17197c478bd9Sstevel@tonic-gate 			TQ_APPEND(bucket->tqbucket_freelist, tqe);
17207c478bd9Sstevel@tonic-gate 			bucket->tqbucket_nalloc--;
17217c478bd9Sstevel@tonic-gate 			bucket->tqbucket_nfree++;
17227c478bd9Sstevel@tonic-gate 			ASSERT(!IS_EMPTY(bucket->tqbucket_freelist));
17237c478bd9Sstevel@tonic-gate 			/*
17247c478bd9Sstevel@tonic-gate 			 * taskq_wait() waits for nalloc to drop to zero on
17257c478bd9Sstevel@tonic-gate 			 * tqbucket_cv.
17267c478bd9Sstevel@tonic-gate 			 */
17277c478bd9Sstevel@tonic-gate 			cv_signal(&bucket->tqbucket_cv);
17287c478bd9Sstevel@tonic-gate 		}
17297c478bd9Sstevel@tonic-gate 
17307c478bd9Sstevel@tonic-gate 		/*
17317c478bd9Sstevel@tonic-gate 		 * At this point the entry must be in the bucket free list -
17327c478bd9Sstevel@tonic-gate 		 * either because it was there initially or because it just
17337c478bd9Sstevel@tonic-gate 		 * finished executing a task and put itself on the free list.
17347c478bd9Sstevel@tonic-gate 		 */
17357c478bd9Sstevel@tonic-gate 		ASSERT(bucket->tqbucket_nfree > 0);
17367c478bd9Sstevel@tonic-gate 		/*
17377c478bd9Sstevel@tonic-gate 		 * Go to sleep unless we are closing.
17387c478bd9Sstevel@tonic-gate 		 * If a thread is sleeping too long, it dies.
17397c478bd9Sstevel@tonic-gate 		 */
17407c478bd9Sstevel@tonic-gate 		if (! (bucket->tqbucket_flags & TQBUCKET_CLOSE)) {
1741e0ad97e3SJonathan Adams 			w = taskq_thread_wait(tq, lock, cv,
1742e0ad97e3SJonathan Adams 			    &cprinfo, taskq_thread_timeout * hz);
17437c478bd9Sstevel@tonic-gate 		}
17447c478bd9Sstevel@tonic-gate 
17457c478bd9Sstevel@tonic-gate 		/*
17467c478bd9Sstevel@tonic-gate 		 * At this point we may be in two different states:
17477c478bd9Sstevel@tonic-gate 		 *
17487c478bd9Sstevel@tonic-gate 		 * (1) tqent_func is set which means that a new task is
17497c478bd9Sstevel@tonic-gate 		 *	dispatched and we need to execute it.
17507c478bd9Sstevel@tonic-gate 		 *
17517c478bd9Sstevel@tonic-gate 		 * (2) Thread is sleeping for too long or we are closing. In
17527c478bd9Sstevel@tonic-gate 		 *	both cases destroy the thread and the entry.
17537c478bd9Sstevel@tonic-gate 		 */
17547c478bd9Sstevel@tonic-gate 
17557c478bd9Sstevel@tonic-gate 		/* If func is NULL we should be on the freelist. */
17567c478bd9Sstevel@tonic-gate 		ASSERT((tqe->tqent_func != NULL) ||
17577c478bd9Sstevel@tonic-gate 		    (bucket->tqbucket_nfree > 0));
17587c478bd9Sstevel@tonic-gate 		/* If func is non-NULL we should be allocated */
17597c478bd9Sstevel@tonic-gate 		ASSERT((tqe->tqent_func == NULL) ||
17607c478bd9Sstevel@tonic-gate 		    (bucket->tqbucket_nalloc > 0));
17617c478bd9Sstevel@tonic-gate 
17627c478bd9Sstevel@tonic-gate 		/* Check freelist consistency */
17637c478bd9Sstevel@tonic-gate 		ASSERT((bucket->tqbucket_nfree > 0) ||
17647c478bd9Sstevel@tonic-gate 		    IS_EMPTY(bucket->tqbucket_freelist));
17657c478bd9Sstevel@tonic-gate 		ASSERT((bucket->tqbucket_nfree == 0) ||
17667c478bd9Sstevel@tonic-gate 		    !IS_EMPTY(bucket->tqbucket_freelist));
17677c478bd9Sstevel@tonic-gate 
17687c478bd9Sstevel@tonic-gate 		if ((tqe->tqent_func == NULL) &&
17697c478bd9Sstevel@tonic-gate 		    ((w == -1) || (bucket->tqbucket_flags & TQBUCKET_CLOSE))) {
17707c478bd9Sstevel@tonic-gate 			/*
17717c478bd9Sstevel@tonic-gate 			 * This thread is sleeping for too long or we are
17727c478bd9Sstevel@tonic-gate 			 * closing - time to die.
17737c478bd9Sstevel@tonic-gate 			 * Thread creation/destruction happens rarely,
17747c478bd9Sstevel@tonic-gate 			 * so grabbing the lock is not a big performance issue.
17757c478bd9Sstevel@tonic-gate 			 * The bucket lock is dropped by CALLB_CPR_EXIT().
17767c478bd9Sstevel@tonic-gate 			 */
17777c478bd9Sstevel@tonic-gate 
17787c478bd9Sstevel@tonic-gate 			/* Remove the entry from the free list. */
17797c478bd9Sstevel@tonic-gate 			tqe->tqent_prev->tqent_next = tqe->tqent_next;
17807c478bd9Sstevel@tonic-gate 			tqe->tqent_next->tqent_prev = tqe->tqent_prev;
17817c478bd9Sstevel@tonic-gate 			ASSERT(bucket->tqbucket_nfree > 0);
17827c478bd9Sstevel@tonic-gate 			bucket->tqbucket_nfree--;
17837c478bd9Sstevel@tonic-gate 
17847c478bd9Sstevel@tonic-gate 			TQ_STAT(bucket, tqs_tdeaths);
17857c478bd9Sstevel@tonic-gate 			cv_signal(&bucket->tqbucket_cv);
17867c478bd9Sstevel@tonic-gate 			tqe->tqent_thread = NULL;
17877c478bd9Sstevel@tonic-gate 			mutex_enter(&tq->tq_lock);
17887c478bd9Sstevel@tonic-gate 			tq->tq_tdeaths++;
17897c478bd9Sstevel@tonic-gate 			mutex_exit(&tq->tq_lock);
17907c478bd9Sstevel@tonic-gate 			CALLB_CPR_EXIT(&cprinfo);
17917c478bd9Sstevel@tonic-gate 			kmem_cache_free(taskq_ent_cache, tqe);
1792c883468aSGordon Ross 			thread_exit();
17937c478bd9Sstevel@tonic-gate 		}
17947c478bd9Sstevel@tonic-gate 	}
17957c478bd9Sstevel@tonic-gate }
17967c478bd9Sstevel@tonic-gate 
17977c478bd9Sstevel@tonic-gate 
17987c478bd9Sstevel@tonic-gate /*
17997c478bd9Sstevel@tonic-gate  * Taskq creation. May sleep for memory.
18007c478bd9Sstevel@tonic-gate  * Always use automatically generated instances to avoid kstat name space
18017c478bd9Sstevel@tonic-gate  * collisions.
18027c478bd9Sstevel@tonic-gate  */
18037c478bd9Sstevel@tonic-gate 
18047c478bd9Sstevel@tonic-gate taskq_t *
18057c478bd9Sstevel@tonic-gate taskq_create(const char *name, int nthreads, pri_t pri, int minalloc,
18067c478bd9Sstevel@tonic-gate     int maxalloc, uint_t flags)
18077c478bd9Sstevel@tonic-gate {
180835a5a358SJonathan Adams 	ASSERT((flags & ~TASKQ_INTERFACE_FLAGS) == 0);
180935a5a358SJonathan Adams 
181035a5a358SJonathan Adams 	return (taskq_create_common(name, 0, nthreads, pri, minalloc,
181135a5a358SJonathan Adams 	    maxalloc, &p0, 0, flags | TASKQ_NOINSTANCE));
18127c478bd9Sstevel@tonic-gate }
18137c478bd9Sstevel@tonic-gate 
18147c478bd9Sstevel@tonic-gate /*
18157c478bd9Sstevel@tonic-gate  * Create an instance of task queue. It is legal to create task queues with the
18167c478bd9Sstevel@tonic-gate  * same name and different instances.
18177c478bd9Sstevel@tonic-gate  *
18187c478bd9Sstevel@tonic-gate  * taskq_create_instance is used by ddi_taskq_create() where it gets the
18197c478bd9Sstevel@tonic-gate  * instance from ddi_get_instance(). In some cases the instance is not
18207c478bd9Sstevel@tonic-gate  * initialized and is set to -1. This case is handled as if no instance was
18217c478bd9Sstevel@tonic-gate  * passed at all.
18227c478bd9Sstevel@tonic-gate  */
18237c478bd9Sstevel@tonic-gate taskq_t *
18247c478bd9Sstevel@tonic-gate taskq_create_instance(const char *name, int instance, int nthreads, pri_t pri,
18257c478bd9Sstevel@tonic-gate     int minalloc, int maxalloc, uint_t flags)
18267c478bd9Sstevel@tonic-gate {
182735a5a358SJonathan Adams 	ASSERT((flags & ~TASKQ_INTERFACE_FLAGS) == 0);
18287c478bd9Sstevel@tonic-gate 	ASSERT((instance >= 0) || (instance == -1));
18297c478bd9Sstevel@tonic-gate 
18307c478bd9Sstevel@tonic-gate 	if (instance < 0) {
18317c478bd9Sstevel@tonic-gate 		flags |= TASKQ_NOINSTANCE;
18327c478bd9Sstevel@tonic-gate 	}
18337c478bd9Sstevel@tonic-gate 
18347c478bd9Sstevel@tonic-gate 	return (taskq_create_common(name, instance, nthreads,
183535a5a358SJonathan Adams 	    pri, minalloc, maxalloc, &p0, 0, flags));
18367c478bd9Sstevel@tonic-gate }
18377c478bd9Sstevel@tonic-gate 
183835a5a358SJonathan Adams taskq_t *
183935a5a358SJonathan Adams taskq_create_proc(const char *name, int nthreads, pri_t pri, int minalloc,
184035a5a358SJonathan Adams     int maxalloc, proc_t *proc, uint_t flags)
184135a5a358SJonathan Adams {
184235a5a358SJonathan Adams 	ASSERT((flags & ~TASKQ_INTERFACE_FLAGS) == 0);
184335a5a358SJonathan Adams 	ASSERT(proc->p_flag & SSYS);
184435a5a358SJonathan Adams 
184535a5a358SJonathan Adams 	return (taskq_create_common(name, 0, nthreads, pri, minalloc,
184635a5a358SJonathan Adams 	    maxalloc, proc, 0, flags | TASKQ_NOINSTANCE));
184735a5a358SJonathan Adams }
184835a5a358SJonathan Adams 
184935a5a358SJonathan Adams taskq_t *
185035a5a358SJonathan Adams taskq_create_sysdc(const char *name, int nthreads, int minalloc,
185135a5a358SJonathan Adams     int maxalloc, proc_t *proc, uint_t dc, uint_t flags)
185235a5a358SJonathan Adams {
185335a5a358SJonathan Adams 	ASSERT((flags & ~TASKQ_INTERFACE_FLAGS) == 0);
185435a5a358SJonathan Adams 	ASSERT(proc->p_flag & SSYS);
185535a5a358SJonathan Adams 
185635a5a358SJonathan Adams 	return (taskq_create_common(name, 0, nthreads, minclsyspri, minalloc,
185735a5a358SJonathan Adams 	    maxalloc, proc, dc, flags | TASKQ_NOINSTANCE | TASKQ_DUTY_CYCLE));
185835a5a358SJonathan Adams }
185935a5a358SJonathan Adams 
18607c478bd9Sstevel@tonic-gate static taskq_t *
18617c478bd9Sstevel@tonic-gate taskq_create_common(const char *name, int instance, int nthreads, pri_t pri,
186235a5a358SJonathan Adams     int minalloc, int maxalloc, proc_t *proc, uint_t dc, uint_t flags)
18637c478bd9Sstevel@tonic-gate {
18647c478bd9Sstevel@tonic-gate 	taskq_t *tq = kmem_cache_alloc(taskq_cache, KM_SLEEP);
18657c478bd9Sstevel@tonic-gate 	uint_t ncpus = ((boot_max_ncpus == -1) ? max_ncpus : boot_max_ncpus);
18667c478bd9Sstevel@tonic-gate 	uint_t bsize;	/* # of buckets - always power of 2 */
18672e0c549eSJonathan Adams 	int max_nthreads;
18687c478bd9Sstevel@tonic-gate 
18697c478bd9Sstevel@tonic-gate 	/*
187035a5a358SJonathan Adams 	 * TASKQ_DYNAMIC, TASKQ_CPR_SAFE and TASKQ_THREADS_CPU_PCT are all
187135a5a358SJonathan Adams 	 * mutually incompatible.
18727c478bd9Sstevel@tonic-gate 	 */
187335a5a358SJonathan Adams 	IMPLY((flags & TASKQ_DYNAMIC), !(flags & TASKQ_CPR_SAFE));
187435a5a358SJonathan Adams 	IMPLY((flags & TASKQ_DYNAMIC), !(flags & TASKQ_THREADS_CPU_PCT));
187535a5a358SJonathan Adams 	IMPLY((flags & TASKQ_CPR_SAFE), !(flags & TASKQ_THREADS_CPU_PCT));
187635a5a358SJonathan Adams 
1877e5488233SGordon Ross 	/* Cannot have DYNAMIC with DUTY_CYCLE */
1878e5488233SGordon Ross 	IMPLY((flags & TASKQ_DYNAMIC), !(flags & TASKQ_DUTY_CYCLE));
1879e5488233SGordon Ross 
1880e5488233SGordon Ross 	/* Cannot have DUTY_CYCLE with a p0 kernel process */
188135a5a358SJonathan Adams 	IMPLY((flags & TASKQ_DUTY_CYCLE), proc != &p0);
18827c478bd9Sstevel@tonic-gate 
188335a5a358SJonathan Adams 	/* Cannot have DC_BATCH without DUTY_CYCLE */
188435a5a358SJonathan Adams 	ASSERT((flags & (TASKQ_DUTY_CYCLE|TASKQ_DC_BATCH)) != TASKQ_DC_BATCH);
188535a5a358SJonathan Adams 
188635a5a358SJonathan Adams 	ASSERT(proc != NULL);
18877c478bd9Sstevel@tonic-gate 
18887c478bd9Sstevel@tonic-gate 	bsize = 1 << (highbit(ncpus) - 1);
18897c478bd9Sstevel@tonic-gate 	ASSERT(bsize >= 1);
18907c478bd9Sstevel@tonic-gate 	bsize = MIN(bsize, taskq_maxbuckets);
18917c478bd9Sstevel@tonic-gate 
18922e0c549eSJonathan Adams 	if (flags & TASKQ_DYNAMIC) {
18932e0c549eSJonathan Adams 		ASSERT3S(nthreads, >=, 1);
18942e0c549eSJonathan Adams 		tq->tq_maxsize = nthreads;
18957c478bd9Sstevel@tonic-gate 
18962e0c549eSJonathan Adams 		/* For dynamic task queues use just one backup thread */
18972e0c549eSJonathan Adams 		nthreads = max_nthreads = 1;
18987c478bd9Sstevel@tonic-gate 
189935a5a358SJonathan Adams 	} else if (flags & TASKQ_THREADS_CPU_PCT) {
19002e0c549eSJonathan Adams 		uint_t pct;
19012e0c549eSJonathan Adams 		ASSERT3S(nthreads, >=, 0);
19022e0c549eSJonathan Adams 		pct = nthreads;
19032e0c549eSJonathan Adams 
19042e0c549eSJonathan Adams 		if (pct > taskq_cpupct_max_percent)
19052e0c549eSJonathan Adams 			pct = taskq_cpupct_max_percent;
19062e0c549eSJonathan Adams 
190735a5a358SJonathan Adams 		/*
190835a5a358SJonathan Adams 		 * If you're using THREADS_CPU_PCT, the process for the
190935a5a358SJonathan Adams 		 * taskq threads must be curproc.  This allows any pset
191035a5a358SJonathan Adams 		 * binding to be inherited correctly.  If proc is &p0,
191135a5a358SJonathan Adams 		 * we won't be creating LWPs, so new threads will be assigned
191235a5a358SJonathan Adams 		 * to the default processor set.
191335a5a358SJonathan Adams 		 */
191435a5a358SJonathan Adams 		ASSERT(curproc == proc || proc == &p0);
19152e0c549eSJonathan Adams 		tq->tq_threads_ncpus_pct = pct;
191635a5a358SJonathan Adams 		nthreads = 1;		/* corrected in taskq_thread_create() */
19172e0c549eSJonathan Adams 		max_nthreads = TASKQ_THREADS_PCT(max_ncpus, pct);
191835a5a358SJonathan Adams 
191935a5a358SJonathan Adams 	} else {
192035a5a358SJonathan Adams 		ASSERT3S(nthreads, >=, 1);
192135a5a358SJonathan Adams 		max_nthreads = nthreads;
19222e0c549eSJonathan Adams 	}
19232e0c549eSJonathan Adams 
19242e0c549eSJonathan Adams 	if (max_nthreads < taskq_minimum_nthreads_max)
19252e0c549eSJonathan Adams 		max_nthreads = taskq_minimum_nthreads_max;
19262e0c549eSJonathan Adams 
19272e0c549eSJonathan Adams 	/*
19282e0c549eSJonathan Adams 	 * Make sure the name is 0-terminated, and conforms to the rules for
19292e0c549eSJonathan Adams 	 * C indentifiers
19302e0c549eSJonathan Adams 	 */
19317c478bd9Sstevel@tonic-gate 	(void) strncpy(tq->tq_name, name, TASKQ_NAMELEN + 1);
19322e0c549eSJonathan Adams 	strident_canon(tq->tq_name, TASKQ_NAMELEN + 1);
19337c478bd9Sstevel@tonic-gate 
19342e0c549eSJonathan Adams 	tq->tq_flags = flags | TASKQ_CHANGING;
19352e0c549eSJonathan Adams 	tq->tq_active = 0;
19367c478bd9Sstevel@tonic-gate 	tq->tq_instance = instance;
19372e0c549eSJonathan Adams 	tq->tq_nthreads_target = nthreads;
19382e0c549eSJonathan Adams 	tq->tq_nthreads_max = max_nthreads;
19397c478bd9Sstevel@tonic-gate 	tq->tq_minalloc = minalloc;
19407c478bd9Sstevel@tonic-gate 	tq->tq_maxalloc = maxalloc;
19417c478bd9Sstevel@tonic-gate 	tq->tq_nbuckets = bsize;
194235a5a358SJonathan Adams 	tq->tq_proc = proc;
19437c478bd9Sstevel@tonic-gate 	tq->tq_pri = pri;
194435a5a358SJonathan Adams 	tq->tq_DC = dc;
194535a5a358SJonathan Adams 	list_link_init(&tq->tq_cpupct_link);
19467c478bd9Sstevel@tonic-gate 
19472e0c549eSJonathan Adams 	if (max_nthreads > 1)
19482e0c549eSJonathan Adams 		tq->tq_threadlist = kmem_alloc(
19492e0c549eSJonathan Adams 		    sizeof (kthread_t *) * max_nthreads, KM_SLEEP);
19502e0c549eSJonathan Adams 
19512e0c549eSJonathan Adams 	mutex_enter(&tq->tq_lock);
19527c478bd9Sstevel@tonic-gate 	if (flags & TASKQ_PREPOPULATE) {
19537c478bd9Sstevel@tonic-gate 		while (minalloc-- > 0)
19547c478bd9Sstevel@tonic-gate 			taskq_ent_free(tq, taskq_ent_alloc(tq, TQ_SLEEP));
19557c478bd9Sstevel@tonic-gate 	}
19567c478bd9Sstevel@tonic-gate 
1957e5488233SGordon Ross 	/*
1958e5488233SGordon Ross 	 * Before we start creating threads for this taskq, take a
1959e5488233SGordon Ross 	 * zone hold so the zone can't go away before taskq_destroy
1960e5488233SGordon Ross 	 * makes sure all the taskq threads are gone.  This hold is
1961e5488233SGordon Ross 	 * similar in purpose to those taken by zthread_create().
1962e5488233SGordon Ross 	 */
1963e5488233SGordon Ross 	zone_hold(tq->tq_proc->p_zone);
1964e5488233SGordon Ross 
196535a5a358SJonathan Adams 	/*
196635a5a358SJonathan Adams 	 * Create the first thread, which will create any other threads
196735a5a358SJonathan Adams 	 * necessary.  taskq_thread_create will not return until we have
196835a5a358SJonathan Adams 	 * enough threads to be able to process requests.
196935a5a358SJonathan Adams 	 */
19702e0c549eSJonathan Adams 	taskq_thread_create(tq);
19712e0c549eSJonathan Adams 	mutex_exit(&tq->tq_lock);
19727c478bd9Sstevel@tonic-gate 
19737c478bd9Sstevel@tonic-gate 	if (flags & TASKQ_DYNAMIC) {
19747c478bd9Sstevel@tonic-gate 		taskq_bucket_t *bucket = kmem_zalloc(sizeof (taskq_bucket_t) *
19757c478bd9Sstevel@tonic-gate 		    bsize, KM_SLEEP);
19767c478bd9Sstevel@tonic-gate 		int b_id;
19777c478bd9Sstevel@tonic-gate 
19787c478bd9Sstevel@tonic-gate 		tq->tq_buckets = bucket;
19797c478bd9Sstevel@tonic-gate 
19807c478bd9Sstevel@tonic-gate 		/* Initialize each bucket */
19817c478bd9Sstevel@tonic-gate 		for (b_id = 0; b_id < bsize; b_id++, bucket++) {
19827c478bd9Sstevel@tonic-gate 			mutex_init(&bucket->tqbucket_lock, NULL, MUTEX_DEFAULT,
19837c478bd9Sstevel@tonic-gate 			    NULL);
19847c478bd9Sstevel@tonic-gate 			cv_init(&bucket->tqbucket_cv, NULL, CV_DEFAULT, NULL);
19857c478bd9Sstevel@tonic-gate 			bucket->tqbucket_taskq = tq;
19867c478bd9Sstevel@tonic-gate 			bucket->tqbucket_freelist.tqent_next =
19877c478bd9Sstevel@tonic-gate 			    bucket->tqbucket_freelist.tqent_prev =
19887c478bd9Sstevel@tonic-gate 			    &bucket->tqbucket_freelist;
19897c478bd9Sstevel@tonic-gate 			if (flags & TASKQ_PREPOPULATE)
19907c478bd9Sstevel@tonic-gate 				taskq_bucket_extend(bucket);
19917c478bd9Sstevel@tonic-gate 		}
19927c478bd9Sstevel@tonic-gate 	}
19937c478bd9Sstevel@tonic-gate 
19947c478bd9Sstevel@tonic-gate 	/*
19957c478bd9Sstevel@tonic-gate 	 * Install kstats.
19967c478bd9Sstevel@tonic-gate 	 * We have two cases:
19977c478bd9Sstevel@tonic-gate 	 *   1) Instance is provided to taskq_create_instance(). In this case it
199835a5a358SJonathan Adams 	 *	should be >= 0 and we use it.
19997c478bd9Sstevel@tonic-gate 	 *
20007c478bd9Sstevel@tonic-gate 	 *   2) Instance is not provided and is automatically generated
20017c478bd9Sstevel@tonic-gate 	 */
20027c478bd9Sstevel@tonic-gate 	if (flags & TASKQ_NOINSTANCE) {
20037c478bd9Sstevel@tonic-gate 		instance = tq->tq_instance =
20047c478bd9Sstevel@tonic-gate 		    (int)(uintptr_t)vmem_alloc(taskq_id_arena, 1, VM_SLEEP);
20057c478bd9Sstevel@tonic-gate 	}
20067c478bd9Sstevel@tonic-gate 
20077c478bd9Sstevel@tonic-gate 	if (flags & TASKQ_DYNAMIC) {
20087c478bd9Sstevel@tonic-gate 		if ((tq->tq_kstat = kstat_create("unix", instance,
20092e0c549eSJonathan Adams 		    tq->tq_name, "taskq_d", KSTAT_TYPE_NAMED,
20102e0c549eSJonathan Adams 		    sizeof (taskq_d_kstat) / sizeof (kstat_named_t),
20112e0c549eSJonathan Adams 		    KSTAT_FLAG_VIRTUAL)) != NULL) {
20127c478bd9Sstevel@tonic-gate 			tq->tq_kstat->ks_lock = &taskq_d_kstat_lock;
20137c478bd9Sstevel@tonic-gate 			tq->tq_kstat->ks_data = &taskq_d_kstat;
20147c478bd9Sstevel@tonic-gate 			tq->tq_kstat->ks_update = taskq_d_kstat_update;
20157c478bd9Sstevel@tonic-gate 			tq->tq_kstat->ks_private = tq;
20167c478bd9Sstevel@tonic-gate 			kstat_install(tq->tq_kstat);
20177c478bd9Sstevel@tonic-gate 		}
20187c478bd9Sstevel@tonic-gate 	} else {
20197c478bd9Sstevel@tonic-gate 		if ((tq->tq_kstat = kstat_create("unix", instance, tq->tq_name,
20202e0c549eSJonathan Adams 		    "taskq", KSTAT_TYPE_NAMED,
20212e0c549eSJonathan Adams 		    sizeof (taskq_kstat) / sizeof (kstat_named_t),
20222e0c549eSJonathan Adams 		    KSTAT_FLAG_VIRTUAL)) != NULL) {
20237c478bd9Sstevel@tonic-gate 			tq->tq_kstat->ks_lock = &taskq_kstat_lock;
20247c478bd9Sstevel@tonic-gate 			tq->tq_kstat->ks_data = &taskq_kstat;
20257c478bd9Sstevel@tonic-gate 			tq->tq_kstat->ks_update = taskq_kstat_update;
20267c478bd9Sstevel@tonic-gate 			tq->tq_kstat->ks_private = tq;
20277c478bd9Sstevel@tonic-gate 			kstat_install(tq->tq_kstat);
20287c478bd9Sstevel@tonic-gate 		}
20297c478bd9Sstevel@tonic-gate 	}
20307c478bd9Sstevel@tonic-gate 
20317c478bd9Sstevel@tonic-gate 	return (tq);
20327c478bd9Sstevel@tonic-gate }
20337c478bd9Sstevel@tonic-gate 
20347c478bd9Sstevel@tonic-gate /*
20357c478bd9Sstevel@tonic-gate  * taskq_destroy().
20367c478bd9Sstevel@tonic-gate  *
20377c478bd9Sstevel@tonic-gate  * Assumes: by the time taskq_destroy is called no one will use this task queue
20387c478bd9Sstevel@tonic-gate  * in any way and no one will try to dispatch entries in it.
20397c478bd9Sstevel@tonic-gate  */
20407c478bd9Sstevel@tonic-gate void
20417c478bd9Sstevel@tonic-gate taskq_destroy(taskq_t *tq)
20427c478bd9Sstevel@tonic-gate {
20437c478bd9Sstevel@tonic-gate 	taskq_bucket_t *b = tq->tq_buckets;
20447c478bd9Sstevel@tonic-gate 	int bid = 0;
20457c478bd9Sstevel@tonic-gate 
20467c478bd9Sstevel@tonic-gate 	ASSERT(! (tq->tq_flags & TASKQ_CPR_SAFE));
20477c478bd9Sstevel@tonic-gate 
20487c478bd9Sstevel@tonic-gate 	/*
20497c478bd9Sstevel@tonic-gate 	 * Destroy kstats.
20507c478bd9Sstevel@tonic-gate 	 */
20517c478bd9Sstevel@tonic-gate 	if (tq->tq_kstat != NULL) {
20527c478bd9Sstevel@tonic-gate 		kstat_delete(tq->tq_kstat);
20537c478bd9Sstevel@tonic-gate 		tq->tq_kstat = NULL;
20547c478bd9Sstevel@tonic-gate 	}
20557c478bd9Sstevel@tonic-gate 
20567c478bd9Sstevel@tonic-gate 	/*
20577c478bd9Sstevel@tonic-gate 	 * Destroy instance if needed.
20587c478bd9Sstevel@tonic-gate 	 */
20597c478bd9Sstevel@tonic-gate 	if (tq->tq_flags & TASKQ_NOINSTANCE) {
20607c478bd9Sstevel@tonic-gate 		vmem_free(taskq_id_arena, (void *)(uintptr_t)(tq->tq_instance),
20617c478bd9Sstevel@tonic-gate 		    1);
20627c478bd9Sstevel@tonic-gate 		tq->tq_instance = 0;
20637c478bd9Sstevel@tonic-gate 	}
20647c478bd9Sstevel@tonic-gate 
20652e0c549eSJonathan Adams 	/*
20662e0c549eSJonathan Adams 	 * Unregister from the cpupct list.
20672e0c549eSJonathan Adams 	 */
20682e0c549eSJonathan Adams 	if (tq->tq_flags & TASKQ_THREADS_CPU_PCT) {
206935a5a358SJonathan Adams 		taskq_cpupct_remove(tq);
20702e0c549eSJonathan Adams 	}
20712e0c549eSJonathan Adams 
20727c478bd9Sstevel@tonic-gate 	/*
20737c478bd9Sstevel@tonic-gate 	 * Wait for any pending entries to complete.
20747c478bd9Sstevel@tonic-gate 	 */
20757c478bd9Sstevel@tonic-gate 	taskq_wait(tq);
20767c478bd9Sstevel@tonic-gate 
20777c478bd9Sstevel@tonic-gate 	mutex_enter(&tq->tq_lock);
20787c478bd9Sstevel@tonic-gate 	ASSERT((tq->tq_task.tqent_next == &tq->tq_task) &&
20797c478bd9Sstevel@tonic-gate 	    (tq->tq_active == 0));
20807c478bd9Sstevel@tonic-gate 
20812e0c549eSJonathan Adams 	/* notify all the threads that they need to exit */
20822e0c549eSJonathan Adams 	tq->tq_nthreads_target = 0;
20837c478bd9Sstevel@tonic-gate 
20842e0c549eSJonathan Adams 	tq->tq_flags |= TASKQ_CHANGING;
20857c478bd9Sstevel@tonic-gate 	cv_broadcast(&tq->tq_dispatch_cv);
20862e0c549eSJonathan Adams 	cv_broadcast(&tq->tq_exit_cv);
20872e0c549eSJonathan Adams 
20887c478bd9Sstevel@tonic-gate 	while (tq->tq_nthreads != 0)
20897c478bd9Sstevel@tonic-gate 		cv_wait(&tq->tq_wait_cv, &tq->tq_lock);
20907c478bd9Sstevel@tonic-gate 
20912e0c549eSJonathan Adams 	if (tq->tq_nthreads_max != 1)
20922e0c549eSJonathan Adams 		kmem_free(tq->tq_threadlist, sizeof (kthread_t *) *
20932e0c549eSJonathan Adams 		    tq->tq_nthreads_max);
20942e0c549eSJonathan Adams 
20957c478bd9Sstevel@tonic-gate 	tq->tq_minalloc = 0;
20967c478bd9Sstevel@tonic-gate 	while (tq->tq_nalloc != 0)
20977c478bd9Sstevel@tonic-gate 		taskq_ent_free(tq, taskq_ent_alloc(tq, TQ_SLEEP));
20987c478bd9Sstevel@tonic-gate 
20997c478bd9Sstevel@tonic-gate 	mutex_exit(&tq->tq_lock);
21007c478bd9Sstevel@tonic-gate 
21017c478bd9Sstevel@tonic-gate 	/*
21027c478bd9Sstevel@tonic-gate 	 * Mark each bucket as closing and wakeup all sleeping threads.
21037c478bd9Sstevel@tonic-gate 	 */
21047c478bd9Sstevel@tonic-gate 	for (; (b != NULL) && (bid < tq->tq_nbuckets); b++, bid++) {
21057c478bd9Sstevel@tonic-gate 		taskq_ent_t *tqe;
21067c478bd9Sstevel@tonic-gate 
21077c478bd9Sstevel@tonic-gate 		mutex_enter(&b->tqbucket_lock);
21087c478bd9Sstevel@tonic-gate 
21097c478bd9Sstevel@tonic-gate 		b->tqbucket_flags |= TQBUCKET_CLOSE;
21107c478bd9Sstevel@tonic-gate 		/* Wakeup all sleeping threads */
21117c478bd9Sstevel@tonic-gate 
21127c478bd9Sstevel@tonic-gate 		for (tqe = b->tqbucket_freelist.tqent_next;
21137c478bd9Sstevel@tonic-gate 		    tqe != &b->tqbucket_freelist; tqe = tqe->tqent_next)
21147c478bd9Sstevel@tonic-gate 			cv_signal(&tqe->tqent_cv);
21157c478bd9Sstevel@tonic-gate 
21167c478bd9Sstevel@tonic-gate 		ASSERT(b->tqbucket_nalloc == 0);
21177c478bd9Sstevel@tonic-gate 
21187c478bd9Sstevel@tonic-gate 		/*
21197c478bd9Sstevel@tonic-gate 		 * At this point we waited for all pending jobs to complete (in
21207c478bd9Sstevel@tonic-gate 		 * both the task queue and the bucket and no new jobs should
21217c478bd9Sstevel@tonic-gate 		 * arrive. Wait for all threads to die.
21227c478bd9Sstevel@tonic-gate 		 */
21237c478bd9Sstevel@tonic-gate 		while (b->tqbucket_nfree > 0)
21247c478bd9Sstevel@tonic-gate 			cv_wait(&b->tqbucket_cv, &b->tqbucket_lock);
21257c478bd9Sstevel@tonic-gate 		mutex_exit(&b->tqbucket_lock);
21267c478bd9Sstevel@tonic-gate 		mutex_destroy(&b->tqbucket_lock);
21277c478bd9Sstevel@tonic-gate 		cv_destroy(&b->tqbucket_cv);
21287c478bd9Sstevel@tonic-gate 	}
21297c478bd9Sstevel@tonic-gate 
21307c478bd9Sstevel@tonic-gate 	if (tq->tq_buckets != NULL) {
21317c478bd9Sstevel@tonic-gate 		ASSERT(tq->tq_flags & TASKQ_DYNAMIC);
21327c478bd9Sstevel@tonic-gate 		kmem_free(tq->tq_buckets,
21337c478bd9Sstevel@tonic-gate 		    sizeof (taskq_bucket_t) * tq->tq_nbuckets);
21347c478bd9Sstevel@tonic-gate 
21357c478bd9Sstevel@tonic-gate 		/* Cleanup fields before returning tq to the cache */
21367c478bd9Sstevel@tonic-gate 		tq->tq_buckets = NULL;
21377c478bd9Sstevel@tonic-gate 		tq->tq_tcreates = 0;
21387c478bd9Sstevel@tonic-gate 		tq->tq_tdeaths = 0;
21397c478bd9Sstevel@tonic-gate 	} else {
21407c478bd9Sstevel@tonic-gate 		ASSERT(!(tq->tq_flags & TASKQ_DYNAMIC));
21417c478bd9Sstevel@tonic-gate 	}
21427c478bd9Sstevel@tonic-gate 
2143e5488233SGordon Ross 	/*
2144e5488233SGordon Ross 	 * Now that all the taskq threads are gone, we can
2145e5488233SGordon Ross 	 * drop the zone hold taken in taskq_create_common
2146e5488233SGordon Ross 	 */
2147e5488233SGordon Ross 	zone_rele(tq->tq_proc->p_zone);
2148e5488233SGordon Ross 
21492e0c549eSJonathan Adams 	tq->tq_threads_ncpus_pct = 0;
21507c478bd9Sstevel@tonic-gate 	tq->tq_totaltime = 0;
21517c478bd9Sstevel@tonic-gate 	tq->tq_tasks = 0;
21527c478bd9Sstevel@tonic-gate 	tq->tq_maxtasks = 0;
21537c478bd9Sstevel@tonic-gate 	tq->tq_executed = 0;
21547c478bd9Sstevel@tonic-gate 	kmem_cache_free(taskq_cache, tq);
21557c478bd9Sstevel@tonic-gate }
21567c478bd9Sstevel@tonic-gate 
21577c478bd9Sstevel@tonic-gate /*
21587c478bd9Sstevel@tonic-gate  * Extend a bucket with a new entry on the free list and attach a worker thread
21597c478bd9Sstevel@tonic-gate  * to it.
21607c478bd9Sstevel@tonic-gate  *
21617c478bd9Sstevel@tonic-gate  * Argument: pointer to the bucket.
21627c478bd9Sstevel@tonic-gate  *
21637c478bd9Sstevel@tonic-gate  * This function may quietly fail. It is only used by taskq_dispatch() which
21647c478bd9Sstevel@tonic-gate  * handles such failures properly.
21657c478bd9Sstevel@tonic-gate  */
21667c478bd9Sstevel@tonic-gate static void
21677c478bd9Sstevel@tonic-gate taskq_bucket_extend(void *arg)
21687c478bd9Sstevel@tonic-gate {
21697c478bd9Sstevel@tonic-gate 	taskq_ent_t *tqe;
21707c478bd9Sstevel@tonic-gate 	taskq_bucket_t *b = (taskq_bucket_t *)arg;
21717c478bd9Sstevel@tonic-gate 	taskq_t *tq = b->tqbucket_taskq;
21727c478bd9Sstevel@tonic-gate 	int nthreads;
21737c478bd9Sstevel@tonic-gate 
21747c478bd9Sstevel@tonic-gate 	if (! ENOUGH_MEMORY()) {
21757c478bd9Sstevel@tonic-gate 		TQ_STAT(b, tqs_nomem);
21767c478bd9Sstevel@tonic-gate 		return;
21777c478bd9Sstevel@tonic-gate 	}
21787c478bd9Sstevel@tonic-gate 
21797c478bd9Sstevel@tonic-gate 	mutex_enter(&tq->tq_lock);
21807c478bd9Sstevel@tonic-gate 
21817c478bd9Sstevel@tonic-gate 	/*
21827c478bd9Sstevel@tonic-gate 	 * Observe global taskq limits on the number of threads.
21837c478bd9Sstevel@tonic-gate 	 */
21847c478bd9Sstevel@tonic-gate 	if (tq->tq_tcreates++ - tq->tq_tdeaths > tq->tq_maxsize) {
21857c478bd9Sstevel@tonic-gate 		tq->tq_tcreates--;
21867c478bd9Sstevel@tonic-gate 		mutex_exit(&tq->tq_lock);
21877c478bd9Sstevel@tonic-gate 		return;
21887c478bd9Sstevel@tonic-gate 	}
21897c478bd9Sstevel@tonic-gate 	mutex_exit(&tq->tq_lock);
21907c478bd9Sstevel@tonic-gate 
21917c478bd9Sstevel@tonic-gate 	tqe = kmem_cache_alloc(taskq_ent_cache, KM_NOSLEEP);
21927c478bd9Sstevel@tonic-gate 
21937c478bd9Sstevel@tonic-gate 	if (tqe == NULL) {
21947c478bd9Sstevel@tonic-gate 		mutex_enter(&tq->tq_lock);
21957c478bd9Sstevel@tonic-gate 		TQ_STAT(b, tqs_nomem);
21967c478bd9Sstevel@tonic-gate 		tq->tq_tcreates--;
21977c478bd9Sstevel@tonic-gate 		mutex_exit(&tq->tq_lock);
21987c478bd9Sstevel@tonic-gate 		return;
21997c478bd9Sstevel@tonic-gate 	}
22007c478bd9Sstevel@tonic-gate 
22017c478bd9Sstevel@tonic-gate 	ASSERT(tqe->tqent_thread == NULL);
22027c478bd9Sstevel@tonic-gate 
22035aeb9474SGarrett D'Amore 	tqe->tqent_un.tqent_bucket = b;
22047c478bd9Sstevel@tonic-gate 
22057c478bd9Sstevel@tonic-gate 	/*
22067c478bd9Sstevel@tonic-gate 	 * Create a thread in a TS_STOPPED state first. If it is successfully
22077c478bd9Sstevel@tonic-gate 	 * created, place the entry on the free list and start the thread.
22087c478bd9Sstevel@tonic-gate 	 */
2209c883468aSGordon Ross 	tqe->tqent_thread = thread_create(NULL, 0, taskq_d_thread, tqe,
2210e5488233SGordon Ross 	    0, tq->tq_proc, TS_STOPPED, tq->tq_pri);
22117c478bd9Sstevel@tonic-gate 
22127c478bd9Sstevel@tonic-gate 	/*
22137c478bd9Sstevel@tonic-gate 	 * Once the entry is ready, link it to the the bucket free list.
22147c478bd9Sstevel@tonic-gate 	 */
22157c478bd9Sstevel@tonic-gate 	mutex_enter(&b->tqbucket_lock);
22167c478bd9Sstevel@tonic-gate 	tqe->tqent_func = NULL;
22177c478bd9Sstevel@tonic-gate 	TQ_APPEND(b->tqbucket_freelist, tqe);
22187c478bd9Sstevel@tonic-gate 	b->tqbucket_nfree++;
22197c478bd9Sstevel@tonic-gate 	TQ_STAT(b, tqs_tcreates);
22207c478bd9Sstevel@tonic-gate 
22217c478bd9Sstevel@tonic-gate #if TASKQ_STATISTIC
22227c478bd9Sstevel@tonic-gate 	nthreads = b->tqbucket_stat.tqs_tcreates -
22237c478bd9Sstevel@tonic-gate 	    b->tqbucket_stat.tqs_tdeaths;
22247c478bd9Sstevel@tonic-gate 	b->tqbucket_stat.tqs_maxthreads = MAX(nthreads,
22257c478bd9Sstevel@tonic-gate 	    b->tqbucket_stat.tqs_maxthreads);
22267c478bd9Sstevel@tonic-gate #endif
22277c478bd9Sstevel@tonic-gate 
22287c478bd9Sstevel@tonic-gate 	mutex_exit(&b->tqbucket_lock);
22297c478bd9Sstevel@tonic-gate 	/*
22307c478bd9Sstevel@tonic-gate 	 * Start the stopped thread.
22317c478bd9Sstevel@tonic-gate 	 */
22327c478bd9Sstevel@tonic-gate 	thread_lock(tqe->tqent_thread);
22337c478bd9Sstevel@tonic-gate 	tqe->tqent_thread->t_taskq = tq;
22347c478bd9Sstevel@tonic-gate 	tqe->tqent_thread->t_schedflag |= TS_ALLSTART;
22357c478bd9Sstevel@tonic-gate 	setrun_locked(tqe->tqent_thread);
22367c478bd9Sstevel@tonic-gate 	thread_unlock(tqe->tqent_thread);
22377c478bd9Sstevel@tonic-gate }
22387c478bd9Sstevel@tonic-gate 
22397c478bd9Sstevel@tonic-gate static int
22407c478bd9Sstevel@tonic-gate taskq_kstat_update(kstat_t *ksp, int rw)
22417c478bd9Sstevel@tonic-gate {
22427c478bd9Sstevel@tonic-gate 	struct taskq_kstat *tqsp = &taskq_kstat;
22437c478bd9Sstevel@tonic-gate 	taskq_t *tq = ksp->ks_private;
22447c478bd9Sstevel@tonic-gate 
22457c478bd9Sstevel@tonic-gate 	if (rw == KSTAT_WRITE)
22467c478bd9Sstevel@tonic-gate 		return (EACCES);
22477c478bd9Sstevel@tonic-gate 
224835a5a358SJonathan Adams 	tqsp->tq_pid.value.ui64 = tq->tq_proc->p_pid;
22497c478bd9Sstevel@tonic-gate 	tqsp->tq_tasks.value.ui64 = tq->tq_tasks;
22507c478bd9Sstevel@tonic-gate 	tqsp->tq_executed.value.ui64 = tq->tq_executed;
22517c478bd9Sstevel@tonic-gate 	tqsp->tq_maxtasks.value.ui64 = tq->tq_maxtasks;
22527c478bd9Sstevel@tonic-gate 	tqsp->tq_totaltime.value.ui64 = tq->tq_totaltime;
22537c478bd9Sstevel@tonic-gate 	tqsp->tq_nactive.value.ui64 = tq->tq_active;
22547c478bd9Sstevel@tonic-gate 	tqsp->tq_nalloc.value.ui64 = tq->tq_nalloc;
22557c478bd9Sstevel@tonic-gate 	tqsp->tq_pri.value.ui64 = tq->tq_pri;
22567c478bd9Sstevel@tonic-gate 	tqsp->tq_nthreads.value.ui64 = tq->tq_nthreads;
22577c478bd9Sstevel@tonic-gate 	return (0);
22587c478bd9Sstevel@tonic-gate }
22597c478bd9Sstevel@tonic-gate 
22607c478bd9Sstevel@tonic-gate static int
22617c478bd9Sstevel@tonic-gate taskq_d_kstat_update(kstat_t *ksp, int rw)
22627c478bd9Sstevel@tonic-gate {
22637c478bd9Sstevel@tonic-gate 	struct taskq_d_kstat *tqsp = &taskq_d_kstat;
22647c478bd9Sstevel@tonic-gate 	taskq_t *tq = ksp->ks_private;
22657c478bd9Sstevel@tonic-gate 	taskq_bucket_t *b = tq->tq_buckets;
22667c478bd9Sstevel@tonic-gate 	int bid = 0;
22677c478bd9Sstevel@tonic-gate 
22687c478bd9Sstevel@tonic-gate 	if (rw == KSTAT_WRITE)
22697c478bd9Sstevel@tonic-gate 		return (EACCES);
22707c478bd9Sstevel@tonic-gate 
22717c478bd9Sstevel@tonic-gate 	ASSERT(tq->tq_flags & TASKQ_DYNAMIC);
22727c478bd9Sstevel@tonic-gate 
22737c478bd9Sstevel@tonic-gate 	tqsp->tqd_btasks.value.ui64 = tq->tq_tasks;
22747c478bd9Sstevel@tonic-gate 	tqsp->tqd_bexecuted.value.ui64 = tq->tq_executed;
22757c478bd9Sstevel@tonic-gate 	tqsp->tqd_bmaxtasks.value.ui64 = tq->tq_maxtasks;
22767c478bd9Sstevel@tonic-gate 	tqsp->tqd_bnalloc.value.ui64 = tq->tq_nalloc;
22777c478bd9Sstevel@tonic-gate 	tqsp->tqd_bnactive.value.ui64 = tq->tq_active;
22787c478bd9Sstevel@tonic-gate 	tqsp->tqd_btotaltime.value.ui64 = tq->tq_totaltime;
22797c478bd9Sstevel@tonic-gate 	tqsp->tqd_pri.value.ui64 = tq->tq_pri;
22807c478bd9Sstevel@tonic-gate 
22817c478bd9Sstevel@tonic-gate 	tqsp->tqd_hits.value.ui64 = 0;
22827c478bd9Sstevel@tonic-gate 	tqsp->tqd_misses.value.ui64 = 0;
22837c478bd9Sstevel@tonic-gate 	tqsp->tqd_overflows.value.ui64 = 0;
22847c478bd9Sstevel@tonic-gate 	tqsp->tqd_tcreates.value.ui64 = 0;
22857c478bd9Sstevel@tonic-gate 	tqsp->tqd_tdeaths.value.ui64 = 0;
22867c478bd9Sstevel@tonic-gate 	tqsp->tqd_maxthreads.value.ui64 = 0;
22877c478bd9Sstevel@tonic-gate 	tqsp->tqd_nomem.value.ui64 = 0;
22887c478bd9Sstevel@tonic-gate 	tqsp->tqd_disptcreates.value.ui64 = 0;
22897c478bd9Sstevel@tonic-gate 	tqsp->tqd_totaltime.value.ui64 = 0;
22907c478bd9Sstevel@tonic-gate 	tqsp->tqd_nalloc.value.ui64 = 0;
22917c478bd9Sstevel@tonic-gate 	tqsp->tqd_nfree.value.ui64 = 0;
22927c478bd9Sstevel@tonic-gate 
22937c478bd9Sstevel@tonic-gate 	for (; (b != NULL) && (bid < tq->tq_nbuckets); b++, bid++) {
22947c478bd9Sstevel@tonic-gate 		tqsp->tqd_hits.value.ui64 += b->tqbucket_stat.tqs_hits;
22957c478bd9Sstevel@tonic-gate 		tqsp->tqd_misses.value.ui64 += b->tqbucket_stat.tqs_misses;
22967c478bd9Sstevel@tonic-gate 		tqsp->tqd_overflows.value.ui64 += b->tqbucket_stat.tqs_overflow;
22977c478bd9Sstevel@tonic-gate 		tqsp->tqd_tcreates.value.ui64 += b->tqbucket_stat.tqs_tcreates;
22987c478bd9Sstevel@tonic-gate 		tqsp->tqd_tdeaths.value.ui64 += b->tqbucket_stat.tqs_tdeaths;
22997c478bd9Sstevel@tonic-gate 		tqsp->tqd_maxthreads.value.ui64 +=
23007c478bd9Sstevel@tonic-gate 		    b->tqbucket_stat.tqs_maxthreads;
23017c478bd9Sstevel@tonic-gate 		tqsp->tqd_nomem.value.ui64 += b->tqbucket_stat.tqs_nomem;
23027c478bd9Sstevel@tonic-gate 		tqsp->tqd_disptcreates.value.ui64 +=
23037c478bd9Sstevel@tonic-gate 		    b->tqbucket_stat.tqs_disptcreates;
23047c478bd9Sstevel@tonic-gate 		tqsp->tqd_totaltime.value.ui64 += b->tqbucket_totaltime;
23057c478bd9Sstevel@tonic-gate 		tqsp->tqd_nalloc.value.ui64 += b->tqbucket_nalloc;
23067c478bd9Sstevel@tonic-gate 		tqsp->tqd_nfree.value.ui64 += b->tqbucket_nfree;
23077c478bd9Sstevel@tonic-gate 	}
23087c478bd9Sstevel@tonic-gate 	return (0);
23097c478bd9Sstevel@tonic-gate }
2310