17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate * CDDL HEADER START
37c478bd9Sstevel@tonic-gate *
47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the
52e0c549eSJonathan Adams * Common Development and Distribution License (the "License").
62e0c549eSJonathan Adams * You may not use this file except in compliance with the License.
77c478bd9Sstevel@tonic-gate *
87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing.
107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions
117c478bd9Sstevel@tonic-gate * and limitations under the License.
127c478bd9Sstevel@tonic-gate *
137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each
147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the
167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying
177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bd9Sstevel@tonic-gate *
197c478bd9Sstevel@tonic-gate * CDDL HEADER END
207c478bd9Sstevel@tonic-gate */
217c478bd9Sstevel@tonic-gate /*
2256f33205SJonathan Adams * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
237c478bd9Sstevel@tonic-gate * Use is subject to license terms.
247c478bd9Sstevel@tonic-gate */
257c478bd9Sstevel@tonic-gate
265aeb9474SGarrett D'Amore /*
27e5488233SGordon Ross * Copyright 2015 Nexenta Systems, Inc. All rights reserved.
28216d7723SPrakash Surya * Copyright (c) 2017 by Delphix. All rights reserved.
294c99ecc3STim Kordas * Copyright 2018, Joyent, Inc.
30*f8d363f3SGordon Ross * Copyright 2023 RackTop Systems, Inc.
315aeb9474SGarrett D'Amore */
325aeb9474SGarrett D'Amore
337c478bd9Sstevel@tonic-gate /*
347c478bd9Sstevel@tonic-gate * Kernel task queues: general-purpose asynchronous task scheduling.
357c478bd9Sstevel@tonic-gate *
367c478bd9Sstevel@tonic-gate * A common problem in kernel programming is the need to schedule tasks
377c478bd9Sstevel@tonic-gate * to be performed later, by another thread. There are several reasons
387c478bd9Sstevel@tonic-gate * you may want or need to do this:
397c478bd9Sstevel@tonic-gate *
407c478bd9Sstevel@tonic-gate * (1) The task isn't time-critical, but your current code path is.
417c478bd9Sstevel@tonic-gate *
427c478bd9Sstevel@tonic-gate * (2) The task may require grabbing locks that you already hold.
437c478bd9Sstevel@tonic-gate *
447c478bd9Sstevel@tonic-gate * (3) The task may need to block (e.g. to wait for memory), but you
457c478bd9Sstevel@tonic-gate * cannot block in your current context.
467c478bd9Sstevel@tonic-gate *
477c478bd9Sstevel@tonic-gate * (4) Your code path can't complete because of some condition, but you can't
487c478bd9Sstevel@tonic-gate * sleep or fail, so you queue the task for later execution when condition
497c478bd9Sstevel@tonic-gate * disappears.
507c478bd9Sstevel@tonic-gate *
517c478bd9Sstevel@tonic-gate * (5) You just want a simple way to launch multiple tasks in parallel.
527c478bd9Sstevel@tonic-gate *
537c478bd9Sstevel@tonic-gate * Task queues provide such a facility. In its simplest form (used when
547c478bd9Sstevel@tonic-gate * performance is not a critical consideration) a task queue consists of a
557c478bd9Sstevel@tonic-gate * single list of tasks, together with one or more threads to service the
567c478bd9Sstevel@tonic-gate * list. There are some cases when this simple queue is not sufficient:
577c478bd9Sstevel@tonic-gate *
587c478bd9Sstevel@tonic-gate * (1) The task queues are very hot and there is a need to avoid data and lock
597c478bd9Sstevel@tonic-gate * contention over global resources.
607c478bd9Sstevel@tonic-gate *
617c478bd9Sstevel@tonic-gate * (2) Some tasks may depend on other tasks to complete, so they can't be put in
627c478bd9Sstevel@tonic-gate * the same list managed by the same thread.
637c478bd9Sstevel@tonic-gate *
647c478bd9Sstevel@tonic-gate * (3) Some tasks may block for a long time, and this should not block other
6535a5a358SJonathan Adams * tasks in the queue.
667c478bd9Sstevel@tonic-gate *
677c478bd9Sstevel@tonic-gate * To provide useful service in such cases we define a "dynamic task queue"
687c478bd9Sstevel@tonic-gate * which has an individual thread for each of the tasks. These threads are
697c478bd9Sstevel@tonic-gate * dynamically created as they are needed and destroyed when they are not in
707c478bd9Sstevel@tonic-gate * use. The API for managing task pools is the same as for managing task queues
717c478bd9Sstevel@tonic-gate * with the exception of a taskq creation flag TASKQ_DYNAMIC which tells that
727c478bd9Sstevel@tonic-gate * dynamic task pool behavior is desired.
737c478bd9Sstevel@tonic-gate *
747c478bd9Sstevel@tonic-gate * Dynamic task queues may also place tasks in the normal queue (called "backing
757c478bd9Sstevel@tonic-gate * queue") when task pool runs out of resources. Users of task queues may
767c478bd9Sstevel@tonic-gate * disallow such queued scheduling by specifying TQ_NOQUEUE in the dispatch
777c478bd9Sstevel@tonic-gate * flags.
787c478bd9Sstevel@tonic-gate *
797c478bd9Sstevel@tonic-gate * The backing task queue is also used for scheduling internal tasks needed for
807c478bd9Sstevel@tonic-gate * dynamic task queue maintenance.
817c478bd9Sstevel@tonic-gate *
822e0c549eSJonathan Adams * INTERFACES ==================================================================
837c478bd9Sstevel@tonic-gate *
84a676a7c9SSergio Aguayo * taskq_t *taskq_create(name, nthreads, pri, minalloc, maxalloc, flags);
857c478bd9Sstevel@tonic-gate *
867c478bd9Sstevel@tonic-gate * Create a taskq with specified properties.
877c478bd9Sstevel@tonic-gate * Possible 'flags':
887c478bd9Sstevel@tonic-gate *
897c478bd9Sstevel@tonic-gate * TASKQ_DYNAMIC: Create task pool for task management. If this flag is
902e0c549eSJonathan Adams * specified, 'nthreads' specifies the maximum number of threads in
917c478bd9Sstevel@tonic-gate * the task queue. Task execution order for dynamic task queues is
927c478bd9Sstevel@tonic-gate * not predictable.
937c478bd9Sstevel@tonic-gate *
947c478bd9Sstevel@tonic-gate * If this flag is not specified (default case) a
952e0c549eSJonathan Adams * single-list task queue is created with 'nthreads' threads
962e0c549eSJonathan Adams * servicing it. Entries in this queue are managed by
972e0c549eSJonathan Adams * taskq_ent_alloc() and taskq_ent_free() which try to keep the
982e0c549eSJonathan Adams * task population between 'minalloc' and 'maxalloc', but the
997c478bd9Sstevel@tonic-gate * latter limit is only advisory for TQ_SLEEP dispatches and the
1007c478bd9Sstevel@tonic-gate * former limit is only advisory for TQ_NOALLOC dispatches. If
1017c478bd9Sstevel@tonic-gate * TASKQ_PREPOPULATE is set in 'flags', the taskq will be
1027c478bd9Sstevel@tonic-gate * prepopulated with 'minalloc' task structures.
1037c478bd9Sstevel@tonic-gate *
1047c478bd9Sstevel@tonic-gate * Since non-DYNAMIC taskqs are queues, tasks are guaranteed to be
1057c478bd9Sstevel@tonic-gate * executed in the order they are scheduled if nthreads == 1.
1067c478bd9Sstevel@tonic-gate * If nthreads > 1, task execution order is not predictable.
1077c478bd9Sstevel@tonic-gate *
1087c478bd9Sstevel@tonic-gate * TASKQ_PREPOPULATE: Prepopulate task queue with threads.
1097c478bd9Sstevel@tonic-gate * Also prepopulate the task queue with 'minalloc' task structures.
1107c478bd9Sstevel@tonic-gate *
1112e0c549eSJonathan Adams * TASKQ_THREADS_CPU_PCT: This flag specifies that 'nthreads' should be
1122e0c549eSJonathan Adams * interpreted as a percentage of the # of online CPUs on the
1132e0c549eSJonathan Adams * system. The taskq subsystem will automatically adjust the
1142e0c549eSJonathan Adams * number of threads in the taskq in response to CPU online
1152e0c549eSJonathan Adams * and offline events, to keep the ratio. nthreads must be in
1162e0c549eSJonathan Adams * the range [0,100].
1172e0c549eSJonathan Adams *
1182e0c549eSJonathan Adams * The calculation used is:
1192e0c549eSJonathan Adams *
1202e0c549eSJonathan Adams * MAX((ncpus_online * percentage)/100, 1)
1212e0c549eSJonathan Adams *
1222e0c549eSJonathan Adams * This flag is not supported for DYNAMIC task queues.
1232e0c549eSJonathan Adams * This flag is not compatible with TASKQ_CPR_SAFE.
1242e0c549eSJonathan Adams *
1257c478bd9Sstevel@tonic-gate * TASKQ_CPR_SAFE: This flag specifies that users of the task queue will
1262e0c549eSJonathan Adams * use their own protocol for handling CPR issues. This flag is not
1272e0c549eSJonathan Adams * supported for DYNAMIC task queues. This flag is not compatible
1282e0c549eSJonathan Adams * with TASKQ_THREADS_CPU_PCT.
1297c478bd9Sstevel@tonic-gate *
1307c478bd9Sstevel@tonic-gate * The 'pri' field specifies the default priority for the threads that
1317c478bd9Sstevel@tonic-gate * service all scheduled tasks.
1327c478bd9Sstevel@tonic-gate *
13335a5a358SJonathan Adams * taskq_t *taskq_create_instance(name, instance, nthreads, pri, minalloc,
134a676a7c9SSergio Aguayo * maxalloc, flags);
13535a5a358SJonathan Adams *
13635a5a358SJonathan Adams * Like taskq_create(), but takes an instance number (or -1 to indicate
13735a5a358SJonathan Adams * no instance).
13835a5a358SJonathan Adams *
139a676a7c9SSergio Aguayo * taskq_t *taskq_create_proc(name, nthreads, pri, minalloc, maxalloc, proc,
14035a5a358SJonathan Adams * flags);
14135a5a358SJonathan Adams *
14235a5a358SJonathan Adams * Like taskq_create(), but creates the taskq threads in the specified
14335a5a358SJonathan Adams * system process. If proc != &p0, this must be called from a thread
14435a5a358SJonathan Adams * in that process.
14535a5a358SJonathan Adams *
146a676a7c9SSergio Aguayo * taskq_t *taskq_create_sysdc(name, nthreads, minalloc, maxalloc, proc,
14735a5a358SJonathan Adams * dc, flags);
14835a5a358SJonathan Adams *
14935a5a358SJonathan Adams * Like taskq_create_proc(), but the taskq threads will use the
15035a5a358SJonathan Adams * System Duty Cycle (SDC) scheduling class with a duty cycle of dc.
15135a5a358SJonathan Adams *
1527c478bd9Sstevel@tonic-gate * void taskq_destroy(tap):
1537c478bd9Sstevel@tonic-gate *
1547c478bd9Sstevel@tonic-gate * Waits for any scheduled tasks to complete, then destroys the taskq.
1557c478bd9Sstevel@tonic-gate * Caller should guarantee that no new tasks are scheduled in the closing
1567c478bd9Sstevel@tonic-gate * taskq.
1577c478bd9Sstevel@tonic-gate *
1587c478bd9Sstevel@tonic-gate * taskqid_t taskq_dispatch(tq, func, arg, flags):
1597c478bd9Sstevel@tonic-gate *
1607c478bd9Sstevel@tonic-gate * Dispatches the task "func(arg)" to taskq. The 'flags' indicates whether
1617c478bd9Sstevel@tonic-gate * the caller is willing to block for memory. The function returns an
1627c478bd9Sstevel@tonic-gate * opaque value which is zero iff dispatch fails. If flags is TQ_NOSLEEP
1637c478bd9Sstevel@tonic-gate * or TQ_NOALLOC and the task can't be dispatched, taskq_dispatch() fails
164fc8ae2ecSToomas Soome * and returns TASKQID_INVALID.
1657c478bd9Sstevel@tonic-gate *
1667c478bd9Sstevel@tonic-gate * ASSUMES: func != NULL.
1677c478bd9Sstevel@tonic-gate *
1687c478bd9Sstevel@tonic-gate * Possible flags:
1697c478bd9Sstevel@tonic-gate * TQ_NOSLEEP: Do not wait for resources; may fail.
1707c478bd9Sstevel@tonic-gate *
1717c478bd9Sstevel@tonic-gate * TQ_NOALLOC: Do not allocate memory; may fail. May only be used with
1727c478bd9Sstevel@tonic-gate * non-dynamic task queues.
1737c478bd9Sstevel@tonic-gate *
1747c478bd9Sstevel@tonic-gate * TQ_NOQUEUE: Do not enqueue a task if it can't dispatch it due to
1757c478bd9Sstevel@tonic-gate * lack of available resources and fail. If this flag is not
17635a5a358SJonathan Adams * set, and the task pool is exhausted, the task may be scheduled
1777c478bd9Sstevel@tonic-gate * in the backing queue. This flag may ONLY be used with dynamic
1787c478bd9Sstevel@tonic-gate * task queues.
1797c478bd9Sstevel@tonic-gate *
1807c478bd9Sstevel@tonic-gate * NOTE: This flag should always be used when a task queue is used
1817c478bd9Sstevel@tonic-gate * for tasks that may depend on each other for completion.
1827c478bd9Sstevel@tonic-gate * Enqueueing dependent tasks may create deadlocks.
1837c478bd9Sstevel@tonic-gate *
1847c478bd9Sstevel@tonic-gate * TQ_SLEEP: May block waiting for resources. May still fail for
18535a5a358SJonathan Adams * dynamic task queues if TQ_NOQUEUE is also specified, otherwise
1867c478bd9Sstevel@tonic-gate * always succeed.
1877c478bd9Sstevel@tonic-gate *
18835a5a358SJonathan Adams * TQ_FRONT: Puts the new task at the front of the queue. Be careful.
18935a5a358SJonathan Adams *
1907c478bd9Sstevel@tonic-gate * NOTE: Dynamic task queues are much more likely to fail in
1917c478bd9Sstevel@tonic-gate * taskq_dispatch() (especially if TQ_NOQUEUE was specified), so it
1927c478bd9Sstevel@tonic-gate * is important to have backup strategies handling such failures.
1937c478bd9Sstevel@tonic-gate *
1945aeb9474SGarrett D'Amore * void taskq_dispatch_ent(tq, func, arg, flags, tqent)
1955aeb9474SGarrett D'Amore *
1965aeb9474SGarrett D'Amore * This is a light-weight form of taskq_dispatch(), that uses a
1975aeb9474SGarrett D'Amore * preallocated taskq_ent_t structure for scheduling. As a
1985aeb9474SGarrett D'Amore * result, it does not perform allocations and cannot ever fail.
1995aeb9474SGarrett D'Amore * Note especially that it cannot be used with TASKQ_DYNAMIC
2005aeb9474SGarrett D'Amore * taskqs. The memory for the tqent must not be modified or used
2015aeb9474SGarrett D'Amore * until the function (func) is called. (However, func itself
2025aeb9474SGarrett D'Amore * may safely modify or free this memory, once it is called.)
2035aeb9474SGarrett D'Amore * Note that the taskq framework will NOT free this memory.
2045aeb9474SGarrett D'Amore *
2054c99ecc3STim Kordas * boolean_t taskq_empty(tq)
2064c99ecc3STim Kordas *
2074c99ecc3STim Kordas * Queries if there are tasks pending on the queue.
2084c99ecc3STim Kordas *
2097c478bd9Sstevel@tonic-gate * void taskq_wait(tq):
2107c478bd9Sstevel@tonic-gate *
2117c478bd9Sstevel@tonic-gate * Waits for all previously scheduled tasks to complete.
2127c478bd9Sstevel@tonic-gate *
2137c478bd9Sstevel@tonic-gate * NOTE: It does not stop any new task dispatches.
2147c478bd9Sstevel@tonic-gate * Do NOT call taskq_wait() from a task: it will cause deadlock.
2157c478bd9Sstevel@tonic-gate *
2167c478bd9Sstevel@tonic-gate * void taskq_suspend(tq)
2177c478bd9Sstevel@tonic-gate *
2187c478bd9Sstevel@tonic-gate * Suspend all task execution. Tasks already scheduled for a dynamic task
2197c478bd9Sstevel@tonic-gate * queue will still be executed, but all new scheduled tasks will be
2207c478bd9Sstevel@tonic-gate * suspended until taskq_resume() is called.
2217c478bd9Sstevel@tonic-gate *
2227c478bd9Sstevel@tonic-gate * int taskq_suspended(tq)
2237c478bd9Sstevel@tonic-gate *
2247c478bd9Sstevel@tonic-gate * Returns 1 if taskq is suspended and 0 otherwise. It is intended to
2257c478bd9Sstevel@tonic-gate * ASSERT that the task queue is suspended.
2267c478bd9Sstevel@tonic-gate *
2277c478bd9Sstevel@tonic-gate * void taskq_resume(tq)
2287c478bd9Sstevel@tonic-gate *
2297c478bd9Sstevel@tonic-gate * Resume task queue execution.
2307c478bd9Sstevel@tonic-gate *
2317c478bd9Sstevel@tonic-gate * int taskq_member(tq, thread)
2327c478bd9Sstevel@tonic-gate *
2337c478bd9Sstevel@tonic-gate * Returns 1 if 'thread' belongs to taskq 'tq' and 0 otherwise. The
2347c478bd9Sstevel@tonic-gate * intended use is to ASSERT that a given function is called in taskq
2357c478bd9Sstevel@tonic-gate * context only.
2367c478bd9Sstevel@tonic-gate *
2377c478bd9Sstevel@tonic-gate * system_taskq
2387c478bd9Sstevel@tonic-gate *
2397c478bd9Sstevel@tonic-gate * Global system-wide dynamic task queue for common uses. It may be used by
2407c478bd9Sstevel@tonic-gate * any subsystem that needs to schedule tasks and does not need to manage
2417c478bd9Sstevel@tonic-gate * its own task queues. It is initialized quite early during system boot.
2427c478bd9Sstevel@tonic-gate *
2432e0c549eSJonathan Adams * IMPLEMENTATION ==============================================================
2447c478bd9Sstevel@tonic-gate *
2457c478bd9Sstevel@tonic-gate * This is schematic representation of the task queue structures.
2467c478bd9Sstevel@tonic-gate *
2477c478bd9Sstevel@tonic-gate * taskq:
2487c478bd9Sstevel@tonic-gate * +-------------+
249e0ad97e3SJonathan Adams * | tq_lock | +---< taskq_ent_free()
2507c478bd9Sstevel@tonic-gate * +-------------+ |
2517c478bd9Sstevel@tonic-gate * |... | | tqent: tqent:
2527c478bd9Sstevel@tonic-gate * +-------------+ | +------------+ +------------+
2537c478bd9Sstevel@tonic-gate * | tq_freelist |-->| tqent_next |--> ... ->| tqent_next |
2547c478bd9Sstevel@tonic-gate * +-------------+ +------------+ +------------+
2557c478bd9Sstevel@tonic-gate * |... | | ... | | ... |
2567c478bd9Sstevel@tonic-gate * +-------------+ +------------+ +------------+
2577c478bd9Sstevel@tonic-gate * | tq_task | |
2587c478bd9Sstevel@tonic-gate * | | +-------------->taskq_ent_alloc()
2597c478bd9Sstevel@tonic-gate * +--------------------------------------------------------------------------+
2607c478bd9Sstevel@tonic-gate * | | | tqent tqent |
2617c478bd9Sstevel@tonic-gate * | +---------------------+ +--> +------------+ +--> +------------+ |
2627c478bd9Sstevel@tonic-gate * | | ... | | | func, arg | | | func, arg | |
2637c478bd9Sstevel@tonic-gate * +>+---------------------+ <---|-+ +------------+ <---|-+ +------------+ |
2647c478bd9Sstevel@tonic-gate * | tq_taskq.tqent_next | ----+ | | tqent_next | --->+ | | tqent_next |--+
2657c478bd9Sstevel@tonic-gate * +---------------------+ | +------------+ ^ | +------------+
2667c478bd9Sstevel@tonic-gate * +-| tq_task.tqent_prev | +--| tqent_prev | | +--| tqent_prev | ^
2677c478bd9Sstevel@tonic-gate * | +---------------------+ +------------+ | +------------+ |
2687c478bd9Sstevel@tonic-gate * | |... | | ... | | | ... | |
2697c478bd9Sstevel@tonic-gate * | +---------------------+ +------------+ | +------------+ |
2707c478bd9Sstevel@tonic-gate * | ^ | |
2717c478bd9Sstevel@tonic-gate * | | | |
2727c478bd9Sstevel@tonic-gate * +--------------------------------------+--------------+ TQ_APPEND() -+
2737c478bd9Sstevel@tonic-gate * | | |
2747c478bd9Sstevel@tonic-gate * |... | taskq_thread()-----+
2757c478bd9Sstevel@tonic-gate * +-------------+
2767c478bd9Sstevel@tonic-gate * | tq_buckets |--+-------> [ NULL ] (for regular task queues)
2777c478bd9Sstevel@tonic-gate * +-------------+ |
2787c478bd9Sstevel@tonic-gate * | DYNAMIC TASK QUEUES:
2797c478bd9Sstevel@tonic-gate * |
28035a5a358SJonathan Adams * +-> taskq_bucket[nCPU] taskq_bucket_dispatch()
2817c478bd9Sstevel@tonic-gate * +-------------------+ ^
2827c478bd9Sstevel@tonic-gate * +--->| tqbucket_lock | |
2837c478bd9Sstevel@tonic-gate * | +-------------------+ +--------+ +--------+
2847c478bd9Sstevel@tonic-gate * | | tqbucket_freelist |-->| tqent |-->...| tqent | ^
2857c478bd9Sstevel@tonic-gate * | +-------------------+<--+--------+<--...+--------+ |
2867c478bd9Sstevel@tonic-gate * | | ... | | thread | | thread | |
2877c478bd9Sstevel@tonic-gate * | +-------------------+ +--------+ +--------+ |
2887c478bd9Sstevel@tonic-gate * | +-------------------+ |
2897c478bd9Sstevel@tonic-gate * taskq_dispatch()--+--->| tqbucket_lock | TQ_APPEND()------+
2907c478bd9Sstevel@tonic-gate * TQ_HASH() | +-------------------+ +--------+ +--------+
2917c478bd9Sstevel@tonic-gate * | | tqbucket_freelist |-->| tqent |-->...| tqent |
2927c478bd9Sstevel@tonic-gate * | +-------------------+<--+--------+<--...+--------+
2937c478bd9Sstevel@tonic-gate * | | ... | | thread | | thread |
2947c478bd9Sstevel@tonic-gate * | +-------------------+ +--------+ +--------+
29535a5a358SJonathan Adams * +---> ...
2967c478bd9Sstevel@tonic-gate *
2977c478bd9Sstevel@tonic-gate *
2987c478bd9Sstevel@tonic-gate * Task queues use tq_task field to link new entry in the queue. The queue is a
2997c478bd9Sstevel@tonic-gate * circular doubly-linked list. Entries are put in the end of the list with
3007c478bd9Sstevel@tonic-gate * TQ_APPEND() and processed from the front of the list by taskq_thread() in
3017c478bd9Sstevel@tonic-gate * FIFO order. Task queue entries are cached in the free list managed by
3027c478bd9Sstevel@tonic-gate * taskq_ent_alloc() and taskq_ent_free() functions.
3037c478bd9Sstevel@tonic-gate *
3047c478bd9Sstevel@tonic-gate * All threads used by task queues mark t_taskq field of the thread to
3057c478bd9Sstevel@tonic-gate * point to the task queue.
3067c478bd9Sstevel@tonic-gate *
3072e0c549eSJonathan Adams * Taskq Thread Management -----------------------------------------------------
3082e0c549eSJonathan Adams *
3092e0c549eSJonathan Adams * Taskq's non-dynamic threads are managed with several variables and flags:
3102e0c549eSJonathan Adams *
3112e0c549eSJonathan Adams * * tq_nthreads - The number of threads in taskq_thread() for the
3122e0c549eSJonathan Adams * taskq.
3132e0c549eSJonathan Adams *
3142e0c549eSJonathan Adams * * tq_active - The number of threads not waiting on a CV in
3152e0c549eSJonathan Adams * taskq_thread(); includes newly created threads
3162e0c549eSJonathan Adams * not yet counted in tq_nthreads.
3172e0c549eSJonathan Adams *
3182e0c549eSJonathan Adams * * tq_nthreads_target
3192e0c549eSJonathan Adams * - The number of threads desired for the taskq.
3202e0c549eSJonathan Adams *
3212e0c549eSJonathan Adams * * tq_flags & TASKQ_CHANGING
3222e0c549eSJonathan Adams * - Indicates that tq_nthreads != tq_nthreads_target.
3232e0c549eSJonathan Adams *
3242e0c549eSJonathan Adams * * tq_flags & TASKQ_THREAD_CREATED
3252e0c549eSJonathan Adams * - Indicates that a thread is being created in the taskq.
3262e0c549eSJonathan Adams *
3272e0c549eSJonathan Adams * During creation, tq_nthreads and tq_active are set to 0, and
3282e0c549eSJonathan Adams * tq_nthreads_target is set to the number of threads desired. The
32935a5a358SJonathan Adams * TASKQ_CHANGING flag is set, and taskq_thread_create() is called to
33035a5a358SJonathan Adams * create the first thread. taskq_thread_create() increments tq_active,
3312e0c549eSJonathan Adams * sets TASKQ_THREAD_CREATED, and creates the new thread.
3322e0c549eSJonathan Adams *
3332e0c549eSJonathan Adams * Each thread starts in taskq_thread(), clears the TASKQ_THREAD_CREATED
3342e0c549eSJonathan Adams * flag, and increments tq_nthreads. It stores the new value of
3352e0c549eSJonathan Adams * tq_nthreads as its "thread_id", and stores its thread pointer in the
3362e0c549eSJonathan Adams * tq_threadlist at the (thread_id - 1). We keep the thread_id space
3372e0c549eSJonathan Adams * densely packed by requiring that only the largest thread_id can exit during
3382e0c549eSJonathan Adams * normal adjustment. The exception is during the destruction of the
3392e0c549eSJonathan Adams * taskq; once tq_nthreads_target is set to zero, no new threads will be created
3402e0c549eSJonathan Adams * for the taskq queue, so every thread can exit without any ordering being
3412e0c549eSJonathan Adams * necessary.
3422e0c549eSJonathan Adams *
3432e0c549eSJonathan Adams * Threads will only process work if their thread id is <= tq_nthreads_target.
3442e0c549eSJonathan Adams *
3452e0c549eSJonathan Adams * When TASKQ_CHANGING is set, threads will check the current thread target
3462e0c549eSJonathan Adams * whenever they wake up, and do whatever they can to apply its effects.
3472e0c549eSJonathan Adams *
3482e0c549eSJonathan Adams * TASKQ_THREAD_CPU_PCT --------------------------------------------------------
3492e0c549eSJonathan Adams *
3502e0c549eSJonathan Adams * When a taskq is created with TASKQ_THREAD_CPU_PCT, we store their requested
3512e0c549eSJonathan Adams * percentage in tq_threads_ncpus_pct, start them off with the correct thread
3522e0c549eSJonathan Adams * target, and add them to the taskq_cpupct_list for later adjustment.
3532e0c549eSJonathan Adams *
3542e0c549eSJonathan Adams * We register taskq_cpu_setup() to be called whenever a CPU changes state. It
3552e0c549eSJonathan Adams * walks the list of TASKQ_THREAD_CPU_PCT taskqs, adjusts their nthread_target
3562e0c549eSJonathan Adams * if need be, and wakes up all of the threads to process the change.
3572e0c549eSJonathan Adams *
3582e0c549eSJonathan Adams * Dynamic Task Queues Implementation ------------------------------------------
3597c478bd9Sstevel@tonic-gate *
3607c478bd9Sstevel@tonic-gate * For a dynamic task queues there is a 1-to-1 mapping between a thread and
3617c478bd9Sstevel@tonic-gate * taskq_ent_structure. Each entry is serviced by its own thread and each thread
3627c478bd9Sstevel@tonic-gate * is controlled by a single entry.
3637c478bd9Sstevel@tonic-gate *
3647c478bd9Sstevel@tonic-gate * Entries are distributed over a set of buckets. To avoid using modulo
3657c478bd9Sstevel@tonic-gate * arithmetics the number of buckets is 2^n and is determined as the nearest
3667c478bd9Sstevel@tonic-gate * power of two roundown of the number of CPUs in the system. Tunable
3677c478bd9Sstevel@tonic-gate * variable 'taskq_maxbuckets' limits the maximum number of buckets. Each entry
3687c478bd9Sstevel@tonic-gate * is attached to a bucket for its lifetime and can't migrate to other buckets.
3697c478bd9Sstevel@tonic-gate *
3707c478bd9Sstevel@tonic-gate * Entries that have scheduled tasks are not placed in any list. The dispatch
3717c478bd9Sstevel@tonic-gate * function sets their "func" and "arg" fields and signals the corresponding
3727c478bd9Sstevel@tonic-gate * thread to execute the task. Once the thread executes the task it clears the
3737c478bd9Sstevel@tonic-gate * "func" field and places an entry on the bucket cache of free entries pointed
3747c478bd9Sstevel@tonic-gate * by "tqbucket_freelist" field. ALL entries on the free list should have "func"
3757c478bd9Sstevel@tonic-gate * field equal to NULL. The free list is a circular doubly-linked list identical
3767c478bd9Sstevel@tonic-gate * in structure to the tq_task list above, but entries are taken from it in LIFO
3777c478bd9Sstevel@tonic-gate * order - the last freed entry is the first to be allocated. The
3787c478bd9Sstevel@tonic-gate * taskq_bucket_dispatch() function gets the most recently used entry from the
3797c478bd9Sstevel@tonic-gate * free list, sets its "func" and "arg" fields and signals a worker thread.
3807c478bd9Sstevel@tonic-gate *
3817c478bd9Sstevel@tonic-gate * After executing each task a per-entry thread taskq_d_thread() places its
3827c478bd9Sstevel@tonic-gate * entry on the bucket free list and goes to a timed sleep. If it wakes up
3837c478bd9Sstevel@tonic-gate * without getting new task it removes the entry from the free list and destroys
3847c478bd9Sstevel@tonic-gate * itself. The thread sleep time is controlled by a tunable variable
3857c478bd9Sstevel@tonic-gate * `taskq_thread_timeout'.
3867c478bd9Sstevel@tonic-gate *
3872e0c549eSJonathan Adams * There are various statistics kept in the bucket which allows for later
3887c478bd9Sstevel@tonic-gate * analysis of taskq usage patterns. Also, a global copy of taskq creation and
3897c478bd9Sstevel@tonic-gate * death statistics is kept in the global taskq data structure. Since thread
3907c478bd9Sstevel@tonic-gate * creation and death happen rarely, updating such global data does not present
3917c478bd9Sstevel@tonic-gate * a performance problem.
3927c478bd9Sstevel@tonic-gate *
3937c478bd9Sstevel@tonic-gate * NOTE: Threads are not bound to any CPU and there is absolutely no association
3947c478bd9Sstevel@tonic-gate * between the bucket and actual thread CPU, so buckets are used only to
3957c478bd9Sstevel@tonic-gate * split resources and reduce resource contention. Having threads attached
3967c478bd9Sstevel@tonic-gate * to the CPU denoted by a bucket may reduce number of times the job
3977c478bd9Sstevel@tonic-gate * switches between CPUs.
3987c478bd9Sstevel@tonic-gate *
3997c478bd9Sstevel@tonic-gate * Current algorithm creates a thread whenever a bucket has no free
4007c478bd9Sstevel@tonic-gate * entries. It would be nice to know how many threads are in the running
4017c478bd9Sstevel@tonic-gate * state and don't create threads if all CPUs are busy with existing
4027c478bd9Sstevel@tonic-gate * tasks, but it is unclear how such strategy can be implemented.
4037c478bd9Sstevel@tonic-gate *
4047c478bd9Sstevel@tonic-gate * Currently buckets are created statically as an array attached to task
4057c478bd9Sstevel@tonic-gate * queue. On some system with nCPUs < max_ncpus it may waste system
4067c478bd9Sstevel@tonic-gate * memory. One solution may be allocation of buckets when they are first
4077c478bd9Sstevel@tonic-gate * touched, but it is not clear how useful it is.
4087c478bd9Sstevel@tonic-gate *
4092e0c549eSJonathan Adams * SUSPEND/RESUME implementation -----------------------------------------------
4107c478bd9Sstevel@tonic-gate *
4117c478bd9Sstevel@tonic-gate * Before executing a task taskq_thread() (executing non-dynamic task
4127c478bd9Sstevel@tonic-gate * queues) obtains taskq's thread lock as a reader. The taskq_suspend()
4137c478bd9Sstevel@tonic-gate * function gets the same lock as a writer blocking all non-dynamic task
4147c478bd9Sstevel@tonic-gate * execution. The taskq_resume() function releases the lock allowing
4157c478bd9Sstevel@tonic-gate * taskq_thread to continue execution.
4167c478bd9Sstevel@tonic-gate *
4177c478bd9Sstevel@tonic-gate * For dynamic task queues, each bucket is marked as TQBUCKET_SUSPEND by
4187c478bd9Sstevel@tonic-gate * taskq_suspend() function. After that taskq_bucket_dispatch() always
4197c478bd9Sstevel@tonic-gate * fails, so that taskq_dispatch() will either enqueue tasks for a
4207c478bd9Sstevel@tonic-gate * suspended backing queue or fail if TQ_NOQUEUE is specified in dispatch
4217c478bd9Sstevel@tonic-gate * flags.
4227c478bd9Sstevel@tonic-gate *
4237c478bd9Sstevel@tonic-gate * NOTE: taskq_suspend() does not immediately block any tasks already
4247c478bd9Sstevel@tonic-gate * scheduled for dynamic task queues. It only suspends new tasks
4257c478bd9Sstevel@tonic-gate * scheduled after taskq_suspend() was called.
4267c478bd9Sstevel@tonic-gate *
4277c478bd9Sstevel@tonic-gate * taskq_member() function works by comparing a thread t_taskq pointer with
4287c478bd9Sstevel@tonic-gate * the passed thread pointer.
4297c478bd9Sstevel@tonic-gate *
4302e0c549eSJonathan Adams * LOCKS and LOCK Hierarchy ----------------------------------------------------
4317c478bd9Sstevel@tonic-gate *
4322e0c549eSJonathan Adams * There are three locks used in task queues:
4337c478bd9Sstevel@tonic-gate *
4342e0c549eSJonathan Adams * 1) The taskq_t's tq_lock, protecting global task queue state.
4357c478bd9Sstevel@tonic-gate *
4367c478bd9Sstevel@tonic-gate * 2) Each per-CPU bucket has a lock for bucket management.
4377c478bd9Sstevel@tonic-gate *
4382e0c549eSJonathan Adams * 3) The global taskq_cpupct_lock, which protects the list of
4392e0c549eSJonathan Adams * TASKQ_THREADS_CPU_PCT taskqs.
4402e0c549eSJonathan Adams *
4412e0c549eSJonathan Adams * If both (1) and (2) are needed, tq_lock should be taken *after* the bucket
4427c478bd9Sstevel@tonic-gate * lock.
4437c478bd9Sstevel@tonic-gate *
4442e0c549eSJonathan Adams * If both (1) and (3) are needed, tq_lock should be taken *after*
4452e0c549eSJonathan Adams * taskq_cpupct_lock.
4462e0c549eSJonathan Adams *
4472e0c549eSJonathan Adams * DEBUG FACILITIES ------------------------------------------------------------
4487c478bd9Sstevel@tonic-gate *
4497c478bd9Sstevel@tonic-gate * For DEBUG kernels it is possible to induce random failures to
4507c478bd9Sstevel@tonic-gate * taskq_dispatch() function when it is given TQ_NOSLEEP argument. The value of
4517c478bd9Sstevel@tonic-gate * taskq_dmtbf and taskq_smtbf tunables control the mean time between induced
4527c478bd9Sstevel@tonic-gate * failures for dynamic and static task queues respectively.
4537c478bd9Sstevel@tonic-gate *
4547c478bd9Sstevel@tonic-gate * Setting TASKQ_STATISTIC to 0 will disable per-bucket statistics.
4557c478bd9Sstevel@tonic-gate *
4562e0c549eSJonathan Adams * TUNABLES --------------------------------------------------------------------
4577c478bd9Sstevel@tonic-gate *
4587c478bd9Sstevel@tonic-gate * system_taskq_size - Size of the global system_taskq.
4597c478bd9Sstevel@tonic-gate * This value is multiplied by nCPUs to determine
4607c478bd9Sstevel@tonic-gate * actual size.
4617c478bd9Sstevel@tonic-gate * Default value: 64
4627c478bd9Sstevel@tonic-gate *
4632e0c549eSJonathan Adams * taskq_minimum_nthreads_max
4642e0c549eSJonathan Adams * - Minimum size of the thread list for a taskq.
4652e0c549eSJonathan Adams * Useful for testing different thread pool
4662e0c549eSJonathan Adams * sizes by overwriting tq_nthreads_target.
4672e0c549eSJonathan Adams *
4687c478bd9Sstevel@tonic-gate * taskq_thread_timeout - Maximum idle time for taskq_d_thread()
4697c478bd9Sstevel@tonic-gate * Default value: 5 minutes
4707c478bd9Sstevel@tonic-gate *
4717c478bd9Sstevel@tonic-gate * taskq_maxbuckets - Maximum number of buckets in any task queue
4727c478bd9Sstevel@tonic-gate * Default value: 128
4737c478bd9Sstevel@tonic-gate *
4747c478bd9Sstevel@tonic-gate * taskq_search_depth - Maximum # of buckets searched for a free entry
4757c478bd9Sstevel@tonic-gate * Default value: 4
4767c478bd9Sstevel@tonic-gate *
4777c478bd9Sstevel@tonic-gate * taskq_dmtbf - Mean time between induced dispatch failures
4787c478bd9Sstevel@tonic-gate * for dynamic task queues.
4797c478bd9Sstevel@tonic-gate * Default value: UINT_MAX (no induced failures)
4807c478bd9Sstevel@tonic-gate *
4817c478bd9Sstevel@tonic-gate * taskq_smtbf - Mean time between induced dispatch failures
4827c478bd9Sstevel@tonic-gate * for static task queues.
4837c478bd9Sstevel@tonic-gate * Default value: UINT_MAX (no induced failures)
4847c478bd9Sstevel@tonic-gate *
4852e0c549eSJonathan Adams * CONDITIONAL compilation -----------------------------------------------------
4867c478bd9Sstevel@tonic-gate *
4877c478bd9Sstevel@tonic-gate * TASKQ_STATISTIC - If set will enable bucket statistic (default).
4887c478bd9Sstevel@tonic-gate *
4897c478bd9Sstevel@tonic-gate */
4907c478bd9Sstevel@tonic-gate
4917c478bd9Sstevel@tonic-gate #include <sys/taskq_impl.h>
4927c478bd9Sstevel@tonic-gate #include <sys/thread.h>
4937c478bd9Sstevel@tonic-gate #include <sys/proc.h>
4947c478bd9Sstevel@tonic-gate #include <sys/kmem.h>
4957c478bd9Sstevel@tonic-gate #include <sys/vmem.h>
4967c478bd9Sstevel@tonic-gate #include <sys/callb.h>
49735a5a358SJonathan Adams #include <sys/class.h>
4987c478bd9Sstevel@tonic-gate #include <sys/systm.h>
4997c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h>
5007c478bd9Sstevel@tonic-gate #include <sys/debug.h>
5017c478bd9Sstevel@tonic-gate #include <sys/vmsystm.h> /* For throttlefree */
5027c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h>
5037c478bd9Sstevel@tonic-gate #include <sys/cpuvar.h>
50435a5a358SJonathan Adams #include <sys/cpupart.h>
5057c478bd9Sstevel@tonic-gate #include <sys/sdt.h>
50635a5a358SJonathan Adams #include <sys/sysdc.h>
5072e0c549eSJonathan Adams #include <sys/note.h>
5087c478bd9Sstevel@tonic-gate
5097c478bd9Sstevel@tonic-gate static kmem_cache_t *taskq_ent_cache, *taskq_cache;
5107c478bd9Sstevel@tonic-gate
5117c478bd9Sstevel@tonic-gate /*
5122e0c549eSJonathan Adams * Pseudo instance numbers for taskqs without explicitly provided instance.
5137c478bd9Sstevel@tonic-gate */
5147c478bd9Sstevel@tonic-gate static vmem_t *taskq_id_arena;
5157c478bd9Sstevel@tonic-gate
5167c478bd9Sstevel@tonic-gate /* Global system task queue for common use */
5177c478bd9Sstevel@tonic-gate taskq_t *system_taskq;
5187c478bd9Sstevel@tonic-gate
5197c478bd9Sstevel@tonic-gate /*
5202e0c549eSJonathan Adams * Maximum number of entries in global system taskq is
52135a5a358SJonathan Adams * system_taskq_size * max_ncpus
5227c478bd9Sstevel@tonic-gate */
5237c478bd9Sstevel@tonic-gate #define SYSTEM_TASKQ_SIZE 64
5247c478bd9Sstevel@tonic-gate int system_taskq_size = SYSTEM_TASKQ_SIZE;
5257c478bd9Sstevel@tonic-gate
5262e0c549eSJonathan Adams /*
5272e0c549eSJonathan Adams * Minimum size for tq_nthreads_max; useful for those who want to play around
5282e0c549eSJonathan Adams * with increasing a taskq's tq_nthreads_target.
5292e0c549eSJonathan Adams */
5302e0c549eSJonathan Adams int taskq_minimum_nthreads_max = 1;
5312e0c549eSJonathan Adams
53235a5a358SJonathan Adams /*
53335a5a358SJonathan Adams * We want to ensure that when taskq_create() returns, there is at least
53435a5a358SJonathan Adams * one thread ready to handle requests. To guarantee this, we have to wait
53535a5a358SJonathan Adams * for the second thread, since the first one cannot process requests until
53635a5a358SJonathan Adams * the second thread has been created.
53735a5a358SJonathan Adams */
53835a5a358SJonathan Adams #define TASKQ_CREATE_ACTIVE_THREADS 2
53935a5a358SJonathan Adams
5402e0c549eSJonathan Adams /* Maximum percentage allowed for TASKQ_THREADS_CPU_PCT */
5412e0c549eSJonathan Adams #define TASKQ_CPUPCT_MAX_PERCENT 1000
5422e0c549eSJonathan Adams int taskq_cpupct_max_percent = TASKQ_CPUPCT_MAX_PERCENT;
5432e0c549eSJonathan Adams
5447c478bd9Sstevel@tonic-gate /*
5457c478bd9Sstevel@tonic-gate * Dynamic task queue threads that don't get any work within
5467c478bd9Sstevel@tonic-gate * taskq_thread_timeout destroy themselves
5477c478bd9Sstevel@tonic-gate */
5487c478bd9Sstevel@tonic-gate #define TASKQ_THREAD_TIMEOUT (60 * 5)
5497c478bd9Sstevel@tonic-gate int taskq_thread_timeout = TASKQ_THREAD_TIMEOUT;
5507c478bd9Sstevel@tonic-gate
5517c478bd9Sstevel@tonic-gate #define TASKQ_MAXBUCKETS 128
5527c478bd9Sstevel@tonic-gate int taskq_maxbuckets = TASKQ_MAXBUCKETS;
5537c478bd9Sstevel@tonic-gate
5547c478bd9Sstevel@tonic-gate /*
5557c478bd9Sstevel@tonic-gate * When a bucket has no available entries another buckets are tried.
5567c478bd9Sstevel@tonic-gate * taskq_search_depth parameter limits the amount of buckets that we search
5577c478bd9Sstevel@tonic-gate * before failing. This is mostly useful in systems with many CPUs where we may
5587c478bd9Sstevel@tonic-gate * spend too much time scanning busy buckets.
5597c478bd9Sstevel@tonic-gate */
5607c478bd9Sstevel@tonic-gate #define TASKQ_SEARCH_DEPTH 4
5617c478bd9Sstevel@tonic-gate int taskq_search_depth = TASKQ_SEARCH_DEPTH;
5627c478bd9Sstevel@tonic-gate
5637c478bd9Sstevel@tonic-gate /*
5647c478bd9Sstevel@tonic-gate * Hashing function: mix various bits of x. May be pretty much anything.
5657c478bd9Sstevel@tonic-gate */
5667c478bd9Sstevel@tonic-gate #define TQ_HASH(x) ((x) ^ ((x) >> 11) ^ ((x) >> 17) ^ ((x) ^ 27))
5677c478bd9Sstevel@tonic-gate
5687c478bd9Sstevel@tonic-gate /*
5697c478bd9Sstevel@tonic-gate * We do not create any new threads when the system is low on memory and start
5707c478bd9Sstevel@tonic-gate * throttling memory allocations. The following macro tries to estimate such
5717c478bd9Sstevel@tonic-gate * condition.
5727c478bd9Sstevel@tonic-gate */
5737c478bd9Sstevel@tonic-gate #define ENOUGH_MEMORY() (freemem > throttlefree)
5747c478bd9Sstevel@tonic-gate
5757c478bd9Sstevel@tonic-gate /*
5767c478bd9Sstevel@tonic-gate * Static functions.
5777c478bd9Sstevel@tonic-gate */
5787c478bd9Sstevel@tonic-gate static taskq_t *taskq_create_common(const char *, int, int, pri_t, int,
57935a5a358SJonathan Adams int, proc_t *, uint_t, uint_t);
5807c478bd9Sstevel@tonic-gate static void taskq_thread(void *);
5817c478bd9Sstevel@tonic-gate static void taskq_d_thread(taskq_ent_t *);
5827c478bd9Sstevel@tonic-gate static void taskq_bucket_extend(void *);
5837c478bd9Sstevel@tonic-gate static int taskq_constructor(void *, void *, int);
5847c478bd9Sstevel@tonic-gate static void taskq_destructor(void *, void *);
5857c478bd9Sstevel@tonic-gate static int taskq_ent_constructor(void *, void *, int);
5867c478bd9Sstevel@tonic-gate static void taskq_ent_destructor(void *, void *);
5877c478bd9Sstevel@tonic-gate static taskq_ent_t *taskq_ent_alloc(taskq_t *, int);
5887c478bd9Sstevel@tonic-gate static void taskq_ent_free(taskq_t *, taskq_ent_t *);
58964109744SChris Horne static int taskq_ent_exists(taskq_t *, task_func_t, void *);
5907c478bd9Sstevel@tonic-gate static taskq_ent_t *taskq_bucket_dispatch(taskq_bucket_t *, task_func_t,
5917c478bd9Sstevel@tonic-gate void *);
5927c478bd9Sstevel@tonic-gate
5937c478bd9Sstevel@tonic-gate /*
5947c478bd9Sstevel@tonic-gate * Task queues kstats.
5957c478bd9Sstevel@tonic-gate */
5967c478bd9Sstevel@tonic-gate struct taskq_kstat {
59735a5a358SJonathan Adams kstat_named_t tq_pid;
5987c478bd9Sstevel@tonic-gate kstat_named_t tq_tasks;
5997c478bd9Sstevel@tonic-gate kstat_named_t tq_executed;
6007c478bd9Sstevel@tonic-gate kstat_named_t tq_maxtasks;
6017c478bd9Sstevel@tonic-gate kstat_named_t tq_totaltime;
6027c478bd9Sstevel@tonic-gate kstat_named_t tq_nalloc;
6037c478bd9Sstevel@tonic-gate kstat_named_t tq_nactive;
6047c478bd9Sstevel@tonic-gate kstat_named_t tq_pri;
6057c478bd9Sstevel@tonic-gate kstat_named_t tq_nthreads;
606216d7723SPrakash Surya kstat_named_t tq_nomem;
6077c478bd9Sstevel@tonic-gate } taskq_kstat = {
60835a5a358SJonathan Adams { "pid", KSTAT_DATA_UINT64 },
6097c478bd9Sstevel@tonic-gate { "tasks", KSTAT_DATA_UINT64 },
6107c478bd9Sstevel@tonic-gate { "executed", KSTAT_DATA_UINT64 },
6117c478bd9Sstevel@tonic-gate { "maxtasks", KSTAT_DATA_UINT64 },
6127c478bd9Sstevel@tonic-gate { "totaltime", KSTAT_DATA_UINT64 },
6137c478bd9Sstevel@tonic-gate { "nalloc", KSTAT_DATA_UINT64 },
6147c5f01c1SRobert Mustacchi { "nactive", KSTAT_DATA_UINT64 },
6157c478bd9Sstevel@tonic-gate { "priority", KSTAT_DATA_UINT64 },
6167c478bd9Sstevel@tonic-gate { "threads", KSTAT_DATA_UINT64 },
617216d7723SPrakash Surya { "nomem", KSTAT_DATA_UINT64 },
6187c478bd9Sstevel@tonic-gate };
6197c478bd9Sstevel@tonic-gate
6207c478bd9Sstevel@tonic-gate struct taskq_d_kstat {
6217c478bd9Sstevel@tonic-gate kstat_named_t tqd_pri;
6227c478bd9Sstevel@tonic-gate kstat_named_t tqd_btasks;
6237c478bd9Sstevel@tonic-gate kstat_named_t tqd_bexecuted;
6247c478bd9Sstevel@tonic-gate kstat_named_t tqd_bmaxtasks;
6257c478bd9Sstevel@tonic-gate kstat_named_t tqd_bnalloc;
6267c478bd9Sstevel@tonic-gate kstat_named_t tqd_bnactive;
6277c478bd9Sstevel@tonic-gate kstat_named_t tqd_btotaltime;
6287c478bd9Sstevel@tonic-gate kstat_named_t tqd_hits;
6297c478bd9Sstevel@tonic-gate kstat_named_t tqd_misses;
6307c478bd9Sstevel@tonic-gate kstat_named_t tqd_overflows;
6317c478bd9Sstevel@tonic-gate kstat_named_t tqd_tcreates;
6327c478bd9Sstevel@tonic-gate kstat_named_t tqd_tdeaths;
6337c478bd9Sstevel@tonic-gate kstat_named_t tqd_maxthreads;
6347c478bd9Sstevel@tonic-gate kstat_named_t tqd_nomem;
6357c478bd9Sstevel@tonic-gate kstat_named_t tqd_disptcreates;
6367c478bd9Sstevel@tonic-gate kstat_named_t tqd_totaltime;
6377c478bd9Sstevel@tonic-gate kstat_named_t tqd_nalloc;
6387c478bd9Sstevel@tonic-gate kstat_named_t tqd_nfree;
6397c478bd9Sstevel@tonic-gate } taskq_d_kstat = {
6407c478bd9Sstevel@tonic-gate { "priority", KSTAT_DATA_UINT64 },
6417c478bd9Sstevel@tonic-gate { "btasks", KSTAT_DATA_UINT64 },
6427c478bd9Sstevel@tonic-gate { "bexecuted", KSTAT_DATA_UINT64 },
6437c478bd9Sstevel@tonic-gate { "bmaxtasks", KSTAT_DATA_UINT64 },
6447c478bd9Sstevel@tonic-gate { "bnalloc", KSTAT_DATA_UINT64 },
6457c478bd9Sstevel@tonic-gate { "bnactive", KSTAT_DATA_UINT64 },
6467c478bd9Sstevel@tonic-gate { "btotaltime", KSTAT_DATA_UINT64 },
6477c478bd9Sstevel@tonic-gate { "hits", KSTAT_DATA_UINT64 },
6487c478bd9Sstevel@tonic-gate { "misses", KSTAT_DATA_UINT64 },
6497c478bd9Sstevel@tonic-gate { "overflows", KSTAT_DATA_UINT64 },
6507c478bd9Sstevel@tonic-gate { "tcreates", KSTAT_DATA_UINT64 },
6517c478bd9Sstevel@tonic-gate { "tdeaths", KSTAT_DATA_UINT64 },
6527c478bd9Sstevel@tonic-gate { "maxthreads", KSTAT_DATA_UINT64 },
6537c478bd9Sstevel@tonic-gate { "nomem", KSTAT_DATA_UINT64 },
6547c478bd9Sstevel@tonic-gate { "disptcreates", KSTAT_DATA_UINT64 },
6557c478bd9Sstevel@tonic-gate { "totaltime", KSTAT_DATA_UINT64 },
6567c478bd9Sstevel@tonic-gate { "nalloc", KSTAT_DATA_UINT64 },
6577c478bd9Sstevel@tonic-gate { "nfree", KSTAT_DATA_UINT64 },
6587c478bd9Sstevel@tonic-gate };
6597c478bd9Sstevel@tonic-gate
6607c478bd9Sstevel@tonic-gate static kmutex_t taskq_kstat_lock;
6617c478bd9Sstevel@tonic-gate static kmutex_t taskq_d_kstat_lock;
6627c478bd9Sstevel@tonic-gate static int taskq_kstat_update(kstat_t *, int);
6637c478bd9Sstevel@tonic-gate static int taskq_d_kstat_update(kstat_t *, int);
6647c478bd9Sstevel@tonic-gate
6652e0c549eSJonathan Adams /*
66635a5a358SJonathan Adams * List of all TASKQ_THREADS_CPU_PCT taskqs.
6672e0c549eSJonathan Adams */
66835a5a358SJonathan Adams static list_t taskq_cpupct_list; /* protected by cpu_lock */
6697c478bd9Sstevel@tonic-gate
6707c478bd9Sstevel@tonic-gate /*
6717c478bd9Sstevel@tonic-gate * Collect per-bucket statistic when TASKQ_STATISTIC is defined.
6727c478bd9Sstevel@tonic-gate */
6737c478bd9Sstevel@tonic-gate #define TASKQ_STATISTIC 1
6747c478bd9Sstevel@tonic-gate
6757c478bd9Sstevel@tonic-gate #if TASKQ_STATISTIC
6767c478bd9Sstevel@tonic-gate #define TQ_STAT(b, x) b->tqbucket_stat.x++
6777c478bd9Sstevel@tonic-gate #else
6787c478bd9Sstevel@tonic-gate #define TQ_STAT(b, x)
6797c478bd9Sstevel@tonic-gate #endif
6807c478bd9Sstevel@tonic-gate
6817c478bd9Sstevel@tonic-gate /*
6827c478bd9Sstevel@tonic-gate * Random fault injection.
6837c478bd9Sstevel@tonic-gate */
6847c478bd9Sstevel@tonic-gate uint_t taskq_random;
6857c478bd9Sstevel@tonic-gate uint_t taskq_dmtbf = UINT_MAX; /* mean time between injected failures */
6867c478bd9Sstevel@tonic-gate uint_t taskq_smtbf = UINT_MAX; /* mean time between injected failures */
6877c478bd9Sstevel@tonic-gate
6887c478bd9Sstevel@tonic-gate /*
6897c478bd9Sstevel@tonic-gate * TQ_NOSLEEP dispatches on dynamic task queues are always allowed to fail.
6907c478bd9Sstevel@tonic-gate *
6917c478bd9Sstevel@tonic-gate * TQ_NOSLEEP dispatches on static task queues can't arbitrarily fail because
6927c478bd9Sstevel@tonic-gate * they could prepopulate the cache and make sure that they do not use more
6937c478bd9Sstevel@tonic-gate * then minalloc entries. So, fault injection in this case insures that
6947c478bd9Sstevel@tonic-gate * either TASKQ_PREPOPULATE is not set or there are more entries allocated
6957c478bd9Sstevel@tonic-gate * than is specified by minalloc. TQ_NOALLOC dispatches are always allowed
6967c478bd9Sstevel@tonic-gate * to fail, but for simplicity we treat them identically to TQ_NOSLEEP
6977c478bd9Sstevel@tonic-gate * dispatches.
6987c478bd9Sstevel@tonic-gate */
6997c478bd9Sstevel@tonic-gate #ifdef DEBUG
7007c478bd9Sstevel@tonic-gate #define TASKQ_D_RANDOM_DISPATCH_FAILURE(tq, flag) \
7017c478bd9Sstevel@tonic-gate taskq_random = (taskq_random * 2416 + 374441) % 1771875;\
7027c478bd9Sstevel@tonic-gate if ((flag & TQ_NOSLEEP) && \
7037c478bd9Sstevel@tonic-gate taskq_random < 1771875 / taskq_dmtbf) { \
704fc8ae2ecSToomas Soome return (TASKQID_INVALID); \
7057c478bd9Sstevel@tonic-gate }
7067c478bd9Sstevel@tonic-gate
7077c478bd9Sstevel@tonic-gate #define TASKQ_S_RANDOM_DISPATCH_FAILURE(tq, flag) \
7087c478bd9Sstevel@tonic-gate taskq_random = (taskq_random * 2416 + 374441) % 1771875;\
7097c478bd9Sstevel@tonic-gate if ((flag & (TQ_NOSLEEP | TQ_NOALLOC)) && \
7107c478bd9Sstevel@tonic-gate (!(tq->tq_flags & TASKQ_PREPOPULATE) || \
7117c478bd9Sstevel@tonic-gate (tq->tq_nalloc > tq->tq_minalloc)) && \
7127c478bd9Sstevel@tonic-gate (taskq_random < (1771875 / taskq_smtbf))) { \
7137c478bd9Sstevel@tonic-gate mutex_exit(&tq->tq_lock); \
714fc8ae2ecSToomas Soome return (TASKQID_INVALID); \
7157c478bd9Sstevel@tonic-gate }
7167c478bd9Sstevel@tonic-gate #else
7177c478bd9Sstevel@tonic-gate #define TASKQ_S_RANDOM_DISPATCH_FAILURE(tq, flag)
7187c478bd9Sstevel@tonic-gate #define TASKQ_D_RANDOM_DISPATCH_FAILURE(tq, flag)
7197c478bd9Sstevel@tonic-gate #endif
7207c478bd9Sstevel@tonic-gate
7217c478bd9Sstevel@tonic-gate #define IS_EMPTY(l) (((l).tqent_prev == (l).tqent_next) && \
7227c478bd9Sstevel@tonic-gate ((l).tqent_prev == &(l)))
7237c478bd9Sstevel@tonic-gate
7247c478bd9Sstevel@tonic-gate /*
7257c478bd9Sstevel@tonic-gate * Append `tqe' in the end of the doubly-linked list denoted by l.
7267c478bd9Sstevel@tonic-gate */
7277c478bd9Sstevel@tonic-gate #define TQ_APPEND(l, tqe) { \
7287c478bd9Sstevel@tonic-gate tqe->tqent_next = &l; \
7297c478bd9Sstevel@tonic-gate tqe->tqent_prev = l.tqent_prev; \
7307c478bd9Sstevel@tonic-gate tqe->tqent_next->tqent_prev = tqe; \
7317c478bd9Sstevel@tonic-gate tqe->tqent_prev->tqent_next = tqe; \
7327c478bd9Sstevel@tonic-gate }
73335a5a358SJonathan Adams /*
73435a5a358SJonathan Adams * Prepend 'tqe' to the beginning of l
73535a5a358SJonathan Adams */
73635a5a358SJonathan Adams #define TQ_PREPEND(l, tqe) { \
73735a5a358SJonathan Adams tqe->tqent_next = l.tqent_next; \
73835a5a358SJonathan Adams tqe->tqent_prev = &l; \
73935a5a358SJonathan Adams tqe->tqent_next->tqent_prev = tqe; \
74035a5a358SJonathan Adams tqe->tqent_prev->tqent_next = tqe; \
74135a5a358SJonathan Adams }
7427c478bd9Sstevel@tonic-gate
7437c478bd9Sstevel@tonic-gate /*
7447c478bd9Sstevel@tonic-gate * Schedule a task specified by func and arg into the task queue entry tqe.
7457c478bd9Sstevel@tonic-gate */
74635a5a358SJonathan Adams #define TQ_DO_ENQUEUE(tq, tqe, func, arg, front) { \
74735a5a358SJonathan Adams ASSERT(MUTEX_HELD(&tq->tq_lock)); \
74835a5a358SJonathan Adams _NOTE(CONSTCOND) \
74935a5a358SJonathan Adams if (front) { \
75035a5a358SJonathan Adams TQ_PREPEND(tq->tq_task, tqe); \
75135a5a358SJonathan Adams } else { \
75235a5a358SJonathan Adams TQ_APPEND(tq->tq_task, tqe); \
75335a5a358SJonathan Adams } \
75435a5a358SJonathan Adams tqe->tqent_func = (func); \
75535a5a358SJonathan Adams tqe->tqent_arg = (arg); \
75635a5a358SJonathan Adams tq->tq_tasks++; \
75735a5a358SJonathan Adams if (tq->tq_tasks - tq->tq_executed > tq->tq_maxtasks) \
7587c478bd9Sstevel@tonic-gate tq->tq_maxtasks = tq->tq_tasks - tq->tq_executed; \
75935a5a358SJonathan Adams cv_signal(&tq->tq_dispatch_cv); \
7607c478bd9Sstevel@tonic-gate DTRACE_PROBE2(taskq__enqueue, taskq_t *, tq, taskq_ent_t *, tqe); \
7617c478bd9Sstevel@tonic-gate }
7627c478bd9Sstevel@tonic-gate
76335a5a358SJonathan Adams #define TQ_ENQUEUE(tq, tqe, func, arg) \
76435a5a358SJonathan Adams TQ_DO_ENQUEUE(tq, tqe, func, arg, 0)
76535a5a358SJonathan Adams
76635a5a358SJonathan Adams #define TQ_ENQUEUE_FRONT(tq, tqe, func, arg) \
76735a5a358SJonathan Adams TQ_DO_ENQUEUE(tq, tqe, func, arg, 1)
76835a5a358SJonathan Adams
7697c478bd9Sstevel@tonic-gate /*
7707c478bd9Sstevel@tonic-gate * Do-nothing task which may be used to prepopulate thread caches.
7717c478bd9Sstevel@tonic-gate */
7727c478bd9Sstevel@tonic-gate /*ARGSUSED*/
7737c478bd9Sstevel@tonic-gate void
nulltask(void * unused)7747c478bd9Sstevel@tonic-gate nulltask(void *unused)
7757c478bd9Sstevel@tonic-gate {
7767c478bd9Sstevel@tonic-gate }
7777c478bd9Sstevel@tonic-gate
7787c478bd9Sstevel@tonic-gate /*ARGSUSED*/
7797c478bd9Sstevel@tonic-gate static int
taskq_constructor(void * buf,void * cdrarg,int kmflags)7807c478bd9Sstevel@tonic-gate taskq_constructor(void *buf, void *cdrarg, int kmflags)
7817c478bd9Sstevel@tonic-gate {
7827c478bd9Sstevel@tonic-gate taskq_t *tq = buf;
7837c478bd9Sstevel@tonic-gate
7847c478bd9Sstevel@tonic-gate bzero(tq, sizeof (taskq_t));
7857c478bd9Sstevel@tonic-gate
7867c478bd9Sstevel@tonic-gate mutex_init(&tq->tq_lock, NULL, MUTEX_DEFAULT, NULL);
7877c478bd9Sstevel@tonic-gate rw_init(&tq->tq_threadlock, NULL, RW_DEFAULT, NULL);
7887c478bd9Sstevel@tonic-gate cv_init(&tq->tq_dispatch_cv, NULL, CV_DEFAULT, NULL);
7892e0c549eSJonathan Adams cv_init(&tq->tq_exit_cv, NULL, CV_DEFAULT, NULL);
7907c478bd9Sstevel@tonic-gate cv_init(&tq->tq_wait_cv, NULL, CV_DEFAULT, NULL);
79164109744SChris Horne cv_init(&tq->tq_maxalloc_cv, NULL, CV_DEFAULT, NULL);
7927c478bd9Sstevel@tonic-gate
7937c478bd9Sstevel@tonic-gate tq->tq_task.tqent_next = &tq->tq_task;
7947c478bd9Sstevel@tonic-gate tq->tq_task.tqent_prev = &tq->tq_task;
7957c478bd9Sstevel@tonic-gate
7967c478bd9Sstevel@tonic-gate return (0);
7977c478bd9Sstevel@tonic-gate }
7987c478bd9Sstevel@tonic-gate
7997c478bd9Sstevel@tonic-gate /*ARGSUSED*/
8007c478bd9Sstevel@tonic-gate static void
taskq_destructor(void * buf,void * cdrarg)8017c478bd9Sstevel@tonic-gate taskq_destructor(void *buf, void *cdrarg)
8027c478bd9Sstevel@tonic-gate {
8037c478bd9Sstevel@tonic-gate taskq_t *tq = buf;
8047c478bd9Sstevel@tonic-gate
8052e0c549eSJonathan Adams ASSERT(tq->tq_nthreads == 0);
8062e0c549eSJonathan Adams ASSERT(tq->tq_buckets == NULL);
8072e0c549eSJonathan Adams ASSERT(tq->tq_tcreates == 0);
8082e0c549eSJonathan Adams ASSERT(tq->tq_tdeaths == 0);
8092e0c549eSJonathan Adams
8107c478bd9Sstevel@tonic-gate mutex_destroy(&tq->tq_lock);
8117c478bd9Sstevel@tonic-gate rw_destroy(&tq->tq_threadlock);
8127c478bd9Sstevel@tonic-gate cv_destroy(&tq->tq_dispatch_cv);
8132e0c549eSJonathan Adams cv_destroy(&tq->tq_exit_cv);
8147c478bd9Sstevel@tonic-gate cv_destroy(&tq->tq_wait_cv);
81564109744SChris Horne cv_destroy(&tq->tq_maxalloc_cv);
8167c478bd9Sstevel@tonic-gate }
8177c478bd9Sstevel@tonic-gate
8187c478bd9Sstevel@tonic-gate /*ARGSUSED*/
8197c478bd9Sstevel@tonic-gate static int
taskq_ent_constructor(void * buf,void * cdrarg,int kmflags)8207c478bd9Sstevel@tonic-gate taskq_ent_constructor(void *buf, void *cdrarg, int kmflags)
8217c478bd9Sstevel@tonic-gate {
8227c478bd9Sstevel@tonic-gate taskq_ent_t *tqe = buf;
8237c478bd9Sstevel@tonic-gate
8247c478bd9Sstevel@tonic-gate tqe->tqent_thread = NULL;
8257c478bd9Sstevel@tonic-gate cv_init(&tqe->tqent_cv, NULL, CV_DEFAULT, NULL);
8267c478bd9Sstevel@tonic-gate
8277c478bd9Sstevel@tonic-gate return (0);
8287c478bd9Sstevel@tonic-gate }
8297c478bd9Sstevel@tonic-gate
8307c478bd9Sstevel@tonic-gate /*ARGSUSED*/
8317c478bd9Sstevel@tonic-gate static void
taskq_ent_destructor(void * buf,void * cdrarg)8327c478bd9Sstevel@tonic-gate taskq_ent_destructor(void *buf, void *cdrarg)
8337c478bd9Sstevel@tonic-gate {
8347c478bd9Sstevel@tonic-gate taskq_ent_t *tqe = buf;
8357c478bd9Sstevel@tonic-gate
8367c478bd9Sstevel@tonic-gate ASSERT(tqe->tqent_thread == NULL);
8377c478bd9Sstevel@tonic-gate cv_destroy(&tqe->tqent_cv);
8387c478bd9Sstevel@tonic-gate }
8397c478bd9Sstevel@tonic-gate
8407c478bd9Sstevel@tonic-gate void
taskq_init(void)8417c478bd9Sstevel@tonic-gate taskq_init(void)
8427c478bd9Sstevel@tonic-gate {
8437c478bd9Sstevel@tonic-gate taskq_ent_cache = kmem_cache_create("taskq_ent_cache",
8447c478bd9Sstevel@tonic-gate sizeof (taskq_ent_t), 0, taskq_ent_constructor,
8457c478bd9Sstevel@tonic-gate taskq_ent_destructor, NULL, NULL, NULL, 0);
8467c478bd9Sstevel@tonic-gate taskq_cache = kmem_cache_create("taskq_cache", sizeof (taskq_t),
8477c478bd9Sstevel@tonic-gate 0, taskq_constructor, taskq_destructor, NULL, NULL, NULL, 0);
8487c478bd9Sstevel@tonic-gate taskq_id_arena = vmem_create("taskq_id_arena",
8497c478bd9Sstevel@tonic-gate (void *)1, INT32_MAX, 1, NULL, NULL, NULL, 0,
8507c478bd9Sstevel@tonic-gate VM_SLEEP | VMC_IDENTIFIER);
8512e0c549eSJonathan Adams
85235a5a358SJonathan Adams list_create(&taskq_cpupct_list, sizeof (taskq_t),
85335a5a358SJonathan Adams offsetof(taskq_t, tq_cpupct_link));
85435a5a358SJonathan Adams }
85535a5a358SJonathan Adams
85635a5a358SJonathan Adams static void
taskq_update_nthreads(taskq_t * tq,uint_t ncpus)85735a5a358SJonathan Adams taskq_update_nthreads(taskq_t *tq, uint_t ncpus)
85835a5a358SJonathan Adams {
85935a5a358SJonathan Adams uint_t newtarget = TASKQ_THREADS_PCT(ncpus, tq->tq_threads_ncpus_pct);
86035a5a358SJonathan Adams
86135a5a358SJonathan Adams ASSERT(MUTEX_HELD(&cpu_lock));
86235a5a358SJonathan Adams ASSERT(MUTEX_HELD(&tq->tq_lock));
86335a5a358SJonathan Adams
86435a5a358SJonathan Adams /* We must be going from non-zero to non-zero; no exiting. */
86535a5a358SJonathan Adams ASSERT3U(tq->tq_nthreads_target, !=, 0);
86635a5a358SJonathan Adams ASSERT3U(newtarget, !=, 0);
86735a5a358SJonathan Adams
86835a5a358SJonathan Adams ASSERT3U(newtarget, <=, tq->tq_nthreads_max);
86935a5a358SJonathan Adams if (newtarget != tq->tq_nthreads_target) {
87035a5a358SJonathan Adams tq->tq_flags |= TASKQ_CHANGING;
87135a5a358SJonathan Adams tq->tq_nthreads_target = newtarget;
87235a5a358SJonathan Adams cv_broadcast(&tq->tq_dispatch_cv);
87335a5a358SJonathan Adams cv_broadcast(&tq->tq_exit_cv);
87435a5a358SJonathan Adams }
87535a5a358SJonathan Adams }
87635a5a358SJonathan Adams
87735a5a358SJonathan Adams /* called during task queue creation */
87835a5a358SJonathan Adams static void
taskq_cpupct_install(taskq_t * tq,cpupart_t * cpup)87935a5a358SJonathan Adams taskq_cpupct_install(taskq_t *tq, cpupart_t *cpup)
88035a5a358SJonathan Adams {
88135a5a358SJonathan Adams ASSERT(tq->tq_flags & TASKQ_THREADS_CPU_PCT);
88235a5a358SJonathan Adams
88335a5a358SJonathan Adams mutex_enter(&cpu_lock);
88435a5a358SJonathan Adams mutex_enter(&tq->tq_lock);
88535a5a358SJonathan Adams tq->tq_cpupart = cpup->cp_id;
88635a5a358SJonathan Adams taskq_update_nthreads(tq, cpup->cp_ncpus);
88735a5a358SJonathan Adams mutex_exit(&tq->tq_lock);
88835a5a358SJonathan Adams
88935a5a358SJonathan Adams list_insert_tail(&taskq_cpupct_list, tq);
89035a5a358SJonathan Adams mutex_exit(&cpu_lock);
89135a5a358SJonathan Adams }
89235a5a358SJonathan Adams
89335a5a358SJonathan Adams static void
taskq_cpupct_remove(taskq_t * tq)89435a5a358SJonathan Adams taskq_cpupct_remove(taskq_t *tq)
89535a5a358SJonathan Adams {
89635a5a358SJonathan Adams ASSERT(tq->tq_flags & TASKQ_THREADS_CPU_PCT);
89735a5a358SJonathan Adams
89835a5a358SJonathan Adams mutex_enter(&cpu_lock);
89935a5a358SJonathan Adams list_remove(&taskq_cpupct_list, tq);
90035a5a358SJonathan Adams mutex_exit(&cpu_lock);
9012e0c549eSJonathan Adams }
9022e0c549eSJonathan Adams
9032e0c549eSJonathan Adams /*ARGSUSED*/
9042e0c549eSJonathan Adams static int
taskq_cpu_setup(cpu_setup_t what,int id,void * arg)9052e0c549eSJonathan Adams taskq_cpu_setup(cpu_setup_t what, int id, void *arg)
9062e0c549eSJonathan Adams {
90735a5a358SJonathan Adams taskq_t *tq;
90835a5a358SJonathan Adams cpupart_t *cp = cpu[id]->cpu_part;
90935a5a358SJonathan Adams uint_t ncpus = cp->cp_ncpus;
91035a5a358SJonathan Adams
91135a5a358SJonathan Adams ASSERT(MUTEX_HELD(&cpu_lock));
91235a5a358SJonathan Adams ASSERT(ncpus > 0);
91335a5a358SJonathan Adams
91435a5a358SJonathan Adams switch (what) {
91535a5a358SJonathan Adams case CPU_OFF:
91635a5a358SJonathan Adams case CPU_CPUPART_OUT:
91735a5a358SJonathan Adams /* offlines are called *before* the cpu is offlined. */
91835a5a358SJonathan Adams if (ncpus > 1)
91935a5a358SJonathan Adams ncpus--;
92035a5a358SJonathan Adams break;
92135a5a358SJonathan Adams
92235a5a358SJonathan Adams case CPU_ON:
92335a5a358SJonathan Adams case CPU_CPUPART_IN:
92435a5a358SJonathan Adams break;
92535a5a358SJonathan Adams
92635a5a358SJonathan Adams default:
92735a5a358SJonathan Adams return (0); /* doesn't affect cpu count */
9282e0c549eSJonathan Adams }
9292e0c549eSJonathan Adams
93035a5a358SJonathan Adams for (tq = list_head(&taskq_cpupct_list); tq != NULL;
93135a5a358SJonathan Adams tq = list_next(&taskq_cpupct_list, tq)) {
9322e0c549eSJonathan Adams
9332e0c549eSJonathan Adams mutex_enter(&tq->tq_lock);
93435a5a358SJonathan Adams /*
93535a5a358SJonathan Adams * If the taskq is part of the cpuset which is changing,
93635a5a358SJonathan Adams * update its nthreads_target.
93735a5a358SJonathan Adams */
93835a5a358SJonathan Adams if (tq->tq_cpupart == cp->cp_id) {
93935a5a358SJonathan Adams taskq_update_nthreads(tq, ncpus);
9402e0c549eSJonathan Adams }
9412e0c549eSJonathan Adams mutex_exit(&tq->tq_lock);
9422e0c549eSJonathan Adams }
9432e0c549eSJonathan Adams return (0);
9442e0c549eSJonathan Adams }
9452e0c549eSJonathan Adams
9462e0c549eSJonathan Adams void
taskq_mp_init(void)9472e0c549eSJonathan Adams taskq_mp_init(void)
9482e0c549eSJonathan Adams {
9492e0c549eSJonathan Adams mutex_enter(&cpu_lock);
9502e0c549eSJonathan Adams register_cpu_setup_func(taskq_cpu_setup, NULL);
95135a5a358SJonathan Adams /*
95235a5a358SJonathan Adams * Make sure we're up to date. At this point in boot, there is only
95335a5a358SJonathan Adams * one processor set, so we only have to update the current CPU.
95435a5a358SJonathan Adams */
95535a5a358SJonathan Adams (void) taskq_cpu_setup(CPU_ON, CPU->cpu_id, NULL);
9562e0c549eSJonathan Adams mutex_exit(&cpu_lock);
9577c478bd9Sstevel@tonic-gate }
9587c478bd9Sstevel@tonic-gate
9597c478bd9Sstevel@tonic-gate /*
9607c478bd9Sstevel@tonic-gate * Create global system dynamic task queue.
9617c478bd9Sstevel@tonic-gate */
9627c478bd9Sstevel@tonic-gate void
system_taskq_init(void)9637c478bd9Sstevel@tonic-gate system_taskq_init(void)
9647c478bd9Sstevel@tonic-gate {
9657c478bd9Sstevel@tonic-gate system_taskq = taskq_create_common("system_taskq", 0,
96635a5a358SJonathan Adams system_taskq_size * max_ncpus, minclsyspri, 4, 512, &p0, 0,
9677c478bd9Sstevel@tonic-gate TASKQ_DYNAMIC | TASKQ_PREPOPULATE);
9687c478bd9Sstevel@tonic-gate }
9697c478bd9Sstevel@tonic-gate
9707c478bd9Sstevel@tonic-gate /*
9717c478bd9Sstevel@tonic-gate * taskq_ent_alloc()
9727c478bd9Sstevel@tonic-gate *
9737c478bd9Sstevel@tonic-gate * Allocates a new taskq_ent_t structure either from the free list or from the
9747c478bd9Sstevel@tonic-gate * cache. Returns NULL if it can't be allocated.
9757c478bd9Sstevel@tonic-gate *
9767c478bd9Sstevel@tonic-gate * Assumes: tq->tq_lock is held.
9777c478bd9Sstevel@tonic-gate */
9787c478bd9Sstevel@tonic-gate static taskq_ent_t *
taskq_ent_alloc(taskq_t * tq,int flags)9797c478bd9Sstevel@tonic-gate taskq_ent_alloc(taskq_t *tq, int flags)
9807c478bd9Sstevel@tonic-gate {
9817c478bd9Sstevel@tonic-gate int kmflags = (flags & TQ_NOSLEEP) ? KM_NOSLEEP : KM_SLEEP;
9827c478bd9Sstevel@tonic-gate taskq_ent_t *tqe;
98364109744SChris Horne clock_t wait_time;
98464109744SChris Horne clock_t wait_rv;
9857c478bd9Sstevel@tonic-gate
9867c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&tq->tq_lock));
9877c478bd9Sstevel@tonic-gate
9887c478bd9Sstevel@tonic-gate /*
9897c478bd9Sstevel@tonic-gate * TQ_NOALLOC allocations are allowed to use the freelist, even if
9907c478bd9Sstevel@tonic-gate * we are below tq_minalloc.
9917c478bd9Sstevel@tonic-gate */
99264109744SChris Horne again: if ((tqe = tq->tq_freelist) != NULL &&
9937c478bd9Sstevel@tonic-gate ((flags & TQ_NOALLOC) || tq->tq_nalloc >= tq->tq_minalloc)) {
9947c478bd9Sstevel@tonic-gate tq->tq_freelist = tqe->tqent_next;
9957c478bd9Sstevel@tonic-gate } else {
9967c478bd9Sstevel@tonic-gate if (flags & TQ_NOALLOC)
9977c478bd9Sstevel@tonic-gate return (NULL);
9987c478bd9Sstevel@tonic-gate
9997c478bd9Sstevel@tonic-gate if (tq->tq_nalloc >= tq->tq_maxalloc) {
100064109744SChris Horne if (kmflags & KM_NOSLEEP)
10017c478bd9Sstevel@tonic-gate return (NULL);
100264109744SChris Horne
10037c478bd9Sstevel@tonic-gate /*
10047c478bd9Sstevel@tonic-gate * We don't want to exceed tq_maxalloc, but we can't
10057c478bd9Sstevel@tonic-gate * wait for other tasks to complete (and thus free up
10067c478bd9Sstevel@tonic-gate * task structures) without risking deadlock with
10077c478bd9Sstevel@tonic-gate * the caller. So, we just delay for one second
100864109744SChris Horne * to throttle the allocation rate. If we have tasks
100964109744SChris Horne * complete before one second timeout expires then
101064109744SChris Horne * taskq_ent_free will signal us and we will
101164109744SChris Horne * immediately retry the allocation (reap free).
10127c478bd9Sstevel@tonic-gate */
101364109744SChris Horne wait_time = ddi_get_lbolt() + hz;
101464109744SChris Horne while (tq->tq_freelist == NULL) {
101564109744SChris Horne tq->tq_maxalloc_wait++;
101664109744SChris Horne wait_rv = cv_timedwait(&tq->tq_maxalloc_cv,
101764109744SChris Horne &tq->tq_lock, wait_time);
101864109744SChris Horne tq->tq_maxalloc_wait--;
101964109744SChris Horne if (wait_rv == -1)
102064109744SChris Horne break;
102164109744SChris Horne }
102264109744SChris Horne if (tq->tq_freelist)
102364109744SChris Horne goto again; /* reap freelist */
102464109744SChris Horne
10257c478bd9Sstevel@tonic-gate }
102664109744SChris Horne mutex_exit(&tq->tq_lock);
102764109744SChris Horne
10287c478bd9Sstevel@tonic-gate tqe = kmem_cache_alloc(taskq_ent_cache, kmflags);
102964109744SChris Horne
10307c478bd9Sstevel@tonic-gate mutex_enter(&tq->tq_lock);
10317c478bd9Sstevel@tonic-gate if (tqe != NULL)
10327c478bd9Sstevel@tonic-gate tq->tq_nalloc++;
10337c478bd9Sstevel@tonic-gate }
10347c478bd9Sstevel@tonic-gate return (tqe);
10357c478bd9Sstevel@tonic-gate }
10367c478bd9Sstevel@tonic-gate
10377c478bd9Sstevel@tonic-gate /*
10387c478bd9Sstevel@tonic-gate * taskq_ent_free()
10397c478bd9Sstevel@tonic-gate *
10407c478bd9Sstevel@tonic-gate * Free taskq_ent_t structure by either putting it on the free list or freeing
10417c478bd9Sstevel@tonic-gate * it to the cache.
10427c478bd9Sstevel@tonic-gate *
10437c478bd9Sstevel@tonic-gate * Assumes: tq->tq_lock is held.
10447c478bd9Sstevel@tonic-gate */
10457c478bd9Sstevel@tonic-gate static void
taskq_ent_free(taskq_t * tq,taskq_ent_t * tqe)10467c478bd9Sstevel@tonic-gate taskq_ent_free(taskq_t *tq, taskq_ent_t *tqe)
10477c478bd9Sstevel@tonic-gate {
10487c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&tq->tq_lock));
10497c478bd9Sstevel@tonic-gate
10507c478bd9Sstevel@tonic-gate if (tq->tq_nalloc <= tq->tq_minalloc) {
10517c478bd9Sstevel@tonic-gate tqe->tqent_next = tq->tq_freelist;
10527c478bd9Sstevel@tonic-gate tq->tq_freelist = tqe;
10537c478bd9Sstevel@tonic-gate } else {
10547c478bd9Sstevel@tonic-gate tq->tq_nalloc--;
10557c478bd9Sstevel@tonic-gate mutex_exit(&tq->tq_lock);
10567c478bd9Sstevel@tonic-gate kmem_cache_free(taskq_ent_cache, tqe);
10577c478bd9Sstevel@tonic-gate mutex_enter(&tq->tq_lock);
10587c478bd9Sstevel@tonic-gate }
105964109744SChris Horne
106064109744SChris Horne if (tq->tq_maxalloc_wait)
106164109744SChris Horne cv_signal(&tq->tq_maxalloc_cv);
106264109744SChris Horne }
106364109744SChris Horne
106464109744SChris Horne /*
106564109744SChris Horne * taskq_ent_exists()
106664109744SChris Horne *
106764109744SChris Horne * Return 1 if taskq already has entry for calling 'func(arg)'.
106864109744SChris Horne *
106964109744SChris Horne * Assumes: tq->tq_lock is held.
107064109744SChris Horne */
107164109744SChris Horne static int
taskq_ent_exists(taskq_t * tq,task_func_t func,void * arg)107264109744SChris Horne taskq_ent_exists(taskq_t *tq, task_func_t func, void *arg)
107364109744SChris Horne {
107464109744SChris Horne taskq_ent_t *tqe;
107564109744SChris Horne
107664109744SChris Horne ASSERT(MUTEX_HELD(&tq->tq_lock));
107764109744SChris Horne
107864109744SChris Horne for (tqe = tq->tq_task.tqent_next; tqe != &tq->tq_task;
107964109744SChris Horne tqe = tqe->tqent_next)
108064109744SChris Horne if ((tqe->tqent_func == func) && (tqe->tqent_arg == arg))
108164109744SChris Horne return (1);
108264109744SChris Horne return (0);
10837c478bd9Sstevel@tonic-gate }
10847c478bd9Sstevel@tonic-gate
10857c478bd9Sstevel@tonic-gate /*
10867c478bd9Sstevel@tonic-gate * Dispatch a task "func(arg)" to a free entry of bucket b.
10877c478bd9Sstevel@tonic-gate *
10887c478bd9Sstevel@tonic-gate * Assumes: no bucket locks is held.
10897c478bd9Sstevel@tonic-gate *
10907c478bd9Sstevel@tonic-gate * Returns: a pointer to an entry if dispatch was successful.
10917c478bd9Sstevel@tonic-gate * NULL if there are no free entries or if the bucket is suspended.
10927c478bd9Sstevel@tonic-gate */
10937c478bd9Sstevel@tonic-gate static taskq_ent_t *
taskq_bucket_dispatch(taskq_bucket_t * b,task_func_t func,void * arg)10947c478bd9Sstevel@tonic-gate taskq_bucket_dispatch(taskq_bucket_t *b, task_func_t func, void *arg)
10957c478bd9Sstevel@tonic-gate {
10967c478bd9Sstevel@tonic-gate taskq_ent_t *tqe;
10977c478bd9Sstevel@tonic-gate
10987c478bd9Sstevel@tonic-gate ASSERT(MUTEX_NOT_HELD(&b->tqbucket_lock));
10997c478bd9Sstevel@tonic-gate ASSERT(func != NULL);
11007c478bd9Sstevel@tonic-gate
11017c478bd9Sstevel@tonic-gate mutex_enter(&b->tqbucket_lock);
11027c478bd9Sstevel@tonic-gate
11037c478bd9Sstevel@tonic-gate ASSERT(b->tqbucket_nfree != 0 || IS_EMPTY(b->tqbucket_freelist));
11047c478bd9Sstevel@tonic-gate ASSERT(b->tqbucket_nfree == 0 || !IS_EMPTY(b->tqbucket_freelist));
11057c478bd9Sstevel@tonic-gate
11067c478bd9Sstevel@tonic-gate /*
11077c478bd9Sstevel@tonic-gate * Get en entry from the freelist if there is one.
11087c478bd9Sstevel@tonic-gate * Schedule task into the entry.
11097c478bd9Sstevel@tonic-gate */
11107c478bd9Sstevel@tonic-gate if ((b->tqbucket_nfree != 0) &&
11117c478bd9Sstevel@tonic-gate !(b->tqbucket_flags & TQBUCKET_SUSPEND)) {
11127c478bd9Sstevel@tonic-gate tqe = b->tqbucket_freelist.tqent_prev;
11137c478bd9Sstevel@tonic-gate
11147c478bd9Sstevel@tonic-gate ASSERT(tqe != &b->tqbucket_freelist);
11157c478bd9Sstevel@tonic-gate ASSERT(tqe->tqent_thread != NULL);
11167c478bd9Sstevel@tonic-gate
11177c478bd9Sstevel@tonic-gate tqe->tqent_prev->tqent_next = tqe->tqent_next;
11187c478bd9Sstevel@tonic-gate tqe->tqent_next->tqent_prev = tqe->tqent_prev;
11197c478bd9Sstevel@tonic-gate b->tqbucket_nalloc++;
11207c478bd9Sstevel@tonic-gate b->tqbucket_nfree--;
11217c478bd9Sstevel@tonic-gate tqe->tqent_func = func;
11227c478bd9Sstevel@tonic-gate tqe->tqent_arg = arg;
11237c478bd9Sstevel@tonic-gate TQ_STAT(b, tqs_hits);
11247c478bd9Sstevel@tonic-gate cv_signal(&tqe->tqent_cv);
11257c478bd9Sstevel@tonic-gate DTRACE_PROBE2(taskq__d__enqueue, taskq_bucket_t *, b,
11267c478bd9Sstevel@tonic-gate taskq_ent_t *, tqe);
11277c478bd9Sstevel@tonic-gate } else {
11287c478bd9Sstevel@tonic-gate tqe = NULL;
11297c478bd9Sstevel@tonic-gate TQ_STAT(b, tqs_misses);
11307c478bd9Sstevel@tonic-gate }
11317c478bd9Sstevel@tonic-gate mutex_exit(&b->tqbucket_lock);
11327c478bd9Sstevel@tonic-gate return (tqe);
11337c478bd9Sstevel@tonic-gate }
11347c478bd9Sstevel@tonic-gate
11357c478bd9Sstevel@tonic-gate /*
11367c478bd9Sstevel@tonic-gate * Dispatch a task.
11377c478bd9Sstevel@tonic-gate *
11387c478bd9Sstevel@tonic-gate * Assumes: func != NULL
11397c478bd9Sstevel@tonic-gate *
11407c478bd9Sstevel@tonic-gate * Returns: NULL if dispatch failed.
11417c478bd9Sstevel@tonic-gate * non-NULL if task dispatched successfully.
11427c478bd9Sstevel@tonic-gate * Actual return value is the pointer to taskq entry that was used to
11437c478bd9Sstevel@tonic-gate * dispatch a task. This is useful for debugging.
11447c478bd9Sstevel@tonic-gate */
11457c478bd9Sstevel@tonic-gate taskqid_t
taskq_dispatch(taskq_t * tq,task_func_t func,void * arg,uint_t flags)11467c478bd9Sstevel@tonic-gate taskq_dispatch(taskq_t *tq, task_func_t func, void *arg, uint_t flags)
11477c478bd9Sstevel@tonic-gate {
11487c478bd9Sstevel@tonic-gate taskq_bucket_t *bucket = NULL; /* Which bucket needs extension */
11497c478bd9Sstevel@tonic-gate taskq_ent_t *tqe = NULL;
11507c478bd9Sstevel@tonic-gate taskq_ent_t *tqe1;
11517c478bd9Sstevel@tonic-gate uint_t bsize;
11527c478bd9Sstevel@tonic-gate
11537c478bd9Sstevel@tonic-gate ASSERT(tq != NULL);
11547c478bd9Sstevel@tonic-gate ASSERT(func != NULL);
11557c478bd9Sstevel@tonic-gate
11567c478bd9Sstevel@tonic-gate if (!(tq->tq_flags & TASKQ_DYNAMIC)) {
11577c478bd9Sstevel@tonic-gate /*
11587c478bd9Sstevel@tonic-gate * TQ_NOQUEUE flag can't be used with non-dynamic task queues.
11597c478bd9Sstevel@tonic-gate */
11605aeb9474SGarrett D'Amore ASSERT(!(flags & TQ_NOQUEUE));
11617c478bd9Sstevel@tonic-gate /*
11627c478bd9Sstevel@tonic-gate * Enqueue the task to the underlying queue.
11637c478bd9Sstevel@tonic-gate */
11647c478bd9Sstevel@tonic-gate mutex_enter(&tq->tq_lock);
11657c478bd9Sstevel@tonic-gate
11667c478bd9Sstevel@tonic-gate TASKQ_S_RANDOM_DISPATCH_FAILURE(tq, flags);
11677c478bd9Sstevel@tonic-gate
11687c478bd9Sstevel@tonic-gate if ((tqe = taskq_ent_alloc(tq, flags)) == NULL) {
1169216d7723SPrakash Surya tq->tq_nomem++;
11707c478bd9Sstevel@tonic-gate mutex_exit(&tq->tq_lock);
11717e12ceb3SToomas Soome return ((taskqid_t)tqe);
11727c478bd9Sstevel@tonic-gate }
11735aeb9474SGarrett D'Amore /* Make sure we start without any flags */
11745aeb9474SGarrett D'Amore tqe->tqent_un.tqent_flags = 0;
11755aeb9474SGarrett D'Amore
117635a5a358SJonathan Adams if (flags & TQ_FRONT) {
117735a5a358SJonathan Adams TQ_ENQUEUE_FRONT(tq, tqe, func, arg);
117835a5a358SJonathan Adams } else {
117935a5a358SJonathan Adams TQ_ENQUEUE(tq, tqe, func, arg);
118035a5a358SJonathan Adams }
11817c478bd9Sstevel@tonic-gate mutex_exit(&tq->tq_lock);
11827c478bd9Sstevel@tonic-gate return ((taskqid_t)tqe);
11837c478bd9Sstevel@tonic-gate }
11847c478bd9Sstevel@tonic-gate
11857c478bd9Sstevel@tonic-gate /*
11867c478bd9Sstevel@tonic-gate * Dynamic taskq dispatching.
11877c478bd9Sstevel@tonic-gate */
118835a5a358SJonathan Adams ASSERT(!(flags & (TQ_NOALLOC | TQ_FRONT)));
11897c478bd9Sstevel@tonic-gate TASKQ_D_RANDOM_DISPATCH_FAILURE(tq, flags);
11907c478bd9Sstevel@tonic-gate
11917c478bd9Sstevel@tonic-gate bsize = tq->tq_nbuckets;
11927c478bd9Sstevel@tonic-gate
11937c478bd9Sstevel@tonic-gate if (bsize == 1) {
11947c478bd9Sstevel@tonic-gate /*
11957c478bd9Sstevel@tonic-gate * In a single-CPU case there is only one bucket, so get
11967c478bd9Sstevel@tonic-gate * entry directly from there.
11977c478bd9Sstevel@tonic-gate */
11987c478bd9Sstevel@tonic-gate if ((tqe = taskq_bucket_dispatch(tq->tq_buckets, func, arg))
11997c478bd9Sstevel@tonic-gate != NULL)
12007c478bd9Sstevel@tonic-gate return ((taskqid_t)tqe); /* Fastpath */
12017c478bd9Sstevel@tonic-gate bucket = tq->tq_buckets;
12027c478bd9Sstevel@tonic-gate } else {
12037c478bd9Sstevel@tonic-gate int loopcount;
12047c478bd9Sstevel@tonic-gate taskq_bucket_t *b;
12057c478bd9Sstevel@tonic-gate uintptr_t h = ((uintptr_t)CPU + (uintptr_t)arg) >> 3;
12067c478bd9Sstevel@tonic-gate
12077c478bd9Sstevel@tonic-gate h = TQ_HASH(h);
12087c478bd9Sstevel@tonic-gate
12097c478bd9Sstevel@tonic-gate /*
12107c478bd9Sstevel@tonic-gate * The 'bucket' points to the original bucket that we hit. If we
12117c478bd9Sstevel@tonic-gate * can't allocate from it, we search other buckets, but only
12127c478bd9Sstevel@tonic-gate * extend this one.
12137c478bd9Sstevel@tonic-gate */
12147c478bd9Sstevel@tonic-gate b = &tq->tq_buckets[h & (bsize - 1)];
12157c478bd9Sstevel@tonic-gate ASSERT(b->tqbucket_taskq == tq); /* Sanity check */
12167c478bd9Sstevel@tonic-gate
12177c478bd9Sstevel@tonic-gate /*
12187c478bd9Sstevel@tonic-gate * Do a quick check before grabbing the lock. If the bucket does
12197c478bd9Sstevel@tonic-gate * not have free entries now, chances are very small that it
12207c478bd9Sstevel@tonic-gate * will after we take the lock, so we just skip it.
12217c478bd9Sstevel@tonic-gate */
12227c478bd9Sstevel@tonic-gate if (b->tqbucket_nfree != 0) {
12237c478bd9Sstevel@tonic-gate if ((tqe = taskq_bucket_dispatch(b, func, arg)) != NULL)
12247c478bd9Sstevel@tonic-gate return ((taskqid_t)tqe); /* Fastpath */
12257c478bd9Sstevel@tonic-gate } else {
12267c478bd9Sstevel@tonic-gate TQ_STAT(b, tqs_misses);
12277c478bd9Sstevel@tonic-gate }
12287c478bd9Sstevel@tonic-gate
12297c478bd9Sstevel@tonic-gate bucket = b;
12307c478bd9Sstevel@tonic-gate loopcount = MIN(taskq_search_depth, bsize);
12317c478bd9Sstevel@tonic-gate /*
12327c478bd9Sstevel@tonic-gate * If bucket dispatch failed, search loopcount number of buckets
12337c478bd9Sstevel@tonic-gate * before we give up and fail.
12347c478bd9Sstevel@tonic-gate */
12357c478bd9Sstevel@tonic-gate do {
12367c478bd9Sstevel@tonic-gate b = &tq->tq_buckets[++h & (bsize - 1)];
12377c478bd9Sstevel@tonic-gate ASSERT(b->tqbucket_taskq == tq); /* Sanity check */
12387c478bd9Sstevel@tonic-gate loopcount--;
12397c478bd9Sstevel@tonic-gate
12407c478bd9Sstevel@tonic-gate if (b->tqbucket_nfree != 0) {
12417c478bd9Sstevel@tonic-gate tqe = taskq_bucket_dispatch(b, func, arg);
12427c478bd9Sstevel@tonic-gate } else {
12437c478bd9Sstevel@tonic-gate TQ_STAT(b, tqs_misses);
12447c478bd9Sstevel@tonic-gate }
12457c478bd9Sstevel@tonic-gate } while ((tqe == NULL) && (loopcount > 0));
12467c478bd9Sstevel@tonic-gate }
12477c478bd9Sstevel@tonic-gate
12487c478bd9Sstevel@tonic-gate /*
12497c478bd9Sstevel@tonic-gate * At this point we either scheduled a task and (tqe != NULL) or failed
12507c478bd9Sstevel@tonic-gate * (tqe == NULL). Try to recover from fails.
12517c478bd9Sstevel@tonic-gate */
12527c478bd9Sstevel@tonic-gate
12537c478bd9Sstevel@tonic-gate /*
12547c478bd9Sstevel@tonic-gate * For KM_SLEEP dispatches, try to extend the bucket and retry dispatch.
12557c478bd9Sstevel@tonic-gate */
12567c478bd9Sstevel@tonic-gate if ((tqe == NULL) && !(flags & TQ_NOSLEEP)) {
12577c478bd9Sstevel@tonic-gate /*
12587c478bd9Sstevel@tonic-gate * taskq_bucket_extend() may fail to do anything, but this is
12597c478bd9Sstevel@tonic-gate * fine - we deal with it later. If the bucket was successfully
12607c478bd9Sstevel@tonic-gate * extended, there is a good chance that taskq_bucket_dispatch()
12617c478bd9Sstevel@tonic-gate * will get this new entry, unless someone is racing with us and
12627c478bd9Sstevel@tonic-gate * stealing the new entry from under our nose.
12637c478bd9Sstevel@tonic-gate * taskq_bucket_extend() may sleep.
12647c478bd9Sstevel@tonic-gate */
12657c478bd9Sstevel@tonic-gate taskq_bucket_extend(bucket);
12667c478bd9Sstevel@tonic-gate TQ_STAT(bucket, tqs_disptcreates);
12677c478bd9Sstevel@tonic-gate if ((tqe = taskq_bucket_dispatch(bucket, func, arg)) != NULL)
12687c478bd9Sstevel@tonic-gate return ((taskqid_t)tqe);
12697c478bd9Sstevel@tonic-gate }
12707c478bd9Sstevel@tonic-gate
12717c478bd9Sstevel@tonic-gate ASSERT(bucket != NULL);
127264109744SChris Horne
12737c478bd9Sstevel@tonic-gate /*
127464109744SChris Horne * Since there are not enough free entries in the bucket, add a
127564109744SChris Horne * taskq entry to extend it in the background using backing queue
127664109744SChris Horne * (unless we already have a taskq entry to perform that extension).
12777c478bd9Sstevel@tonic-gate */
12787c478bd9Sstevel@tonic-gate mutex_enter(&tq->tq_lock);
127964109744SChris Horne if (!taskq_ent_exists(tq, taskq_bucket_extend, bucket)) {
128064109744SChris Horne if ((tqe1 = taskq_ent_alloc(tq, TQ_NOSLEEP)) != NULL) {
128164109744SChris Horne TQ_ENQUEUE_FRONT(tq, tqe1, taskq_bucket_extend, bucket);
128264109744SChris Horne } else {
1283216d7723SPrakash Surya tq->tq_nomem++;
128464109744SChris Horne }
12857c478bd9Sstevel@tonic-gate }
12867c478bd9Sstevel@tonic-gate
12877c478bd9Sstevel@tonic-gate /*
12887c478bd9Sstevel@tonic-gate * Dispatch failed and we can't find an entry to schedule a task.
12897c478bd9Sstevel@tonic-gate * Revert to the backing queue unless TQ_NOQUEUE was asked.
12907c478bd9Sstevel@tonic-gate */
12917c478bd9Sstevel@tonic-gate if ((tqe == NULL) && !(flags & TQ_NOQUEUE)) {
12927c478bd9Sstevel@tonic-gate if ((tqe = taskq_ent_alloc(tq, flags)) != NULL) {
12937c478bd9Sstevel@tonic-gate TQ_ENQUEUE(tq, tqe, func, arg);
12947c478bd9Sstevel@tonic-gate } else {
1295216d7723SPrakash Surya tq->tq_nomem++;
12967c478bd9Sstevel@tonic-gate }
12977c478bd9Sstevel@tonic-gate }
12987c478bd9Sstevel@tonic-gate mutex_exit(&tq->tq_lock);
12997c478bd9Sstevel@tonic-gate
13007c478bd9Sstevel@tonic-gate return ((taskqid_t)tqe);
13017c478bd9Sstevel@tonic-gate }
13027c478bd9Sstevel@tonic-gate
13035aeb9474SGarrett D'Amore void
taskq_dispatch_ent(taskq_t * tq,task_func_t func,void * arg,uint_t flags,taskq_ent_t * tqe)13045aeb9474SGarrett D'Amore taskq_dispatch_ent(taskq_t *tq, task_func_t func, void *arg, uint_t flags,
13055aeb9474SGarrett D'Amore taskq_ent_t *tqe)
13065aeb9474SGarrett D'Amore {
13075aeb9474SGarrett D'Amore ASSERT(func != NULL);
13085aeb9474SGarrett D'Amore ASSERT(!(tq->tq_flags & TASKQ_DYNAMIC));
13095aeb9474SGarrett D'Amore
13105aeb9474SGarrett D'Amore /*
13115aeb9474SGarrett D'Amore * Mark it as a prealloc'd task. This is important
13125aeb9474SGarrett D'Amore * to ensure that we don't free it later.
13135aeb9474SGarrett D'Amore */
13145aeb9474SGarrett D'Amore tqe->tqent_un.tqent_flags |= TQENT_FLAG_PREALLOC;
13155aeb9474SGarrett D'Amore /*
13165aeb9474SGarrett D'Amore * Enqueue the task to the underlying queue.
13175aeb9474SGarrett D'Amore */
13185aeb9474SGarrett D'Amore mutex_enter(&tq->tq_lock);
13195aeb9474SGarrett D'Amore
13205aeb9474SGarrett D'Amore if (flags & TQ_FRONT) {
13215aeb9474SGarrett D'Amore TQ_ENQUEUE_FRONT(tq, tqe, func, arg);
13225aeb9474SGarrett D'Amore } else {
13235aeb9474SGarrett D'Amore TQ_ENQUEUE(tq, tqe, func, arg);
13245aeb9474SGarrett D'Amore }
13255aeb9474SGarrett D'Amore mutex_exit(&tq->tq_lock);
13265aeb9474SGarrett D'Amore }
13275aeb9474SGarrett D'Amore
13284c99ecc3STim Kordas /*
13294c99ecc3STim Kordas * Allow our caller to ask if there are tasks pending on the queue.
13304c99ecc3STim Kordas */
13314c99ecc3STim Kordas boolean_t
taskq_empty(taskq_t * tq)13324c99ecc3STim Kordas taskq_empty(taskq_t *tq)
13334c99ecc3STim Kordas {
13344c99ecc3STim Kordas boolean_t rv;
13354c99ecc3STim Kordas
13364c99ecc3STim Kordas ASSERT3P(tq, !=, curthread->t_taskq);
13374c99ecc3STim Kordas mutex_enter(&tq->tq_lock);
13384c99ecc3STim Kordas rv = (tq->tq_task.tqent_next == &tq->tq_task) && (tq->tq_active == 0);
13394c99ecc3STim Kordas mutex_exit(&tq->tq_lock);
13404c99ecc3STim Kordas
13414c99ecc3STim Kordas return (rv);
13424c99ecc3STim Kordas }
13434c99ecc3STim Kordas
13447c478bd9Sstevel@tonic-gate /*
13457c478bd9Sstevel@tonic-gate * Wait for all pending tasks to complete.
13467c478bd9Sstevel@tonic-gate * Calling taskq_wait from a task will cause deadlock.
13477c478bd9Sstevel@tonic-gate */
13487c478bd9Sstevel@tonic-gate void
taskq_wait(taskq_t * tq)13497c478bd9Sstevel@tonic-gate taskq_wait(taskq_t *tq)
13507c478bd9Sstevel@tonic-gate {
13517c478bd9Sstevel@tonic-gate ASSERT(tq != curthread->t_taskq);
13527c478bd9Sstevel@tonic-gate
13537c478bd9Sstevel@tonic-gate mutex_enter(&tq->tq_lock);
13547c478bd9Sstevel@tonic-gate while (tq->tq_task.tqent_next != &tq->tq_task || tq->tq_active != 0)
13557c478bd9Sstevel@tonic-gate cv_wait(&tq->tq_wait_cv, &tq->tq_lock);
13567c478bd9Sstevel@tonic-gate mutex_exit(&tq->tq_lock);
13577c478bd9Sstevel@tonic-gate
13587c478bd9Sstevel@tonic-gate if (tq->tq_flags & TASKQ_DYNAMIC) {
13597c478bd9Sstevel@tonic-gate taskq_bucket_t *b = tq->tq_buckets;
13607c478bd9Sstevel@tonic-gate int bid = 0;
13617c478bd9Sstevel@tonic-gate for (; (b != NULL) && (bid < tq->tq_nbuckets); b++, bid++) {
13627c478bd9Sstevel@tonic-gate mutex_enter(&b->tqbucket_lock);
13637c478bd9Sstevel@tonic-gate while (b->tqbucket_nalloc > 0)
13647c478bd9Sstevel@tonic-gate cv_wait(&b->tqbucket_cv, &b->tqbucket_lock);
13657c478bd9Sstevel@tonic-gate mutex_exit(&b->tqbucket_lock);
13667c478bd9Sstevel@tonic-gate }
13677c478bd9Sstevel@tonic-gate }
13687c478bd9Sstevel@tonic-gate }
13697c478bd9Sstevel@tonic-gate
1370a3874b8bSToomas Soome void
taskq_wait_id(taskq_t * tq,taskqid_t id __unused)1371a3874b8bSToomas Soome taskq_wait_id(taskq_t *tq, taskqid_t id __unused)
1372a3874b8bSToomas Soome {
1373a3874b8bSToomas Soome taskq_wait(tq);
1374a3874b8bSToomas Soome }
1375a3874b8bSToomas Soome
13767c478bd9Sstevel@tonic-gate /*
13777c478bd9Sstevel@tonic-gate * Suspend execution of tasks.
13787c478bd9Sstevel@tonic-gate *
13797c478bd9Sstevel@tonic-gate * Tasks in the queue part will be suspended immediately upon return from this
13807c478bd9Sstevel@tonic-gate * function. Pending tasks in the dynamic part will continue to execute, but all
13817c478bd9Sstevel@tonic-gate * new tasks will be suspended.
13827c478bd9Sstevel@tonic-gate */
13837c478bd9Sstevel@tonic-gate void
taskq_suspend(taskq_t * tq)13847c478bd9Sstevel@tonic-gate taskq_suspend(taskq_t *tq)
13857c478bd9Sstevel@tonic-gate {
13867c478bd9Sstevel@tonic-gate rw_enter(&tq->tq_threadlock, RW_WRITER);
13877c478bd9Sstevel@tonic-gate
13887c478bd9Sstevel@tonic-gate if (tq->tq_flags & TASKQ_DYNAMIC) {
13897c478bd9Sstevel@tonic-gate taskq_bucket_t *b = tq->tq_buckets;
13907c478bd9Sstevel@tonic-gate int bid = 0;
13917c478bd9Sstevel@tonic-gate for (; (b != NULL) && (bid < tq->tq_nbuckets); b++, bid++) {
13927c478bd9Sstevel@tonic-gate mutex_enter(&b->tqbucket_lock);
13937c478bd9Sstevel@tonic-gate b->tqbucket_flags |= TQBUCKET_SUSPEND;
13947c478bd9Sstevel@tonic-gate mutex_exit(&b->tqbucket_lock);
13957c478bd9Sstevel@tonic-gate }
13967c478bd9Sstevel@tonic-gate }
13977c478bd9Sstevel@tonic-gate /*
13987c478bd9Sstevel@tonic-gate * Mark task queue as being suspended. Needed for taskq_suspended().
13997c478bd9Sstevel@tonic-gate */
14007c478bd9Sstevel@tonic-gate mutex_enter(&tq->tq_lock);
14017c478bd9Sstevel@tonic-gate ASSERT(!(tq->tq_flags & TASKQ_SUSPENDED));
14027c478bd9Sstevel@tonic-gate tq->tq_flags |= TASKQ_SUSPENDED;
14037c478bd9Sstevel@tonic-gate mutex_exit(&tq->tq_lock);
14047c478bd9Sstevel@tonic-gate }
14057c478bd9Sstevel@tonic-gate
14067c478bd9Sstevel@tonic-gate /*
14077c478bd9Sstevel@tonic-gate * returns: 1 if tq is suspended, 0 otherwise.
14087c478bd9Sstevel@tonic-gate */
14097c478bd9Sstevel@tonic-gate int
taskq_suspended(taskq_t * tq)14107c478bd9Sstevel@tonic-gate taskq_suspended(taskq_t *tq)
14117c478bd9Sstevel@tonic-gate {
14127c478bd9Sstevel@tonic-gate return ((tq->tq_flags & TASKQ_SUSPENDED) != 0);
14137c478bd9Sstevel@tonic-gate }
14147c478bd9Sstevel@tonic-gate
14157c478bd9Sstevel@tonic-gate /*
14167c478bd9Sstevel@tonic-gate * Resume taskq execution.
14177c478bd9Sstevel@tonic-gate */
14187c478bd9Sstevel@tonic-gate void
taskq_resume(taskq_t * tq)14197c478bd9Sstevel@tonic-gate taskq_resume(taskq_t *tq)
14207c478bd9Sstevel@tonic-gate {
14217c478bd9Sstevel@tonic-gate ASSERT(RW_WRITE_HELD(&tq->tq_threadlock));
14227c478bd9Sstevel@tonic-gate
14237c478bd9Sstevel@tonic-gate if (tq->tq_flags & TASKQ_DYNAMIC) {
14247c478bd9Sstevel@tonic-gate taskq_bucket_t *b = tq->tq_buckets;
14257c478bd9Sstevel@tonic-gate int bid = 0;
14267c478bd9Sstevel@tonic-gate for (; (b != NULL) && (bid < tq->tq_nbuckets); b++, bid++) {
14277c478bd9Sstevel@tonic-gate mutex_enter(&b->tqbucket_lock);
14287c478bd9Sstevel@tonic-gate b->tqbucket_flags &= ~TQBUCKET_SUSPEND;
14297c478bd9Sstevel@tonic-gate mutex_exit(&b->tqbucket_lock);
14307c478bd9Sstevel@tonic-gate }
14317c478bd9Sstevel@tonic-gate }
14327c478bd9Sstevel@tonic-gate mutex_enter(&tq->tq_lock);
14337c478bd9Sstevel@tonic-gate ASSERT(tq->tq_flags & TASKQ_SUSPENDED);
14347c478bd9Sstevel@tonic-gate tq->tq_flags &= ~TASKQ_SUSPENDED;
14357c478bd9Sstevel@tonic-gate mutex_exit(&tq->tq_lock);
14367c478bd9Sstevel@tonic-gate
14377c478bd9Sstevel@tonic-gate rw_exit(&tq->tq_threadlock);
14387c478bd9Sstevel@tonic-gate }
14397c478bd9Sstevel@tonic-gate
14407c478bd9Sstevel@tonic-gate int
taskq_member(taskq_t * tq,kthread_t * thread)14417c478bd9Sstevel@tonic-gate taskq_member(taskq_t *tq, kthread_t *thread)
14427c478bd9Sstevel@tonic-gate {
14437c478bd9Sstevel@tonic-gate return (thread->t_taskq == tq);
14447c478bd9Sstevel@tonic-gate }
14457c478bd9Sstevel@tonic-gate
144635a5a358SJonathan Adams /*
144735a5a358SJonathan Adams * Creates a thread in the taskq. We only allow one outstanding create at
144835a5a358SJonathan Adams * a time. We drop and reacquire the tq_lock in order to avoid blocking other
144935a5a358SJonathan Adams * taskq activity while thread_create() or lwp_kernel_create() run.
145035a5a358SJonathan Adams *
145135a5a358SJonathan Adams * The first time we're called, we do some additional setup, and do not
145235a5a358SJonathan Adams * return until there are enough threads to start servicing requests.
145335a5a358SJonathan Adams */
14542e0c549eSJonathan Adams static void
taskq_thread_create(taskq_t * tq)14552e0c549eSJonathan Adams taskq_thread_create(taskq_t *tq)
14562e0c549eSJonathan Adams {
145735a5a358SJonathan Adams kthread_t *t;
145835a5a358SJonathan Adams const boolean_t first = (tq->tq_nthreads == 0);
14592e0c549eSJonathan Adams
14602e0c549eSJonathan Adams ASSERT(MUTEX_HELD(&tq->tq_lock));
146135a5a358SJonathan Adams ASSERT(tq->tq_flags & TASKQ_CHANGING);
146235a5a358SJonathan Adams ASSERT(tq->tq_nthreads < tq->tq_nthreads_target);
14632e0c549eSJonathan Adams ASSERT(!(tq->tq_flags & TASKQ_THREAD_CREATED));
14642e0c549eSJonathan Adams
146535a5a358SJonathan Adams
14662e0c549eSJonathan Adams tq->tq_flags |= TASKQ_THREAD_CREATED;
14672e0c549eSJonathan Adams tq->tq_active++;
146835a5a358SJonathan Adams mutex_exit(&tq->tq_lock);
146935a5a358SJonathan Adams
1470e5488233SGordon Ross /*
1471e5488233SGordon Ross * With TASKQ_DUTY_CYCLE the new thread must have an LWP
1472e5488233SGordon Ross * as explained in ../disp/sysdc.c (for the msacct data).
1473*f8d363f3SGordon Ross * Normally simple kthreads are preferred, unless the
1474*f8d363f3SGordon Ross * caller has asked for LWPs for other reasons.
1475e5488233SGordon Ross */
1476*f8d363f3SGordon Ross if ((tq->tq_flags & (TASKQ_DUTY_CYCLE | TASKQ_THREADS_LWP)) != 0) {
1477e5488233SGordon Ross /* Enforced in taskq_create_common */
1478e5488233SGordon Ross ASSERT3P(tq->tq_proc, !=, &p0);
147935a5a358SJonathan Adams t = lwp_kernel_create(tq->tq_proc, taskq_thread, tq, TS_RUN,
148035a5a358SJonathan Adams tq->tq_pri);
148135a5a358SJonathan Adams } else {
1482e5488233SGordon Ross t = thread_create(NULL, 0, taskq_thread, tq, 0, tq->tq_proc,
1483e5488233SGordon Ross TS_RUN, tq->tq_pri);
148435a5a358SJonathan Adams }
148535a5a358SJonathan Adams
148635a5a358SJonathan Adams if (!first) {
148735a5a358SJonathan Adams mutex_enter(&tq->tq_lock);
148835a5a358SJonathan Adams return;
148935a5a358SJonathan Adams }
149035a5a358SJonathan Adams
149135a5a358SJonathan Adams /*
149235a5a358SJonathan Adams * We know the thread cannot go away, since tq cannot be
149335a5a358SJonathan Adams * destroyed until creation has completed. We can therefore
149435a5a358SJonathan Adams * safely dereference t.
149535a5a358SJonathan Adams */
149635a5a358SJonathan Adams if (tq->tq_flags & TASKQ_THREADS_CPU_PCT) {
149735a5a358SJonathan Adams taskq_cpupct_install(tq, t->t_cpupart);
149835a5a358SJonathan Adams }
149935a5a358SJonathan Adams mutex_enter(&tq->tq_lock);
150035a5a358SJonathan Adams
150135a5a358SJonathan Adams /* Wait until we can service requests. */
150235a5a358SJonathan Adams while (tq->tq_nthreads != tq->tq_nthreads_target &&
150335a5a358SJonathan Adams tq->tq_nthreads < TASKQ_CREATE_ACTIVE_THREADS) {
150435a5a358SJonathan Adams cv_wait(&tq->tq_wait_cv, &tq->tq_lock);
150535a5a358SJonathan Adams }
15062e0c549eSJonathan Adams }
15072e0c549eSJonathan Adams
1508e0ad97e3SJonathan Adams /*
1509e0ad97e3SJonathan Adams * Common "sleep taskq thread" function, which handles CPR stuff, as well
1510e0ad97e3SJonathan Adams * as giving a nice common point for debuggers to find inactive threads.
1511e0ad97e3SJonathan Adams */
1512e0ad97e3SJonathan Adams static clock_t
taskq_thread_wait(taskq_t * tq,kmutex_t * mx,kcondvar_t * cv,callb_cpr_t * cprinfo,clock_t timeout)1513e0ad97e3SJonathan Adams taskq_thread_wait(taskq_t *tq, kmutex_t *mx, kcondvar_t *cv,
1514e0ad97e3SJonathan Adams callb_cpr_t *cprinfo, clock_t timeout)
15152e0c549eSJonathan Adams {
1516e0ad97e3SJonathan Adams clock_t ret = 0;
1517e0ad97e3SJonathan Adams
1518e0ad97e3SJonathan Adams if (!(tq->tq_flags & TASKQ_CPR_SAFE)) {
15192e0c549eSJonathan Adams CALLB_CPR_SAFE_BEGIN(cprinfo);
15202e0c549eSJonathan Adams }
1521e0ad97e3SJonathan Adams if (timeout < 0)
1522e0ad97e3SJonathan Adams cv_wait(cv, mx);
1523e0ad97e3SJonathan Adams else
1524d3d50737SRafael Vanoni ret = cv_reltimedwait(cv, mx, timeout, TR_CLOCK_TICK);
1525e0ad97e3SJonathan Adams
1526e0ad97e3SJonathan Adams if (!(tq->tq_flags & TASKQ_CPR_SAFE)) {
1527e0ad97e3SJonathan Adams CALLB_CPR_SAFE_END(cprinfo, mx);
1528e0ad97e3SJonathan Adams }
1529e0ad97e3SJonathan Adams
1530e0ad97e3SJonathan Adams return (ret);
15312e0c549eSJonathan Adams }
15322e0c549eSJonathan Adams
15337c478bd9Sstevel@tonic-gate /*
15347c478bd9Sstevel@tonic-gate * Worker thread for processing task queue.
15357c478bd9Sstevel@tonic-gate */
15367c478bd9Sstevel@tonic-gate static void
taskq_thread(void * arg)15377c478bd9Sstevel@tonic-gate taskq_thread(void *arg)
15387c478bd9Sstevel@tonic-gate {
15392e0c549eSJonathan Adams int thread_id;
15402e0c549eSJonathan Adams
15417c478bd9Sstevel@tonic-gate taskq_t *tq = arg;
15427c478bd9Sstevel@tonic-gate taskq_ent_t *tqe;
15437c478bd9Sstevel@tonic-gate callb_cpr_t cprinfo;
15447c478bd9Sstevel@tonic-gate hrtime_t start, end;
15455aeb9474SGarrett D'Amore boolean_t freeit;
15467c478bd9Sstevel@tonic-gate
154735a5a358SJonathan Adams curthread->t_taskq = tq; /* mark ourselves for taskq_member() */
154835a5a358SJonathan Adams
154935a5a358SJonathan Adams if (curproc != &p0 && (tq->tq_flags & TASKQ_DUTY_CYCLE)) {
155035a5a358SJonathan Adams sysdc_thread_enter(curthread, tq->tq_DC,
155135a5a358SJonathan Adams (tq->tq_flags & TASKQ_DC_BATCH) ? SYSDC_THREAD_BATCH : 0);
155235a5a358SJonathan Adams }
155335a5a358SJonathan Adams
15547c478bd9Sstevel@tonic-gate if (tq->tq_flags & TASKQ_CPR_SAFE) {
15557c478bd9Sstevel@tonic-gate CALLB_CPR_INIT_SAFE(curthread, tq->tq_name);
15567c478bd9Sstevel@tonic-gate } else {
15577c478bd9Sstevel@tonic-gate CALLB_CPR_INIT(&cprinfo, &tq->tq_lock, callb_generic_cpr,
15587c478bd9Sstevel@tonic-gate tq->tq_name);
15597c478bd9Sstevel@tonic-gate }
15607c478bd9Sstevel@tonic-gate mutex_enter(&tq->tq_lock);
15612e0c549eSJonathan Adams thread_id = ++tq->tq_nthreads;
15622e0c549eSJonathan Adams ASSERT(tq->tq_flags & TASKQ_THREAD_CREATED);
156335a5a358SJonathan Adams ASSERT(tq->tq_flags & TASKQ_CHANGING);
15642e0c549eSJonathan Adams tq->tq_flags &= ~TASKQ_THREAD_CREATED;
15652e0c549eSJonathan Adams
15662e0c549eSJonathan Adams VERIFY3S(thread_id, <=, tq->tq_nthreads_max);
15672e0c549eSJonathan Adams
15682e0c549eSJonathan Adams if (tq->tq_nthreads_max == 1)
15692e0c549eSJonathan Adams tq->tq_thread = curthread;
15702e0c549eSJonathan Adams else
15712e0c549eSJonathan Adams tq->tq_threadlist[thread_id - 1] = curthread;
15722e0c549eSJonathan Adams
157335a5a358SJonathan Adams /* Allow taskq_create_common()'s taskq_thread_create() to return. */
157435a5a358SJonathan Adams if (tq->tq_nthreads == TASKQ_CREATE_ACTIVE_THREADS)
157535a5a358SJonathan Adams cv_broadcast(&tq->tq_wait_cv);
157635a5a358SJonathan Adams
15772e0c549eSJonathan Adams for (;;) {
15782e0c549eSJonathan Adams if (tq->tq_flags & TASKQ_CHANGING) {
157935a5a358SJonathan Adams /* See if we're no longer needed */
15802e0c549eSJonathan Adams if (thread_id > tq->tq_nthreads_target) {
15812e0c549eSJonathan Adams /*
15822e0c549eSJonathan Adams * To preserve the one-to-one mapping between
15832e0c549eSJonathan Adams * thread_id and thread, we must exit from
15842e0c549eSJonathan Adams * highest thread ID to least.
15852e0c549eSJonathan Adams *
15862e0c549eSJonathan Adams * However, if everyone is exiting, the order
15872e0c549eSJonathan Adams * doesn't matter, so just exit immediately.
15882e0c549eSJonathan Adams * (this is safe, since you must wait for
15892e0c549eSJonathan Adams * nthreads to reach 0 after setting
15902e0c549eSJonathan Adams * tq_nthreads_target to 0)
15912e0c549eSJonathan Adams */
15922e0c549eSJonathan Adams if (thread_id == tq->tq_nthreads ||
15932e0c549eSJonathan Adams tq->tq_nthreads_target == 0)
15942e0c549eSJonathan Adams break;
15952e0c549eSJonathan Adams
15962e0c549eSJonathan Adams /* Wait for higher thread_ids to exit */
1597e0ad97e3SJonathan Adams (void) taskq_thread_wait(tq, &tq->tq_lock,
1598e0ad97e3SJonathan Adams &tq->tq_exit_cv, &cprinfo, -1);
15992e0c549eSJonathan Adams continue;
16002e0c549eSJonathan Adams }
160135a5a358SJonathan Adams
160235a5a358SJonathan Adams /*
160335a5a358SJonathan Adams * If no thread is starting taskq_thread(), we can
160435a5a358SJonathan Adams * do some bookkeeping.
160535a5a358SJonathan Adams */
160635a5a358SJonathan Adams if (!(tq->tq_flags & TASKQ_THREAD_CREATED)) {
160735a5a358SJonathan Adams /* Check if we've reached our target */
160835a5a358SJonathan Adams if (tq->tq_nthreads == tq->tq_nthreads_target) {
160935a5a358SJonathan Adams tq->tq_flags &= ~TASKQ_CHANGING;
161035a5a358SJonathan Adams cv_broadcast(&tq->tq_wait_cv);
161135a5a358SJonathan Adams }
161235a5a358SJonathan Adams /* Check if we need to create a thread */
161335a5a358SJonathan Adams if (tq->tq_nthreads < tq->tq_nthreads_target) {
161435a5a358SJonathan Adams taskq_thread_create(tq);
161535a5a358SJonathan Adams continue; /* tq_lock was dropped */
161635a5a358SJonathan Adams }
161735a5a358SJonathan Adams }
16182e0c549eSJonathan Adams }
16197c478bd9Sstevel@tonic-gate if ((tqe = tq->tq_task.tqent_next) == &tq->tq_task) {
16207c478bd9Sstevel@tonic-gate if (--tq->tq_active == 0)
16217c478bd9Sstevel@tonic-gate cv_broadcast(&tq->tq_wait_cv);
1622e0ad97e3SJonathan Adams (void) taskq_thread_wait(tq, &tq->tq_lock,
1623e0ad97e3SJonathan Adams &tq->tq_dispatch_cv, &cprinfo, -1);
16247c478bd9Sstevel@tonic-gate tq->tq_active++;
16257c478bd9Sstevel@tonic-gate continue;
16267c478bd9Sstevel@tonic-gate }
162735a5a358SJonathan Adams
16287c478bd9Sstevel@tonic-gate tqe->tqent_prev->tqent_next = tqe->tqent_next;
16297c478bd9Sstevel@tonic-gate tqe->tqent_next->tqent_prev = tqe->tqent_prev;
16307c478bd9Sstevel@tonic-gate mutex_exit(&tq->tq_lock);
16317c478bd9Sstevel@tonic-gate
16325aeb9474SGarrett D'Amore /*
16335aeb9474SGarrett D'Amore * For prealloc'd tasks, we don't free anything. We
16345aeb9474SGarrett D'Amore * have to check this now, because once we call the
16355aeb9474SGarrett D'Amore * function for a prealloc'd taskq, we can't touch the
16365aeb9474SGarrett D'Amore * tqent any longer (calling the function returns the
16375aeb9474SGarrett D'Amore * ownershp of the tqent back to caller of
16385aeb9474SGarrett D'Amore * taskq_dispatch.)
16395aeb9474SGarrett D'Amore */
16405aeb9474SGarrett D'Amore if ((!(tq->tq_flags & TASKQ_DYNAMIC)) &&
16415aeb9474SGarrett D'Amore (tqe->tqent_un.tqent_flags & TQENT_FLAG_PREALLOC)) {
16425aeb9474SGarrett D'Amore /* clear pointers to assist assertion checks */
16435aeb9474SGarrett D'Amore tqe->tqent_next = tqe->tqent_prev = NULL;
16445aeb9474SGarrett D'Amore freeit = B_FALSE;
16455aeb9474SGarrett D'Amore } else {
16465aeb9474SGarrett D'Amore freeit = B_TRUE;
16475aeb9474SGarrett D'Amore }
16485aeb9474SGarrett D'Amore
16497c478bd9Sstevel@tonic-gate rw_enter(&tq->tq_threadlock, RW_READER);
16507c478bd9Sstevel@tonic-gate start = gethrtime();
16517c478bd9Sstevel@tonic-gate DTRACE_PROBE2(taskq__exec__start, taskq_t *, tq,
16527c478bd9Sstevel@tonic-gate taskq_ent_t *, tqe);
16537c478bd9Sstevel@tonic-gate tqe->tqent_func(tqe->tqent_arg);
16547c478bd9Sstevel@tonic-gate DTRACE_PROBE2(taskq__exec__end, taskq_t *, tq,
16557c478bd9Sstevel@tonic-gate taskq_ent_t *, tqe);
16567c478bd9Sstevel@tonic-gate end = gethrtime();
16577c478bd9Sstevel@tonic-gate rw_exit(&tq->tq_threadlock);
16587c478bd9Sstevel@tonic-gate
16597c478bd9Sstevel@tonic-gate mutex_enter(&tq->tq_lock);
16607c478bd9Sstevel@tonic-gate tq->tq_totaltime += end - start;
16617c478bd9Sstevel@tonic-gate tq->tq_executed++;
16627c478bd9Sstevel@tonic-gate
16635aeb9474SGarrett D'Amore if (freeit)
16645aeb9474SGarrett D'Amore taskq_ent_free(tq, tqe);
16657c478bd9Sstevel@tonic-gate }
16662e0c549eSJonathan Adams
16672e0c549eSJonathan Adams if (tq->tq_nthreads_max == 1)
16682e0c549eSJonathan Adams tq->tq_thread = NULL;
16692e0c549eSJonathan Adams else
16702e0c549eSJonathan Adams tq->tq_threadlist[thread_id - 1] = NULL;
16712e0c549eSJonathan Adams
167235a5a358SJonathan Adams /* We're exiting, and therefore no longer active */
167335a5a358SJonathan Adams ASSERT(tq->tq_active > 0);
167435a5a358SJonathan Adams tq->tq_active--;
167535a5a358SJonathan Adams
16762e0c549eSJonathan Adams ASSERT(tq->tq_nthreads > 0);
167735a5a358SJonathan Adams tq->tq_nthreads--;
16782e0c549eSJonathan Adams
167935a5a358SJonathan Adams /* Wake up anyone waiting for us to exit */
16802e0c549eSJonathan Adams cv_broadcast(&tq->tq_exit_cv);
168135a5a358SJonathan Adams if (tq->tq_nthreads == tq->tq_nthreads_target) {
168235a5a358SJonathan Adams if (!(tq->tq_flags & TASKQ_THREAD_CREATED))
168335a5a358SJonathan Adams tq->tq_flags &= ~TASKQ_CHANGING;
16842e0c549eSJonathan Adams
168535a5a358SJonathan Adams cv_broadcast(&tq->tq_wait_cv);
168635a5a358SJonathan Adams }
16872e0c549eSJonathan Adams
16887c478bd9Sstevel@tonic-gate ASSERT(!(tq->tq_flags & TASKQ_CPR_SAFE));
168935a5a358SJonathan Adams CALLB_CPR_EXIT(&cprinfo); /* drops tq->tq_lock */
169035a5a358SJonathan Adams if (curthread->t_lwp != NULL) {
169135a5a358SJonathan Adams mutex_enter(&curproc->p_lock);
169235a5a358SJonathan Adams lwp_exit();
169335a5a358SJonathan Adams } else {
169435a5a358SJonathan Adams thread_exit();
169535a5a358SJonathan Adams }
16967c478bd9Sstevel@tonic-gate }
16977c478bd9Sstevel@tonic-gate
16987c478bd9Sstevel@tonic-gate /*
16997c478bd9Sstevel@tonic-gate * Worker per-entry thread for dynamic dispatches.
17007c478bd9Sstevel@tonic-gate */
17017c478bd9Sstevel@tonic-gate static void
taskq_d_thread(taskq_ent_t * tqe)17027c478bd9Sstevel@tonic-gate taskq_d_thread(taskq_ent_t *tqe)
17037c478bd9Sstevel@tonic-gate {
17045aeb9474SGarrett D'Amore taskq_bucket_t *bucket = tqe->tqent_un.tqent_bucket;
17057c478bd9Sstevel@tonic-gate taskq_t *tq = bucket->tqbucket_taskq;
17067c478bd9Sstevel@tonic-gate kmutex_t *lock = &bucket->tqbucket_lock;
17077c478bd9Sstevel@tonic-gate kcondvar_t *cv = &tqe->tqent_cv;
17087c478bd9Sstevel@tonic-gate callb_cpr_t cprinfo;
1709c6f039c7SToomas Soome clock_t w = 0;
17107c478bd9Sstevel@tonic-gate
17117c478bd9Sstevel@tonic-gate CALLB_CPR_INIT(&cprinfo, lock, callb_generic_cpr, tq->tq_name);
17127c478bd9Sstevel@tonic-gate
17137c478bd9Sstevel@tonic-gate mutex_enter(lock);
17147c478bd9Sstevel@tonic-gate
17157c478bd9Sstevel@tonic-gate for (;;) {
17167c478bd9Sstevel@tonic-gate /*
17177c478bd9Sstevel@tonic-gate * If a task is scheduled (func != NULL), execute it, otherwise
17187c478bd9Sstevel@tonic-gate * sleep, waiting for a job.
17197c478bd9Sstevel@tonic-gate */
17207c478bd9Sstevel@tonic-gate if (tqe->tqent_func != NULL) {
17217c478bd9Sstevel@tonic-gate hrtime_t start;
17227c478bd9Sstevel@tonic-gate hrtime_t end;
17237c478bd9Sstevel@tonic-gate
17247c478bd9Sstevel@tonic-gate ASSERT(bucket->tqbucket_nalloc > 0);
17257c478bd9Sstevel@tonic-gate
17267c478bd9Sstevel@tonic-gate /*
17277c478bd9Sstevel@tonic-gate * It is possible to free the entry right away before
17287c478bd9Sstevel@tonic-gate * actually executing the task so that subsequent
17297c478bd9Sstevel@tonic-gate * dispatches may immediately reuse it. But this,
17307c478bd9Sstevel@tonic-gate * effectively, creates a two-length queue in the entry
17317c478bd9Sstevel@tonic-gate * and may lead to a deadlock if the execution of the
17327c478bd9Sstevel@tonic-gate * current task depends on the execution of the next
17337c478bd9Sstevel@tonic-gate * scheduled task. So, we keep the entry busy until the
17347c478bd9Sstevel@tonic-gate * task is processed.
17357c478bd9Sstevel@tonic-gate */
17367c478bd9Sstevel@tonic-gate
17377c478bd9Sstevel@tonic-gate mutex_exit(lock);
17387c478bd9Sstevel@tonic-gate start = gethrtime();
17397c478bd9Sstevel@tonic-gate DTRACE_PROBE3(taskq__d__exec__start, taskq_t *, tq,
17407c478bd9Sstevel@tonic-gate taskq_bucket_t *, bucket, taskq_ent_t *, tqe);
17417c478bd9Sstevel@tonic-gate tqe->tqent_func(tqe->tqent_arg);
17427c478bd9Sstevel@tonic-gate DTRACE_PROBE3(taskq__d__exec__end, taskq_t *, tq,
17437c478bd9Sstevel@tonic-gate taskq_bucket_t *, bucket, taskq_ent_t *, tqe);
17447c478bd9Sstevel@tonic-gate end = gethrtime();
17457c478bd9Sstevel@tonic-gate mutex_enter(lock);
17467c478bd9Sstevel@tonic-gate bucket->tqbucket_totaltime += end - start;
17477c478bd9Sstevel@tonic-gate
17487c478bd9Sstevel@tonic-gate /*
17497c478bd9Sstevel@tonic-gate * Return the entry to the bucket free list.
17507c478bd9Sstevel@tonic-gate */
17517c478bd9Sstevel@tonic-gate tqe->tqent_func = NULL;
17527c478bd9Sstevel@tonic-gate TQ_APPEND(bucket->tqbucket_freelist, tqe);
17537c478bd9Sstevel@tonic-gate bucket->tqbucket_nalloc--;
17547c478bd9Sstevel@tonic-gate bucket->tqbucket_nfree++;
17557c478bd9Sstevel@tonic-gate ASSERT(!IS_EMPTY(bucket->tqbucket_freelist));
17567c478bd9Sstevel@tonic-gate /*
17577c478bd9Sstevel@tonic-gate * taskq_wait() waits for nalloc to drop to zero on
17587c478bd9Sstevel@tonic-gate * tqbucket_cv.
17597c478bd9Sstevel@tonic-gate */
17607c478bd9Sstevel@tonic-gate cv_signal(&bucket->tqbucket_cv);
17617c478bd9Sstevel@tonic-gate }
17627c478bd9Sstevel@tonic-gate
17637c478bd9Sstevel@tonic-gate /*
17647c478bd9Sstevel@tonic-gate * At this point the entry must be in the bucket free list -
17657c478bd9Sstevel@tonic-gate * either because it was there initially or because it just
17667c478bd9Sstevel@tonic-gate * finished executing a task and put itself on the free list.
17677c478bd9Sstevel@tonic-gate */
17687c478bd9Sstevel@tonic-gate ASSERT(bucket->tqbucket_nfree > 0);
17697c478bd9Sstevel@tonic-gate /*
17707c478bd9Sstevel@tonic-gate * Go to sleep unless we are closing.
17717c478bd9Sstevel@tonic-gate * If a thread is sleeping too long, it dies.
17727c478bd9Sstevel@tonic-gate */
17737c478bd9Sstevel@tonic-gate if (! (bucket->tqbucket_flags & TQBUCKET_CLOSE)) {
1774e0ad97e3SJonathan Adams w = taskq_thread_wait(tq, lock, cv,
1775e0ad97e3SJonathan Adams &cprinfo, taskq_thread_timeout * hz);
17767c478bd9Sstevel@tonic-gate }
17777c478bd9Sstevel@tonic-gate
17787c478bd9Sstevel@tonic-gate /*
17797c478bd9Sstevel@tonic-gate * At this point we may be in two different states:
17807c478bd9Sstevel@tonic-gate *
17817c478bd9Sstevel@tonic-gate * (1) tqent_func is set which means that a new task is
17827c478bd9Sstevel@tonic-gate * dispatched and we need to execute it.
17837c478bd9Sstevel@tonic-gate *
17847c478bd9Sstevel@tonic-gate * (2) Thread is sleeping for too long or we are closing. In
17857c478bd9Sstevel@tonic-gate * both cases destroy the thread and the entry.
17867c478bd9Sstevel@tonic-gate */
17877c478bd9Sstevel@tonic-gate
17887c478bd9Sstevel@tonic-gate /* If func is NULL we should be on the freelist. */
17897c478bd9Sstevel@tonic-gate ASSERT((tqe->tqent_func != NULL) ||
17907c478bd9Sstevel@tonic-gate (bucket->tqbucket_nfree > 0));
17917c478bd9Sstevel@tonic-gate /* If func is non-NULL we should be allocated */
17927c478bd9Sstevel@tonic-gate ASSERT((tqe->tqent_func == NULL) ||
17937c478bd9Sstevel@tonic-gate (bucket->tqbucket_nalloc > 0));
17947c478bd9Sstevel@tonic-gate
17957c478bd9Sstevel@tonic-gate /* Check freelist consistency */
17967c478bd9Sstevel@tonic-gate ASSERT((bucket->tqbucket_nfree > 0) ||
17977c478bd9Sstevel@tonic-gate IS_EMPTY(bucket->tqbucket_freelist));
17987c478bd9Sstevel@tonic-gate ASSERT((bucket->tqbucket_nfree == 0) ||
17997c478bd9Sstevel@tonic-gate !IS_EMPTY(bucket->tqbucket_freelist));
18007c478bd9Sstevel@tonic-gate
18017c478bd9Sstevel@tonic-gate if ((tqe->tqent_func == NULL) &&
18027c478bd9Sstevel@tonic-gate ((w == -1) || (bucket->tqbucket_flags & TQBUCKET_CLOSE))) {
18037c478bd9Sstevel@tonic-gate /*
18047c478bd9Sstevel@tonic-gate * This thread is sleeping for too long or we are
18057c478bd9Sstevel@tonic-gate * closing - time to die.
18067c478bd9Sstevel@tonic-gate */
1807*f8d363f3SGordon Ross break;
1808*f8d363f3SGordon Ross }
1809*f8d363f3SGordon Ross }
18107c478bd9Sstevel@tonic-gate
1811*f8d363f3SGordon Ross /*
1812*f8d363f3SGordon Ross * Thread creation/destruction happens rarely,
1813*f8d363f3SGordon Ross * so grabbing the lock is not a big performance issue.
1814*f8d363f3SGordon Ross * The bucket lock is dropped by CALLB_CPR_EXIT().
1815*f8d363f3SGordon Ross */
18167c478bd9Sstevel@tonic-gate
1817*f8d363f3SGordon Ross /* Remove the entry from the free list. */
1818*f8d363f3SGordon Ross tqe->tqent_prev->tqent_next = tqe->tqent_next;
1819*f8d363f3SGordon Ross tqe->tqent_next->tqent_prev = tqe->tqent_prev;
1820*f8d363f3SGordon Ross ASSERT(bucket->tqbucket_nfree > 0);
1821*f8d363f3SGordon Ross bucket->tqbucket_nfree--;
1822*f8d363f3SGordon Ross
1823*f8d363f3SGordon Ross TQ_STAT(bucket, tqs_tdeaths);
1824*f8d363f3SGordon Ross cv_signal(&bucket->tqbucket_cv);
1825*f8d363f3SGordon Ross tqe->tqent_thread = NULL;
1826*f8d363f3SGordon Ross mutex_enter(&tq->tq_lock);
1827*f8d363f3SGordon Ross tq->tq_tdeaths++;
1828*f8d363f3SGordon Ross mutex_exit(&tq->tq_lock);
1829*f8d363f3SGordon Ross CALLB_CPR_EXIT(&cprinfo); /* mutex_exit(lock) */
1830*f8d363f3SGordon Ross
1831*f8d363f3SGordon Ross kmem_cache_free(taskq_ent_cache, tqe);
1832*f8d363f3SGordon Ross
1833*f8d363f3SGordon Ross if (curthread->t_lwp != NULL) {
1834*f8d363f3SGordon Ross mutex_enter(&curproc->p_lock);
1835*f8d363f3SGordon Ross lwp_exit(); /* noreturn. drops p_lock */
1836*f8d363f3SGordon Ross } else {
1837*f8d363f3SGordon Ross thread_exit();
18387c478bd9Sstevel@tonic-gate }
18397c478bd9Sstevel@tonic-gate }
18407c478bd9Sstevel@tonic-gate
18417c478bd9Sstevel@tonic-gate
18427c478bd9Sstevel@tonic-gate /*
18437c478bd9Sstevel@tonic-gate * Taskq creation. May sleep for memory.
18447c478bd9Sstevel@tonic-gate * Always use automatically generated instances to avoid kstat name space
18457c478bd9Sstevel@tonic-gate * collisions.
18467c478bd9Sstevel@tonic-gate */
18477c478bd9Sstevel@tonic-gate
18487c478bd9Sstevel@tonic-gate taskq_t *
taskq_create(const char * name,int nthreads,pri_t pri,int minalloc,int maxalloc,uint_t flags)18497c478bd9Sstevel@tonic-gate taskq_create(const char *name, int nthreads, pri_t pri, int minalloc,
18507c478bd9Sstevel@tonic-gate int maxalloc, uint_t flags)
18517c478bd9Sstevel@tonic-gate {
185235a5a358SJonathan Adams ASSERT((flags & ~TASKQ_INTERFACE_FLAGS) == 0);
185335a5a358SJonathan Adams
185435a5a358SJonathan Adams return (taskq_create_common(name, 0, nthreads, pri, minalloc,
185535a5a358SJonathan Adams maxalloc, &p0, 0, flags | TASKQ_NOINSTANCE));
18567c478bd9Sstevel@tonic-gate }
18577c478bd9Sstevel@tonic-gate
18587c478bd9Sstevel@tonic-gate /*
18597c478bd9Sstevel@tonic-gate * Create an instance of task queue. It is legal to create task queues with the
18607c478bd9Sstevel@tonic-gate * same name and different instances.
18617c478bd9Sstevel@tonic-gate *
18627c478bd9Sstevel@tonic-gate * taskq_create_instance is used by ddi_taskq_create() where it gets the
18637c478bd9Sstevel@tonic-gate * instance from ddi_get_instance(). In some cases the instance is not
18647c478bd9Sstevel@tonic-gate * initialized and is set to -1. This case is handled as if no instance was
18657c478bd9Sstevel@tonic-gate * passed at all.
18667c478bd9Sstevel@tonic-gate */
18677c478bd9Sstevel@tonic-gate taskq_t *
taskq_create_instance(const char * name,int instance,int nthreads,pri_t pri,int minalloc,int maxalloc,uint_t flags)18687c478bd9Sstevel@tonic-gate taskq_create_instance(const char *name, int instance, int nthreads, pri_t pri,
18697c478bd9Sstevel@tonic-gate int minalloc, int maxalloc, uint_t flags)
18707c478bd9Sstevel@tonic-gate {
187135a5a358SJonathan Adams ASSERT((flags & ~TASKQ_INTERFACE_FLAGS) == 0);
18727c478bd9Sstevel@tonic-gate ASSERT((instance >= 0) || (instance == -1));
18737c478bd9Sstevel@tonic-gate
18747c478bd9Sstevel@tonic-gate if (instance < 0) {
18757c478bd9Sstevel@tonic-gate flags |= TASKQ_NOINSTANCE;
18767c478bd9Sstevel@tonic-gate }
18777c478bd9Sstevel@tonic-gate
18787c478bd9Sstevel@tonic-gate return (taskq_create_common(name, instance, nthreads,
187935a5a358SJonathan Adams pri, minalloc, maxalloc, &p0, 0, flags));
18807c478bd9Sstevel@tonic-gate }
18817c478bd9Sstevel@tonic-gate
188235a5a358SJonathan Adams taskq_t *
taskq_create_proc(const char * name,int nthreads,pri_t pri,int minalloc,int maxalloc,proc_t * proc,uint_t flags)188335a5a358SJonathan Adams taskq_create_proc(const char *name, int nthreads, pri_t pri, int minalloc,
188435a5a358SJonathan Adams int maxalloc, proc_t *proc, uint_t flags)
188535a5a358SJonathan Adams {
188635a5a358SJonathan Adams ASSERT((flags & ~TASKQ_INTERFACE_FLAGS) == 0);
188735a5a358SJonathan Adams ASSERT(proc->p_flag & SSYS);
188835a5a358SJonathan Adams
188935a5a358SJonathan Adams return (taskq_create_common(name, 0, nthreads, pri, minalloc,
189035a5a358SJonathan Adams maxalloc, proc, 0, flags | TASKQ_NOINSTANCE));
189135a5a358SJonathan Adams }
189235a5a358SJonathan Adams
189335a5a358SJonathan Adams taskq_t *
taskq_create_sysdc(const char * name,int nthreads,int minalloc,int maxalloc,proc_t * proc,uint_t dc,uint_t flags)189435a5a358SJonathan Adams taskq_create_sysdc(const char *name, int nthreads, int minalloc,
189535a5a358SJonathan Adams int maxalloc, proc_t *proc, uint_t dc, uint_t flags)
189635a5a358SJonathan Adams {
189735a5a358SJonathan Adams ASSERT((flags & ~TASKQ_INTERFACE_FLAGS) == 0);
189835a5a358SJonathan Adams ASSERT(proc->p_flag & SSYS);
189935a5a358SJonathan Adams
190035a5a358SJonathan Adams return (taskq_create_common(name, 0, nthreads, minclsyspri, minalloc,
190135a5a358SJonathan Adams maxalloc, proc, dc, flags | TASKQ_NOINSTANCE | TASKQ_DUTY_CYCLE));
190235a5a358SJonathan Adams }
190335a5a358SJonathan Adams
19047c478bd9Sstevel@tonic-gate static taskq_t *
taskq_create_common(const char * name,int instance,int nthreads,pri_t pri,int minalloc,int maxalloc,proc_t * proc,uint_t dc,uint_t flags)19057c478bd9Sstevel@tonic-gate taskq_create_common(const char *name, int instance, int nthreads, pri_t pri,
190635a5a358SJonathan Adams int minalloc, int maxalloc, proc_t *proc, uint_t dc, uint_t flags)
19077c478bd9Sstevel@tonic-gate {
19087c478bd9Sstevel@tonic-gate taskq_t *tq = kmem_cache_alloc(taskq_cache, KM_SLEEP);
19097c478bd9Sstevel@tonic-gate uint_t ncpus = ((boot_max_ncpus == -1) ? max_ncpus : boot_max_ncpus);
19107c478bd9Sstevel@tonic-gate uint_t bsize; /* # of buckets - always power of 2 */
19112e0c549eSJonathan Adams int max_nthreads;
19127c478bd9Sstevel@tonic-gate
19137c478bd9Sstevel@tonic-gate /*
191435a5a358SJonathan Adams * TASKQ_DYNAMIC, TASKQ_CPR_SAFE and TASKQ_THREADS_CPU_PCT are all
191535a5a358SJonathan Adams * mutually incompatible.
19167c478bd9Sstevel@tonic-gate */
191735a5a358SJonathan Adams IMPLY((flags & TASKQ_DYNAMIC), !(flags & TASKQ_CPR_SAFE));
191835a5a358SJonathan Adams IMPLY((flags & TASKQ_DYNAMIC), !(flags & TASKQ_THREADS_CPU_PCT));
191935a5a358SJonathan Adams IMPLY((flags & TASKQ_CPR_SAFE), !(flags & TASKQ_THREADS_CPU_PCT));
192035a5a358SJonathan Adams
1921e5488233SGordon Ross /* Cannot have DYNAMIC with DUTY_CYCLE */
1922e5488233SGordon Ross IMPLY((flags & TASKQ_DYNAMIC), !(flags & TASKQ_DUTY_CYCLE));
1923e5488233SGordon Ross
1924e5488233SGordon Ross /* Cannot have DUTY_CYCLE with a p0 kernel process */
192535a5a358SJonathan Adams IMPLY((flags & TASKQ_DUTY_CYCLE), proc != &p0);
19267c478bd9Sstevel@tonic-gate
1927*f8d363f3SGordon Ross /* Cannot have THREADS_LWP with a p0 kernel process */
1928*f8d363f3SGordon Ross IMPLY((flags & TASKQ_THREADS_LWP), proc != &p0);
1929*f8d363f3SGordon Ross
193035a5a358SJonathan Adams /* Cannot have DC_BATCH without DUTY_CYCLE */
193135a5a358SJonathan Adams ASSERT((flags & (TASKQ_DUTY_CYCLE|TASKQ_DC_BATCH)) != TASKQ_DC_BATCH);
193235a5a358SJonathan Adams
193335a5a358SJonathan Adams ASSERT(proc != NULL);
19347c478bd9Sstevel@tonic-gate
19357c478bd9Sstevel@tonic-gate bsize = 1 << (highbit(ncpus) - 1);
19367c478bd9Sstevel@tonic-gate ASSERT(bsize >= 1);
19377c478bd9Sstevel@tonic-gate bsize = MIN(bsize, taskq_maxbuckets);
19387c478bd9Sstevel@tonic-gate
19392e0c549eSJonathan Adams if (flags & TASKQ_DYNAMIC) {
19402e0c549eSJonathan Adams ASSERT3S(nthreads, >=, 1);
19412e0c549eSJonathan Adams tq->tq_maxsize = nthreads;
19427c478bd9Sstevel@tonic-gate
19432e0c549eSJonathan Adams /* For dynamic task queues use just one backup thread */
19442e0c549eSJonathan Adams nthreads = max_nthreads = 1;
19457c478bd9Sstevel@tonic-gate
194635a5a358SJonathan Adams } else if (flags & TASKQ_THREADS_CPU_PCT) {
19472e0c549eSJonathan Adams uint_t pct;
19482e0c549eSJonathan Adams ASSERT3S(nthreads, >=, 0);
19492e0c549eSJonathan Adams pct = nthreads;
19502e0c549eSJonathan Adams
19512e0c549eSJonathan Adams if (pct > taskq_cpupct_max_percent)
19522e0c549eSJonathan Adams pct = taskq_cpupct_max_percent;
19532e0c549eSJonathan Adams
195435a5a358SJonathan Adams /*
195535a5a358SJonathan Adams * If you're using THREADS_CPU_PCT, the process for the
195635a5a358SJonathan Adams * taskq threads must be curproc. This allows any pset
195735a5a358SJonathan Adams * binding to be inherited correctly. If proc is &p0,
195835a5a358SJonathan Adams * we won't be creating LWPs, so new threads will be assigned
195935a5a358SJonathan Adams * to the default processor set.
196035a5a358SJonathan Adams */
196135a5a358SJonathan Adams ASSERT(curproc == proc || proc == &p0);
19622e0c549eSJonathan Adams tq->tq_threads_ncpus_pct = pct;
196335a5a358SJonathan Adams nthreads = 1; /* corrected in taskq_thread_create() */
19642e0c549eSJonathan Adams max_nthreads = TASKQ_THREADS_PCT(max_ncpus, pct);
196535a5a358SJonathan Adams
196635a5a358SJonathan Adams } else {
196735a5a358SJonathan Adams ASSERT3S(nthreads, >=, 1);
196835a5a358SJonathan Adams max_nthreads = nthreads;
19692e0c549eSJonathan Adams }
19702e0c549eSJonathan Adams
19712e0c549eSJonathan Adams if (max_nthreads < taskq_minimum_nthreads_max)
19722e0c549eSJonathan Adams max_nthreads = taskq_minimum_nthreads_max;
19732e0c549eSJonathan Adams
19742e0c549eSJonathan Adams /*
19752e0c549eSJonathan Adams * Make sure the name is 0-terminated, and conforms to the rules for
19762e0c549eSJonathan Adams * C indentifiers
19772e0c549eSJonathan Adams */
19787c478bd9Sstevel@tonic-gate (void) strncpy(tq->tq_name, name, TASKQ_NAMELEN + 1);
19792e0c549eSJonathan Adams strident_canon(tq->tq_name, TASKQ_NAMELEN + 1);
19807c478bd9Sstevel@tonic-gate
19812e0c549eSJonathan Adams tq->tq_flags = flags | TASKQ_CHANGING;
19822e0c549eSJonathan Adams tq->tq_active = 0;
19837c478bd9Sstevel@tonic-gate tq->tq_instance = instance;
19842e0c549eSJonathan Adams tq->tq_nthreads_target = nthreads;
19852e0c549eSJonathan Adams tq->tq_nthreads_max = max_nthreads;
19867c478bd9Sstevel@tonic-gate tq->tq_minalloc = minalloc;
19877c478bd9Sstevel@tonic-gate tq->tq_maxalloc = maxalloc;
19887c478bd9Sstevel@tonic-gate tq->tq_nbuckets = bsize;
198935a5a358SJonathan Adams tq->tq_proc = proc;
19907c478bd9Sstevel@tonic-gate tq->tq_pri = pri;
199135a5a358SJonathan Adams tq->tq_DC = dc;
199235a5a358SJonathan Adams list_link_init(&tq->tq_cpupct_link);
19937c478bd9Sstevel@tonic-gate
19942e0c549eSJonathan Adams if (max_nthreads > 1)
19952e0c549eSJonathan Adams tq->tq_threadlist = kmem_alloc(
19962e0c549eSJonathan Adams sizeof (kthread_t *) * max_nthreads, KM_SLEEP);
19972e0c549eSJonathan Adams
19982e0c549eSJonathan Adams mutex_enter(&tq->tq_lock);
19997c478bd9Sstevel@tonic-gate if (flags & TASKQ_PREPOPULATE) {
20007c478bd9Sstevel@tonic-gate while (minalloc-- > 0)
20017c478bd9Sstevel@tonic-gate taskq_ent_free(tq, taskq_ent_alloc(tq, TQ_SLEEP));
20027c478bd9Sstevel@tonic-gate }
20037c478bd9Sstevel@tonic-gate
2004e5488233SGordon Ross /*
2005e5488233SGordon Ross * Before we start creating threads for this taskq, take a
2006e5488233SGordon Ross * zone hold so the zone can't go away before taskq_destroy
2007e5488233SGordon Ross * makes sure all the taskq threads are gone. This hold is
2008e5488233SGordon Ross * similar in purpose to those taken by zthread_create().
2009e5488233SGordon Ross */
2010e5488233SGordon Ross zone_hold(tq->tq_proc->p_zone);
2011e5488233SGordon Ross
201235a5a358SJonathan Adams /*
201335a5a358SJonathan Adams * Create the first thread, which will create any other threads
201435a5a358SJonathan Adams * necessary. taskq_thread_create will not return until we have
201535a5a358SJonathan Adams * enough threads to be able to process requests.
201635a5a358SJonathan Adams */
20172e0c549eSJonathan Adams taskq_thread_create(tq);
20182e0c549eSJonathan Adams mutex_exit(&tq->tq_lock);
20197c478bd9Sstevel@tonic-gate
20207c478bd9Sstevel@tonic-gate if (flags & TASKQ_DYNAMIC) {
20217c478bd9Sstevel@tonic-gate taskq_bucket_t *bucket = kmem_zalloc(sizeof (taskq_bucket_t) *
20227c478bd9Sstevel@tonic-gate bsize, KM_SLEEP);
20237c478bd9Sstevel@tonic-gate int b_id;
20247c478bd9Sstevel@tonic-gate
20257c478bd9Sstevel@tonic-gate tq->tq_buckets = bucket;
20267c478bd9Sstevel@tonic-gate
20277c478bd9Sstevel@tonic-gate /* Initialize each bucket */
20287c478bd9Sstevel@tonic-gate for (b_id = 0; b_id < bsize; b_id++, bucket++) {
20297c478bd9Sstevel@tonic-gate mutex_init(&bucket->tqbucket_lock, NULL, MUTEX_DEFAULT,
20307c478bd9Sstevel@tonic-gate NULL);
20317c478bd9Sstevel@tonic-gate cv_init(&bucket->tqbucket_cv, NULL, CV_DEFAULT, NULL);
20327c478bd9Sstevel@tonic-gate bucket->tqbucket_taskq = tq;
20337c478bd9Sstevel@tonic-gate bucket->tqbucket_freelist.tqent_next =
20347c478bd9Sstevel@tonic-gate bucket->tqbucket_freelist.tqent_prev =
20357c478bd9Sstevel@tonic-gate &bucket->tqbucket_freelist;
20367c478bd9Sstevel@tonic-gate if (flags & TASKQ_PREPOPULATE)
20377c478bd9Sstevel@tonic-gate taskq_bucket_extend(bucket);
20387c478bd9Sstevel@tonic-gate }
20397c478bd9Sstevel@tonic-gate }
20407c478bd9Sstevel@tonic-gate
20417c478bd9Sstevel@tonic-gate /*
20427c478bd9Sstevel@tonic-gate * Install kstats.
20437c478bd9Sstevel@tonic-gate * We have two cases:
20447c478bd9Sstevel@tonic-gate * 1) Instance is provided to taskq_create_instance(). In this case it
204535a5a358SJonathan Adams * should be >= 0 and we use it.
20467c478bd9Sstevel@tonic-gate *
20477c478bd9Sstevel@tonic-gate * 2) Instance is not provided and is automatically generated
20487c478bd9Sstevel@tonic-gate */
20497c478bd9Sstevel@tonic-gate if (flags & TASKQ_NOINSTANCE) {
20507c478bd9Sstevel@tonic-gate instance = tq->tq_instance =
20517c478bd9Sstevel@tonic-gate (int)(uintptr_t)vmem_alloc(taskq_id_arena, 1, VM_SLEEP);
20527c478bd9Sstevel@tonic-gate }
20537c478bd9Sstevel@tonic-gate
20547c478bd9Sstevel@tonic-gate if (flags & TASKQ_DYNAMIC) {
20557c478bd9Sstevel@tonic-gate if ((tq->tq_kstat = kstat_create("unix", instance,
20562e0c549eSJonathan Adams tq->tq_name, "taskq_d", KSTAT_TYPE_NAMED,
20572e0c549eSJonathan Adams sizeof (taskq_d_kstat) / sizeof (kstat_named_t),
20582e0c549eSJonathan Adams KSTAT_FLAG_VIRTUAL)) != NULL) {
20597c478bd9Sstevel@tonic-gate tq->tq_kstat->ks_lock = &taskq_d_kstat_lock;
20607c478bd9Sstevel@tonic-gate tq->tq_kstat->ks_data = &taskq_d_kstat;
20617c478bd9Sstevel@tonic-gate tq->tq_kstat->ks_update = taskq_d_kstat_update;
20627c478bd9Sstevel@tonic-gate tq->tq_kstat->ks_private = tq;
20637c478bd9Sstevel@tonic-gate kstat_install(tq->tq_kstat);
20647c478bd9Sstevel@tonic-gate }
20657c478bd9Sstevel@tonic-gate } else {
20667c478bd9Sstevel@tonic-gate if ((tq->tq_kstat = kstat_create("unix", instance, tq->tq_name,
20672e0c549eSJonathan Adams "taskq", KSTAT_TYPE_NAMED,
20682e0c549eSJonathan Adams sizeof (taskq_kstat) / sizeof (kstat_named_t),
20692e0c549eSJonathan Adams KSTAT_FLAG_VIRTUAL)) != NULL) {
20707c478bd9Sstevel@tonic-gate tq->tq_kstat->ks_lock = &taskq_kstat_lock;
20717c478bd9Sstevel@tonic-gate tq->tq_kstat->ks_data = &taskq_kstat;
20727c478bd9Sstevel@tonic-gate tq->tq_kstat->ks_update = taskq_kstat_update;
20737c478bd9Sstevel@tonic-gate tq->tq_kstat->ks_private = tq;
20747c478bd9Sstevel@tonic-gate kstat_install(tq->tq_kstat);
20757c478bd9Sstevel@tonic-gate }
20767c478bd9Sstevel@tonic-gate }
20777c478bd9Sstevel@tonic-gate
20787c478bd9Sstevel@tonic-gate return (tq);
20797c478bd9Sstevel@tonic-gate }
20807c478bd9Sstevel@tonic-gate
20817c478bd9Sstevel@tonic-gate /*
20827c478bd9Sstevel@tonic-gate * taskq_destroy().
20837c478bd9Sstevel@tonic-gate *
20847c478bd9Sstevel@tonic-gate * Assumes: by the time taskq_destroy is called no one will use this task queue
20857c478bd9Sstevel@tonic-gate * in any way and no one will try to dispatch entries in it.
20867c478bd9Sstevel@tonic-gate */
20877c478bd9Sstevel@tonic-gate void
taskq_destroy(taskq_t * tq)20887c478bd9Sstevel@tonic-gate taskq_destroy(taskq_t *tq)
20897c478bd9Sstevel@tonic-gate {
20907c478bd9Sstevel@tonic-gate taskq_bucket_t *b = tq->tq_buckets;
20917c478bd9Sstevel@tonic-gate int bid = 0;
20927c478bd9Sstevel@tonic-gate
20937c478bd9Sstevel@tonic-gate ASSERT(! (tq->tq_flags & TASKQ_CPR_SAFE));
20947c478bd9Sstevel@tonic-gate
20957c478bd9Sstevel@tonic-gate /*
20967c478bd9Sstevel@tonic-gate * Destroy kstats.
20977c478bd9Sstevel@tonic-gate */
20987c478bd9Sstevel@tonic-gate if (tq->tq_kstat != NULL) {
20997c478bd9Sstevel@tonic-gate kstat_delete(tq->tq_kstat);
21007c478bd9Sstevel@tonic-gate tq->tq_kstat = NULL;
21017c478bd9Sstevel@tonic-gate }
21027c478bd9Sstevel@tonic-gate
21037c478bd9Sstevel@tonic-gate /*
21047c478bd9Sstevel@tonic-gate * Destroy instance if needed.
21057c478bd9Sstevel@tonic-gate */
21067c478bd9Sstevel@tonic-gate if (tq->tq_flags & TASKQ_NOINSTANCE) {
21077c478bd9Sstevel@tonic-gate vmem_free(taskq_id_arena, (void *)(uintptr_t)(tq->tq_instance),
21087c478bd9Sstevel@tonic-gate 1);
21097c478bd9Sstevel@tonic-gate tq->tq_instance = 0;
21107c478bd9Sstevel@tonic-gate }
21117c478bd9Sstevel@tonic-gate
21122e0c549eSJonathan Adams /*
21132e0c549eSJonathan Adams * Unregister from the cpupct list.
21142e0c549eSJonathan Adams */
21152e0c549eSJonathan Adams if (tq->tq_flags & TASKQ_THREADS_CPU_PCT) {
211635a5a358SJonathan Adams taskq_cpupct_remove(tq);
21172e0c549eSJonathan Adams }
21182e0c549eSJonathan Adams
21197c478bd9Sstevel@tonic-gate /*
21207c478bd9Sstevel@tonic-gate * Wait for any pending entries to complete.
21217c478bd9Sstevel@tonic-gate */
21227c478bd9Sstevel@tonic-gate taskq_wait(tq);
21237c478bd9Sstevel@tonic-gate
21247c478bd9Sstevel@tonic-gate mutex_enter(&tq->tq_lock);
21257c478bd9Sstevel@tonic-gate ASSERT((tq->tq_task.tqent_next == &tq->tq_task) &&
21267c478bd9Sstevel@tonic-gate (tq->tq_active == 0));
21277c478bd9Sstevel@tonic-gate
21282e0c549eSJonathan Adams /* notify all the threads that they need to exit */
21292e0c549eSJonathan Adams tq->tq_nthreads_target = 0;
21307c478bd9Sstevel@tonic-gate
21312e0c549eSJonathan Adams tq->tq_flags |= TASKQ_CHANGING;
21327c478bd9Sstevel@tonic-gate cv_broadcast(&tq->tq_dispatch_cv);
21332e0c549eSJonathan Adams cv_broadcast(&tq->tq_exit_cv);
21342e0c549eSJonathan Adams
21357c478bd9Sstevel@tonic-gate while (tq->tq_nthreads != 0)
21367c478bd9Sstevel@tonic-gate cv_wait(&tq->tq_wait_cv, &tq->tq_lock);
21377c478bd9Sstevel@tonic-gate
21382e0c549eSJonathan Adams if (tq->tq_nthreads_max != 1)
21392e0c549eSJonathan Adams kmem_free(tq->tq_threadlist, sizeof (kthread_t *) *
21402e0c549eSJonathan Adams tq->tq_nthreads_max);
21412e0c549eSJonathan Adams
21427c478bd9Sstevel@tonic-gate tq->tq_minalloc = 0;
21437c478bd9Sstevel@tonic-gate while (tq->tq_nalloc != 0)
21447c478bd9Sstevel@tonic-gate taskq_ent_free(tq, taskq_ent_alloc(tq, TQ_SLEEP));
21457c478bd9Sstevel@tonic-gate
21467c478bd9Sstevel@tonic-gate mutex_exit(&tq->tq_lock);
21477c478bd9Sstevel@tonic-gate
21487c478bd9Sstevel@tonic-gate /*
21497c478bd9Sstevel@tonic-gate * Mark each bucket as closing and wakeup all sleeping threads.
21507c478bd9Sstevel@tonic-gate */
21517c478bd9Sstevel@tonic-gate for (; (b != NULL) && (bid < tq->tq_nbuckets); b++, bid++) {
21527c478bd9Sstevel@tonic-gate taskq_ent_t *tqe;
21537c478bd9Sstevel@tonic-gate
21547c478bd9Sstevel@tonic-gate mutex_enter(&b->tqbucket_lock);
21557c478bd9Sstevel@tonic-gate
21567c478bd9Sstevel@tonic-gate b->tqbucket_flags |= TQBUCKET_CLOSE;
21577c478bd9Sstevel@tonic-gate /* Wakeup all sleeping threads */
21587c478bd9Sstevel@tonic-gate
21597c478bd9Sstevel@tonic-gate for (tqe = b->tqbucket_freelist.tqent_next;
21607c478bd9Sstevel@tonic-gate tqe != &b->tqbucket_freelist; tqe = tqe->tqent_next)
21617c478bd9Sstevel@tonic-gate cv_signal(&tqe->tqent_cv);
21627c478bd9Sstevel@tonic-gate
21637c478bd9Sstevel@tonic-gate ASSERT(b->tqbucket_nalloc == 0);
21647c478bd9Sstevel@tonic-gate
21657c478bd9Sstevel@tonic-gate /*
21667c478bd9Sstevel@tonic-gate * At this point we waited for all pending jobs to complete (in
21677c478bd9Sstevel@tonic-gate * both the task queue and the bucket and no new jobs should
21687c478bd9Sstevel@tonic-gate * arrive. Wait for all threads to die.
21697c478bd9Sstevel@tonic-gate */
21707c478bd9Sstevel@tonic-gate while (b->tqbucket_nfree > 0)
21717c478bd9Sstevel@tonic-gate cv_wait(&b->tqbucket_cv, &b->tqbucket_lock);
21727c478bd9Sstevel@tonic-gate mutex_exit(&b->tqbucket_lock);
21737c478bd9Sstevel@tonic-gate mutex_destroy(&b->tqbucket_lock);
21747c478bd9Sstevel@tonic-gate cv_destroy(&b->tqbucket_cv);
21757c478bd9Sstevel@tonic-gate }
21767c478bd9Sstevel@tonic-gate
21777c478bd9Sstevel@tonic-gate if (tq->tq_buckets != NULL) {
21787c478bd9Sstevel@tonic-gate ASSERT(tq->tq_flags & TASKQ_DYNAMIC);
21797c478bd9Sstevel@tonic-gate kmem_free(tq->tq_buckets,
21807c478bd9Sstevel@tonic-gate sizeof (taskq_bucket_t) * tq->tq_nbuckets);
21817c478bd9Sstevel@tonic-gate
21827c478bd9Sstevel@tonic-gate /* Cleanup fields before returning tq to the cache */
21837c478bd9Sstevel@tonic-gate tq->tq_buckets = NULL;
21847c478bd9Sstevel@tonic-gate tq->tq_tcreates = 0;
21857c478bd9Sstevel@tonic-gate tq->tq_tdeaths = 0;
21867c478bd9Sstevel@tonic-gate } else {
21877c478bd9Sstevel@tonic-gate ASSERT(!(tq->tq_flags & TASKQ_DYNAMIC));
21887c478bd9Sstevel@tonic-gate }
21897c478bd9Sstevel@tonic-gate
2190e5488233SGordon Ross /*
2191e5488233SGordon Ross * Now that all the taskq threads are gone, we can
2192e5488233SGordon Ross * drop the zone hold taken in taskq_create_common
2193e5488233SGordon Ross */
2194e5488233SGordon Ross zone_rele(tq->tq_proc->p_zone);
2195e5488233SGordon Ross
21962e0c549eSJonathan Adams tq->tq_threads_ncpus_pct = 0;
21977c478bd9Sstevel@tonic-gate tq->tq_totaltime = 0;
21987c478bd9Sstevel@tonic-gate tq->tq_tasks = 0;
21997c478bd9Sstevel@tonic-gate tq->tq_maxtasks = 0;
22007c478bd9Sstevel@tonic-gate tq->tq_executed = 0;
22017c478bd9Sstevel@tonic-gate kmem_cache_free(taskq_cache, tq);
22027c478bd9Sstevel@tonic-gate }
22037c478bd9Sstevel@tonic-gate
22047c478bd9Sstevel@tonic-gate /*
22057c478bd9Sstevel@tonic-gate * Extend a bucket with a new entry on the free list and attach a worker thread
22067c478bd9Sstevel@tonic-gate * to it.
22077c478bd9Sstevel@tonic-gate *
22087c478bd9Sstevel@tonic-gate * Argument: pointer to the bucket.
22097c478bd9Sstevel@tonic-gate *
22107c478bd9Sstevel@tonic-gate * This function may quietly fail. It is only used by taskq_dispatch() which
22117c478bd9Sstevel@tonic-gate * handles such failures properly.
22127c478bd9Sstevel@tonic-gate */
22137c478bd9Sstevel@tonic-gate static void
taskq_bucket_extend(void * arg)22147c478bd9Sstevel@tonic-gate taskq_bucket_extend(void *arg)
22157c478bd9Sstevel@tonic-gate {
22167c478bd9Sstevel@tonic-gate taskq_ent_t *tqe;
22177c478bd9Sstevel@tonic-gate taskq_bucket_t *b = (taskq_bucket_t *)arg;
22187c478bd9Sstevel@tonic-gate taskq_t *tq = b->tqbucket_taskq;
2219*f8d363f3SGordon Ross kthread_t *t;
22207c478bd9Sstevel@tonic-gate int nthreads;
22217c478bd9Sstevel@tonic-gate
2222216d7723SPrakash Surya mutex_enter(&tq->tq_lock);
2223216d7723SPrakash Surya
22247c478bd9Sstevel@tonic-gate if (! ENOUGH_MEMORY()) {
2225216d7723SPrakash Surya tq->tq_nomem++;
2226216d7723SPrakash Surya mutex_exit(&tq->tq_lock);
22277c478bd9Sstevel@tonic-gate return;
22287c478bd9Sstevel@tonic-gate }
22297c478bd9Sstevel@tonic-gate
22307c478bd9Sstevel@tonic-gate /*
22317c478bd9Sstevel@tonic-gate * Observe global taskq limits on the number of threads.
22327c478bd9Sstevel@tonic-gate */
22337c478bd9Sstevel@tonic-gate if (tq->tq_tcreates++ - tq->tq_tdeaths > tq->tq_maxsize) {
22347c478bd9Sstevel@tonic-gate tq->tq_tcreates--;
22357c478bd9Sstevel@tonic-gate mutex_exit(&tq->tq_lock);
22367c478bd9Sstevel@tonic-gate return;
22377c478bd9Sstevel@tonic-gate }
22387c478bd9Sstevel@tonic-gate mutex_exit(&tq->tq_lock);
22397c478bd9Sstevel@tonic-gate
22407c478bd9Sstevel@tonic-gate tqe = kmem_cache_alloc(taskq_ent_cache, KM_NOSLEEP);
22417c478bd9Sstevel@tonic-gate
22427c478bd9Sstevel@tonic-gate if (tqe == NULL) {
22437c478bd9Sstevel@tonic-gate mutex_enter(&tq->tq_lock);
2244216d7723SPrakash Surya tq->tq_nomem++;
22457c478bd9Sstevel@tonic-gate tq->tq_tcreates--;
22467c478bd9Sstevel@tonic-gate mutex_exit(&tq->tq_lock);
22477c478bd9Sstevel@tonic-gate return;
22487c478bd9Sstevel@tonic-gate }
22497c478bd9Sstevel@tonic-gate
22507c478bd9Sstevel@tonic-gate ASSERT(tqe->tqent_thread == NULL);
22517c478bd9Sstevel@tonic-gate
22525aeb9474SGarrett D'Amore tqe->tqent_un.tqent_bucket = b;
22537c478bd9Sstevel@tonic-gate
22547c478bd9Sstevel@tonic-gate /*
22557c478bd9Sstevel@tonic-gate * Create a thread in a TS_STOPPED state first. If it is successfully
22567c478bd9Sstevel@tonic-gate * created, place the entry on the free list and start the thread.
22577c478bd9Sstevel@tonic-gate */
2258*f8d363f3SGordon Ross if ((tq->tq_flags & TASKQ_THREADS_LWP) != 0) {
2259*f8d363f3SGordon Ross /* Enforced in taskq_create_common */
2260*f8d363f3SGordon Ross ASSERT3P(tq->tq_proc, !=, &p0);
2261*f8d363f3SGordon Ross t = lwp_kernel_create(tq->tq_proc, taskq_d_thread,
2262*f8d363f3SGordon Ross tqe, TS_STOPPED, tq->tq_pri);
2263*f8d363f3SGordon Ross } else {
2264*f8d363f3SGordon Ross t = thread_create(NULL, 0, taskq_d_thread, tqe,
2265*f8d363f3SGordon Ross 0, tq->tq_proc, TS_STOPPED, tq->tq_pri);
2266*f8d363f3SGordon Ross }
2267*f8d363f3SGordon Ross tqe->tqent_thread = t;
2268*f8d363f3SGordon Ross t->t_taskq = tq; /* mark thread as a taskq_member() */
22697c478bd9Sstevel@tonic-gate
22707c478bd9Sstevel@tonic-gate /*
22717c478bd9Sstevel@tonic-gate * Once the entry is ready, link it to the the bucket free list.
22727c478bd9Sstevel@tonic-gate */
22737c478bd9Sstevel@tonic-gate mutex_enter(&b->tqbucket_lock);
22747c478bd9Sstevel@tonic-gate tqe->tqent_func = NULL;
22757c478bd9Sstevel@tonic-gate TQ_APPEND(b->tqbucket_freelist, tqe);
22767c478bd9Sstevel@tonic-gate b->tqbucket_nfree++;
22777c478bd9Sstevel@tonic-gate TQ_STAT(b, tqs_tcreates);
22787c478bd9Sstevel@tonic-gate
22797c478bd9Sstevel@tonic-gate #if TASKQ_STATISTIC
22807c478bd9Sstevel@tonic-gate nthreads = b->tqbucket_stat.tqs_tcreates -
22817c478bd9Sstevel@tonic-gate b->tqbucket_stat.tqs_tdeaths;
22827c478bd9Sstevel@tonic-gate b->tqbucket_stat.tqs_maxthreads = MAX(nthreads,
22837c478bd9Sstevel@tonic-gate b->tqbucket_stat.tqs_maxthreads);
22847c478bd9Sstevel@tonic-gate #endif
22857c478bd9Sstevel@tonic-gate
22867c478bd9Sstevel@tonic-gate mutex_exit(&b->tqbucket_lock);
2287*f8d363f3SGordon Ross
22887c478bd9Sstevel@tonic-gate /*
22897c478bd9Sstevel@tonic-gate * Start the stopped thread.
22907c478bd9Sstevel@tonic-gate */
2291*f8d363f3SGordon Ross if (t->t_lwp != NULL) {
2292*f8d363f3SGordon Ross proc_t *p = tq->tq_proc;
2293*f8d363f3SGordon Ross mutex_enter(&p->p_lock);
2294*f8d363f3SGordon Ross t->t_proc_flag &= ~TP_HOLDLWP;
2295*f8d363f3SGordon Ross lwp_create_done(t); /* Sets TS_ALLSTART etc. */
2296*f8d363f3SGordon Ross mutex_exit(&p->p_lock);
2297*f8d363f3SGordon Ross } else {
2298*f8d363f3SGordon Ross thread_lock(t);
2299*f8d363f3SGordon Ross t->t_schedflag |= TS_ALLSTART;
2300*f8d363f3SGordon Ross setrun_locked(t);
2301*f8d363f3SGordon Ross thread_unlock(t);
2302*f8d363f3SGordon Ross }
23037c478bd9Sstevel@tonic-gate }
23047c478bd9Sstevel@tonic-gate
23057c478bd9Sstevel@tonic-gate static int
taskq_kstat_update(kstat_t * ksp,int rw)23067c478bd9Sstevel@tonic-gate taskq_kstat_update(kstat_t *ksp, int rw)
23077c478bd9Sstevel@tonic-gate {
23087c478bd9Sstevel@tonic-gate struct taskq_kstat *tqsp = &taskq_kstat;
23097c478bd9Sstevel@tonic-gate taskq_t *tq = ksp->ks_private;
23107c478bd9Sstevel@tonic-gate
23117c478bd9Sstevel@tonic-gate if (rw == KSTAT_WRITE)
23127c478bd9Sstevel@tonic-gate return (EACCES);
23137c478bd9Sstevel@tonic-gate
231435a5a358SJonathan Adams tqsp->tq_pid.value.ui64 = tq->tq_proc->p_pid;
23157c478bd9Sstevel@tonic-gate tqsp->tq_tasks.value.ui64 = tq->tq_tasks;
23167c478bd9Sstevel@tonic-gate tqsp->tq_executed.value.ui64 = tq->tq_executed;
23177c478bd9Sstevel@tonic-gate tqsp->tq_maxtasks.value.ui64 = tq->tq_maxtasks;
23187c478bd9Sstevel@tonic-gate tqsp->tq_totaltime.value.ui64 = tq->tq_totaltime;
23197c478bd9Sstevel@tonic-gate tqsp->tq_nactive.value.ui64 = tq->tq_active;
23207c478bd9Sstevel@tonic-gate tqsp->tq_nalloc.value.ui64 = tq->tq_nalloc;
23217c478bd9Sstevel@tonic-gate tqsp->tq_pri.value.ui64 = tq->tq_pri;
23227c478bd9Sstevel@tonic-gate tqsp->tq_nthreads.value.ui64 = tq->tq_nthreads;
2323216d7723SPrakash Surya tqsp->tq_nomem.value.ui64 = tq->tq_nomem;
23247c478bd9Sstevel@tonic-gate return (0);
23257c478bd9Sstevel@tonic-gate }
23267c478bd9Sstevel@tonic-gate
23277c478bd9Sstevel@tonic-gate static int
taskq_d_kstat_update(kstat_t * ksp,int rw)23287c478bd9Sstevel@tonic-gate taskq_d_kstat_update(kstat_t *ksp, int rw)
23297c478bd9Sstevel@tonic-gate {
23307c478bd9Sstevel@tonic-gate struct taskq_d_kstat *tqsp = &taskq_d_kstat;
23317c478bd9Sstevel@tonic-gate taskq_t *tq = ksp->ks_private;
23327c478bd9Sstevel@tonic-gate taskq_bucket_t *b = tq->tq_buckets;
23337c478bd9Sstevel@tonic-gate int bid = 0;
23347c478bd9Sstevel@tonic-gate
23357c478bd9Sstevel@tonic-gate if (rw == KSTAT_WRITE)
23367c478bd9Sstevel@tonic-gate return (EACCES);
23377c478bd9Sstevel@tonic-gate
23387c478bd9Sstevel@tonic-gate ASSERT(tq->tq_flags & TASKQ_DYNAMIC);
23397c478bd9Sstevel@tonic-gate
23407c478bd9Sstevel@tonic-gate tqsp->tqd_btasks.value.ui64 = tq->tq_tasks;
23417c478bd9Sstevel@tonic-gate tqsp->tqd_bexecuted.value.ui64 = tq->tq_executed;
23427c478bd9Sstevel@tonic-gate tqsp->tqd_bmaxtasks.value.ui64 = tq->tq_maxtasks;
23437c478bd9Sstevel@tonic-gate tqsp->tqd_bnalloc.value.ui64 = tq->tq_nalloc;
23447c478bd9Sstevel@tonic-gate tqsp->tqd_bnactive.value.ui64 = tq->tq_active;
23457c478bd9Sstevel@tonic-gate tqsp->tqd_btotaltime.value.ui64 = tq->tq_totaltime;
23467c478bd9Sstevel@tonic-gate tqsp->tqd_pri.value.ui64 = tq->tq_pri;
2347216d7723SPrakash Surya tqsp->tqd_nomem.value.ui64 = tq->tq_nomem;
23487c478bd9Sstevel@tonic-gate
23497c478bd9Sstevel@tonic-gate tqsp->tqd_hits.value.ui64 = 0;
23507c478bd9Sstevel@tonic-gate tqsp->tqd_misses.value.ui64 = 0;
23517c478bd9Sstevel@tonic-gate tqsp->tqd_overflows.value.ui64 = 0;
23527c478bd9Sstevel@tonic-gate tqsp->tqd_tcreates.value.ui64 = 0;
23537c478bd9Sstevel@tonic-gate tqsp->tqd_tdeaths.value.ui64 = 0;
23547c478bd9Sstevel@tonic-gate tqsp->tqd_maxthreads.value.ui64 = 0;
23557c478bd9Sstevel@tonic-gate tqsp->tqd_nomem.value.ui64 = 0;
23567c478bd9Sstevel@tonic-gate tqsp->tqd_disptcreates.value.ui64 = 0;
23577c478bd9Sstevel@tonic-gate tqsp->tqd_totaltime.value.ui64 = 0;
23587c478bd9Sstevel@tonic-gate tqsp->tqd_nalloc.value.ui64 = 0;
23597c478bd9Sstevel@tonic-gate tqsp->tqd_nfree.value.ui64 = 0;
23607c478bd9Sstevel@tonic-gate
23617c478bd9Sstevel@tonic-gate for (; (b != NULL) && (bid < tq->tq_nbuckets); b++, bid++) {
23627c478bd9Sstevel@tonic-gate tqsp->tqd_hits.value.ui64 += b->tqbucket_stat.tqs_hits;
23637c478bd9Sstevel@tonic-gate tqsp->tqd_misses.value.ui64 += b->tqbucket_stat.tqs_misses;
23647c478bd9Sstevel@tonic-gate tqsp->tqd_overflows.value.ui64 += b->tqbucket_stat.tqs_overflow;
23657c478bd9Sstevel@tonic-gate tqsp->tqd_tcreates.value.ui64 += b->tqbucket_stat.tqs_tcreates;
23667c478bd9Sstevel@tonic-gate tqsp->tqd_tdeaths.value.ui64 += b->tqbucket_stat.tqs_tdeaths;
23677c478bd9Sstevel@tonic-gate tqsp->tqd_maxthreads.value.ui64 +=
23687c478bd9Sstevel@tonic-gate b->tqbucket_stat.tqs_maxthreads;
23697c478bd9Sstevel@tonic-gate tqsp->tqd_disptcreates.value.ui64 +=
23707c478bd9Sstevel@tonic-gate b->tqbucket_stat.tqs_disptcreates;
23717c478bd9Sstevel@tonic-gate tqsp->tqd_totaltime.value.ui64 += b->tqbucket_totaltime;
23727c478bd9Sstevel@tonic-gate tqsp->tqd_nalloc.value.ui64 += b->tqbucket_nalloc;
23737c478bd9Sstevel@tonic-gate tqsp->tqd_nfree.value.ui64 += b->tqbucket_nfree;
23747c478bd9Sstevel@tonic-gate }
23757c478bd9Sstevel@tonic-gate return (0);
23767c478bd9Sstevel@tonic-gate }
2377