17c478bd9Sstevel@tonic-gate /* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 5*2e0c549eSJonathan Adams * Common Development and Distribution License (the "License"). 6*2e0c549eSJonathan Adams * You may not use this file except in compliance with the License. 77c478bd9Sstevel@tonic-gate * 87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 117c478bd9Sstevel@tonic-gate * and limitations under the License. 127c478bd9Sstevel@tonic-gate * 137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 187c478bd9Sstevel@tonic-gate * 197c478bd9Sstevel@tonic-gate * CDDL HEADER END 207c478bd9Sstevel@tonic-gate */ 217c478bd9Sstevel@tonic-gate /* 22*2e0c549eSJonathan Adams * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 237c478bd9Sstevel@tonic-gate * Use is subject to license terms. 247c478bd9Sstevel@tonic-gate */ 257c478bd9Sstevel@tonic-gate 267c478bd9Sstevel@tonic-gate /* 277c478bd9Sstevel@tonic-gate * Kernel task queues: general-purpose asynchronous task scheduling. 287c478bd9Sstevel@tonic-gate * 297c478bd9Sstevel@tonic-gate * A common problem in kernel programming is the need to schedule tasks 307c478bd9Sstevel@tonic-gate * to be performed later, by another thread. There are several reasons 317c478bd9Sstevel@tonic-gate * you may want or need to do this: 327c478bd9Sstevel@tonic-gate * 337c478bd9Sstevel@tonic-gate * (1) The task isn't time-critical, but your current code path is. 347c478bd9Sstevel@tonic-gate * 357c478bd9Sstevel@tonic-gate * (2) The task may require grabbing locks that you already hold. 367c478bd9Sstevel@tonic-gate * 377c478bd9Sstevel@tonic-gate * (3) The task may need to block (e.g. to wait for memory), but you 387c478bd9Sstevel@tonic-gate * cannot block in your current context. 397c478bd9Sstevel@tonic-gate * 407c478bd9Sstevel@tonic-gate * (4) Your code path can't complete because of some condition, but you can't 417c478bd9Sstevel@tonic-gate * sleep or fail, so you queue the task for later execution when condition 427c478bd9Sstevel@tonic-gate * disappears. 437c478bd9Sstevel@tonic-gate * 447c478bd9Sstevel@tonic-gate * (5) You just want a simple way to launch multiple tasks in parallel. 457c478bd9Sstevel@tonic-gate * 467c478bd9Sstevel@tonic-gate * Task queues provide such a facility. In its simplest form (used when 477c478bd9Sstevel@tonic-gate * performance is not a critical consideration) a task queue consists of a 487c478bd9Sstevel@tonic-gate * single list of tasks, together with one or more threads to service the 497c478bd9Sstevel@tonic-gate * list. There are some cases when this simple queue is not sufficient: 507c478bd9Sstevel@tonic-gate * 517c478bd9Sstevel@tonic-gate * (1) The task queues are very hot and there is a need to avoid data and lock 527c478bd9Sstevel@tonic-gate * contention over global resources. 537c478bd9Sstevel@tonic-gate * 547c478bd9Sstevel@tonic-gate * (2) Some tasks may depend on other tasks to complete, so they can't be put in 557c478bd9Sstevel@tonic-gate * the same list managed by the same thread. 567c478bd9Sstevel@tonic-gate * 577c478bd9Sstevel@tonic-gate * (3) Some tasks may block for a long time, and this should not block other 587c478bd9Sstevel@tonic-gate * tasks in the queue. 597c478bd9Sstevel@tonic-gate * 607c478bd9Sstevel@tonic-gate * To provide useful service in such cases we define a "dynamic task queue" 617c478bd9Sstevel@tonic-gate * which has an individual thread for each of the tasks. These threads are 627c478bd9Sstevel@tonic-gate * dynamically created as they are needed and destroyed when they are not in 637c478bd9Sstevel@tonic-gate * use. The API for managing task pools is the same as for managing task queues 647c478bd9Sstevel@tonic-gate * with the exception of a taskq creation flag TASKQ_DYNAMIC which tells that 657c478bd9Sstevel@tonic-gate * dynamic task pool behavior is desired. 667c478bd9Sstevel@tonic-gate * 677c478bd9Sstevel@tonic-gate * Dynamic task queues may also place tasks in the normal queue (called "backing 687c478bd9Sstevel@tonic-gate * queue") when task pool runs out of resources. Users of task queues may 697c478bd9Sstevel@tonic-gate * disallow such queued scheduling by specifying TQ_NOQUEUE in the dispatch 707c478bd9Sstevel@tonic-gate * flags. 717c478bd9Sstevel@tonic-gate * 727c478bd9Sstevel@tonic-gate * The backing task queue is also used for scheduling internal tasks needed for 737c478bd9Sstevel@tonic-gate * dynamic task queue maintenance. 747c478bd9Sstevel@tonic-gate * 75*2e0c549eSJonathan Adams * INTERFACES ================================================================== 767c478bd9Sstevel@tonic-gate * 777c478bd9Sstevel@tonic-gate * taskq_t *taskq_create(name, nthreads, pri_t pri, minalloc, maxall, flags); 787c478bd9Sstevel@tonic-gate * 797c478bd9Sstevel@tonic-gate * Create a taskq with specified properties. 807c478bd9Sstevel@tonic-gate * Possible 'flags': 817c478bd9Sstevel@tonic-gate * 827c478bd9Sstevel@tonic-gate * TASKQ_DYNAMIC: Create task pool for task management. If this flag is 83*2e0c549eSJonathan Adams * specified, 'nthreads' specifies the maximum number of threads in 847c478bd9Sstevel@tonic-gate * the task queue. Task execution order for dynamic task queues is 857c478bd9Sstevel@tonic-gate * not predictable. 867c478bd9Sstevel@tonic-gate * 877c478bd9Sstevel@tonic-gate * If this flag is not specified (default case) a 88*2e0c549eSJonathan Adams * single-list task queue is created with 'nthreads' threads 89*2e0c549eSJonathan Adams * servicing it. Entries in this queue are managed by 90*2e0c549eSJonathan Adams * taskq_ent_alloc() and taskq_ent_free() which try to keep the 91*2e0c549eSJonathan Adams * task population between 'minalloc' and 'maxalloc', but the 927c478bd9Sstevel@tonic-gate * latter limit is only advisory for TQ_SLEEP dispatches and the 937c478bd9Sstevel@tonic-gate * former limit is only advisory for TQ_NOALLOC dispatches. If 947c478bd9Sstevel@tonic-gate * TASKQ_PREPOPULATE is set in 'flags', the taskq will be 957c478bd9Sstevel@tonic-gate * prepopulated with 'minalloc' task structures. 967c478bd9Sstevel@tonic-gate * 977c478bd9Sstevel@tonic-gate * Since non-DYNAMIC taskqs are queues, tasks are guaranteed to be 987c478bd9Sstevel@tonic-gate * executed in the order they are scheduled if nthreads == 1. 997c478bd9Sstevel@tonic-gate * If nthreads > 1, task execution order is not predictable. 1007c478bd9Sstevel@tonic-gate * 1017c478bd9Sstevel@tonic-gate * TASKQ_PREPOPULATE: Prepopulate task queue with threads. 1027c478bd9Sstevel@tonic-gate * Also prepopulate the task queue with 'minalloc' task structures. 1037c478bd9Sstevel@tonic-gate * 104*2e0c549eSJonathan Adams * TASKQ_THREADS_CPU_PCT: This flag specifies that 'nthreads' should be 105*2e0c549eSJonathan Adams * interpreted as a percentage of the # of online CPUs on the 106*2e0c549eSJonathan Adams * system. The taskq subsystem will automatically adjust the 107*2e0c549eSJonathan Adams * number of threads in the taskq in response to CPU online 108*2e0c549eSJonathan Adams * and offline events, to keep the ratio. nthreads must be in 109*2e0c549eSJonathan Adams * the range [0,100]. 110*2e0c549eSJonathan Adams * 111*2e0c549eSJonathan Adams * The calculation used is: 112*2e0c549eSJonathan Adams * 113*2e0c549eSJonathan Adams * MAX((ncpus_online * percentage)/100, 1) 114*2e0c549eSJonathan Adams * 115*2e0c549eSJonathan Adams * This flag is not supported for DYNAMIC task queues. 116*2e0c549eSJonathan Adams * This flag is not compatible with TASKQ_CPR_SAFE. 117*2e0c549eSJonathan Adams * 1187c478bd9Sstevel@tonic-gate * TASKQ_CPR_SAFE: This flag specifies that users of the task queue will 119*2e0c549eSJonathan Adams * use their own protocol for handling CPR issues. This flag is not 120*2e0c549eSJonathan Adams * supported for DYNAMIC task queues. This flag is not compatible 121*2e0c549eSJonathan Adams * with TASKQ_THREADS_CPU_PCT. 1227c478bd9Sstevel@tonic-gate * 1237c478bd9Sstevel@tonic-gate * The 'pri' field specifies the default priority for the threads that 1247c478bd9Sstevel@tonic-gate * service all scheduled tasks. 1257c478bd9Sstevel@tonic-gate * 1267c478bd9Sstevel@tonic-gate * void taskq_destroy(tap): 1277c478bd9Sstevel@tonic-gate * 1287c478bd9Sstevel@tonic-gate * Waits for any scheduled tasks to complete, then destroys the taskq. 1297c478bd9Sstevel@tonic-gate * Caller should guarantee that no new tasks are scheduled in the closing 1307c478bd9Sstevel@tonic-gate * taskq. 1317c478bd9Sstevel@tonic-gate * 1327c478bd9Sstevel@tonic-gate * taskqid_t taskq_dispatch(tq, func, arg, flags): 1337c478bd9Sstevel@tonic-gate * 1347c478bd9Sstevel@tonic-gate * Dispatches the task "func(arg)" to taskq. The 'flags' indicates whether 1357c478bd9Sstevel@tonic-gate * the caller is willing to block for memory. The function returns an 1367c478bd9Sstevel@tonic-gate * opaque value which is zero iff dispatch fails. If flags is TQ_NOSLEEP 1377c478bd9Sstevel@tonic-gate * or TQ_NOALLOC and the task can't be dispatched, taskq_dispatch() fails 1387c478bd9Sstevel@tonic-gate * and returns (taskqid_t)0. 1397c478bd9Sstevel@tonic-gate * 1407c478bd9Sstevel@tonic-gate * ASSUMES: func != NULL. 1417c478bd9Sstevel@tonic-gate * 1427c478bd9Sstevel@tonic-gate * Possible flags: 1437c478bd9Sstevel@tonic-gate * TQ_NOSLEEP: Do not wait for resources; may fail. 1447c478bd9Sstevel@tonic-gate * 1457c478bd9Sstevel@tonic-gate * TQ_NOALLOC: Do not allocate memory; may fail. May only be used with 1467c478bd9Sstevel@tonic-gate * non-dynamic task queues. 1477c478bd9Sstevel@tonic-gate * 1487c478bd9Sstevel@tonic-gate * TQ_NOQUEUE: Do not enqueue a task if it can't dispatch it due to 1497c478bd9Sstevel@tonic-gate * lack of available resources and fail. If this flag is not 1507c478bd9Sstevel@tonic-gate * set, and the task pool is exhausted, the task may be scheduled 1517c478bd9Sstevel@tonic-gate * in the backing queue. This flag may ONLY be used with dynamic 1527c478bd9Sstevel@tonic-gate * task queues. 1537c478bd9Sstevel@tonic-gate * 1547c478bd9Sstevel@tonic-gate * NOTE: This flag should always be used when a task queue is used 1557c478bd9Sstevel@tonic-gate * for tasks that may depend on each other for completion. 1567c478bd9Sstevel@tonic-gate * Enqueueing dependent tasks may create deadlocks. 1577c478bd9Sstevel@tonic-gate * 1587c478bd9Sstevel@tonic-gate * TQ_SLEEP: May block waiting for resources. May still fail for 1597c478bd9Sstevel@tonic-gate * dynamic task queues if TQ_NOQUEUE is also specified, otherwise 1607c478bd9Sstevel@tonic-gate * always succeed. 1617c478bd9Sstevel@tonic-gate * 1627c478bd9Sstevel@tonic-gate * NOTE: Dynamic task queues are much more likely to fail in 1637c478bd9Sstevel@tonic-gate * taskq_dispatch() (especially if TQ_NOQUEUE was specified), so it 1647c478bd9Sstevel@tonic-gate * is important to have backup strategies handling such failures. 1657c478bd9Sstevel@tonic-gate * 1667c478bd9Sstevel@tonic-gate * void taskq_wait(tq): 1677c478bd9Sstevel@tonic-gate * 1687c478bd9Sstevel@tonic-gate * Waits for all previously scheduled tasks to complete. 1697c478bd9Sstevel@tonic-gate * 1707c478bd9Sstevel@tonic-gate * NOTE: It does not stop any new task dispatches. 1717c478bd9Sstevel@tonic-gate * Do NOT call taskq_wait() from a task: it will cause deadlock. 1727c478bd9Sstevel@tonic-gate * 1737c478bd9Sstevel@tonic-gate * void taskq_suspend(tq) 1747c478bd9Sstevel@tonic-gate * 1757c478bd9Sstevel@tonic-gate * Suspend all task execution. Tasks already scheduled for a dynamic task 1767c478bd9Sstevel@tonic-gate * queue will still be executed, but all new scheduled tasks will be 1777c478bd9Sstevel@tonic-gate * suspended until taskq_resume() is called. 1787c478bd9Sstevel@tonic-gate * 1797c478bd9Sstevel@tonic-gate * int taskq_suspended(tq) 1807c478bd9Sstevel@tonic-gate * 1817c478bd9Sstevel@tonic-gate * Returns 1 if taskq is suspended and 0 otherwise. It is intended to 1827c478bd9Sstevel@tonic-gate * ASSERT that the task queue is suspended. 1837c478bd9Sstevel@tonic-gate * 1847c478bd9Sstevel@tonic-gate * void taskq_resume(tq) 1857c478bd9Sstevel@tonic-gate * 1867c478bd9Sstevel@tonic-gate * Resume task queue execution. 1877c478bd9Sstevel@tonic-gate * 1887c478bd9Sstevel@tonic-gate * int taskq_member(tq, thread) 1897c478bd9Sstevel@tonic-gate * 1907c478bd9Sstevel@tonic-gate * Returns 1 if 'thread' belongs to taskq 'tq' and 0 otherwise. The 1917c478bd9Sstevel@tonic-gate * intended use is to ASSERT that a given function is called in taskq 1927c478bd9Sstevel@tonic-gate * context only. 1937c478bd9Sstevel@tonic-gate * 1947c478bd9Sstevel@tonic-gate * system_taskq 1957c478bd9Sstevel@tonic-gate * 1967c478bd9Sstevel@tonic-gate * Global system-wide dynamic task queue for common uses. It may be used by 1977c478bd9Sstevel@tonic-gate * any subsystem that needs to schedule tasks and does not need to manage 1987c478bd9Sstevel@tonic-gate * its own task queues. It is initialized quite early during system boot. 1997c478bd9Sstevel@tonic-gate * 200*2e0c549eSJonathan Adams * IMPLEMENTATION ============================================================== 2017c478bd9Sstevel@tonic-gate * 2027c478bd9Sstevel@tonic-gate * This is schematic representation of the task queue structures. 2037c478bd9Sstevel@tonic-gate * 2047c478bd9Sstevel@tonic-gate * taskq: 2057c478bd9Sstevel@tonic-gate * +-------------+ 2067c478bd9Sstevel@tonic-gate * |tq_lock | +---< taskq_ent_free() 2077c478bd9Sstevel@tonic-gate * +-------------+ | 2087c478bd9Sstevel@tonic-gate * |... | | tqent: tqent: 2097c478bd9Sstevel@tonic-gate * +-------------+ | +------------+ +------------+ 2107c478bd9Sstevel@tonic-gate * | tq_freelist |-->| tqent_next |--> ... ->| tqent_next | 2117c478bd9Sstevel@tonic-gate * +-------------+ +------------+ +------------+ 2127c478bd9Sstevel@tonic-gate * |... | | ... | | ... | 2137c478bd9Sstevel@tonic-gate * +-------------+ +------------+ +------------+ 2147c478bd9Sstevel@tonic-gate * | tq_task | | 2157c478bd9Sstevel@tonic-gate * | | +-------------->taskq_ent_alloc() 2167c478bd9Sstevel@tonic-gate * +--------------------------------------------------------------------------+ 2177c478bd9Sstevel@tonic-gate * | | | tqent tqent | 2187c478bd9Sstevel@tonic-gate * | +---------------------+ +--> +------------+ +--> +------------+ | 2197c478bd9Sstevel@tonic-gate * | | ... | | | func, arg | | | func, arg | | 2207c478bd9Sstevel@tonic-gate * +>+---------------------+ <---|-+ +------------+ <---|-+ +------------+ | 2217c478bd9Sstevel@tonic-gate * | tq_taskq.tqent_next | ----+ | | tqent_next | --->+ | | tqent_next |--+ 2227c478bd9Sstevel@tonic-gate * +---------------------+ | +------------+ ^ | +------------+ 2237c478bd9Sstevel@tonic-gate * +-| tq_task.tqent_prev | +--| tqent_prev | | +--| tqent_prev | ^ 2247c478bd9Sstevel@tonic-gate * | +---------------------+ +------------+ | +------------+ | 2257c478bd9Sstevel@tonic-gate * | |... | | ... | | | ... | | 2267c478bd9Sstevel@tonic-gate * | +---------------------+ +------------+ | +------------+ | 2277c478bd9Sstevel@tonic-gate * | ^ | | 2287c478bd9Sstevel@tonic-gate * | | | | 2297c478bd9Sstevel@tonic-gate * +--------------------------------------+--------------+ TQ_APPEND() -+ 2307c478bd9Sstevel@tonic-gate * | | | 2317c478bd9Sstevel@tonic-gate * |... | taskq_thread()-----+ 2327c478bd9Sstevel@tonic-gate * +-------------+ 2337c478bd9Sstevel@tonic-gate * | tq_buckets |--+-------> [ NULL ] (for regular task queues) 2347c478bd9Sstevel@tonic-gate * +-------------+ | 2357c478bd9Sstevel@tonic-gate * | DYNAMIC TASK QUEUES: 2367c478bd9Sstevel@tonic-gate * | 2377c478bd9Sstevel@tonic-gate * +-> taskq_bucket[nCPU] taskq_bucket_dispatch() 2387c478bd9Sstevel@tonic-gate * +-------------------+ ^ 2397c478bd9Sstevel@tonic-gate * +--->| tqbucket_lock | | 2407c478bd9Sstevel@tonic-gate * | +-------------------+ +--------+ +--------+ 2417c478bd9Sstevel@tonic-gate * | | tqbucket_freelist |-->| tqent |-->...| tqent | ^ 2427c478bd9Sstevel@tonic-gate * | +-------------------+<--+--------+<--...+--------+ | 2437c478bd9Sstevel@tonic-gate * | | ... | | thread | | thread | | 2447c478bd9Sstevel@tonic-gate * | +-------------------+ +--------+ +--------+ | 2457c478bd9Sstevel@tonic-gate * | +-------------------+ | 2467c478bd9Sstevel@tonic-gate * taskq_dispatch()--+--->| tqbucket_lock | TQ_APPEND()------+ 2477c478bd9Sstevel@tonic-gate * TQ_HASH() | +-------------------+ +--------+ +--------+ 2487c478bd9Sstevel@tonic-gate * | | tqbucket_freelist |-->| tqent |-->...| tqent | 2497c478bd9Sstevel@tonic-gate * | +-------------------+<--+--------+<--...+--------+ 2507c478bd9Sstevel@tonic-gate * | | ... | | thread | | thread | 2517c478bd9Sstevel@tonic-gate * | +-------------------+ +--------+ +--------+ 2527c478bd9Sstevel@tonic-gate * +---> ... 2537c478bd9Sstevel@tonic-gate * 2547c478bd9Sstevel@tonic-gate * 2557c478bd9Sstevel@tonic-gate * Task queues use tq_task field to link new entry in the queue. The queue is a 2567c478bd9Sstevel@tonic-gate * circular doubly-linked list. Entries are put in the end of the list with 2577c478bd9Sstevel@tonic-gate * TQ_APPEND() and processed from the front of the list by taskq_thread() in 2587c478bd9Sstevel@tonic-gate * FIFO order. Task queue entries are cached in the free list managed by 2597c478bd9Sstevel@tonic-gate * taskq_ent_alloc() and taskq_ent_free() functions. 2607c478bd9Sstevel@tonic-gate * 2617c478bd9Sstevel@tonic-gate * All threads used by task queues mark t_taskq field of the thread to 2627c478bd9Sstevel@tonic-gate * point to the task queue. 2637c478bd9Sstevel@tonic-gate * 264*2e0c549eSJonathan Adams * Taskq Thread Management ----------------------------------------------------- 265*2e0c549eSJonathan Adams * 266*2e0c549eSJonathan Adams * Taskq's non-dynamic threads are managed with several variables and flags: 267*2e0c549eSJonathan Adams * 268*2e0c549eSJonathan Adams * * tq_nthreads - The number of threads in taskq_thread() for the 269*2e0c549eSJonathan Adams * taskq. 270*2e0c549eSJonathan Adams * 271*2e0c549eSJonathan Adams * * tq_active - The number of threads not waiting on a CV in 272*2e0c549eSJonathan Adams * taskq_thread(); includes newly created threads 273*2e0c549eSJonathan Adams * not yet counted in tq_nthreads. 274*2e0c549eSJonathan Adams * 275*2e0c549eSJonathan Adams * * tq_nthreads_target 276*2e0c549eSJonathan Adams * - The number of threads desired for the taskq. 277*2e0c549eSJonathan Adams * 278*2e0c549eSJonathan Adams * * tq_flags & TASKQ_CHANGING 279*2e0c549eSJonathan Adams * - Indicates that tq_nthreads != tq_nthreads_target. 280*2e0c549eSJonathan Adams * 281*2e0c549eSJonathan Adams * * tq_flags & TASKQ_THREAD_CREATED 282*2e0c549eSJonathan Adams * - Indicates that a thread is being created in the taskq. 283*2e0c549eSJonathan Adams * 284*2e0c549eSJonathan Adams * During creation, tq_nthreads and tq_active are set to 0, and 285*2e0c549eSJonathan Adams * tq_nthreads_target is set to the number of threads desired. The 286*2e0c549eSJonathan Adams * TASKQ_CHANGING flag is set, and taskq_create_thread() is called to 287*2e0c549eSJonathan Adams * create the first thread. taskq_create_thread() increments tq_active, 288*2e0c549eSJonathan Adams * sets TASKQ_THREAD_CREATED, and creates the new thread. 289*2e0c549eSJonathan Adams * 290*2e0c549eSJonathan Adams * Each thread starts in taskq_thread(), clears the TASKQ_THREAD_CREATED 291*2e0c549eSJonathan Adams * flag, and increments tq_nthreads. It stores the new value of 292*2e0c549eSJonathan Adams * tq_nthreads as its "thread_id", and stores its thread pointer in the 293*2e0c549eSJonathan Adams * tq_threadlist at the (thread_id - 1). We keep the thread_id space 294*2e0c549eSJonathan Adams * densely packed by requiring that only the largest thread_id can exit during 295*2e0c549eSJonathan Adams * normal adjustment. The exception is during the destruction of the 296*2e0c549eSJonathan Adams * taskq; once tq_nthreads_target is set to zero, no new threads will be created 297*2e0c549eSJonathan Adams * for the taskq queue, so every thread can exit without any ordering being 298*2e0c549eSJonathan Adams * necessary. 299*2e0c549eSJonathan Adams * 300*2e0c549eSJonathan Adams * Threads will only process work if their thread id is <= tq_nthreads_target. 301*2e0c549eSJonathan Adams * 302*2e0c549eSJonathan Adams * When TASKQ_CHANGING is set, threads will check the current thread target 303*2e0c549eSJonathan Adams * whenever they wake up, and do whatever they can to apply its effects. 304*2e0c549eSJonathan Adams * 305*2e0c549eSJonathan Adams * TASKQ_THREAD_CPU_PCT -------------------------------------------------------- 306*2e0c549eSJonathan Adams * 307*2e0c549eSJonathan Adams * When a taskq is created with TASKQ_THREAD_CPU_PCT, we store their requested 308*2e0c549eSJonathan Adams * percentage in tq_threads_ncpus_pct, start them off with the correct thread 309*2e0c549eSJonathan Adams * target, and add them to the taskq_cpupct_list for later adjustment. 310*2e0c549eSJonathan Adams * 311*2e0c549eSJonathan Adams * We register taskq_cpu_setup() to be called whenever a CPU changes state. It 312*2e0c549eSJonathan Adams * walks the list of TASKQ_THREAD_CPU_PCT taskqs, adjusts their nthread_target 313*2e0c549eSJonathan Adams * if need be, and wakes up all of the threads to process the change. 314*2e0c549eSJonathan Adams * 315*2e0c549eSJonathan Adams * Dynamic Task Queues Implementation ------------------------------------------ 3167c478bd9Sstevel@tonic-gate * 3177c478bd9Sstevel@tonic-gate * For a dynamic task queues there is a 1-to-1 mapping between a thread and 3187c478bd9Sstevel@tonic-gate * taskq_ent_structure. Each entry is serviced by its own thread and each thread 3197c478bd9Sstevel@tonic-gate * is controlled by a single entry. 3207c478bd9Sstevel@tonic-gate * 3217c478bd9Sstevel@tonic-gate * Entries are distributed over a set of buckets. To avoid using modulo 3227c478bd9Sstevel@tonic-gate * arithmetics the number of buckets is 2^n and is determined as the nearest 3237c478bd9Sstevel@tonic-gate * power of two roundown of the number of CPUs in the system. Tunable 3247c478bd9Sstevel@tonic-gate * variable 'taskq_maxbuckets' limits the maximum number of buckets. Each entry 3257c478bd9Sstevel@tonic-gate * is attached to a bucket for its lifetime and can't migrate to other buckets. 3267c478bd9Sstevel@tonic-gate * 3277c478bd9Sstevel@tonic-gate * Entries that have scheduled tasks are not placed in any list. The dispatch 3287c478bd9Sstevel@tonic-gate * function sets their "func" and "arg" fields and signals the corresponding 3297c478bd9Sstevel@tonic-gate * thread to execute the task. Once the thread executes the task it clears the 3307c478bd9Sstevel@tonic-gate * "func" field and places an entry on the bucket cache of free entries pointed 3317c478bd9Sstevel@tonic-gate * by "tqbucket_freelist" field. ALL entries on the free list should have "func" 3327c478bd9Sstevel@tonic-gate * field equal to NULL. The free list is a circular doubly-linked list identical 3337c478bd9Sstevel@tonic-gate * in structure to the tq_task list above, but entries are taken from it in LIFO 3347c478bd9Sstevel@tonic-gate * order - the last freed entry is the first to be allocated. The 3357c478bd9Sstevel@tonic-gate * taskq_bucket_dispatch() function gets the most recently used entry from the 3367c478bd9Sstevel@tonic-gate * free list, sets its "func" and "arg" fields and signals a worker thread. 3377c478bd9Sstevel@tonic-gate * 3387c478bd9Sstevel@tonic-gate * After executing each task a per-entry thread taskq_d_thread() places its 3397c478bd9Sstevel@tonic-gate * entry on the bucket free list and goes to a timed sleep. If it wakes up 3407c478bd9Sstevel@tonic-gate * without getting new task it removes the entry from the free list and destroys 3417c478bd9Sstevel@tonic-gate * itself. The thread sleep time is controlled by a tunable variable 3427c478bd9Sstevel@tonic-gate * `taskq_thread_timeout'. 3437c478bd9Sstevel@tonic-gate * 344*2e0c549eSJonathan Adams * There are various statistics kept in the bucket which allows for later 3457c478bd9Sstevel@tonic-gate * analysis of taskq usage patterns. Also, a global copy of taskq creation and 3467c478bd9Sstevel@tonic-gate * death statistics is kept in the global taskq data structure. Since thread 3477c478bd9Sstevel@tonic-gate * creation and death happen rarely, updating such global data does not present 3487c478bd9Sstevel@tonic-gate * a performance problem. 3497c478bd9Sstevel@tonic-gate * 3507c478bd9Sstevel@tonic-gate * NOTE: Threads are not bound to any CPU and there is absolutely no association 3517c478bd9Sstevel@tonic-gate * between the bucket and actual thread CPU, so buckets are used only to 3527c478bd9Sstevel@tonic-gate * split resources and reduce resource contention. Having threads attached 3537c478bd9Sstevel@tonic-gate * to the CPU denoted by a bucket may reduce number of times the job 3547c478bd9Sstevel@tonic-gate * switches between CPUs. 3557c478bd9Sstevel@tonic-gate * 3567c478bd9Sstevel@tonic-gate * Current algorithm creates a thread whenever a bucket has no free 3577c478bd9Sstevel@tonic-gate * entries. It would be nice to know how many threads are in the running 3587c478bd9Sstevel@tonic-gate * state and don't create threads if all CPUs are busy with existing 3597c478bd9Sstevel@tonic-gate * tasks, but it is unclear how such strategy can be implemented. 3607c478bd9Sstevel@tonic-gate * 3617c478bd9Sstevel@tonic-gate * Currently buckets are created statically as an array attached to task 3627c478bd9Sstevel@tonic-gate * queue. On some system with nCPUs < max_ncpus it may waste system 3637c478bd9Sstevel@tonic-gate * memory. One solution may be allocation of buckets when they are first 3647c478bd9Sstevel@tonic-gate * touched, but it is not clear how useful it is. 3657c478bd9Sstevel@tonic-gate * 366*2e0c549eSJonathan Adams * SUSPEND/RESUME implementation ----------------------------------------------- 3677c478bd9Sstevel@tonic-gate * 3687c478bd9Sstevel@tonic-gate * Before executing a task taskq_thread() (executing non-dynamic task 3697c478bd9Sstevel@tonic-gate * queues) obtains taskq's thread lock as a reader. The taskq_suspend() 3707c478bd9Sstevel@tonic-gate * function gets the same lock as a writer blocking all non-dynamic task 3717c478bd9Sstevel@tonic-gate * execution. The taskq_resume() function releases the lock allowing 3727c478bd9Sstevel@tonic-gate * taskq_thread to continue execution. 3737c478bd9Sstevel@tonic-gate * 3747c478bd9Sstevel@tonic-gate * For dynamic task queues, each bucket is marked as TQBUCKET_SUSPEND by 3757c478bd9Sstevel@tonic-gate * taskq_suspend() function. After that taskq_bucket_dispatch() always 3767c478bd9Sstevel@tonic-gate * fails, so that taskq_dispatch() will either enqueue tasks for a 3777c478bd9Sstevel@tonic-gate * suspended backing queue or fail if TQ_NOQUEUE is specified in dispatch 3787c478bd9Sstevel@tonic-gate * flags. 3797c478bd9Sstevel@tonic-gate * 3807c478bd9Sstevel@tonic-gate * NOTE: taskq_suspend() does not immediately block any tasks already 3817c478bd9Sstevel@tonic-gate * scheduled for dynamic task queues. It only suspends new tasks 3827c478bd9Sstevel@tonic-gate * scheduled after taskq_suspend() was called. 3837c478bd9Sstevel@tonic-gate * 3847c478bd9Sstevel@tonic-gate * taskq_member() function works by comparing a thread t_taskq pointer with 3857c478bd9Sstevel@tonic-gate * the passed thread pointer. 3867c478bd9Sstevel@tonic-gate * 387*2e0c549eSJonathan Adams * LOCKS and LOCK Hierarchy ---------------------------------------------------- 3887c478bd9Sstevel@tonic-gate * 389*2e0c549eSJonathan Adams * There are three locks used in task queues: 3907c478bd9Sstevel@tonic-gate * 391*2e0c549eSJonathan Adams * 1) The taskq_t's tq_lock, protecting global task queue state. 3927c478bd9Sstevel@tonic-gate * 3937c478bd9Sstevel@tonic-gate * 2) Each per-CPU bucket has a lock for bucket management. 3947c478bd9Sstevel@tonic-gate * 395*2e0c549eSJonathan Adams * 3) The global taskq_cpupct_lock, which protects the list of 396*2e0c549eSJonathan Adams * TASKQ_THREADS_CPU_PCT taskqs. 397*2e0c549eSJonathan Adams * 398*2e0c549eSJonathan Adams * If both (1) and (2) are needed, tq_lock should be taken *after* the bucket 3997c478bd9Sstevel@tonic-gate * lock. 4007c478bd9Sstevel@tonic-gate * 401*2e0c549eSJonathan Adams * If both (1) and (3) are needed, tq_lock should be taken *after* 402*2e0c549eSJonathan Adams * taskq_cpupct_lock. 403*2e0c549eSJonathan Adams * 404*2e0c549eSJonathan Adams * DEBUG FACILITIES ------------------------------------------------------------ 4057c478bd9Sstevel@tonic-gate * 4067c478bd9Sstevel@tonic-gate * For DEBUG kernels it is possible to induce random failures to 4077c478bd9Sstevel@tonic-gate * taskq_dispatch() function when it is given TQ_NOSLEEP argument. The value of 4087c478bd9Sstevel@tonic-gate * taskq_dmtbf and taskq_smtbf tunables control the mean time between induced 4097c478bd9Sstevel@tonic-gate * failures for dynamic and static task queues respectively. 4107c478bd9Sstevel@tonic-gate * 4117c478bd9Sstevel@tonic-gate * Setting TASKQ_STATISTIC to 0 will disable per-bucket statistics. 4127c478bd9Sstevel@tonic-gate * 413*2e0c549eSJonathan Adams * TUNABLES -------------------------------------------------------------------- 4147c478bd9Sstevel@tonic-gate * 4157c478bd9Sstevel@tonic-gate * system_taskq_size - Size of the global system_taskq. 4167c478bd9Sstevel@tonic-gate * This value is multiplied by nCPUs to determine 4177c478bd9Sstevel@tonic-gate * actual size. 4187c478bd9Sstevel@tonic-gate * Default value: 64 4197c478bd9Sstevel@tonic-gate * 420*2e0c549eSJonathan Adams * taskq_minimum_nthreads_max 421*2e0c549eSJonathan Adams * - Minimum size of the thread list for a taskq. 422*2e0c549eSJonathan Adams * Useful for testing different thread pool 423*2e0c549eSJonathan Adams * sizes by overwriting tq_nthreads_target. 424*2e0c549eSJonathan Adams * 4257c478bd9Sstevel@tonic-gate * taskq_thread_timeout - Maximum idle time for taskq_d_thread() 4267c478bd9Sstevel@tonic-gate * Default value: 5 minutes 4277c478bd9Sstevel@tonic-gate * 4287c478bd9Sstevel@tonic-gate * taskq_maxbuckets - Maximum number of buckets in any task queue 4297c478bd9Sstevel@tonic-gate * Default value: 128 4307c478bd9Sstevel@tonic-gate * 4317c478bd9Sstevel@tonic-gate * taskq_search_depth - Maximum # of buckets searched for a free entry 4327c478bd9Sstevel@tonic-gate * Default value: 4 4337c478bd9Sstevel@tonic-gate * 4347c478bd9Sstevel@tonic-gate * taskq_dmtbf - Mean time between induced dispatch failures 4357c478bd9Sstevel@tonic-gate * for dynamic task queues. 4367c478bd9Sstevel@tonic-gate * Default value: UINT_MAX (no induced failures) 4377c478bd9Sstevel@tonic-gate * 4387c478bd9Sstevel@tonic-gate * taskq_smtbf - Mean time between induced dispatch failures 4397c478bd9Sstevel@tonic-gate * for static task queues. 4407c478bd9Sstevel@tonic-gate * Default value: UINT_MAX (no induced failures) 4417c478bd9Sstevel@tonic-gate * 442*2e0c549eSJonathan Adams * CONDITIONAL compilation ----------------------------------------------------- 4437c478bd9Sstevel@tonic-gate * 4447c478bd9Sstevel@tonic-gate * TASKQ_STATISTIC - If set will enable bucket statistic (default). 4457c478bd9Sstevel@tonic-gate * 4467c478bd9Sstevel@tonic-gate */ 4477c478bd9Sstevel@tonic-gate 4487c478bd9Sstevel@tonic-gate #include <sys/taskq_impl.h> 4497c478bd9Sstevel@tonic-gate #include <sys/thread.h> 4507c478bd9Sstevel@tonic-gate #include <sys/proc.h> 4517c478bd9Sstevel@tonic-gate #include <sys/kmem.h> 4527c478bd9Sstevel@tonic-gate #include <sys/vmem.h> 4537c478bd9Sstevel@tonic-gate #include <sys/callb.h> 4547c478bd9Sstevel@tonic-gate #include <sys/systm.h> 4557c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h> 4567c478bd9Sstevel@tonic-gate #include <sys/debug.h> 4577c478bd9Sstevel@tonic-gate #include <sys/vmsystm.h> /* For throttlefree */ 4587c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h> 4597c478bd9Sstevel@tonic-gate #include <sys/cpuvar.h> 4607c478bd9Sstevel@tonic-gate #include <sys/sdt.h> 461*2e0c549eSJonathan Adams #include <sys/note.h> 4627c478bd9Sstevel@tonic-gate 4637c478bd9Sstevel@tonic-gate static kmem_cache_t *taskq_ent_cache, *taskq_cache; 4647c478bd9Sstevel@tonic-gate 4657c478bd9Sstevel@tonic-gate /* 466*2e0c549eSJonathan Adams * Pseudo instance numbers for taskqs without explicitly provided instance. 4677c478bd9Sstevel@tonic-gate */ 4687c478bd9Sstevel@tonic-gate static vmem_t *taskq_id_arena; 4697c478bd9Sstevel@tonic-gate 4707c478bd9Sstevel@tonic-gate /* Global system task queue for common use */ 4717c478bd9Sstevel@tonic-gate taskq_t *system_taskq; 4727c478bd9Sstevel@tonic-gate 4737c478bd9Sstevel@tonic-gate /* 474*2e0c549eSJonathan Adams * Maximum number of entries in global system taskq is 4757c478bd9Sstevel@tonic-gate * system_taskq_size * max_ncpus 4767c478bd9Sstevel@tonic-gate */ 4777c478bd9Sstevel@tonic-gate #define SYSTEM_TASKQ_SIZE 64 4787c478bd9Sstevel@tonic-gate int system_taskq_size = SYSTEM_TASKQ_SIZE; 4797c478bd9Sstevel@tonic-gate 480*2e0c549eSJonathan Adams /* 481*2e0c549eSJonathan Adams * Minimum size for tq_nthreads_max; useful for those who want to play around 482*2e0c549eSJonathan Adams * with increasing a taskq's tq_nthreads_target. 483*2e0c549eSJonathan Adams */ 484*2e0c549eSJonathan Adams int taskq_minimum_nthreads_max = 1; 485*2e0c549eSJonathan Adams 486*2e0c549eSJonathan Adams /* Maximum percentage allowed for TASKQ_THREADS_CPU_PCT */ 487*2e0c549eSJonathan Adams #define TASKQ_CPUPCT_MAX_PERCENT 1000 488*2e0c549eSJonathan Adams int taskq_cpupct_max_percent = TASKQ_CPUPCT_MAX_PERCENT; 489*2e0c549eSJonathan Adams 4907c478bd9Sstevel@tonic-gate /* 4917c478bd9Sstevel@tonic-gate * Dynamic task queue threads that don't get any work within 4927c478bd9Sstevel@tonic-gate * taskq_thread_timeout destroy themselves 4937c478bd9Sstevel@tonic-gate */ 4947c478bd9Sstevel@tonic-gate #define TASKQ_THREAD_TIMEOUT (60 * 5) 4957c478bd9Sstevel@tonic-gate int taskq_thread_timeout = TASKQ_THREAD_TIMEOUT; 4967c478bd9Sstevel@tonic-gate 4977c478bd9Sstevel@tonic-gate #define TASKQ_MAXBUCKETS 128 4987c478bd9Sstevel@tonic-gate int taskq_maxbuckets = TASKQ_MAXBUCKETS; 4997c478bd9Sstevel@tonic-gate 5007c478bd9Sstevel@tonic-gate /* 5017c478bd9Sstevel@tonic-gate * When a bucket has no available entries another buckets are tried. 5027c478bd9Sstevel@tonic-gate * taskq_search_depth parameter limits the amount of buckets that we search 5037c478bd9Sstevel@tonic-gate * before failing. This is mostly useful in systems with many CPUs where we may 5047c478bd9Sstevel@tonic-gate * spend too much time scanning busy buckets. 5057c478bd9Sstevel@tonic-gate */ 5067c478bd9Sstevel@tonic-gate #define TASKQ_SEARCH_DEPTH 4 5077c478bd9Sstevel@tonic-gate int taskq_search_depth = TASKQ_SEARCH_DEPTH; 5087c478bd9Sstevel@tonic-gate 5097c478bd9Sstevel@tonic-gate /* 5107c478bd9Sstevel@tonic-gate * Hashing function: mix various bits of x. May be pretty much anything. 5117c478bd9Sstevel@tonic-gate */ 5127c478bd9Sstevel@tonic-gate #define TQ_HASH(x) ((x) ^ ((x) >> 11) ^ ((x) >> 17) ^ ((x) ^ 27)) 5137c478bd9Sstevel@tonic-gate 5147c478bd9Sstevel@tonic-gate /* 5157c478bd9Sstevel@tonic-gate * We do not create any new threads when the system is low on memory and start 5167c478bd9Sstevel@tonic-gate * throttling memory allocations. The following macro tries to estimate such 5177c478bd9Sstevel@tonic-gate * condition. 5187c478bd9Sstevel@tonic-gate */ 5197c478bd9Sstevel@tonic-gate #define ENOUGH_MEMORY() (freemem > throttlefree) 5207c478bd9Sstevel@tonic-gate 5217c478bd9Sstevel@tonic-gate /* 5227c478bd9Sstevel@tonic-gate * Static functions. 5237c478bd9Sstevel@tonic-gate */ 5247c478bd9Sstevel@tonic-gate static taskq_t *taskq_create_common(const char *, int, int, pri_t, int, 5257c478bd9Sstevel@tonic-gate int, uint_t); 5267c478bd9Sstevel@tonic-gate static void taskq_thread(void *); 5277c478bd9Sstevel@tonic-gate static void taskq_d_thread(taskq_ent_t *); 5287c478bd9Sstevel@tonic-gate static void taskq_bucket_extend(void *); 5297c478bd9Sstevel@tonic-gate static int taskq_constructor(void *, void *, int); 5307c478bd9Sstevel@tonic-gate static void taskq_destructor(void *, void *); 5317c478bd9Sstevel@tonic-gate static int taskq_ent_constructor(void *, void *, int); 5327c478bd9Sstevel@tonic-gate static void taskq_ent_destructor(void *, void *); 5337c478bd9Sstevel@tonic-gate static taskq_ent_t *taskq_ent_alloc(taskq_t *, int); 5347c478bd9Sstevel@tonic-gate static void taskq_ent_free(taskq_t *, taskq_ent_t *); 5357c478bd9Sstevel@tonic-gate static taskq_ent_t *taskq_bucket_dispatch(taskq_bucket_t *, task_func_t, 5367c478bd9Sstevel@tonic-gate void *); 5377c478bd9Sstevel@tonic-gate 5387c478bd9Sstevel@tonic-gate /* 5397c478bd9Sstevel@tonic-gate * Task queues kstats. 5407c478bd9Sstevel@tonic-gate */ 5417c478bd9Sstevel@tonic-gate struct taskq_kstat { 5427c478bd9Sstevel@tonic-gate kstat_named_t tq_tasks; 5437c478bd9Sstevel@tonic-gate kstat_named_t tq_executed; 5447c478bd9Sstevel@tonic-gate kstat_named_t tq_maxtasks; 5457c478bd9Sstevel@tonic-gate kstat_named_t tq_totaltime; 5467c478bd9Sstevel@tonic-gate kstat_named_t tq_nalloc; 5477c478bd9Sstevel@tonic-gate kstat_named_t tq_nactive; 5487c478bd9Sstevel@tonic-gate kstat_named_t tq_pri; 5497c478bd9Sstevel@tonic-gate kstat_named_t tq_nthreads; 5507c478bd9Sstevel@tonic-gate } taskq_kstat = { 5517c478bd9Sstevel@tonic-gate { "tasks", KSTAT_DATA_UINT64 }, 5527c478bd9Sstevel@tonic-gate { "executed", KSTAT_DATA_UINT64 }, 5537c478bd9Sstevel@tonic-gate { "maxtasks", KSTAT_DATA_UINT64 }, 5547c478bd9Sstevel@tonic-gate { "totaltime", KSTAT_DATA_UINT64 }, 5557c478bd9Sstevel@tonic-gate { "nactive", KSTAT_DATA_UINT64 }, 5567c478bd9Sstevel@tonic-gate { "nalloc", KSTAT_DATA_UINT64 }, 5577c478bd9Sstevel@tonic-gate { "priority", KSTAT_DATA_UINT64 }, 5587c478bd9Sstevel@tonic-gate { "threads", KSTAT_DATA_UINT64 }, 5597c478bd9Sstevel@tonic-gate }; 5607c478bd9Sstevel@tonic-gate 5617c478bd9Sstevel@tonic-gate struct taskq_d_kstat { 5627c478bd9Sstevel@tonic-gate kstat_named_t tqd_pri; 5637c478bd9Sstevel@tonic-gate kstat_named_t tqd_btasks; 5647c478bd9Sstevel@tonic-gate kstat_named_t tqd_bexecuted; 5657c478bd9Sstevel@tonic-gate kstat_named_t tqd_bmaxtasks; 5667c478bd9Sstevel@tonic-gate kstat_named_t tqd_bnalloc; 5677c478bd9Sstevel@tonic-gate kstat_named_t tqd_bnactive; 5687c478bd9Sstevel@tonic-gate kstat_named_t tqd_btotaltime; 5697c478bd9Sstevel@tonic-gate kstat_named_t tqd_hits; 5707c478bd9Sstevel@tonic-gate kstat_named_t tqd_misses; 5717c478bd9Sstevel@tonic-gate kstat_named_t tqd_overflows; 5727c478bd9Sstevel@tonic-gate kstat_named_t tqd_tcreates; 5737c478bd9Sstevel@tonic-gate kstat_named_t tqd_tdeaths; 5747c478bd9Sstevel@tonic-gate kstat_named_t tqd_maxthreads; 5757c478bd9Sstevel@tonic-gate kstat_named_t tqd_nomem; 5767c478bd9Sstevel@tonic-gate kstat_named_t tqd_disptcreates; 5777c478bd9Sstevel@tonic-gate kstat_named_t tqd_totaltime; 5787c478bd9Sstevel@tonic-gate kstat_named_t tqd_nalloc; 5797c478bd9Sstevel@tonic-gate kstat_named_t tqd_nfree; 5807c478bd9Sstevel@tonic-gate } taskq_d_kstat = { 5817c478bd9Sstevel@tonic-gate { "priority", KSTAT_DATA_UINT64 }, 5827c478bd9Sstevel@tonic-gate { "btasks", KSTAT_DATA_UINT64 }, 5837c478bd9Sstevel@tonic-gate { "bexecuted", KSTAT_DATA_UINT64 }, 5847c478bd9Sstevel@tonic-gate { "bmaxtasks", KSTAT_DATA_UINT64 }, 5857c478bd9Sstevel@tonic-gate { "bnalloc", KSTAT_DATA_UINT64 }, 5867c478bd9Sstevel@tonic-gate { "bnactive", KSTAT_DATA_UINT64 }, 5877c478bd9Sstevel@tonic-gate { "btotaltime", KSTAT_DATA_UINT64 }, 5887c478bd9Sstevel@tonic-gate { "hits", KSTAT_DATA_UINT64 }, 5897c478bd9Sstevel@tonic-gate { "misses", KSTAT_DATA_UINT64 }, 5907c478bd9Sstevel@tonic-gate { "overflows", KSTAT_DATA_UINT64 }, 5917c478bd9Sstevel@tonic-gate { "tcreates", KSTAT_DATA_UINT64 }, 5927c478bd9Sstevel@tonic-gate { "tdeaths", KSTAT_DATA_UINT64 }, 5937c478bd9Sstevel@tonic-gate { "maxthreads", KSTAT_DATA_UINT64 }, 5947c478bd9Sstevel@tonic-gate { "nomem", KSTAT_DATA_UINT64 }, 5957c478bd9Sstevel@tonic-gate { "disptcreates", KSTAT_DATA_UINT64 }, 5967c478bd9Sstevel@tonic-gate { "totaltime", KSTAT_DATA_UINT64 }, 5977c478bd9Sstevel@tonic-gate { "nalloc", KSTAT_DATA_UINT64 }, 5987c478bd9Sstevel@tonic-gate { "nfree", KSTAT_DATA_UINT64 }, 5997c478bd9Sstevel@tonic-gate }; 6007c478bd9Sstevel@tonic-gate 6017c478bd9Sstevel@tonic-gate static kmutex_t taskq_kstat_lock; 6027c478bd9Sstevel@tonic-gate static kmutex_t taskq_d_kstat_lock; 6037c478bd9Sstevel@tonic-gate static int taskq_kstat_update(kstat_t *, int); 6047c478bd9Sstevel@tonic-gate static int taskq_d_kstat_update(kstat_t *, int); 6057c478bd9Sstevel@tonic-gate 606*2e0c549eSJonathan Adams /* 607*2e0c549eSJonathan Adams * State for THREAD_CPU_PCT management 608*2e0c549eSJonathan Adams */ 609*2e0c549eSJonathan Adams typedef struct taskq_cpupct_ent { 610*2e0c549eSJonathan Adams list_node_t tp_link; 611*2e0c549eSJonathan Adams taskq_t *tp_taskq; 612*2e0c549eSJonathan Adams } taskq_cpupct_ent_t; 613*2e0c549eSJonathan Adams 614*2e0c549eSJonathan Adams static kmutex_t taskq_cpupct_lock; 615*2e0c549eSJonathan Adams static list_t taskq_cpupct_list; 616*2e0c549eSJonathan Adams static int taskq_cpupct_ncpus_online; 6177c478bd9Sstevel@tonic-gate 6187c478bd9Sstevel@tonic-gate /* 6197c478bd9Sstevel@tonic-gate * Collect per-bucket statistic when TASKQ_STATISTIC is defined. 6207c478bd9Sstevel@tonic-gate */ 6217c478bd9Sstevel@tonic-gate #define TASKQ_STATISTIC 1 6227c478bd9Sstevel@tonic-gate 6237c478bd9Sstevel@tonic-gate #if TASKQ_STATISTIC 6247c478bd9Sstevel@tonic-gate #define TQ_STAT(b, x) b->tqbucket_stat.x++ 6257c478bd9Sstevel@tonic-gate #else 6267c478bd9Sstevel@tonic-gate #define TQ_STAT(b, x) 6277c478bd9Sstevel@tonic-gate #endif 6287c478bd9Sstevel@tonic-gate 6297c478bd9Sstevel@tonic-gate /* 6307c478bd9Sstevel@tonic-gate * Random fault injection. 6317c478bd9Sstevel@tonic-gate */ 6327c478bd9Sstevel@tonic-gate uint_t taskq_random; 6337c478bd9Sstevel@tonic-gate uint_t taskq_dmtbf = UINT_MAX; /* mean time between injected failures */ 6347c478bd9Sstevel@tonic-gate uint_t taskq_smtbf = UINT_MAX; /* mean time between injected failures */ 6357c478bd9Sstevel@tonic-gate 6367c478bd9Sstevel@tonic-gate /* 6377c478bd9Sstevel@tonic-gate * TQ_NOSLEEP dispatches on dynamic task queues are always allowed to fail. 6387c478bd9Sstevel@tonic-gate * 6397c478bd9Sstevel@tonic-gate * TQ_NOSLEEP dispatches on static task queues can't arbitrarily fail because 6407c478bd9Sstevel@tonic-gate * they could prepopulate the cache and make sure that they do not use more 6417c478bd9Sstevel@tonic-gate * then minalloc entries. So, fault injection in this case insures that 6427c478bd9Sstevel@tonic-gate * either TASKQ_PREPOPULATE is not set or there are more entries allocated 6437c478bd9Sstevel@tonic-gate * than is specified by minalloc. TQ_NOALLOC dispatches are always allowed 6447c478bd9Sstevel@tonic-gate * to fail, but for simplicity we treat them identically to TQ_NOSLEEP 6457c478bd9Sstevel@tonic-gate * dispatches. 6467c478bd9Sstevel@tonic-gate */ 6477c478bd9Sstevel@tonic-gate #ifdef DEBUG 6487c478bd9Sstevel@tonic-gate #define TASKQ_D_RANDOM_DISPATCH_FAILURE(tq, flag) \ 6497c478bd9Sstevel@tonic-gate taskq_random = (taskq_random * 2416 + 374441) % 1771875;\ 6507c478bd9Sstevel@tonic-gate if ((flag & TQ_NOSLEEP) && \ 6517c478bd9Sstevel@tonic-gate taskq_random < 1771875 / taskq_dmtbf) { \ 6527c478bd9Sstevel@tonic-gate return (NULL); \ 6537c478bd9Sstevel@tonic-gate } 6547c478bd9Sstevel@tonic-gate 6557c478bd9Sstevel@tonic-gate #define TASKQ_S_RANDOM_DISPATCH_FAILURE(tq, flag) \ 6567c478bd9Sstevel@tonic-gate taskq_random = (taskq_random * 2416 + 374441) % 1771875;\ 6577c478bd9Sstevel@tonic-gate if ((flag & (TQ_NOSLEEP | TQ_NOALLOC)) && \ 6587c478bd9Sstevel@tonic-gate (!(tq->tq_flags & TASKQ_PREPOPULATE) || \ 6597c478bd9Sstevel@tonic-gate (tq->tq_nalloc > tq->tq_minalloc)) && \ 6607c478bd9Sstevel@tonic-gate (taskq_random < (1771875 / taskq_smtbf))) { \ 6617c478bd9Sstevel@tonic-gate mutex_exit(&tq->tq_lock); \ 6627c478bd9Sstevel@tonic-gate return (NULL); \ 6637c478bd9Sstevel@tonic-gate } 6647c478bd9Sstevel@tonic-gate #else 6657c478bd9Sstevel@tonic-gate #define TASKQ_S_RANDOM_DISPATCH_FAILURE(tq, flag) 6667c478bd9Sstevel@tonic-gate #define TASKQ_D_RANDOM_DISPATCH_FAILURE(tq, flag) 6677c478bd9Sstevel@tonic-gate #endif 6687c478bd9Sstevel@tonic-gate 6697c478bd9Sstevel@tonic-gate #define IS_EMPTY(l) (((l).tqent_prev == (l).tqent_next) && \ 6707c478bd9Sstevel@tonic-gate ((l).tqent_prev == &(l))) 6717c478bd9Sstevel@tonic-gate 6727c478bd9Sstevel@tonic-gate /* 6737c478bd9Sstevel@tonic-gate * Append `tqe' in the end of the doubly-linked list denoted by l. 6747c478bd9Sstevel@tonic-gate */ 6757c478bd9Sstevel@tonic-gate #define TQ_APPEND(l, tqe) { \ 6767c478bd9Sstevel@tonic-gate tqe->tqent_next = &l; \ 6777c478bd9Sstevel@tonic-gate tqe->tqent_prev = l.tqent_prev; \ 6787c478bd9Sstevel@tonic-gate tqe->tqent_next->tqent_prev = tqe; \ 6797c478bd9Sstevel@tonic-gate tqe->tqent_prev->tqent_next = tqe; \ 6807c478bd9Sstevel@tonic-gate } 6817c478bd9Sstevel@tonic-gate 6827c478bd9Sstevel@tonic-gate /* 6837c478bd9Sstevel@tonic-gate * Schedule a task specified by func and arg into the task queue entry tqe. 6847c478bd9Sstevel@tonic-gate */ 6857c478bd9Sstevel@tonic-gate #define TQ_ENQUEUE(tq, tqe, func, arg) { \ 6867c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&tq->tq_lock)); \ 6877c478bd9Sstevel@tonic-gate TQ_APPEND(tq->tq_task, tqe); \ 6887c478bd9Sstevel@tonic-gate tqe->tqent_func = (func); \ 6897c478bd9Sstevel@tonic-gate tqe->tqent_arg = (arg); \ 6907c478bd9Sstevel@tonic-gate tq->tq_tasks++; \ 6917c478bd9Sstevel@tonic-gate if (tq->tq_tasks - tq->tq_executed > tq->tq_maxtasks) \ 6927c478bd9Sstevel@tonic-gate tq->tq_maxtasks = tq->tq_tasks - tq->tq_executed; \ 6937c478bd9Sstevel@tonic-gate cv_signal(&tq->tq_dispatch_cv); \ 6947c478bd9Sstevel@tonic-gate DTRACE_PROBE2(taskq__enqueue, taskq_t *, tq, taskq_ent_t *, tqe); \ 6957c478bd9Sstevel@tonic-gate } 6967c478bd9Sstevel@tonic-gate 6977c478bd9Sstevel@tonic-gate /* 6987c478bd9Sstevel@tonic-gate * Do-nothing task which may be used to prepopulate thread caches. 6997c478bd9Sstevel@tonic-gate */ 7007c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 7017c478bd9Sstevel@tonic-gate void 7027c478bd9Sstevel@tonic-gate nulltask(void *unused) 7037c478bd9Sstevel@tonic-gate { 7047c478bd9Sstevel@tonic-gate } 7057c478bd9Sstevel@tonic-gate 7067c478bd9Sstevel@tonic-gate 7077c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 7087c478bd9Sstevel@tonic-gate static int 7097c478bd9Sstevel@tonic-gate taskq_constructor(void *buf, void *cdrarg, int kmflags) 7107c478bd9Sstevel@tonic-gate { 7117c478bd9Sstevel@tonic-gate taskq_t *tq = buf; 7127c478bd9Sstevel@tonic-gate 7137c478bd9Sstevel@tonic-gate bzero(tq, sizeof (taskq_t)); 7147c478bd9Sstevel@tonic-gate 7157c478bd9Sstevel@tonic-gate mutex_init(&tq->tq_lock, NULL, MUTEX_DEFAULT, NULL); 7167c478bd9Sstevel@tonic-gate rw_init(&tq->tq_threadlock, NULL, RW_DEFAULT, NULL); 7177c478bd9Sstevel@tonic-gate cv_init(&tq->tq_dispatch_cv, NULL, CV_DEFAULT, NULL); 718*2e0c549eSJonathan Adams cv_init(&tq->tq_exit_cv, NULL, CV_DEFAULT, NULL); 7197c478bd9Sstevel@tonic-gate cv_init(&tq->tq_wait_cv, NULL, CV_DEFAULT, NULL); 7207c478bd9Sstevel@tonic-gate 7217c478bd9Sstevel@tonic-gate tq->tq_task.tqent_next = &tq->tq_task; 7227c478bd9Sstevel@tonic-gate tq->tq_task.tqent_prev = &tq->tq_task; 7237c478bd9Sstevel@tonic-gate 7247c478bd9Sstevel@tonic-gate return (0); 7257c478bd9Sstevel@tonic-gate } 7267c478bd9Sstevel@tonic-gate 7277c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 7287c478bd9Sstevel@tonic-gate static void 7297c478bd9Sstevel@tonic-gate taskq_destructor(void *buf, void *cdrarg) 7307c478bd9Sstevel@tonic-gate { 7317c478bd9Sstevel@tonic-gate taskq_t *tq = buf; 7327c478bd9Sstevel@tonic-gate 733*2e0c549eSJonathan Adams ASSERT(tq->tq_nthreads == 0); 734*2e0c549eSJonathan Adams ASSERT(tq->tq_buckets == NULL); 735*2e0c549eSJonathan Adams ASSERT(tq->tq_tcreates == 0); 736*2e0c549eSJonathan Adams ASSERT(tq->tq_tdeaths == 0); 737*2e0c549eSJonathan Adams 7387c478bd9Sstevel@tonic-gate mutex_destroy(&tq->tq_lock); 7397c478bd9Sstevel@tonic-gate rw_destroy(&tq->tq_threadlock); 7407c478bd9Sstevel@tonic-gate cv_destroy(&tq->tq_dispatch_cv); 741*2e0c549eSJonathan Adams cv_destroy(&tq->tq_exit_cv); 7427c478bd9Sstevel@tonic-gate cv_destroy(&tq->tq_wait_cv); 7437c478bd9Sstevel@tonic-gate } 7447c478bd9Sstevel@tonic-gate 7457c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 7467c478bd9Sstevel@tonic-gate static int 7477c478bd9Sstevel@tonic-gate taskq_ent_constructor(void *buf, void *cdrarg, int kmflags) 7487c478bd9Sstevel@tonic-gate { 7497c478bd9Sstevel@tonic-gate taskq_ent_t *tqe = buf; 7507c478bd9Sstevel@tonic-gate 7517c478bd9Sstevel@tonic-gate tqe->tqent_thread = NULL; 7527c478bd9Sstevel@tonic-gate cv_init(&tqe->tqent_cv, NULL, CV_DEFAULT, NULL); 7537c478bd9Sstevel@tonic-gate 7547c478bd9Sstevel@tonic-gate return (0); 7557c478bd9Sstevel@tonic-gate } 7567c478bd9Sstevel@tonic-gate 7577c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 7587c478bd9Sstevel@tonic-gate static void 7597c478bd9Sstevel@tonic-gate taskq_ent_destructor(void *buf, void *cdrarg) 7607c478bd9Sstevel@tonic-gate { 7617c478bd9Sstevel@tonic-gate taskq_ent_t *tqe = buf; 7627c478bd9Sstevel@tonic-gate 7637c478bd9Sstevel@tonic-gate ASSERT(tqe->tqent_thread == NULL); 7647c478bd9Sstevel@tonic-gate cv_destroy(&tqe->tqent_cv); 7657c478bd9Sstevel@tonic-gate } 7667c478bd9Sstevel@tonic-gate 7677c478bd9Sstevel@tonic-gate void 7687c478bd9Sstevel@tonic-gate taskq_init(void) 7697c478bd9Sstevel@tonic-gate { 7707c478bd9Sstevel@tonic-gate taskq_ent_cache = kmem_cache_create("taskq_ent_cache", 7717c478bd9Sstevel@tonic-gate sizeof (taskq_ent_t), 0, taskq_ent_constructor, 7727c478bd9Sstevel@tonic-gate taskq_ent_destructor, NULL, NULL, NULL, 0); 7737c478bd9Sstevel@tonic-gate taskq_cache = kmem_cache_create("taskq_cache", sizeof (taskq_t), 7747c478bd9Sstevel@tonic-gate 0, taskq_constructor, taskq_destructor, NULL, NULL, NULL, 0); 7757c478bd9Sstevel@tonic-gate taskq_id_arena = vmem_create("taskq_id_arena", 7767c478bd9Sstevel@tonic-gate (void *)1, INT32_MAX, 1, NULL, NULL, NULL, 0, 7777c478bd9Sstevel@tonic-gate VM_SLEEP | VMC_IDENTIFIER); 778*2e0c549eSJonathan Adams 779*2e0c549eSJonathan Adams list_create(&taskq_cpupct_list, sizeof (taskq_cpupct_ent_t), 780*2e0c549eSJonathan Adams offsetof(taskq_cpupct_ent_t, tp_link)); 781*2e0c549eSJonathan Adams } 782*2e0c549eSJonathan Adams 783*2e0c549eSJonathan Adams /*ARGSUSED*/ 784*2e0c549eSJonathan Adams static int 785*2e0c549eSJonathan Adams taskq_cpu_setup(cpu_setup_t what, int id, void *arg) 786*2e0c549eSJonathan Adams { 787*2e0c549eSJonathan Adams taskq_cpupct_ent_t *tpp; 788*2e0c549eSJonathan Adams int cpus_online = ncpus_online; 789*2e0c549eSJonathan Adams 790*2e0c549eSJonathan Adams /* offlines are called *before* the cpu is offlined. */ 791*2e0c549eSJonathan Adams if (what == CPU_OFF) 792*2e0c549eSJonathan Adams cpus_online--; 793*2e0c549eSJonathan Adams if (cpus_online < 1) 794*2e0c549eSJonathan Adams cpus_online = 1; 795*2e0c549eSJonathan Adams 796*2e0c549eSJonathan Adams mutex_enter(&taskq_cpupct_lock); 797*2e0c549eSJonathan Adams if (cpus_online == taskq_cpupct_ncpus_online) { 798*2e0c549eSJonathan Adams mutex_exit(&taskq_cpupct_lock); 799*2e0c549eSJonathan Adams return (0); 800*2e0c549eSJonathan Adams } 801*2e0c549eSJonathan Adams 802*2e0c549eSJonathan Adams for (tpp = list_head(&taskq_cpupct_list); tpp != NULL; 803*2e0c549eSJonathan Adams tpp = list_next(&taskq_cpupct_list, tpp)) { 804*2e0c549eSJonathan Adams taskq_t *tq = tpp->tp_taskq; 805*2e0c549eSJonathan Adams int newtarget; 806*2e0c549eSJonathan Adams 807*2e0c549eSJonathan Adams mutex_enter(&tq->tq_lock); 808*2e0c549eSJonathan Adams newtarget = 809*2e0c549eSJonathan Adams TASKQ_THREADS_PCT(cpus_online, tq->tq_threads_ncpus_pct); 810*2e0c549eSJonathan Adams ASSERT3S(newtarget, <=, tq->tq_nthreads_max); 811*2e0c549eSJonathan Adams if (newtarget != tq->tq_nthreads_target) { 812*2e0c549eSJonathan Adams /* The taskq must not be exiting */ 813*2e0c549eSJonathan Adams ASSERT3S(tq->tq_nthreads_target, !=, 0); 814*2e0c549eSJonathan Adams tq->tq_flags |= TASKQ_CHANGING; 815*2e0c549eSJonathan Adams tq->tq_nthreads_target = newtarget; 816*2e0c549eSJonathan Adams cv_broadcast(&tq->tq_dispatch_cv); 817*2e0c549eSJonathan Adams cv_broadcast(&tq->tq_exit_cv); 818*2e0c549eSJonathan Adams } 819*2e0c549eSJonathan Adams mutex_exit(&tq->tq_lock); 820*2e0c549eSJonathan Adams } 821*2e0c549eSJonathan Adams 822*2e0c549eSJonathan Adams taskq_cpupct_ncpus_online = cpus_online; 823*2e0c549eSJonathan Adams mutex_exit(&taskq_cpupct_lock); 824*2e0c549eSJonathan Adams return (0); 825*2e0c549eSJonathan Adams } 826*2e0c549eSJonathan Adams 827*2e0c549eSJonathan Adams void 828*2e0c549eSJonathan Adams taskq_mp_init(void) 829*2e0c549eSJonathan Adams { 830*2e0c549eSJonathan Adams mutex_enter(&cpu_lock); 831*2e0c549eSJonathan Adams register_cpu_setup_func(taskq_cpu_setup, NULL); 832*2e0c549eSJonathan Adams (void) taskq_cpu_setup(CPU_ON, 0, NULL); 833*2e0c549eSJonathan Adams mutex_exit(&cpu_lock); 8347c478bd9Sstevel@tonic-gate } 8357c478bd9Sstevel@tonic-gate 8367c478bd9Sstevel@tonic-gate /* 8377c478bd9Sstevel@tonic-gate * Create global system dynamic task queue. 8387c478bd9Sstevel@tonic-gate */ 8397c478bd9Sstevel@tonic-gate void 8407c478bd9Sstevel@tonic-gate system_taskq_init(void) 8417c478bd9Sstevel@tonic-gate { 8427c478bd9Sstevel@tonic-gate system_taskq = taskq_create_common("system_taskq", 0, 8437c478bd9Sstevel@tonic-gate system_taskq_size * max_ncpus, minclsyspri, 4, 512, 8447c478bd9Sstevel@tonic-gate TASKQ_DYNAMIC | TASKQ_PREPOPULATE); 8457c478bd9Sstevel@tonic-gate } 8467c478bd9Sstevel@tonic-gate 8477c478bd9Sstevel@tonic-gate /* 8487c478bd9Sstevel@tonic-gate * taskq_ent_alloc() 8497c478bd9Sstevel@tonic-gate * 8507c478bd9Sstevel@tonic-gate * Allocates a new taskq_ent_t structure either from the free list or from the 8517c478bd9Sstevel@tonic-gate * cache. Returns NULL if it can't be allocated. 8527c478bd9Sstevel@tonic-gate * 8537c478bd9Sstevel@tonic-gate * Assumes: tq->tq_lock is held. 8547c478bd9Sstevel@tonic-gate */ 8557c478bd9Sstevel@tonic-gate static taskq_ent_t * 8567c478bd9Sstevel@tonic-gate taskq_ent_alloc(taskq_t *tq, int flags) 8577c478bd9Sstevel@tonic-gate { 8587c478bd9Sstevel@tonic-gate int kmflags = (flags & TQ_NOSLEEP) ? KM_NOSLEEP : KM_SLEEP; 8597c478bd9Sstevel@tonic-gate 8607c478bd9Sstevel@tonic-gate taskq_ent_t *tqe; 8617c478bd9Sstevel@tonic-gate 8627c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&tq->tq_lock)); 8637c478bd9Sstevel@tonic-gate 8647c478bd9Sstevel@tonic-gate /* 8657c478bd9Sstevel@tonic-gate * TQ_NOALLOC allocations are allowed to use the freelist, even if 8667c478bd9Sstevel@tonic-gate * we are below tq_minalloc. 8677c478bd9Sstevel@tonic-gate */ 8687c478bd9Sstevel@tonic-gate if ((tqe = tq->tq_freelist) != NULL && 8697c478bd9Sstevel@tonic-gate ((flags & TQ_NOALLOC) || tq->tq_nalloc >= tq->tq_minalloc)) { 8707c478bd9Sstevel@tonic-gate tq->tq_freelist = tqe->tqent_next; 8717c478bd9Sstevel@tonic-gate } else { 8727c478bd9Sstevel@tonic-gate if (flags & TQ_NOALLOC) 8737c478bd9Sstevel@tonic-gate return (NULL); 8747c478bd9Sstevel@tonic-gate 8757c478bd9Sstevel@tonic-gate mutex_exit(&tq->tq_lock); 8767c478bd9Sstevel@tonic-gate if (tq->tq_nalloc >= tq->tq_maxalloc) { 8777c478bd9Sstevel@tonic-gate if (kmflags & KM_NOSLEEP) { 8787c478bd9Sstevel@tonic-gate mutex_enter(&tq->tq_lock); 8797c478bd9Sstevel@tonic-gate return (NULL); 8807c478bd9Sstevel@tonic-gate } 8817c478bd9Sstevel@tonic-gate /* 8827c478bd9Sstevel@tonic-gate * We don't want to exceed tq_maxalloc, but we can't 8837c478bd9Sstevel@tonic-gate * wait for other tasks to complete (and thus free up 8847c478bd9Sstevel@tonic-gate * task structures) without risking deadlock with 8857c478bd9Sstevel@tonic-gate * the caller. So, we just delay for one second 8867c478bd9Sstevel@tonic-gate * to throttle the allocation rate. 8877c478bd9Sstevel@tonic-gate */ 8887c478bd9Sstevel@tonic-gate delay(hz); 8897c478bd9Sstevel@tonic-gate } 8907c478bd9Sstevel@tonic-gate tqe = kmem_cache_alloc(taskq_ent_cache, kmflags); 8917c478bd9Sstevel@tonic-gate mutex_enter(&tq->tq_lock); 8927c478bd9Sstevel@tonic-gate if (tqe != NULL) 8937c478bd9Sstevel@tonic-gate tq->tq_nalloc++; 8947c478bd9Sstevel@tonic-gate } 8957c478bd9Sstevel@tonic-gate return (tqe); 8967c478bd9Sstevel@tonic-gate } 8977c478bd9Sstevel@tonic-gate 8987c478bd9Sstevel@tonic-gate /* 8997c478bd9Sstevel@tonic-gate * taskq_ent_free() 9007c478bd9Sstevel@tonic-gate * 9017c478bd9Sstevel@tonic-gate * Free taskq_ent_t structure by either putting it on the free list or freeing 9027c478bd9Sstevel@tonic-gate * it to the cache. 9037c478bd9Sstevel@tonic-gate * 9047c478bd9Sstevel@tonic-gate * Assumes: tq->tq_lock is held. 9057c478bd9Sstevel@tonic-gate */ 9067c478bd9Sstevel@tonic-gate static void 9077c478bd9Sstevel@tonic-gate taskq_ent_free(taskq_t *tq, taskq_ent_t *tqe) 9087c478bd9Sstevel@tonic-gate { 9097c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&tq->tq_lock)); 9107c478bd9Sstevel@tonic-gate 9117c478bd9Sstevel@tonic-gate if (tq->tq_nalloc <= tq->tq_minalloc) { 9127c478bd9Sstevel@tonic-gate tqe->tqent_next = tq->tq_freelist; 9137c478bd9Sstevel@tonic-gate tq->tq_freelist = tqe; 9147c478bd9Sstevel@tonic-gate } else { 9157c478bd9Sstevel@tonic-gate tq->tq_nalloc--; 9167c478bd9Sstevel@tonic-gate mutex_exit(&tq->tq_lock); 9177c478bd9Sstevel@tonic-gate kmem_cache_free(taskq_ent_cache, tqe); 9187c478bd9Sstevel@tonic-gate mutex_enter(&tq->tq_lock); 9197c478bd9Sstevel@tonic-gate } 9207c478bd9Sstevel@tonic-gate } 9217c478bd9Sstevel@tonic-gate 9227c478bd9Sstevel@tonic-gate /* 9237c478bd9Sstevel@tonic-gate * Dispatch a task "func(arg)" to a free entry of bucket b. 9247c478bd9Sstevel@tonic-gate * 9257c478bd9Sstevel@tonic-gate * Assumes: no bucket locks is held. 9267c478bd9Sstevel@tonic-gate * 9277c478bd9Sstevel@tonic-gate * Returns: a pointer to an entry if dispatch was successful. 9287c478bd9Sstevel@tonic-gate * NULL if there are no free entries or if the bucket is suspended. 9297c478bd9Sstevel@tonic-gate */ 9307c478bd9Sstevel@tonic-gate static taskq_ent_t * 9317c478bd9Sstevel@tonic-gate taskq_bucket_dispatch(taskq_bucket_t *b, task_func_t func, void *arg) 9327c478bd9Sstevel@tonic-gate { 9337c478bd9Sstevel@tonic-gate taskq_ent_t *tqe; 9347c478bd9Sstevel@tonic-gate 9357c478bd9Sstevel@tonic-gate ASSERT(MUTEX_NOT_HELD(&b->tqbucket_lock)); 9367c478bd9Sstevel@tonic-gate ASSERT(func != NULL); 9377c478bd9Sstevel@tonic-gate 9387c478bd9Sstevel@tonic-gate mutex_enter(&b->tqbucket_lock); 9397c478bd9Sstevel@tonic-gate 9407c478bd9Sstevel@tonic-gate ASSERT(b->tqbucket_nfree != 0 || IS_EMPTY(b->tqbucket_freelist)); 9417c478bd9Sstevel@tonic-gate ASSERT(b->tqbucket_nfree == 0 || !IS_EMPTY(b->tqbucket_freelist)); 9427c478bd9Sstevel@tonic-gate 9437c478bd9Sstevel@tonic-gate /* 9447c478bd9Sstevel@tonic-gate * Get en entry from the freelist if there is one. 9457c478bd9Sstevel@tonic-gate * Schedule task into the entry. 9467c478bd9Sstevel@tonic-gate */ 9477c478bd9Sstevel@tonic-gate if ((b->tqbucket_nfree != 0) && 9487c478bd9Sstevel@tonic-gate !(b->tqbucket_flags & TQBUCKET_SUSPEND)) { 9497c478bd9Sstevel@tonic-gate tqe = b->tqbucket_freelist.tqent_prev; 9507c478bd9Sstevel@tonic-gate 9517c478bd9Sstevel@tonic-gate ASSERT(tqe != &b->tqbucket_freelist); 9527c478bd9Sstevel@tonic-gate ASSERT(tqe->tqent_thread != NULL); 9537c478bd9Sstevel@tonic-gate 9547c478bd9Sstevel@tonic-gate tqe->tqent_prev->tqent_next = tqe->tqent_next; 9557c478bd9Sstevel@tonic-gate tqe->tqent_next->tqent_prev = tqe->tqent_prev; 9567c478bd9Sstevel@tonic-gate b->tqbucket_nalloc++; 9577c478bd9Sstevel@tonic-gate b->tqbucket_nfree--; 9587c478bd9Sstevel@tonic-gate tqe->tqent_func = func; 9597c478bd9Sstevel@tonic-gate tqe->tqent_arg = arg; 9607c478bd9Sstevel@tonic-gate TQ_STAT(b, tqs_hits); 9617c478bd9Sstevel@tonic-gate cv_signal(&tqe->tqent_cv); 9627c478bd9Sstevel@tonic-gate DTRACE_PROBE2(taskq__d__enqueue, taskq_bucket_t *, b, 9637c478bd9Sstevel@tonic-gate taskq_ent_t *, tqe); 9647c478bd9Sstevel@tonic-gate } else { 9657c478bd9Sstevel@tonic-gate tqe = NULL; 9667c478bd9Sstevel@tonic-gate TQ_STAT(b, tqs_misses); 9677c478bd9Sstevel@tonic-gate } 9687c478bd9Sstevel@tonic-gate mutex_exit(&b->tqbucket_lock); 9697c478bd9Sstevel@tonic-gate return (tqe); 9707c478bd9Sstevel@tonic-gate } 9717c478bd9Sstevel@tonic-gate 9727c478bd9Sstevel@tonic-gate /* 9737c478bd9Sstevel@tonic-gate * Dispatch a task. 9747c478bd9Sstevel@tonic-gate * 9757c478bd9Sstevel@tonic-gate * Assumes: func != NULL 9767c478bd9Sstevel@tonic-gate * 9777c478bd9Sstevel@tonic-gate * Returns: NULL if dispatch failed. 9787c478bd9Sstevel@tonic-gate * non-NULL if task dispatched successfully. 9797c478bd9Sstevel@tonic-gate * Actual return value is the pointer to taskq entry that was used to 9807c478bd9Sstevel@tonic-gate * dispatch a task. This is useful for debugging. 9817c478bd9Sstevel@tonic-gate */ 9827c478bd9Sstevel@tonic-gate /* ARGSUSED */ 9837c478bd9Sstevel@tonic-gate taskqid_t 9847c478bd9Sstevel@tonic-gate taskq_dispatch(taskq_t *tq, task_func_t func, void *arg, uint_t flags) 9857c478bd9Sstevel@tonic-gate { 9867c478bd9Sstevel@tonic-gate taskq_bucket_t *bucket = NULL; /* Which bucket needs extension */ 9877c478bd9Sstevel@tonic-gate taskq_ent_t *tqe = NULL; 9887c478bd9Sstevel@tonic-gate taskq_ent_t *tqe1; 9897c478bd9Sstevel@tonic-gate uint_t bsize; 9907c478bd9Sstevel@tonic-gate 9917c478bd9Sstevel@tonic-gate ASSERT(tq != NULL); 9927c478bd9Sstevel@tonic-gate ASSERT(func != NULL); 9937c478bd9Sstevel@tonic-gate 9947c478bd9Sstevel@tonic-gate if (!(tq->tq_flags & TASKQ_DYNAMIC)) { 9957c478bd9Sstevel@tonic-gate /* 9967c478bd9Sstevel@tonic-gate * TQ_NOQUEUE flag can't be used with non-dynamic task queues. 9977c478bd9Sstevel@tonic-gate */ 9987c478bd9Sstevel@tonic-gate ASSERT(! (flags & TQ_NOQUEUE)); 9997c478bd9Sstevel@tonic-gate /* 10007c478bd9Sstevel@tonic-gate * Enqueue the task to the underlying queue. 10017c478bd9Sstevel@tonic-gate */ 10027c478bd9Sstevel@tonic-gate mutex_enter(&tq->tq_lock); 10037c478bd9Sstevel@tonic-gate 10047c478bd9Sstevel@tonic-gate TASKQ_S_RANDOM_DISPATCH_FAILURE(tq, flags); 10057c478bd9Sstevel@tonic-gate 10067c478bd9Sstevel@tonic-gate if ((tqe = taskq_ent_alloc(tq, flags)) == NULL) { 10077c478bd9Sstevel@tonic-gate mutex_exit(&tq->tq_lock); 10087c478bd9Sstevel@tonic-gate return (NULL); 10097c478bd9Sstevel@tonic-gate } 10107c478bd9Sstevel@tonic-gate TQ_ENQUEUE(tq, tqe, func, arg); 10117c478bd9Sstevel@tonic-gate mutex_exit(&tq->tq_lock); 10127c478bd9Sstevel@tonic-gate return ((taskqid_t)tqe); 10137c478bd9Sstevel@tonic-gate } 10147c478bd9Sstevel@tonic-gate 10157c478bd9Sstevel@tonic-gate /* 10167c478bd9Sstevel@tonic-gate * Dynamic taskq dispatching. 10177c478bd9Sstevel@tonic-gate */ 10187c478bd9Sstevel@tonic-gate ASSERT(!(flags & TQ_NOALLOC)); 10197c478bd9Sstevel@tonic-gate TASKQ_D_RANDOM_DISPATCH_FAILURE(tq, flags); 10207c478bd9Sstevel@tonic-gate 10217c478bd9Sstevel@tonic-gate bsize = tq->tq_nbuckets; 10227c478bd9Sstevel@tonic-gate 10237c478bd9Sstevel@tonic-gate if (bsize == 1) { 10247c478bd9Sstevel@tonic-gate /* 10257c478bd9Sstevel@tonic-gate * In a single-CPU case there is only one bucket, so get 10267c478bd9Sstevel@tonic-gate * entry directly from there. 10277c478bd9Sstevel@tonic-gate */ 10287c478bd9Sstevel@tonic-gate if ((tqe = taskq_bucket_dispatch(tq->tq_buckets, func, arg)) 10297c478bd9Sstevel@tonic-gate != NULL) 10307c478bd9Sstevel@tonic-gate return ((taskqid_t)tqe); /* Fastpath */ 10317c478bd9Sstevel@tonic-gate bucket = tq->tq_buckets; 10327c478bd9Sstevel@tonic-gate } else { 10337c478bd9Sstevel@tonic-gate int loopcount; 10347c478bd9Sstevel@tonic-gate taskq_bucket_t *b; 10357c478bd9Sstevel@tonic-gate uintptr_t h = ((uintptr_t)CPU + (uintptr_t)arg) >> 3; 10367c478bd9Sstevel@tonic-gate 10377c478bd9Sstevel@tonic-gate h = TQ_HASH(h); 10387c478bd9Sstevel@tonic-gate 10397c478bd9Sstevel@tonic-gate /* 10407c478bd9Sstevel@tonic-gate * The 'bucket' points to the original bucket that we hit. If we 10417c478bd9Sstevel@tonic-gate * can't allocate from it, we search other buckets, but only 10427c478bd9Sstevel@tonic-gate * extend this one. 10437c478bd9Sstevel@tonic-gate */ 10447c478bd9Sstevel@tonic-gate b = &tq->tq_buckets[h & (bsize - 1)]; 10457c478bd9Sstevel@tonic-gate ASSERT(b->tqbucket_taskq == tq); /* Sanity check */ 10467c478bd9Sstevel@tonic-gate 10477c478bd9Sstevel@tonic-gate /* 10487c478bd9Sstevel@tonic-gate * Do a quick check before grabbing the lock. If the bucket does 10497c478bd9Sstevel@tonic-gate * not have free entries now, chances are very small that it 10507c478bd9Sstevel@tonic-gate * will after we take the lock, so we just skip it. 10517c478bd9Sstevel@tonic-gate */ 10527c478bd9Sstevel@tonic-gate if (b->tqbucket_nfree != 0) { 10537c478bd9Sstevel@tonic-gate if ((tqe = taskq_bucket_dispatch(b, func, arg)) != NULL) 10547c478bd9Sstevel@tonic-gate return ((taskqid_t)tqe); /* Fastpath */ 10557c478bd9Sstevel@tonic-gate } else { 10567c478bd9Sstevel@tonic-gate TQ_STAT(b, tqs_misses); 10577c478bd9Sstevel@tonic-gate } 10587c478bd9Sstevel@tonic-gate 10597c478bd9Sstevel@tonic-gate bucket = b; 10607c478bd9Sstevel@tonic-gate loopcount = MIN(taskq_search_depth, bsize); 10617c478bd9Sstevel@tonic-gate /* 10627c478bd9Sstevel@tonic-gate * If bucket dispatch failed, search loopcount number of buckets 10637c478bd9Sstevel@tonic-gate * before we give up and fail. 10647c478bd9Sstevel@tonic-gate */ 10657c478bd9Sstevel@tonic-gate do { 10667c478bd9Sstevel@tonic-gate b = &tq->tq_buckets[++h & (bsize - 1)]; 10677c478bd9Sstevel@tonic-gate ASSERT(b->tqbucket_taskq == tq); /* Sanity check */ 10687c478bd9Sstevel@tonic-gate loopcount--; 10697c478bd9Sstevel@tonic-gate 10707c478bd9Sstevel@tonic-gate if (b->tqbucket_nfree != 0) { 10717c478bd9Sstevel@tonic-gate tqe = taskq_bucket_dispatch(b, func, arg); 10727c478bd9Sstevel@tonic-gate } else { 10737c478bd9Sstevel@tonic-gate TQ_STAT(b, tqs_misses); 10747c478bd9Sstevel@tonic-gate } 10757c478bd9Sstevel@tonic-gate } while ((tqe == NULL) && (loopcount > 0)); 10767c478bd9Sstevel@tonic-gate } 10777c478bd9Sstevel@tonic-gate 10787c478bd9Sstevel@tonic-gate /* 10797c478bd9Sstevel@tonic-gate * At this point we either scheduled a task and (tqe != NULL) or failed 10807c478bd9Sstevel@tonic-gate * (tqe == NULL). Try to recover from fails. 10817c478bd9Sstevel@tonic-gate */ 10827c478bd9Sstevel@tonic-gate 10837c478bd9Sstevel@tonic-gate /* 10847c478bd9Sstevel@tonic-gate * For KM_SLEEP dispatches, try to extend the bucket and retry dispatch. 10857c478bd9Sstevel@tonic-gate */ 10867c478bd9Sstevel@tonic-gate if ((tqe == NULL) && !(flags & TQ_NOSLEEP)) { 10877c478bd9Sstevel@tonic-gate /* 10887c478bd9Sstevel@tonic-gate * taskq_bucket_extend() may fail to do anything, but this is 10897c478bd9Sstevel@tonic-gate * fine - we deal with it later. If the bucket was successfully 10907c478bd9Sstevel@tonic-gate * extended, there is a good chance that taskq_bucket_dispatch() 10917c478bd9Sstevel@tonic-gate * will get this new entry, unless someone is racing with us and 10927c478bd9Sstevel@tonic-gate * stealing the new entry from under our nose. 10937c478bd9Sstevel@tonic-gate * taskq_bucket_extend() may sleep. 10947c478bd9Sstevel@tonic-gate */ 10957c478bd9Sstevel@tonic-gate taskq_bucket_extend(bucket); 10967c478bd9Sstevel@tonic-gate TQ_STAT(bucket, tqs_disptcreates); 10977c478bd9Sstevel@tonic-gate if ((tqe = taskq_bucket_dispatch(bucket, func, arg)) != NULL) 10987c478bd9Sstevel@tonic-gate return ((taskqid_t)tqe); 10997c478bd9Sstevel@tonic-gate } 11007c478bd9Sstevel@tonic-gate 11017c478bd9Sstevel@tonic-gate ASSERT(bucket != NULL); 11027c478bd9Sstevel@tonic-gate /* 11037c478bd9Sstevel@tonic-gate * Since there are not enough free entries in the bucket, extend it 11047c478bd9Sstevel@tonic-gate * in the background using backing queue. 11057c478bd9Sstevel@tonic-gate */ 11067c478bd9Sstevel@tonic-gate mutex_enter(&tq->tq_lock); 11077c478bd9Sstevel@tonic-gate if ((tqe1 = taskq_ent_alloc(tq, TQ_NOSLEEP)) != NULL) { 11087c478bd9Sstevel@tonic-gate TQ_ENQUEUE(tq, tqe1, taskq_bucket_extend, 11097c478bd9Sstevel@tonic-gate bucket); 11107c478bd9Sstevel@tonic-gate } else { 11117c478bd9Sstevel@tonic-gate TQ_STAT(bucket, tqs_nomem); 11127c478bd9Sstevel@tonic-gate } 11137c478bd9Sstevel@tonic-gate 11147c478bd9Sstevel@tonic-gate /* 11157c478bd9Sstevel@tonic-gate * Dispatch failed and we can't find an entry to schedule a task. 11167c478bd9Sstevel@tonic-gate * Revert to the backing queue unless TQ_NOQUEUE was asked. 11177c478bd9Sstevel@tonic-gate */ 11187c478bd9Sstevel@tonic-gate if ((tqe == NULL) && !(flags & TQ_NOQUEUE)) { 11197c478bd9Sstevel@tonic-gate if ((tqe = taskq_ent_alloc(tq, flags)) != NULL) { 11207c478bd9Sstevel@tonic-gate TQ_ENQUEUE(tq, tqe, func, arg); 11217c478bd9Sstevel@tonic-gate } else { 11227c478bd9Sstevel@tonic-gate TQ_STAT(bucket, tqs_nomem); 11237c478bd9Sstevel@tonic-gate } 11247c478bd9Sstevel@tonic-gate } 11257c478bd9Sstevel@tonic-gate mutex_exit(&tq->tq_lock); 11267c478bd9Sstevel@tonic-gate 11277c478bd9Sstevel@tonic-gate return ((taskqid_t)tqe); 11287c478bd9Sstevel@tonic-gate } 11297c478bd9Sstevel@tonic-gate 11307c478bd9Sstevel@tonic-gate /* 11317c478bd9Sstevel@tonic-gate * Wait for all pending tasks to complete. 11327c478bd9Sstevel@tonic-gate * Calling taskq_wait from a task will cause deadlock. 11337c478bd9Sstevel@tonic-gate */ 11347c478bd9Sstevel@tonic-gate void 11357c478bd9Sstevel@tonic-gate taskq_wait(taskq_t *tq) 11367c478bd9Sstevel@tonic-gate { 11377c478bd9Sstevel@tonic-gate ASSERT(tq != curthread->t_taskq); 11387c478bd9Sstevel@tonic-gate 11397c478bd9Sstevel@tonic-gate mutex_enter(&tq->tq_lock); 11407c478bd9Sstevel@tonic-gate while (tq->tq_task.tqent_next != &tq->tq_task || tq->tq_active != 0) 11417c478bd9Sstevel@tonic-gate cv_wait(&tq->tq_wait_cv, &tq->tq_lock); 11427c478bd9Sstevel@tonic-gate mutex_exit(&tq->tq_lock); 11437c478bd9Sstevel@tonic-gate 11447c478bd9Sstevel@tonic-gate if (tq->tq_flags & TASKQ_DYNAMIC) { 11457c478bd9Sstevel@tonic-gate taskq_bucket_t *b = tq->tq_buckets; 11467c478bd9Sstevel@tonic-gate int bid = 0; 11477c478bd9Sstevel@tonic-gate for (; (b != NULL) && (bid < tq->tq_nbuckets); b++, bid++) { 11487c478bd9Sstevel@tonic-gate mutex_enter(&b->tqbucket_lock); 11497c478bd9Sstevel@tonic-gate while (b->tqbucket_nalloc > 0) 11507c478bd9Sstevel@tonic-gate cv_wait(&b->tqbucket_cv, &b->tqbucket_lock); 11517c478bd9Sstevel@tonic-gate mutex_exit(&b->tqbucket_lock); 11527c478bd9Sstevel@tonic-gate } 11537c478bd9Sstevel@tonic-gate } 11547c478bd9Sstevel@tonic-gate } 11557c478bd9Sstevel@tonic-gate 11567c478bd9Sstevel@tonic-gate /* 11577c478bd9Sstevel@tonic-gate * Suspend execution of tasks. 11587c478bd9Sstevel@tonic-gate * 11597c478bd9Sstevel@tonic-gate * Tasks in the queue part will be suspended immediately upon return from this 11607c478bd9Sstevel@tonic-gate * function. Pending tasks in the dynamic part will continue to execute, but all 11617c478bd9Sstevel@tonic-gate * new tasks will be suspended. 11627c478bd9Sstevel@tonic-gate */ 11637c478bd9Sstevel@tonic-gate void 11647c478bd9Sstevel@tonic-gate taskq_suspend(taskq_t *tq) 11657c478bd9Sstevel@tonic-gate { 11667c478bd9Sstevel@tonic-gate rw_enter(&tq->tq_threadlock, RW_WRITER); 11677c478bd9Sstevel@tonic-gate 11687c478bd9Sstevel@tonic-gate if (tq->tq_flags & TASKQ_DYNAMIC) { 11697c478bd9Sstevel@tonic-gate taskq_bucket_t *b = tq->tq_buckets; 11707c478bd9Sstevel@tonic-gate int bid = 0; 11717c478bd9Sstevel@tonic-gate for (; (b != NULL) && (bid < tq->tq_nbuckets); b++, bid++) { 11727c478bd9Sstevel@tonic-gate mutex_enter(&b->tqbucket_lock); 11737c478bd9Sstevel@tonic-gate b->tqbucket_flags |= TQBUCKET_SUSPEND; 11747c478bd9Sstevel@tonic-gate mutex_exit(&b->tqbucket_lock); 11757c478bd9Sstevel@tonic-gate } 11767c478bd9Sstevel@tonic-gate } 11777c478bd9Sstevel@tonic-gate /* 11787c478bd9Sstevel@tonic-gate * Mark task queue as being suspended. Needed for taskq_suspended(). 11797c478bd9Sstevel@tonic-gate */ 11807c478bd9Sstevel@tonic-gate mutex_enter(&tq->tq_lock); 11817c478bd9Sstevel@tonic-gate ASSERT(!(tq->tq_flags & TASKQ_SUSPENDED)); 11827c478bd9Sstevel@tonic-gate tq->tq_flags |= TASKQ_SUSPENDED; 11837c478bd9Sstevel@tonic-gate mutex_exit(&tq->tq_lock); 11847c478bd9Sstevel@tonic-gate } 11857c478bd9Sstevel@tonic-gate 11867c478bd9Sstevel@tonic-gate /* 11877c478bd9Sstevel@tonic-gate * returns: 1 if tq is suspended, 0 otherwise. 11887c478bd9Sstevel@tonic-gate */ 11897c478bd9Sstevel@tonic-gate int 11907c478bd9Sstevel@tonic-gate taskq_suspended(taskq_t *tq) 11917c478bd9Sstevel@tonic-gate { 11927c478bd9Sstevel@tonic-gate return ((tq->tq_flags & TASKQ_SUSPENDED) != 0); 11937c478bd9Sstevel@tonic-gate } 11947c478bd9Sstevel@tonic-gate 11957c478bd9Sstevel@tonic-gate /* 11967c478bd9Sstevel@tonic-gate * Resume taskq execution. 11977c478bd9Sstevel@tonic-gate */ 11987c478bd9Sstevel@tonic-gate void 11997c478bd9Sstevel@tonic-gate taskq_resume(taskq_t *tq) 12007c478bd9Sstevel@tonic-gate { 12017c478bd9Sstevel@tonic-gate ASSERT(RW_WRITE_HELD(&tq->tq_threadlock)); 12027c478bd9Sstevel@tonic-gate 12037c478bd9Sstevel@tonic-gate if (tq->tq_flags & TASKQ_DYNAMIC) { 12047c478bd9Sstevel@tonic-gate taskq_bucket_t *b = tq->tq_buckets; 12057c478bd9Sstevel@tonic-gate int bid = 0; 12067c478bd9Sstevel@tonic-gate for (; (b != NULL) && (bid < tq->tq_nbuckets); b++, bid++) { 12077c478bd9Sstevel@tonic-gate mutex_enter(&b->tqbucket_lock); 12087c478bd9Sstevel@tonic-gate b->tqbucket_flags &= ~TQBUCKET_SUSPEND; 12097c478bd9Sstevel@tonic-gate mutex_exit(&b->tqbucket_lock); 12107c478bd9Sstevel@tonic-gate } 12117c478bd9Sstevel@tonic-gate } 12127c478bd9Sstevel@tonic-gate mutex_enter(&tq->tq_lock); 12137c478bd9Sstevel@tonic-gate ASSERT(tq->tq_flags & TASKQ_SUSPENDED); 12147c478bd9Sstevel@tonic-gate tq->tq_flags &= ~TASKQ_SUSPENDED; 12157c478bd9Sstevel@tonic-gate mutex_exit(&tq->tq_lock); 12167c478bd9Sstevel@tonic-gate 12177c478bd9Sstevel@tonic-gate rw_exit(&tq->tq_threadlock); 12187c478bd9Sstevel@tonic-gate } 12197c478bd9Sstevel@tonic-gate 12207c478bd9Sstevel@tonic-gate int 12217c478bd9Sstevel@tonic-gate taskq_member(taskq_t *tq, kthread_t *thread) 12227c478bd9Sstevel@tonic-gate { 12237c478bd9Sstevel@tonic-gate return (thread->t_taskq == tq); 12247c478bd9Sstevel@tonic-gate } 12257c478bd9Sstevel@tonic-gate 1226*2e0c549eSJonathan Adams static void 1227*2e0c549eSJonathan Adams taskq_thread_create(taskq_t *tq) 1228*2e0c549eSJonathan Adams { 1229*2e0c549eSJonathan Adams kthread_t *t; 1230*2e0c549eSJonathan Adams 1231*2e0c549eSJonathan Adams ASSERT(MUTEX_HELD(&tq->tq_lock)); 1232*2e0c549eSJonathan Adams ASSERT(!(tq->tq_flags & TASKQ_THREAD_CREATED)); 1233*2e0c549eSJonathan Adams 1234*2e0c549eSJonathan Adams tq->tq_flags |= TASKQ_THREAD_CREATED; 1235*2e0c549eSJonathan Adams tq->tq_active++; 1236*2e0c549eSJonathan Adams t = thread_create(NULL, 0, taskq_thread, tq, 0, &p0, TS_RUN, 1237*2e0c549eSJonathan Adams tq->tq_pri); 1238*2e0c549eSJonathan Adams t->t_taskq = tq; 1239*2e0c549eSJonathan Adams } 1240*2e0c549eSJonathan Adams 1241*2e0c549eSJonathan Adams static void 1242*2e0c549eSJonathan Adams taskq_thread_wait(taskq_t *tq, kcondvar_t *cv, callb_cpr_t *cprinfo) 1243*2e0c549eSJonathan Adams { 1244*2e0c549eSJonathan Adams if (tq->tq_flags & TASKQ_CPR_SAFE) { 1245*2e0c549eSJonathan Adams cv_wait(cv, &tq->tq_lock); 1246*2e0c549eSJonathan Adams } else { 1247*2e0c549eSJonathan Adams CALLB_CPR_SAFE_BEGIN(cprinfo); 1248*2e0c549eSJonathan Adams cv_wait(cv, &tq->tq_lock); 1249*2e0c549eSJonathan Adams CALLB_CPR_SAFE_END(cprinfo, &tq->tq_lock); 1250*2e0c549eSJonathan Adams } 1251*2e0c549eSJonathan Adams } 1252*2e0c549eSJonathan Adams 12537c478bd9Sstevel@tonic-gate /* 12547c478bd9Sstevel@tonic-gate * Worker thread for processing task queue. 12557c478bd9Sstevel@tonic-gate */ 12567c478bd9Sstevel@tonic-gate static void 12577c478bd9Sstevel@tonic-gate taskq_thread(void *arg) 12587c478bd9Sstevel@tonic-gate { 1259*2e0c549eSJonathan Adams int thread_id; 1260*2e0c549eSJonathan Adams 12617c478bd9Sstevel@tonic-gate taskq_t *tq = arg; 12627c478bd9Sstevel@tonic-gate taskq_ent_t *tqe; 12637c478bd9Sstevel@tonic-gate callb_cpr_t cprinfo; 12647c478bd9Sstevel@tonic-gate hrtime_t start, end; 12657c478bd9Sstevel@tonic-gate 12667c478bd9Sstevel@tonic-gate if (tq->tq_flags & TASKQ_CPR_SAFE) { 12677c478bd9Sstevel@tonic-gate CALLB_CPR_INIT_SAFE(curthread, tq->tq_name); 12687c478bd9Sstevel@tonic-gate } else { 12697c478bd9Sstevel@tonic-gate CALLB_CPR_INIT(&cprinfo, &tq->tq_lock, callb_generic_cpr, 12707c478bd9Sstevel@tonic-gate tq->tq_name); 12717c478bd9Sstevel@tonic-gate } 12727c478bd9Sstevel@tonic-gate mutex_enter(&tq->tq_lock); 1273*2e0c549eSJonathan Adams thread_id = ++tq->tq_nthreads; 1274*2e0c549eSJonathan Adams ASSERT(tq->tq_flags & TASKQ_THREAD_CREATED); 1275*2e0c549eSJonathan Adams tq->tq_flags &= ~TASKQ_THREAD_CREATED; 1276*2e0c549eSJonathan Adams 1277*2e0c549eSJonathan Adams VERIFY3S(thread_id, <=, tq->tq_nthreads_max); 1278*2e0c549eSJonathan Adams 1279*2e0c549eSJonathan Adams if (tq->tq_nthreads_max == 1) 1280*2e0c549eSJonathan Adams tq->tq_thread = curthread; 1281*2e0c549eSJonathan Adams else 1282*2e0c549eSJonathan Adams tq->tq_threadlist[thread_id - 1] = curthread; 1283*2e0c549eSJonathan Adams 1284*2e0c549eSJonathan Adams for (;;) { 1285*2e0c549eSJonathan Adams if (tq->tq_flags & TASKQ_CHANGING) { 1286*2e0c549eSJonathan Adams /* we're done; clear the CHANGING flag */ 1287*2e0c549eSJonathan Adams if (tq->tq_nthreads == tq->tq_nthreads_target) { 1288*2e0c549eSJonathan Adams tq->tq_flags &= ~TASKQ_CHANGING; 1289*2e0c549eSJonathan Adams continue; 1290*2e0c549eSJonathan Adams } 1291*2e0c549eSJonathan Adams /* We're low on threads and none have been created */ 1292*2e0c549eSJonathan Adams if (tq->tq_nthreads < tq->tq_nthreads_target && 1293*2e0c549eSJonathan Adams !(tq->tq_flags & TASKQ_THREAD_CREATED)) { 1294*2e0c549eSJonathan Adams taskq_thread_create(tq); 1295*2e0c549eSJonathan Adams continue; 1296*2e0c549eSJonathan Adams } 1297*2e0c549eSJonathan Adams /* We're no longer needed */ 1298*2e0c549eSJonathan Adams if (thread_id > tq->tq_nthreads_target) { 1299*2e0c549eSJonathan Adams /* 1300*2e0c549eSJonathan Adams * To preserve the one-to-one mapping between 1301*2e0c549eSJonathan Adams * thread_id and thread, we must exit from 1302*2e0c549eSJonathan Adams * highest thread ID to least. 1303*2e0c549eSJonathan Adams * 1304*2e0c549eSJonathan Adams * However, if everyone is exiting, the order 1305*2e0c549eSJonathan Adams * doesn't matter, so just exit immediately. 1306*2e0c549eSJonathan Adams * (this is safe, since you must wait for 1307*2e0c549eSJonathan Adams * nthreads to reach 0 after setting 1308*2e0c549eSJonathan Adams * tq_nthreads_target to 0) 1309*2e0c549eSJonathan Adams */ 1310*2e0c549eSJonathan Adams if (thread_id == tq->tq_nthreads || 1311*2e0c549eSJonathan Adams tq->tq_nthreads_target == 0) 1312*2e0c549eSJonathan Adams break; 1313*2e0c549eSJonathan Adams 1314*2e0c549eSJonathan Adams /* Wait for higher thread_ids to exit */ 1315*2e0c549eSJonathan Adams taskq_thread_wait(tq, &tq->tq_exit_cv, 1316*2e0c549eSJonathan Adams &cprinfo); 1317*2e0c549eSJonathan Adams continue; 1318*2e0c549eSJonathan Adams } 1319*2e0c549eSJonathan Adams } 13207c478bd9Sstevel@tonic-gate if ((tqe = tq->tq_task.tqent_next) == &tq->tq_task) { 13217c478bd9Sstevel@tonic-gate if (--tq->tq_active == 0) 13227c478bd9Sstevel@tonic-gate cv_broadcast(&tq->tq_wait_cv); 1323*2e0c549eSJonathan Adams taskq_thread_wait(tq, &tq->tq_dispatch_cv, &cprinfo); 13247c478bd9Sstevel@tonic-gate tq->tq_active++; 13257c478bd9Sstevel@tonic-gate continue; 13267c478bd9Sstevel@tonic-gate } 13277c478bd9Sstevel@tonic-gate tqe->tqent_prev->tqent_next = tqe->tqent_next; 13287c478bd9Sstevel@tonic-gate tqe->tqent_next->tqent_prev = tqe->tqent_prev; 13297c478bd9Sstevel@tonic-gate mutex_exit(&tq->tq_lock); 13307c478bd9Sstevel@tonic-gate 13317c478bd9Sstevel@tonic-gate rw_enter(&tq->tq_threadlock, RW_READER); 13327c478bd9Sstevel@tonic-gate start = gethrtime(); 13337c478bd9Sstevel@tonic-gate DTRACE_PROBE2(taskq__exec__start, taskq_t *, tq, 13347c478bd9Sstevel@tonic-gate taskq_ent_t *, tqe); 13357c478bd9Sstevel@tonic-gate tqe->tqent_func(tqe->tqent_arg); 13367c478bd9Sstevel@tonic-gate DTRACE_PROBE2(taskq__exec__end, taskq_t *, tq, 13377c478bd9Sstevel@tonic-gate taskq_ent_t *, tqe); 13387c478bd9Sstevel@tonic-gate end = gethrtime(); 13397c478bd9Sstevel@tonic-gate rw_exit(&tq->tq_threadlock); 13407c478bd9Sstevel@tonic-gate 13417c478bd9Sstevel@tonic-gate mutex_enter(&tq->tq_lock); 13427c478bd9Sstevel@tonic-gate tq->tq_totaltime += end - start; 13437c478bd9Sstevel@tonic-gate tq->tq_executed++; 13447c478bd9Sstevel@tonic-gate 13457c478bd9Sstevel@tonic-gate taskq_ent_free(tq, tqe); 13467c478bd9Sstevel@tonic-gate } 1347*2e0c549eSJonathan Adams 1348*2e0c549eSJonathan Adams if (tq->tq_nthreads_max == 1) 1349*2e0c549eSJonathan Adams tq->tq_thread = NULL; 1350*2e0c549eSJonathan Adams else 1351*2e0c549eSJonathan Adams tq->tq_threadlist[thread_id - 1] = NULL; 1352*2e0c549eSJonathan Adams 1353*2e0c549eSJonathan Adams ASSERT(tq->tq_nthreads > 0); 1354*2e0c549eSJonathan Adams if (--tq->tq_nthreads == 0) 1355*2e0c549eSJonathan Adams cv_broadcast(&tq->tq_wait_cv); 1356*2e0c549eSJonathan Adams 1357*2e0c549eSJonathan Adams /* let the other threads which need to exit know we're done */ 1358*2e0c549eSJonathan Adams cv_broadcast(&tq->tq_exit_cv); 1359*2e0c549eSJonathan Adams 1360*2e0c549eSJonathan Adams /* We're exiting, and therefore no longer active */ 1361*2e0c549eSJonathan Adams tq->tq_active--; 1362*2e0c549eSJonathan Adams 13637c478bd9Sstevel@tonic-gate ASSERT(!(tq->tq_flags & TASKQ_CPR_SAFE)); 13647c478bd9Sstevel@tonic-gate CALLB_CPR_EXIT(&cprinfo); 13657c478bd9Sstevel@tonic-gate thread_exit(); 13667c478bd9Sstevel@tonic-gate } 13677c478bd9Sstevel@tonic-gate 13687c478bd9Sstevel@tonic-gate /* 13697c478bd9Sstevel@tonic-gate * Worker per-entry thread for dynamic dispatches. 13707c478bd9Sstevel@tonic-gate */ 13717c478bd9Sstevel@tonic-gate static void 13727c478bd9Sstevel@tonic-gate taskq_d_thread(taskq_ent_t *tqe) 13737c478bd9Sstevel@tonic-gate { 13747c478bd9Sstevel@tonic-gate taskq_bucket_t *bucket = tqe->tqent_bucket; 13757c478bd9Sstevel@tonic-gate taskq_t *tq = bucket->tqbucket_taskq; 13767c478bd9Sstevel@tonic-gate kmutex_t *lock = &bucket->tqbucket_lock; 13777c478bd9Sstevel@tonic-gate kcondvar_t *cv = &tqe->tqent_cv; 13787c478bd9Sstevel@tonic-gate callb_cpr_t cprinfo; 13797c478bd9Sstevel@tonic-gate clock_t w; 13807c478bd9Sstevel@tonic-gate 13817c478bd9Sstevel@tonic-gate CALLB_CPR_INIT(&cprinfo, lock, callb_generic_cpr, tq->tq_name); 13827c478bd9Sstevel@tonic-gate 13837c478bd9Sstevel@tonic-gate mutex_enter(lock); 13847c478bd9Sstevel@tonic-gate 13857c478bd9Sstevel@tonic-gate for (;;) { 13867c478bd9Sstevel@tonic-gate /* 13877c478bd9Sstevel@tonic-gate * If a task is scheduled (func != NULL), execute it, otherwise 13887c478bd9Sstevel@tonic-gate * sleep, waiting for a job. 13897c478bd9Sstevel@tonic-gate */ 13907c478bd9Sstevel@tonic-gate if (tqe->tqent_func != NULL) { 13917c478bd9Sstevel@tonic-gate hrtime_t start; 13927c478bd9Sstevel@tonic-gate hrtime_t end; 13937c478bd9Sstevel@tonic-gate 13947c478bd9Sstevel@tonic-gate ASSERT(bucket->tqbucket_nalloc > 0); 13957c478bd9Sstevel@tonic-gate 13967c478bd9Sstevel@tonic-gate /* 13977c478bd9Sstevel@tonic-gate * It is possible to free the entry right away before 13987c478bd9Sstevel@tonic-gate * actually executing the task so that subsequent 13997c478bd9Sstevel@tonic-gate * dispatches may immediately reuse it. But this, 14007c478bd9Sstevel@tonic-gate * effectively, creates a two-length queue in the entry 14017c478bd9Sstevel@tonic-gate * and may lead to a deadlock if the execution of the 14027c478bd9Sstevel@tonic-gate * current task depends on the execution of the next 14037c478bd9Sstevel@tonic-gate * scheduled task. So, we keep the entry busy until the 14047c478bd9Sstevel@tonic-gate * task is processed. 14057c478bd9Sstevel@tonic-gate */ 14067c478bd9Sstevel@tonic-gate 14077c478bd9Sstevel@tonic-gate mutex_exit(lock); 14087c478bd9Sstevel@tonic-gate start = gethrtime(); 14097c478bd9Sstevel@tonic-gate DTRACE_PROBE3(taskq__d__exec__start, taskq_t *, tq, 14107c478bd9Sstevel@tonic-gate taskq_bucket_t *, bucket, taskq_ent_t *, tqe); 14117c478bd9Sstevel@tonic-gate tqe->tqent_func(tqe->tqent_arg); 14127c478bd9Sstevel@tonic-gate DTRACE_PROBE3(taskq__d__exec__end, taskq_t *, tq, 14137c478bd9Sstevel@tonic-gate taskq_bucket_t *, bucket, taskq_ent_t *, tqe); 14147c478bd9Sstevel@tonic-gate end = gethrtime(); 14157c478bd9Sstevel@tonic-gate mutex_enter(lock); 14167c478bd9Sstevel@tonic-gate bucket->tqbucket_totaltime += end - start; 14177c478bd9Sstevel@tonic-gate 14187c478bd9Sstevel@tonic-gate /* 14197c478bd9Sstevel@tonic-gate * Return the entry to the bucket free list. 14207c478bd9Sstevel@tonic-gate */ 14217c478bd9Sstevel@tonic-gate tqe->tqent_func = NULL; 14227c478bd9Sstevel@tonic-gate TQ_APPEND(bucket->tqbucket_freelist, tqe); 14237c478bd9Sstevel@tonic-gate bucket->tqbucket_nalloc--; 14247c478bd9Sstevel@tonic-gate bucket->tqbucket_nfree++; 14257c478bd9Sstevel@tonic-gate ASSERT(!IS_EMPTY(bucket->tqbucket_freelist)); 14267c478bd9Sstevel@tonic-gate /* 14277c478bd9Sstevel@tonic-gate * taskq_wait() waits for nalloc to drop to zero on 14287c478bd9Sstevel@tonic-gate * tqbucket_cv. 14297c478bd9Sstevel@tonic-gate */ 14307c478bd9Sstevel@tonic-gate cv_signal(&bucket->tqbucket_cv); 14317c478bd9Sstevel@tonic-gate } 14327c478bd9Sstevel@tonic-gate 14337c478bd9Sstevel@tonic-gate /* 14347c478bd9Sstevel@tonic-gate * At this point the entry must be in the bucket free list - 14357c478bd9Sstevel@tonic-gate * either because it was there initially or because it just 14367c478bd9Sstevel@tonic-gate * finished executing a task and put itself on the free list. 14377c478bd9Sstevel@tonic-gate */ 14387c478bd9Sstevel@tonic-gate ASSERT(bucket->tqbucket_nfree > 0); 14397c478bd9Sstevel@tonic-gate /* 14407c478bd9Sstevel@tonic-gate * Go to sleep unless we are closing. 14417c478bd9Sstevel@tonic-gate * If a thread is sleeping too long, it dies. 14427c478bd9Sstevel@tonic-gate */ 14437c478bd9Sstevel@tonic-gate if (! (bucket->tqbucket_flags & TQBUCKET_CLOSE)) { 14447c478bd9Sstevel@tonic-gate CALLB_CPR_SAFE_BEGIN(&cprinfo); 14457c478bd9Sstevel@tonic-gate w = cv_timedwait(cv, lock, lbolt + 14467c478bd9Sstevel@tonic-gate taskq_thread_timeout * hz); 14477c478bd9Sstevel@tonic-gate CALLB_CPR_SAFE_END(&cprinfo, lock); 14487c478bd9Sstevel@tonic-gate } 14497c478bd9Sstevel@tonic-gate 14507c478bd9Sstevel@tonic-gate /* 14517c478bd9Sstevel@tonic-gate * At this point we may be in two different states: 14527c478bd9Sstevel@tonic-gate * 14537c478bd9Sstevel@tonic-gate * (1) tqent_func is set which means that a new task is 14547c478bd9Sstevel@tonic-gate * dispatched and we need to execute it. 14557c478bd9Sstevel@tonic-gate * 14567c478bd9Sstevel@tonic-gate * (2) Thread is sleeping for too long or we are closing. In 14577c478bd9Sstevel@tonic-gate * both cases destroy the thread and the entry. 14587c478bd9Sstevel@tonic-gate */ 14597c478bd9Sstevel@tonic-gate 14607c478bd9Sstevel@tonic-gate /* If func is NULL we should be on the freelist. */ 14617c478bd9Sstevel@tonic-gate ASSERT((tqe->tqent_func != NULL) || 14627c478bd9Sstevel@tonic-gate (bucket->tqbucket_nfree > 0)); 14637c478bd9Sstevel@tonic-gate /* If func is non-NULL we should be allocated */ 14647c478bd9Sstevel@tonic-gate ASSERT((tqe->tqent_func == NULL) || 14657c478bd9Sstevel@tonic-gate (bucket->tqbucket_nalloc > 0)); 14667c478bd9Sstevel@tonic-gate 14677c478bd9Sstevel@tonic-gate /* Check freelist consistency */ 14687c478bd9Sstevel@tonic-gate ASSERT((bucket->tqbucket_nfree > 0) || 14697c478bd9Sstevel@tonic-gate IS_EMPTY(bucket->tqbucket_freelist)); 14707c478bd9Sstevel@tonic-gate ASSERT((bucket->tqbucket_nfree == 0) || 14717c478bd9Sstevel@tonic-gate !IS_EMPTY(bucket->tqbucket_freelist)); 14727c478bd9Sstevel@tonic-gate 14737c478bd9Sstevel@tonic-gate if ((tqe->tqent_func == NULL) && 14747c478bd9Sstevel@tonic-gate ((w == -1) || (bucket->tqbucket_flags & TQBUCKET_CLOSE))) { 14757c478bd9Sstevel@tonic-gate /* 14767c478bd9Sstevel@tonic-gate * This thread is sleeping for too long or we are 14777c478bd9Sstevel@tonic-gate * closing - time to die. 14787c478bd9Sstevel@tonic-gate * Thread creation/destruction happens rarely, 14797c478bd9Sstevel@tonic-gate * so grabbing the lock is not a big performance issue. 14807c478bd9Sstevel@tonic-gate * The bucket lock is dropped by CALLB_CPR_EXIT(). 14817c478bd9Sstevel@tonic-gate */ 14827c478bd9Sstevel@tonic-gate 14837c478bd9Sstevel@tonic-gate /* Remove the entry from the free list. */ 14847c478bd9Sstevel@tonic-gate tqe->tqent_prev->tqent_next = tqe->tqent_next; 14857c478bd9Sstevel@tonic-gate tqe->tqent_next->tqent_prev = tqe->tqent_prev; 14867c478bd9Sstevel@tonic-gate ASSERT(bucket->tqbucket_nfree > 0); 14877c478bd9Sstevel@tonic-gate bucket->tqbucket_nfree--; 14887c478bd9Sstevel@tonic-gate 14897c478bd9Sstevel@tonic-gate TQ_STAT(bucket, tqs_tdeaths); 14907c478bd9Sstevel@tonic-gate cv_signal(&bucket->tqbucket_cv); 14917c478bd9Sstevel@tonic-gate tqe->tqent_thread = NULL; 14927c478bd9Sstevel@tonic-gate mutex_enter(&tq->tq_lock); 14937c478bd9Sstevel@tonic-gate tq->tq_tdeaths++; 14947c478bd9Sstevel@tonic-gate mutex_exit(&tq->tq_lock); 14957c478bd9Sstevel@tonic-gate CALLB_CPR_EXIT(&cprinfo); 14967c478bd9Sstevel@tonic-gate kmem_cache_free(taskq_ent_cache, tqe); 14977c478bd9Sstevel@tonic-gate thread_exit(); 14987c478bd9Sstevel@tonic-gate } 14997c478bd9Sstevel@tonic-gate } 15007c478bd9Sstevel@tonic-gate } 15017c478bd9Sstevel@tonic-gate 15027c478bd9Sstevel@tonic-gate 15037c478bd9Sstevel@tonic-gate /* 15047c478bd9Sstevel@tonic-gate * Taskq creation. May sleep for memory. 15057c478bd9Sstevel@tonic-gate * Always use automatically generated instances to avoid kstat name space 15067c478bd9Sstevel@tonic-gate * collisions. 15077c478bd9Sstevel@tonic-gate */ 15087c478bd9Sstevel@tonic-gate 15097c478bd9Sstevel@tonic-gate taskq_t * 15107c478bd9Sstevel@tonic-gate taskq_create(const char *name, int nthreads, pri_t pri, int minalloc, 15117c478bd9Sstevel@tonic-gate int maxalloc, uint_t flags) 15127c478bd9Sstevel@tonic-gate { 15137c478bd9Sstevel@tonic-gate return taskq_create_common(name, 0, nthreads, pri, minalloc, 15147c478bd9Sstevel@tonic-gate maxalloc, flags | TASKQ_NOINSTANCE); 15157c478bd9Sstevel@tonic-gate } 15167c478bd9Sstevel@tonic-gate 15177c478bd9Sstevel@tonic-gate /* 15187c478bd9Sstevel@tonic-gate * Create an instance of task queue. It is legal to create task queues with the 15197c478bd9Sstevel@tonic-gate * same name and different instances. 15207c478bd9Sstevel@tonic-gate * 15217c478bd9Sstevel@tonic-gate * taskq_create_instance is used by ddi_taskq_create() where it gets the 15227c478bd9Sstevel@tonic-gate * instance from ddi_get_instance(). In some cases the instance is not 15237c478bd9Sstevel@tonic-gate * initialized and is set to -1. This case is handled as if no instance was 15247c478bd9Sstevel@tonic-gate * passed at all. 15257c478bd9Sstevel@tonic-gate */ 15267c478bd9Sstevel@tonic-gate taskq_t * 15277c478bd9Sstevel@tonic-gate taskq_create_instance(const char *name, int instance, int nthreads, pri_t pri, 15287c478bd9Sstevel@tonic-gate int minalloc, int maxalloc, uint_t flags) 15297c478bd9Sstevel@tonic-gate { 15307c478bd9Sstevel@tonic-gate ASSERT((instance >= 0) || (instance == -1)); 15317c478bd9Sstevel@tonic-gate 15327c478bd9Sstevel@tonic-gate if (instance < 0) { 15337c478bd9Sstevel@tonic-gate flags |= TASKQ_NOINSTANCE; 15347c478bd9Sstevel@tonic-gate } 15357c478bd9Sstevel@tonic-gate 15367c478bd9Sstevel@tonic-gate return (taskq_create_common(name, instance, nthreads, 15377c478bd9Sstevel@tonic-gate pri, minalloc, maxalloc, flags)); 15387c478bd9Sstevel@tonic-gate } 15397c478bd9Sstevel@tonic-gate 15407c478bd9Sstevel@tonic-gate static taskq_t * 15417c478bd9Sstevel@tonic-gate taskq_create_common(const char *name, int instance, int nthreads, pri_t pri, 15427c478bd9Sstevel@tonic-gate int minalloc, int maxalloc, uint_t flags) 15437c478bd9Sstevel@tonic-gate { 15447c478bd9Sstevel@tonic-gate taskq_t *tq = kmem_cache_alloc(taskq_cache, KM_SLEEP); 15457c478bd9Sstevel@tonic-gate uint_t ncpus = ((boot_max_ncpus == -1) ? max_ncpus : boot_max_ncpus); 15467c478bd9Sstevel@tonic-gate uint_t bsize; /* # of buckets - always power of 2 */ 1547*2e0c549eSJonathan Adams int max_nthreads; 15487c478bd9Sstevel@tonic-gate 15497c478bd9Sstevel@tonic-gate /* 1550*2e0c549eSJonathan Adams * TASKQ_DYNAMIC is incompatible with TASKQ_CPR_SAFE and 1551*2e0c549eSJonathan Adams * TASKQ_THREADS_CPU_PCT. 15527c478bd9Sstevel@tonic-gate */ 1553*2e0c549eSJonathan Adams ASSERT(!(flags & TASKQ_DYNAMIC) || 1554*2e0c549eSJonathan Adams !(flags & (TASKQ_CPR_SAFE | TASKQ_THREADS_CPU_PCT))); 1555*2e0c549eSJonathan Adams /* TASKQ_CPR_SAFE is incompatible with TASKQ_THREADS_CPU_PCT */ 15567c478bd9Sstevel@tonic-gate 1557*2e0c549eSJonathan Adams ASSERT(!(flags & TASKQ_CPR_SAFE) || !(flags & TASKQ_THREADS_CPU_PCT)); 15587c478bd9Sstevel@tonic-gate 15597c478bd9Sstevel@tonic-gate bsize = 1 << (highbit(ncpus) - 1); 15607c478bd9Sstevel@tonic-gate ASSERT(bsize >= 1); 15617c478bd9Sstevel@tonic-gate bsize = MIN(bsize, taskq_maxbuckets); 15627c478bd9Sstevel@tonic-gate 1563*2e0c549eSJonathan Adams if (flags & TASKQ_DYNAMIC) { 1564*2e0c549eSJonathan Adams ASSERT3S(nthreads, >=, 1); 1565*2e0c549eSJonathan Adams tq->tq_maxsize = nthreads; 15667c478bd9Sstevel@tonic-gate 1567*2e0c549eSJonathan Adams /* For dynamic task queues use just one backup thread */ 1568*2e0c549eSJonathan Adams nthreads = max_nthreads = 1; 15697c478bd9Sstevel@tonic-gate 1570*2e0c549eSJonathan Adams } else if (!(flags & TASKQ_THREADS_CPU_PCT)) { 1571*2e0c549eSJonathan Adams ASSERT3S(nthreads, >=, 1); 1572*2e0c549eSJonathan Adams max_nthreads = nthreads; 1573*2e0c549eSJonathan Adams } else { 1574*2e0c549eSJonathan Adams uint_t pct; 1575*2e0c549eSJonathan Adams ASSERT3S(nthreads, >=, 0); 1576*2e0c549eSJonathan Adams pct = nthreads; 1577*2e0c549eSJonathan Adams 1578*2e0c549eSJonathan Adams if (pct > taskq_cpupct_max_percent) 1579*2e0c549eSJonathan Adams pct = taskq_cpupct_max_percent; 1580*2e0c549eSJonathan Adams 1581*2e0c549eSJonathan Adams tq->tq_threads_ncpus_pct = pct; 1582*2e0c549eSJonathan Adams nthreads = TASKQ_THREADS_PCT(ncpus_online, pct); 1583*2e0c549eSJonathan Adams max_nthreads = TASKQ_THREADS_PCT(max_ncpus, pct); 1584*2e0c549eSJonathan Adams } 1585*2e0c549eSJonathan Adams 1586*2e0c549eSJonathan Adams if (max_nthreads < taskq_minimum_nthreads_max) 1587*2e0c549eSJonathan Adams max_nthreads = taskq_minimum_nthreads_max; 1588*2e0c549eSJonathan Adams 1589*2e0c549eSJonathan Adams /* 1590*2e0c549eSJonathan Adams * Make sure the name is 0-terminated, and conforms to the rules for 1591*2e0c549eSJonathan Adams * C indentifiers 1592*2e0c549eSJonathan Adams */ 15937c478bd9Sstevel@tonic-gate (void) strncpy(tq->tq_name, name, TASKQ_NAMELEN + 1); 1594*2e0c549eSJonathan Adams strident_canon(tq->tq_name, TASKQ_NAMELEN + 1); 15957c478bd9Sstevel@tonic-gate 1596*2e0c549eSJonathan Adams tq->tq_flags = flags | TASKQ_CHANGING; 1597*2e0c549eSJonathan Adams tq->tq_active = 0; 15987c478bd9Sstevel@tonic-gate tq->tq_instance = instance; 1599*2e0c549eSJonathan Adams tq->tq_nthreads_target = nthreads; 1600*2e0c549eSJonathan Adams tq->tq_nthreads_max = max_nthreads; 16017c478bd9Sstevel@tonic-gate tq->tq_minalloc = minalloc; 16027c478bd9Sstevel@tonic-gate tq->tq_maxalloc = maxalloc; 16037c478bd9Sstevel@tonic-gate tq->tq_nbuckets = bsize; 16047c478bd9Sstevel@tonic-gate tq->tq_pri = pri; 16057c478bd9Sstevel@tonic-gate 1606*2e0c549eSJonathan Adams if (max_nthreads > 1) 1607*2e0c549eSJonathan Adams tq->tq_threadlist = kmem_alloc( 1608*2e0c549eSJonathan Adams sizeof (kthread_t *) * max_nthreads, KM_SLEEP); 1609*2e0c549eSJonathan Adams 1610*2e0c549eSJonathan Adams /* Add the taskq to the list of CPU_PCT taskqs */ 1611*2e0c549eSJonathan Adams if (flags & TASKQ_THREADS_CPU_PCT) { 1612*2e0c549eSJonathan Adams taskq_cpupct_ent_t *tpp = kmem_zalloc(sizeof (*tpp), KM_SLEEP); 1613*2e0c549eSJonathan Adams 1614*2e0c549eSJonathan Adams list_link_init(&tpp->tp_link); 1615*2e0c549eSJonathan Adams tpp->tp_taskq = tq; 1616*2e0c549eSJonathan Adams 1617*2e0c549eSJonathan Adams mutex_enter(&taskq_cpupct_lock); 1618*2e0c549eSJonathan Adams list_insert_tail(&taskq_cpupct_list, tpp); 1619*2e0c549eSJonathan Adams /* reset our target, to avoid race conditions */ 1620*2e0c549eSJonathan Adams tq->tq_nthreads_target = TASKQ_THREADS_PCT(ncpus_online, 1621*2e0c549eSJonathan Adams tq->tq_threads_ncpus_pct); 1622*2e0c549eSJonathan Adams mutex_exit(&taskq_cpupct_lock); 1623*2e0c549eSJonathan Adams } 1624*2e0c549eSJonathan Adams 1625*2e0c549eSJonathan Adams mutex_enter(&tq->tq_lock); 16267c478bd9Sstevel@tonic-gate if (flags & TASKQ_PREPOPULATE) { 16277c478bd9Sstevel@tonic-gate while (minalloc-- > 0) 16287c478bd9Sstevel@tonic-gate taskq_ent_free(tq, taskq_ent_alloc(tq, TQ_SLEEP)); 16297c478bd9Sstevel@tonic-gate } 16307c478bd9Sstevel@tonic-gate 1631*2e0c549eSJonathan Adams /* create the first thread; if more are needed, it'll create them */ 1632*2e0c549eSJonathan Adams taskq_thread_create(tq); 1633*2e0c549eSJonathan Adams mutex_exit(&tq->tq_lock); 16347c478bd9Sstevel@tonic-gate 16357c478bd9Sstevel@tonic-gate if (flags & TASKQ_DYNAMIC) { 16367c478bd9Sstevel@tonic-gate taskq_bucket_t *bucket = kmem_zalloc(sizeof (taskq_bucket_t) * 16377c478bd9Sstevel@tonic-gate bsize, KM_SLEEP); 16387c478bd9Sstevel@tonic-gate int b_id; 16397c478bd9Sstevel@tonic-gate 16407c478bd9Sstevel@tonic-gate tq->tq_buckets = bucket; 16417c478bd9Sstevel@tonic-gate 16427c478bd9Sstevel@tonic-gate /* Initialize each bucket */ 16437c478bd9Sstevel@tonic-gate for (b_id = 0; b_id < bsize; b_id++, bucket++) { 16447c478bd9Sstevel@tonic-gate mutex_init(&bucket->tqbucket_lock, NULL, MUTEX_DEFAULT, 16457c478bd9Sstevel@tonic-gate NULL); 16467c478bd9Sstevel@tonic-gate cv_init(&bucket->tqbucket_cv, NULL, CV_DEFAULT, NULL); 16477c478bd9Sstevel@tonic-gate bucket->tqbucket_taskq = tq; 16487c478bd9Sstevel@tonic-gate bucket->tqbucket_freelist.tqent_next = 16497c478bd9Sstevel@tonic-gate bucket->tqbucket_freelist.tqent_prev = 16507c478bd9Sstevel@tonic-gate &bucket->tqbucket_freelist; 16517c478bd9Sstevel@tonic-gate if (flags & TASKQ_PREPOPULATE) 16527c478bd9Sstevel@tonic-gate taskq_bucket_extend(bucket); 16537c478bd9Sstevel@tonic-gate } 16547c478bd9Sstevel@tonic-gate } 16557c478bd9Sstevel@tonic-gate 16567c478bd9Sstevel@tonic-gate /* 16577c478bd9Sstevel@tonic-gate * Install kstats. 16587c478bd9Sstevel@tonic-gate * We have two cases: 16597c478bd9Sstevel@tonic-gate * 1) Instance is provided to taskq_create_instance(). In this case it 16607c478bd9Sstevel@tonic-gate * should be >= 0 and we use it. 16617c478bd9Sstevel@tonic-gate * 16627c478bd9Sstevel@tonic-gate * 2) Instance is not provided and is automatically generated 16637c478bd9Sstevel@tonic-gate */ 16647c478bd9Sstevel@tonic-gate if (flags & TASKQ_NOINSTANCE) { 16657c478bd9Sstevel@tonic-gate instance = tq->tq_instance = 16667c478bd9Sstevel@tonic-gate (int)(uintptr_t)vmem_alloc(taskq_id_arena, 1, VM_SLEEP); 16677c478bd9Sstevel@tonic-gate } 16687c478bd9Sstevel@tonic-gate 16697c478bd9Sstevel@tonic-gate if (flags & TASKQ_DYNAMIC) { 16707c478bd9Sstevel@tonic-gate if ((tq->tq_kstat = kstat_create("unix", instance, 1671*2e0c549eSJonathan Adams tq->tq_name, "taskq_d", KSTAT_TYPE_NAMED, 1672*2e0c549eSJonathan Adams sizeof (taskq_d_kstat) / sizeof (kstat_named_t), 1673*2e0c549eSJonathan Adams KSTAT_FLAG_VIRTUAL)) != NULL) { 16747c478bd9Sstevel@tonic-gate tq->tq_kstat->ks_lock = &taskq_d_kstat_lock; 16757c478bd9Sstevel@tonic-gate tq->tq_kstat->ks_data = &taskq_d_kstat; 16767c478bd9Sstevel@tonic-gate tq->tq_kstat->ks_update = taskq_d_kstat_update; 16777c478bd9Sstevel@tonic-gate tq->tq_kstat->ks_private = tq; 16787c478bd9Sstevel@tonic-gate kstat_install(tq->tq_kstat); 16797c478bd9Sstevel@tonic-gate } 16807c478bd9Sstevel@tonic-gate } else { 16817c478bd9Sstevel@tonic-gate if ((tq->tq_kstat = kstat_create("unix", instance, tq->tq_name, 1682*2e0c549eSJonathan Adams "taskq", KSTAT_TYPE_NAMED, 1683*2e0c549eSJonathan Adams sizeof (taskq_kstat) / sizeof (kstat_named_t), 1684*2e0c549eSJonathan Adams KSTAT_FLAG_VIRTUAL)) != NULL) { 16857c478bd9Sstevel@tonic-gate tq->tq_kstat->ks_lock = &taskq_kstat_lock; 16867c478bd9Sstevel@tonic-gate tq->tq_kstat->ks_data = &taskq_kstat; 16877c478bd9Sstevel@tonic-gate tq->tq_kstat->ks_update = taskq_kstat_update; 16887c478bd9Sstevel@tonic-gate tq->tq_kstat->ks_private = tq; 16897c478bd9Sstevel@tonic-gate kstat_install(tq->tq_kstat); 16907c478bd9Sstevel@tonic-gate } 16917c478bd9Sstevel@tonic-gate } 16927c478bd9Sstevel@tonic-gate 16937c478bd9Sstevel@tonic-gate return (tq); 16947c478bd9Sstevel@tonic-gate } 16957c478bd9Sstevel@tonic-gate 16967c478bd9Sstevel@tonic-gate /* 16977c478bd9Sstevel@tonic-gate * taskq_destroy(). 16987c478bd9Sstevel@tonic-gate * 16997c478bd9Sstevel@tonic-gate * Assumes: by the time taskq_destroy is called no one will use this task queue 17007c478bd9Sstevel@tonic-gate * in any way and no one will try to dispatch entries in it. 17017c478bd9Sstevel@tonic-gate */ 17027c478bd9Sstevel@tonic-gate void 17037c478bd9Sstevel@tonic-gate taskq_destroy(taskq_t *tq) 17047c478bd9Sstevel@tonic-gate { 17057c478bd9Sstevel@tonic-gate taskq_bucket_t *b = tq->tq_buckets; 17067c478bd9Sstevel@tonic-gate int bid = 0; 17077c478bd9Sstevel@tonic-gate 17087c478bd9Sstevel@tonic-gate ASSERT(! (tq->tq_flags & TASKQ_CPR_SAFE)); 17097c478bd9Sstevel@tonic-gate 17107c478bd9Sstevel@tonic-gate /* 17117c478bd9Sstevel@tonic-gate * Destroy kstats. 17127c478bd9Sstevel@tonic-gate */ 17137c478bd9Sstevel@tonic-gate if (tq->tq_kstat != NULL) { 17147c478bd9Sstevel@tonic-gate kstat_delete(tq->tq_kstat); 17157c478bd9Sstevel@tonic-gate tq->tq_kstat = NULL; 17167c478bd9Sstevel@tonic-gate } 17177c478bd9Sstevel@tonic-gate 17187c478bd9Sstevel@tonic-gate /* 17197c478bd9Sstevel@tonic-gate * Destroy instance if needed. 17207c478bd9Sstevel@tonic-gate */ 17217c478bd9Sstevel@tonic-gate if (tq->tq_flags & TASKQ_NOINSTANCE) { 17227c478bd9Sstevel@tonic-gate vmem_free(taskq_id_arena, (void *)(uintptr_t)(tq->tq_instance), 17237c478bd9Sstevel@tonic-gate 1); 17247c478bd9Sstevel@tonic-gate tq->tq_instance = 0; 17257c478bd9Sstevel@tonic-gate } 17267c478bd9Sstevel@tonic-gate 1727*2e0c549eSJonathan Adams /* 1728*2e0c549eSJonathan Adams * Unregister from the cpupct list. 1729*2e0c549eSJonathan Adams */ 1730*2e0c549eSJonathan Adams if (tq->tq_flags & TASKQ_THREADS_CPU_PCT) { 1731*2e0c549eSJonathan Adams taskq_cpupct_ent_t *tpp; 1732*2e0c549eSJonathan Adams 1733*2e0c549eSJonathan Adams mutex_enter(&taskq_cpupct_lock); 1734*2e0c549eSJonathan Adams for (tpp = list_head(&taskq_cpupct_list); tpp != NULL; 1735*2e0c549eSJonathan Adams tpp = list_next(&taskq_cpupct_list, tpp)) { 1736*2e0c549eSJonathan Adams if (tpp->tp_taskq == tq) 1737*2e0c549eSJonathan Adams break; 1738*2e0c549eSJonathan Adams } 1739*2e0c549eSJonathan Adams ASSERT3P(tpp, !=, NULL); 1740*2e0c549eSJonathan Adams 1741*2e0c549eSJonathan Adams list_remove(&taskq_cpupct_list, tpp); 1742*2e0c549eSJonathan Adams mutex_exit(&taskq_cpupct_lock); 1743*2e0c549eSJonathan Adams 1744*2e0c549eSJonathan Adams kmem_free(tpp, sizeof (*tpp)); 1745*2e0c549eSJonathan Adams } 1746*2e0c549eSJonathan Adams 17477c478bd9Sstevel@tonic-gate /* 17487c478bd9Sstevel@tonic-gate * Wait for any pending entries to complete. 17497c478bd9Sstevel@tonic-gate */ 17507c478bd9Sstevel@tonic-gate taskq_wait(tq); 17517c478bd9Sstevel@tonic-gate 17527c478bd9Sstevel@tonic-gate mutex_enter(&tq->tq_lock); 17537c478bd9Sstevel@tonic-gate ASSERT((tq->tq_task.tqent_next == &tq->tq_task) && 17547c478bd9Sstevel@tonic-gate (tq->tq_active == 0)); 17557c478bd9Sstevel@tonic-gate 1756*2e0c549eSJonathan Adams /* notify all the threads that they need to exit */ 1757*2e0c549eSJonathan Adams tq->tq_nthreads_target = 0; 17587c478bd9Sstevel@tonic-gate 1759*2e0c549eSJonathan Adams tq->tq_flags |= TASKQ_CHANGING; 17607c478bd9Sstevel@tonic-gate cv_broadcast(&tq->tq_dispatch_cv); 1761*2e0c549eSJonathan Adams cv_broadcast(&tq->tq_exit_cv); 1762*2e0c549eSJonathan Adams 17637c478bd9Sstevel@tonic-gate while (tq->tq_nthreads != 0) 17647c478bd9Sstevel@tonic-gate cv_wait(&tq->tq_wait_cv, &tq->tq_lock); 17657c478bd9Sstevel@tonic-gate 1766*2e0c549eSJonathan Adams if (tq->tq_nthreads_max != 1) 1767*2e0c549eSJonathan Adams kmem_free(tq->tq_threadlist, sizeof (kthread_t *) * 1768*2e0c549eSJonathan Adams tq->tq_nthreads_max); 1769*2e0c549eSJonathan Adams 17707c478bd9Sstevel@tonic-gate tq->tq_minalloc = 0; 17717c478bd9Sstevel@tonic-gate while (tq->tq_nalloc != 0) 17727c478bd9Sstevel@tonic-gate taskq_ent_free(tq, taskq_ent_alloc(tq, TQ_SLEEP)); 17737c478bd9Sstevel@tonic-gate 17747c478bd9Sstevel@tonic-gate mutex_exit(&tq->tq_lock); 17757c478bd9Sstevel@tonic-gate 17767c478bd9Sstevel@tonic-gate /* 17777c478bd9Sstevel@tonic-gate * Mark each bucket as closing and wakeup all sleeping threads. 17787c478bd9Sstevel@tonic-gate */ 17797c478bd9Sstevel@tonic-gate for (; (b != NULL) && (bid < tq->tq_nbuckets); b++, bid++) { 17807c478bd9Sstevel@tonic-gate taskq_ent_t *tqe; 17817c478bd9Sstevel@tonic-gate 17827c478bd9Sstevel@tonic-gate mutex_enter(&b->tqbucket_lock); 17837c478bd9Sstevel@tonic-gate 17847c478bd9Sstevel@tonic-gate b->tqbucket_flags |= TQBUCKET_CLOSE; 17857c478bd9Sstevel@tonic-gate /* Wakeup all sleeping threads */ 17867c478bd9Sstevel@tonic-gate 17877c478bd9Sstevel@tonic-gate for (tqe = b->tqbucket_freelist.tqent_next; 17887c478bd9Sstevel@tonic-gate tqe != &b->tqbucket_freelist; tqe = tqe->tqent_next) 17897c478bd9Sstevel@tonic-gate cv_signal(&tqe->tqent_cv); 17907c478bd9Sstevel@tonic-gate 17917c478bd9Sstevel@tonic-gate ASSERT(b->tqbucket_nalloc == 0); 17927c478bd9Sstevel@tonic-gate 17937c478bd9Sstevel@tonic-gate /* 17947c478bd9Sstevel@tonic-gate * At this point we waited for all pending jobs to complete (in 17957c478bd9Sstevel@tonic-gate * both the task queue and the bucket and no new jobs should 17967c478bd9Sstevel@tonic-gate * arrive. Wait for all threads to die. 17977c478bd9Sstevel@tonic-gate */ 17987c478bd9Sstevel@tonic-gate while (b->tqbucket_nfree > 0) 17997c478bd9Sstevel@tonic-gate cv_wait(&b->tqbucket_cv, &b->tqbucket_lock); 18007c478bd9Sstevel@tonic-gate mutex_exit(&b->tqbucket_lock); 18017c478bd9Sstevel@tonic-gate mutex_destroy(&b->tqbucket_lock); 18027c478bd9Sstevel@tonic-gate cv_destroy(&b->tqbucket_cv); 18037c478bd9Sstevel@tonic-gate } 18047c478bd9Sstevel@tonic-gate 18057c478bd9Sstevel@tonic-gate if (tq->tq_buckets != NULL) { 18067c478bd9Sstevel@tonic-gate ASSERT(tq->tq_flags & TASKQ_DYNAMIC); 18077c478bd9Sstevel@tonic-gate kmem_free(tq->tq_buckets, 18087c478bd9Sstevel@tonic-gate sizeof (taskq_bucket_t) * tq->tq_nbuckets); 18097c478bd9Sstevel@tonic-gate 18107c478bd9Sstevel@tonic-gate /* Cleanup fields before returning tq to the cache */ 18117c478bd9Sstevel@tonic-gate tq->tq_buckets = NULL; 18127c478bd9Sstevel@tonic-gate tq->tq_tcreates = 0; 18137c478bd9Sstevel@tonic-gate tq->tq_tdeaths = 0; 18147c478bd9Sstevel@tonic-gate } else { 18157c478bd9Sstevel@tonic-gate ASSERT(!(tq->tq_flags & TASKQ_DYNAMIC)); 18167c478bd9Sstevel@tonic-gate } 18177c478bd9Sstevel@tonic-gate 1818*2e0c549eSJonathan Adams tq->tq_threads_ncpus_pct = 0; 18197c478bd9Sstevel@tonic-gate tq->tq_totaltime = 0; 18207c478bd9Sstevel@tonic-gate tq->tq_tasks = 0; 18217c478bd9Sstevel@tonic-gate tq->tq_maxtasks = 0; 18227c478bd9Sstevel@tonic-gate tq->tq_executed = 0; 18237c478bd9Sstevel@tonic-gate kmem_cache_free(taskq_cache, tq); 18247c478bd9Sstevel@tonic-gate } 18257c478bd9Sstevel@tonic-gate 18267c478bd9Sstevel@tonic-gate /* 18277c478bd9Sstevel@tonic-gate * Extend a bucket with a new entry on the free list and attach a worker thread 18287c478bd9Sstevel@tonic-gate * to it. 18297c478bd9Sstevel@tonic-gate * 18307c478bd9Sstevel@tonic-gate * Argument: pointer to the bucket. 18317c478bd9Sstevel@tonic-gate * 18327c478bd9Sstevel@tonic-gate * This function may quietly fail. It is only used by taskq_dispatch() which 18337c478bd9Sstevel@tonic-gate * handles such failures properly. 18347c478bd9Sstevel@tonic-gate */ 18357c478bd9Sstevel@tonic-gate static void 18367c478bd9Sstevel@tonic-gate taskq_bucket_extend(void *arg) 18377c478bd9Sstevel@tonic-gate { 18387c478bd9Sstevel@tonic-gate taskq_ent_t *tqe; 18397c478bd9Sstevel@tonic-gate taskq_bucket_t *b = (taskq_bucket_t *)arg; 18407c478bd9Sstevel@tonic-gate taskq_t *tq = b->tqbucket_taskq; 18417c478bd9Sstevel@tonic-gate int nthreads; 18427c478bd9Sstevel@tonic-gate 18437c478bd9Sstevel@tonic-gate if (! ENOUGH_MEMORY()) { 18447c478bd9Sstevel@tonic-gate TQ_STAT(b, tqs_nomem); 18457c478bd9Sstevel@tonic-gate return; 18467c478bd9Sstevel@tonic-gate } 18477c478bd9Sstevel@tonic-gate 18487c478bd9Sstevel@tonic-gate mutex_enter(&tq->tq_lock); 18497c478bd9Sstevel@tonic-gate 18507c478bd9Sstevel@tonic-gate /* 18517c478bd9Sstevel@tonic-gate * Observe global taskq limits on the number of threads. 18527c478bd9Sstevel@tonic-gate */ 18537c478bd9Sstevel@tonic-gate if (tq->tq_tcreates++ - tq->tq_tdeaths > tq->tq_maxsize) { 18547c478bd9Sstevel@tonic-gate tq->tq_tcreates--; 18557c478bd9Sstevel@tonic-gate mutex_exit(&tq->tq_lock); 18567c478bd9Sstevel@tonic-gate return; 18577c478bd9Sstevel@tonic-gate } 18587c478bd9Sstevel@tonic-gate mutex_exit(&tq->tq_lock); 18597c478bd9Sstevel@tonic-gate 18607c478bd9Sstevel@tonic-gate tqe = kmem_cache_alloc(taskq_ent_cache, KM_NOSLEEP); 18617c478bd9Sstevel@tonic-gate 18627c478bd9Sstevel@tonic-gate if (tqe == NULL) { 18637c478bd9Sstevel@tonic-gate mutex_enter(&tq->tq_lock); 18647c478bd9Sstevel@tonic-gate TQ_STAT(b, tqs_nomem); 18657c478bd9Sstevel@tonic-gate tq->tq_tcreates--; 18667c478bd9Sstevel@tonic-gate mutex_exit(&tq->tq_lock); 18677c478bd9Sstevel@tonic-gate return; 18687c478bd9Sstevel@tonic-gate } 18697c478bd9Sstevel@tonic-gate 18707c478bd9Sstevel@tonic-gate ASSERT(tqe->tqent_thread == NULL); 18717c478bd9Sstevel@tonic-gate 18727c478bd9Sstevel@tonic-gate tqe->tqent_bucket = b; 18737c478bd9Sstevel@tonic-gate 18747c478bd9Sstevel@tonic-gate /* 18757c478bd9Sstevel@tonic-gate * Create a thread in a TS_STOPPED state first. If it is successfully 18767c478bd9Sstevel@tonic-gate * created, place the entry on the free list and start the thread. 18777c478bd9Sstevel@tonic-gate */ 18787c478bd9Sstevel@tonic-gate tqe->tqent_thread = thread_create(NULL, 0, taskq_d_thread, tqe, 18797c478bd9Sstevel@tonic-gate 0, &p0, TS_STOPPED, tq->tq_pri); 18807c478bd9Sstevel@tonic-gate 18817c478bd9Sstevel@tonic-gate /* 18827c478bd9Sstevel@tonic-gate * Once the entry is ready, link it to the the bucket free list. 18837c478bd9Sstevel@tonic-gate */ 18847c478bd9Sstevel@tonic-gate mutex_enter(&b->tqbucket_lock); 18857c478bd9Sstevel@tonic-gate tqe->tqent_func = NULL; 18867c478bd9Sstevel@tonic-gate TQ_APPEND(b->tqbucket_freelist, tqe); 18877c478bd9Sstevel@tonic-gate b->tqbucket_nfree++; 18887c478bd9Sstevel@tonic-gate TQ_STAT(b, tqs_tcreates); 18897c478bd9Sstevel@tonic-gate 18907c478bd9Sstevel@tonic-gate #if TASKQ_STATISTIC 18917c478bd9Sstevel@tonic-gate nthreads = b->tqbucket_stat.tqs_tcreates - 18927c478bd9Sstevel@tonic-gate b->tqbucket_stat.tqs_tdeaths; 18937c478bd9Sstevel@tonic-gate b->tqbucket_stat.tqs_maxthreads = MAX(nthreads, 18947c478bd9Sstevel@tonic-gate b->tqbucket_stat.tqs_maxthreads); 18957c478bd9Sstevel@tonic-gate #endif 18967c478bd9Sstevel@tonic-gate 18977c478bd9Sstevel@tonic-gate mutex_exit(&b->tqbucket_lock); 18987c478bd9Sstevel@tonic-gate /* 18997c478bd9Sstevel@tonic-gate * Start the stopped thread. 19007c478bd9Sstevel@tonic-gate */ 19017c478bd9Sstevel@tonic-gate thread_lock(tqe->tqent_thread); 19027c478bd9Sstevel@tonic-gate tqe->tqent_thread->t_taskq = tq; 19037c478bd9Sstevel@tonic-gate tqe->tqent_thread->t_schedflag |= TS_ALLSTART; 19047c478bd9Sstevel@tonic-gate setrun_locked(tqe->tqent_thread); 19057c478bd9Sstevel@tonic-gate thread_unlock(tqe->tqent_thread); 19067c478bd9Sstevel@tonic-gate } 19077c478bd9Sstevel@tonic-gate 19087c478bd9Sstevel@tonic-gate static int 19097c478bd9Sstevel@tonic-gate taskq_kstat_update(kstat_t *ksp, int rw) 19107c478bd9Sstevel@tonic-gate { 19117c478bd9Sstevel@tonic-gate struct taskq_kstat *tqsp = &taskq_kstat; 19127c478bd9Sstevel@tonic-gate taskq_t *tq = ksp->ks_private; 19137c478bd9Sstevel@tonic-gate 19147c478bd9Sstevel@tonic-gate if (rw == KSTAT_WRITE) 19157c478bd9Sstevel@tonic-gate return (EACCES); 19167c478bd9Sstevel@tonic-gate 19177c478bd9Sstevel@tonic-gate tqsp->tq_tasks.value.ui64 = tq->tq_tasks; 19187c478bd9Sstevel@tonic-gate tqsp->tq_executed.value.ui64 = tq->tq_executed; 19197c478bd9Sstevel@tonic-gate tqsp->tq_maxtasks.value.ui64 = tq->tq_maxtasks; 19207c478bd9Sstevel@tonic-gate tqsp->tq_totaltime.value.ui64 = tq->tq_totaltime; 19217c478bd9Sstevel@tonic-gate tqsp->tq_nactive.value.ui64 = tq->tq_active; 19227c478bd9Sstevel@tonic-gate tqsp->tq_nalloc.value.ui64 = tq->tq_nalloc; 19237c478bd9Sstevel@tonic-gate tqsp->tq_pri.value.ui64 = tq->tq_pri; 19247c478bd9Sstevel@tonic-gate tqsp->tq_nthreads.value.ui64 = tq->tq_nthreads; 19257c478bd9Sstevel@tonic-gate return (0); 19267c478bd9Sstevel@tonic-gate } 19277c478bd9Sstevel@tonic-gate 19287c478bd9Sstevel@tonic-gate static int 19297c478bd9Sstevel@tonic-gate taskq_d_kstat_update(kstat_t *ksp, int rw) 19307c478bd9Sstevel@tonic-gate { 19317c478bd9Sstevel@tonic-gate struct taskq_d_kstat *tqsp = &taskq_d_kstat; 19327c478bd9Sstevel@tonic-gate taskq_t *tq = ksp->ks_private; 19337c478bd9Sstevel@tonic-gate taskq_bucket_t *b = tq->tq_buckets; 19347c478bd9Sstevel@tonic-gate int bid = 0; 19357c478bd9Sstevel@tonic-gate 19367c478bd9Sstevel@tonic-gate if (rw == KSTAT_WRITE) 19377c478bd9Sstevel@tonic-gate return (EACCES); 19387c478bd9Sstevel@tonic-gate 19397c478bd9Sstevel@tonic-gate ASSERT(tq->tq_flags & TASKQ_DYNAMIC); 19407c478bd9Sstevel@tonic-gate 19417c478bd9Sstevel@tonic-gate tqsp->tqd_btasks.value.ui64 = tq->tq_tasks; 19427c478bd9Sstevel@tonic-gate tqsp->tqd_bexecuted.value.ui64 = tq->tq_executed; 19437c478bd9Sstevel@tonic-gate tqsp->tqd_bmaxtasks.value.ui64 = tq->tq_maxtasks; 19447c478bd9Sstevel@tonic-gate tqsp->tqd_bnalloc.value.ui64 = tq->tq_nalloc; 19457c478bd9Sstevel@tonic-gate tqsp->tqd_bnactive.value.ui64 = tq->tq_active; 19467c478bd9Sstevel@tonic-gate tqsp->tqd_btotaltime.value.ui64 = tq->tq_totaltime; 19477c478bd9Sstevel@tonic-gate tqsp->tqd_pri.value.ui64 = tq->tq_pri; 19487c478bd9Sstevel@tonic-gate 19497c478bd9Sstevel@tonic-gate tqsp->tqd_hits.value.ui64 = 0; 19507c478bd9Sstevel@tonic-gate tqsp->tqd_misses.value.ui64 = 0; 19517c478bd9Sstevel@tonic-gate tqsp->tqd_overflows.value.ui64 = 0; 19527c478bd9Sstevel@tonic-gate tqsp->tqd_tcreates.value.ui64 = 0; 19537c478bd9Sstevel@tonic-gate tqsp->tqd_tdeaths.value.ui64 = 0; 19547c478bd9Sstevel@tonic-gate tqsp->tqd_maxthreads.value.ui64 = 0; 19557c478bd9Sstevel@tonic-gate tqsp->tqd_nomem.value.ui64 = 0; 19567c478bd9Sstevel@tonic-gate tqsp->tqd_disptcreates.value.ui64 = 0; 19577c478bd9Sstevel@tonic-gate tqsp->tqd_totaltime.value.ui64 = 0; 19587c478bd9Sstevel@tonic-gate tqsp->tqd_nalloc.value.ui64 = 0; 19597c478bd9Sstevel@tonic-gate tqsp->tqd_nfree.value.ui64 = 0; 19607c478bd9Sstevel@tonic-gate 19617c478bd9Sstevel@tonic-gate for (; (b != NULL) && (bid < tq->tq_nbuckets); b++, bid++) { 19627c478bd9Sstevel@tonic-gate tqsp->tqd_hits.value.ui64 += b->tqbucket_stat.tqs_hits; 19637c478bd9Sstevel@tonic-gate tqsp->tqd_misses.value.ui64 += b->tqbucket_stat.tqs_misses; 19647c478bd9Sstevel@tonic-gate tqsp->tqd_overflows.value.ui64 += b->tqbucket_stat.tqs_overflow; 19657c478bd9Sstevel@tonic-gate tqsp->tqd_tcreates.value.ui64 += b->tqbucket_stat.tqs_tcreates; 19667c478bd9Sstevel@tonic-gate tqsp->tqd_tdeaths.value.ui64 += b->tqbucket_stat.tqs_tdeaths; 19677c478bd9Sstevel@tonic-gate tqsp->tqd_maxthreads.value.ui64 += 19687c478bd9Sstevel@tonic-gate b->tqbucket_stat.tqs_maxthreads; 19697c478bd9Sstevel@tonic-gate tqsp->tqd_nomem.value.ui64 += b->tqbucket_stat.tqs_nomem; 19707c478bd9Sstevel@tonic-gate tqsp->tqd_disptcreates.value.ui64 += 19717c478bd9Sstevel@tonic-gate b->tqbucket_stat.tqs_disptcreates; 19727c478bd9Sstevel@tonic-gate tqsp->tqd_totaltime.value.ui64 += b->tqbucket_totaltime; 19737c478bd9Sstevel@tonic-gate tqsp->tqd_nalloc.value.ui64 += b->tqbucket_nalloc; 19747c478bd9Sstevel@tonic-gate tqsp->tqd_nfree.value.ui64 += b->tqbucket_nfree; 19757c478bd9Sstevel@tonic-gate } 19767c478bd9Sstevel@tonic-gate return (0); 19777c478bd9Sstevel@tonic-gate } 1978