1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21/*
22 * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
23 * Use is subject to license terms.
24 */
25/*
26 * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
27 * Copyright (c) 2017 by Delphix. All rights reserved.
28 * Copyright 2017 RackTop Systems.
29 */
30
31#ifndef	_SYS_TASKQ_IMPL_H
32#define	_SYS_TASKQ_IMPL_H
33
34#include <sys/taskq.h>
35#include <sys/inttypes.h>
36#include <sys/vmem.h>
37#include <sys/list.h>
38#include <sys/kstat.h>
39#include <sys/rwlock.h>
40
41#ifdef	__cplusplus
42extern "C" {
43#endif
44
45typedef struct taskq_bucket taskq_bucket_t;
46
47typedef struct taskq_ent {
48	struct taskq_ent	*tqent_next;
49	struct taskq_ent	*tqent_prev;
50	task_func_t		*tqent_func;
51	void			*tqent_arg;
52	union {
53		taskq_bucket_t	*tqent_bucket;
54		uintptr_t	tqent_flags;
55	}			tqent_un;
56	kthread_t		*tqent_thread;
57	kcondvar_t		tqent_cv;
58} taskq_ent_t;
59
60#define	TQENT_FLAG_PREALLOC	0x1
61
62/*
63 * Taskq Statistics fields are not protected by any locks.
64 */
65typedef struct tqstat {
66	uint_t		tqs_hits;
67	uint_t		tqs_misses;
68	uint_t		tqs_overflow;	/* no threads to allocate   */
69	uint_t		tqs_tcreates;	/* threads created 	*/
70	uint_t		tqs_tdeaths;	/* threads died		*/
71	uint_t		tqs_maxthreads;	/* max # of alive threads */
72	uint_t		tqs_disptcreates;
73} tqstat_t;
74
75/*
76 * Per-CPU hash bucket manages taskq_bent_t structures using freelist.
77 */
78struct taskq_bucket {
79	kmutex_t	tqbucket_lock;
80	taskq_t		*tqbucket_taskq;	/* Enclosing taskq */
81	taskq_ent_t	tqbucket_freelist;
82	uint_t		tqbucket_nalloc;	/* # of allocated entries */
83	uint_t		tqbucket_nfree;		/* # of free entries */
84	kcondvar_t	tqbucket_cv;
85	ushort_t	tqbucket_flags;
86	hrtime_t	tqbucket_totaltime;
87	tqstat_t	tqbucket_stat;
88};
89
90/*
91 * Bucket flags.
92 */
93#define	TQBUCKET_CLOSE		0x01
94#define	TQBUCKET_SUSPEND	0x02
95
96#define	TASKQ_INTERFACE_FLAGS	0x0000ffff	/* defined in <sys/taskq.h> */
97
98/*
99 * taskq implementation flags: bit range 16-31
100 */
101#define	TASKQ_CHANGING		0x00010000	/* nthreads != target */
102#define	TASKQ_SUSPENDED		0x00020000	/* taskq is suspended */
103#define	TASKQ_NOINSTANCE	0x00040000	/* no instance number */
104#define	TASKQ_THREAD_CREATED	0x00080000	/* a thread has been created */
105#define	TASKQ_DUTY_CYCLE	0x00100000	/* using the SDC class */
106
107struct taskq {
108	char		tq_name[TASKQ_NAMELEN + 1];
109	kmutex_t	tq_lock;
110	krwlock_t	tq_threadlock;
111	kcondvar_t	tq_dispatch_cv;
112	kcondvar_t	tq_wait_cv;
113	kcondvar_t	tq_exit_cv;
114	pri_t		tq_pri;		/* Scheduling priority */
115	uint_t		tq_flags;
116	int		tq_active;
117	int		tq_nthreads;
118	int		tq_nthreads_target;
119	int		tq_nthreads_max;
120	int		tq_threads_ncpus_pct;
121	int		tq_nalloc;
122	int		tq_minalloc;
123	int		tq_maxalloc;
124	kcondvar_t	tq_maxalloc_cv;
125	int		tq_maxalloc_wait;
126	taskq_ent_t	*tq_freelist;
127	taskq_ent_t	tq_task;
128	int		tq_maxsize;
129	taskq_bucket_t	*tq_buckets;	/* Per-cpu array of buckets */
130	int		tq_instance;
131	uint_t		tq_nbuckets;	/* # of buckets	(2^n)	    */
132	union {
133		kthread_t *_tq_thread;
134		kthread_t **_tq_threadlist;
135	}		tq_thr;
136
137	list_node_t	tq_cpupct_link;	/* linkage for taskq_cpupct_list */
138	struct proc	*tq_proc;	/* process for taskq threads */
139	int		tq_cpupart;	/* cpupart id bound to */
140	uint_t		tq_DC;		/* duty cycle for SDC */
141
142	/*
143	 * Statistics.
144	 */
145	kstat_t		*tq_kstat;	/* Exported statistics */
146	hrtime_t	tq_totaltime;	/* Time spent processing tasks */
147	uint64_t	tq_nomem;	/* # of times there was no memory */
148	uint64_t	tq_tasks;	/* Total # of tasks posted */
149	uint64_t	tq_executed;	/* Total # of tasks executed */
150	int		tq_maxtasks;	/* Max number of tasks in the queue */
151	int		tq_tcreates;
152	int		tq_tdeaths;
153};
154
155/* Special form of taskq dispatch that uses preallocated entries. */
156void taskq_dispatch_ent(taskq_t *, task_func_t, void *, uint_t, taskq_ent_t *);
157
158
159#define	tq_thread tq_thr._tq_thread
160#define	tq_threadlist tq_thr._tq_threadlist
161
162/* The MAX guarantees we have at least one thread */
163#define	TASKQ_THREADS_PCT(ncpus, pct)	MAX(((ncpus) * (pct)) / 100, 1)
164
165#ifdef	__cplusplus
166}
167#endif
168
169#endif	/* _SYS_TASKQ_IMPL_H */
170