17c478bdstevel@tonic-gate/*
27c478bdstevel@tonic-gate * CDDL HEADER START
37c478bdstevel@tonic-gate *
47c478bdstevel@tonic-gate * The contents of this file are subject to the terms of the
52e0c549Jonathan Adams * Common Development and Distribution License (the "License").
62e0c549Jonathan Adams * You may not use this file except in compliance with the License.
77c478bdstevel@tonic-gate *
87c478bdstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bdstevel@tonic-gate * or http://www.opensolaris.org/os/licensing.
107c478bdstevel@tonic-gate * See the License for the specific language governing permissions
117c478bdstevel@tonic-gate * and limitations under the License.
127c478bdstevel@tonic-gate *
137c478bdstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each
147c478bdstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bdstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the
167c478bdstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying
177c478bdstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bdstevel@tonic-gate *
197c478bdstevel@tonic-gate * CDDL HEADER END
207c478bdstevel@tonic-gate */
217c478bdstevel@tonic-gate/*
226410974Chris Horne * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
237c478bdstevel@tonic-gate * Use is subject to license terms.
247c478bdstevel@tonic-gate */
255aeb947Garrett D'Amore/*
265aeb947Garrett D'Amore * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
27216d772Prakash Surya * Copyright (c) 2017 by Delphix. All rights reserved.
28f06dce2Andrew Stormont * Copyright 2017 RackTop Systems.
295aeb947Garrett D'Amore */
307c478bdstevel@tonic-gate
317c478bdstevel@tonic-gate#ifndef	_SYS_TASKQ_IMPL_H
327c478bdstevel@tonic-gate#define	_SYS_TASKQ_IMPL_H
337c478bdstevel@tonic-gate
347c478bdstevel@tonic-gate#include <sys/taskq.h>
3535a5a35Jonathan Adams#include <sys/inttypes.h>
367c478bdstevel@tonic-gate#include <sys/vmem.h>
3735a5a35Jonathan Adams#include <sys/list.h>
387c478bdstevel@tonic-gate#include <sys/kstat.h>
39f06dce2Andrew Stormont#include <sys/rwlock.h>
407c478bdstevel@tonic-gate
417c478bdstevel@tonic-gate#ifdef	__cplusplus
427c478bdstevel@tonic-gateextern "C" {
437c478bdstevel@tonic-gate#endif
447c478bdstevel@tonic-gate
457c478bdstevel@tonic-gatetypedef struct taskq_bucket taskq_bucket_t;
467c478bdstevel@tonic-gate
477c478bdstevel@tonic-gatetypedef struct taskq_ent {
487c478bdstevel@tonic-gate	struct taskq_ent	*tqent_next;
497c478bdstevel@tonic-gate	struct taskq_ent	*tqent_prev;
507c478bdstevel@tonic-gate	task_func_t		*tqent_func;
517c478bdstevel@tonic-gate	void			*tqent_arg;
525aeb947Garrett D'Amore	union {
535aeb947Garrett D'Amore		taskq_bucket_t	*tqent_bucket;
545aeb947Garrett D'Amore		uintptr_t	tqent_flags;
555aeb947Garrett D'Amore	}			tqent_un;
567c478bdstevel@tonic-gate	kthread_t		*tqent_thread;
577c478bdstevel@tonic-gate	kcondvar_t		tqent_cv;
587c478bdstevel@tonic-gate} taskq_ent_t;
597c478bdstevel@tonic-gate
605aeb947Garrett D'Amore#define	TQENT_FLAG_PREALLOC	0x1
615aeb947Garrett D'Amore
627c478bdstevel@tonic-gate/*
637c478bdstevel@tonic-gate * Taskq Statistics fields are not protected by any locks.
647c478bdstevel@tonic-gate */
657c478bdstevel@tonic-gatetypedef struct tqstat {
667c478bdstevel@tonic-gate	uint_t		tqs_hits;
677c478bdstevel@tonic-gate	uint_t		tqs_misses;
687c478bdstevel@tonic-gate	uint_t		tqs_overflow;	/* no threads to allocate   */
697c478bdstevel@tonic-gate	uint_t		tqs_tcreates;	/* threads created 	*/
707c478bdstevel@tonic-gate	uint_t		tqs_tdeaths;	/* threads died		*/
717c478bdstevel@tonic-gate	uint_t		tqs_maxthreads;	/* max # of alive threads */
727c478bdstevel@tonic-gate	uint_t		tqs_disptcreates;
737c478bdstevel@tonic-gate} tqstat_t;
747c478bdstevel@tonic-gate
757c478bdstevel@tonic-gate/*
767c478bdstevel@tonic-gate * Per-CPU hash bucket manages taskq_bent_t structures using freelist.
777c478bdstevel@tonic-gate */
787c478bdstevel@tonic-gatestruct taskq_bucket {
797c478bdstevel@tonic-gate	kmutex_t	tqbucket_lock;
807c478bdstevel@tonic-gate	taskq_t		*tqbucket_taskq;	/* Enclosing taskq */
817c478bdstevel@tonic-gate	taskq_ent_t	tqbucket_freelist;
827c478bdstevel@tonic-gate	uint_t		tqbucket_nalloc;	/* # of allocated entries */
837c478bdstevel@tonic-gate	uint_t		tqbucket_nfree;		/* # of free entries */
847c478bdstevel@tonic-gate	kcondvar_t	tqbucket_cv;
857c478bdstevel@tonic-gate	ushort_t	tqbucket_flags;
867c478bdstevel@tonic-gate	hrtime_t	tqbucket_totaltime;
877c478bdstevel@tonic-gate	tqstat_t	tqbucket_stat;
887c478bdstevel@tonic-gate};
897c478bdstevel@tonic-gate
907c478bdstevel@tonic-gate/*
917c478bdstevel@tonic-gate * Bucket flags.
927c478bdstevel@tonic-gate */
937c478bdstevel@tonic-gate#define	TQBUCKET_CLOSE		0x01
947c478bdstevel@tonic-gate#define	TQBUCKET_SUSPEND	0x02
957c478bdstevel@tonic-gate
9635a5a35Jonathan Adams#define	TASKQ_INTERFACE_FLAGS	0x0000ffff	/* defined in <sys/taskq.h> */
9735a5a35Jonathan Adams
987c478bdstevel@tonic-gate/*
997c478bdstevel@tonic-gate * taskq implementation flags: bit range 16-31
1007c478bdstevel@tonic-gate */
10135a5a35Jonathan Adams#define	TASKQ_CHANGING		0x00010000	/* nthreads != target */
10235a5a35Jonathan Adams#define	TASKQ_SUSPENDED		0x00020000	/* taskq is suspended */
10335a5a35Jonathan Adams#define	TASKQ_NOINSTANCE	0x00040000	/* no instance number */
10435a5a35Jonathan Adams#define	TASKQ_THREAD_CREATED	0x00080000	/* a thread has been created */
10535a5a35Jonathan Adams#define	TASKQ_DUTY_CYCLE	0x00100000	/* using the SDC class */
1067c478bdstevel@tonic-gate
1077c478bdstevel@tonic-gatestruct taskq {
1087c478bdstevel@tonic-gate	char		tq_name[TASKQ_NAMELEN + 1];
1097c478bdstevel@tonic-gate	kmutex_t	tq_lock;
1107c478bdstevel@tonic-gate	krwlock_t	tq_threadlock;
1117c478bdstevel@tonic-gate	kcondvar_t	tq_dispatch_cv;
1127c478bdstevel@tonic-gate	kcondvar_t	tq_wait_cv;
1132e0c549Jonathan Adams	kcondvar_t	tq_exit_cv;
1142e0c549Jonathan Adams	pri_t		tq_pri;		/* Scheduling priority */
1157c478bdstevel@tonic-gate	uint_t		tq_flags;
1167c478bdstevel@tonic-gate	int		tq_active;
1177c478bdstevel@tonic-gate	int		tq_nthreads;
1182e0c549Jonathan Adams	int		tq_nthreads_target;
1192e0c549Jonathan Adams	int		tq_nthreads_max;
1202e0c549Jonathan Adams	int		tq_threads_ncpus_pct;
1217c478bdstevel@tonic-gate	int		tq_nalloc;
1227c478bdstevel@tonic-gate	int		tq_minalloc;
1237c478bdstevel@tonic-gate	int		tq_maxalloc;
1246410974Chris Horne	kcondvar_t	tq_maxalloc_cv;
1256410974Chris Horne	int		tq_maxalloc_wait;
1267c478bdstevel@tonic-gate	taskq_ent_t	*tq_freelist;
1277c478bdstevel@tonic-gate	taskq_ent_t	tq_task;
1287c478bdstevel@tonic-gate	int		tq_maxsize;
1297c478bdstevel@tonic-gate	taskq_bucket_t	*tq_buckets;	/* Per-cpu array of buckets */
1307c478bdstevel@tonic-gate	int		tq_instance;
1317c478bdstevel@tonic-gate	uint_t		tq_nbuckets;	/* # of buckets	(2^n)	    */
1327c478bdstevel@tonic-gate	union {
1337c478bdstevel@tonic-gate		kthread_t *_tq_thread;
1347c478bdstevel@tonic-gate		kthread_t **_tq_threadlist;
1357c478bdstevel@tonic-gate	}		tq_thr;
13635a5a35Jonathan Adams
13735a5a35Jonathan Adams	list_node_t	tq_cpupct_link;	/* linkage for taskq_cpupct_list */
13835a5a35Jonathan Adams	struct proc	*tq_proc;	/* process for taskq threads */
13935a5a35Jonathan Adams	int		tq_cpupart;	/* cpupart id bound to */
14035a5a35Jonathan Adams	uint_t		tq_DC;		/* duty cycle for SDC */
14135a5a35Jonathan Adams
1427c478bdstevel@tonic-gate	/*
1437c478bdstevel@tonic-gate	 * Statistics.
1447c478bdstevel@tonic-gate	 */
1457c478bdstevel@tonic-gate	kstat_t		*tq_kstat;	/* Exported statistics */
1467c478bdstevel@tonic-gate	hrtime_t	tq_totaltime;	/* Time spent processing tasks */
147216d772Prakash Surya	uint64_t	tq_nomem;	/* # of times there was no memory */
14835a5a35Jonathan Adams	uint64_t	tq_tasks;	/* Total # of tasks posted */
14935a5a35Jonathan Adams	uint64_t	tq_executed;	/* Total # of tasks executed */
1507c478bdstevel@tonic-gate	int		tq_maxtasks;	/* Max number of tasks in the queue */
1517c478bdstevel@tonic-gate	int		tq_tcreates;
1527c478bdstevel@tonic-gate	int		tq_tdeaths;
1537c478bdstevel@tonic-gate};
1547c478bdstevel@tonic-gate
1555aeb947Garrett D'Amore/* Special form of taskq dispatch that uses preallocated entries. */
1565aeb947Garrett D'Amorevoid taskq_dispatch_ent(taskq_t *, task_func_t, void *, uint_t, taskq_ent_t *);
1575aeb947Garrett D'Amore
1585aeb947Garrett D'Amore
1597c478bdstevel@tonic-gate#define	tq_thread tq_thr._tq_thread
1607c478bdstevel@tonic-gate#define	tq_threadlist tq_thr._tq_threadlist
1617c478bdstevel@tonic-gate
1622e0c549Jonathan Adams/* The MAX guarantees we have at least one thread */
1632e0c549Jonathan Adams#define	TASKQ_THREADS_PCT(ncpus, pct)	MAX(((ncpus) * (pct)) / 100, 1)
1642e0c549Jonathan Adams
1657c478bdstevel@tonic-gate#ifdef	__cplusplus
1667c478bdstevel@tonic-gate}
1677c478bdstevel@tonic-gate#endif
1687c478bdstevel@tonic-gate
1697c478bdstevel@tonic-gate#endif	/* _SYS_TASKQ_IMPL_H */
170