17c478bdstevel@tonic-gate/*
27c478bdstevel@tonic-gate * CDDL HEADER START
37c478bdstevel@tonic-gate *
47c478bdstevel@tonic-gate * The contents of this file are subject to the terms of the
5fb2f18fesaxe * Common Development and Distribution License (the "License").
6fb2f18fesaxe * You may not use this file except in compliance with the License.
77c478bdstevel@tonic-gate *
87c478bdstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bdstevel@tonic-gate * or http://www.opensolaris.org/os/licensing.
107c478bdstevel@tonic-gate * See the License for the specific language governing permissions
117c478bdstevel@tonic-gate * and limitations under the License.
127c478bdstevel@tonic-gate *
137c478bdstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each
147c478bdstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bdstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the
167c478bdstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying
177c478bdstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bdstevel@tonic-gate *
197c478bdstevel@tonic-gate * CDDL HEADER END
207c478bdstevel@tonic-gate */
217c478bdstevel@tonic-gate/*
22fb2f18fesaxe * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
237c478bdstevel@tonic-gate * Use is subject to license terms.
24b819ceaGordon Ross *
25b819ceaGordon Ross * Copyright 2013 Nexenta Systems, Inc.  All rights reserved.
26455e370John Levon *
27455e370John Levon * Copyright 2018 Joyent, Inc.
287c478bdstevel@tonic-gate */
297c478bdstevel@tonic-gate
307c478bdstevel@tonic-gate/*	Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T	*/
31b4203d7Marcel Telka/*	  All Rights Reserved	*/
327c478bdstevel@tonic-gate
337c478bdstevel@tonic-gate
347c478bdstevel@tonic-gate#ifndef _SYS_DISP_H
357c478bdstevel@tonic-gate#define	_SYS_DISP_H
367c478bdstevel@tonic-gate
377c478bdstevel@tonic-gate#include <sys/priocntl.h>
387c478bdstevel@tonic-gate#include <sys/thread.h>
397c478bdstevel@tonic-gate#include <sys/class.h>
407c478bdstevel@tonic-gate
417c478bdstevel@tonic-gate#ifdef	__cplusplus
427c478bdstevel@tonic-gateextern "C" {
437c478bdstevel@tonic-gate#endif
447c478bdstevel@tonic-gate
457c478bdstevel@tonic-gate/*
467c478bdstevel@tonic-gate * The following is the format of a dispatcher queue entry.
477c478bdstevel@tonic-gate */
487c478bdstevel@tonic-gatetypedef struct dispq {
497c478bdstevel@tonic-gate	kthread_t	*dq_first;	/* first thread on queue or NULL */
507c478bdstevel@tonic-gate	kthread_t	*dq_last;	/* last thread on queue or NULL */
517c478bdstevel@tonic-gate	int		dq_sruncnt;	/* number of loaded, runnable */
527c478bdstevel@tonic-gate					/*    threads on queue */
537c478bdstevel@tonic-gate} dispq_t;
547c478bdstevel@tonic-gate
557c478bdstevel@tonic-gate/*
567c478bdstevel@tonic-gate * Dispatch queue structure.
577c478bdstevel@tonic-gate */
587c478bdstevel@tonic-gatetypedef struct _disp {
597c478bdstevel@tonic-gate	disp_lock_t	disp_lock;	/* protects dispatching fields */
607c478bdstevel@tonic-gate	pri_t		disp_npri;	/* # of priority levels in queue */
617c478bdstevel@tonic-gate	dispq_t		*disp_q;		/* the dispatch queue */
627c478bdstevel@tonic-gate	dispq_t		*disp_q_limit;	/* ptr past end of dispatch queue */
637c478bdstevel@tonic-gate	ulong_t		*disp_qactmap;	/* bitmap of active dispatch queues */
647c478bdstevel@tonic-gate
657c478bdstevel@tonic-gate	/*
667c478bdstevel@tonic-gate	 * Priorities:
677c478bdstevel@tonic-gate	 *	disp_maxrunpri is the maximum run priority of runnable threads
68455e370John Levon	 *	on this queue.  It is -1 if nothing is runnable.
697c478bdstevel@tonic-gate	 *
707c478bdstevel@tonic-gate	 *	disp_max_unbound_pri is the maximum run priority of threads on
717c478bdstevel@tonic-gate	 *	this dispatch queue but runnable by any CPU.  This may be left
72455e370John Levon	 *	artificially high, then corrected when some CPU tries to take
737c478bdstevel@tonic-gate	 *	an unbound thread.  It is -1 if nothing is runnable.
747c478bdstevel@tonic-gate	 */
757c478bdstevel@tonic-gate	pri_t		disp_maxrunpri;	/* maximum run priority */
767c478bdstevel@tonic-gate	pri_t		disp_max_unbound_pri;	/* max pri of unbound threads */
777c478bdstevel@tonic-gate
787c478bdstevel@tonic-gate	volatile int	disp_nrunnable;	/* runnable threads in cpu dispq */
797c478bdstevel@tonic-gate
807c478bdstevel@tonic-gate	struct cpu	*disp_cpu;	/* cpu owning this queue or NULL */
81685679fakolb	hrtime_t	disp_steal;	/* time when threads become stealable */
827c478bdstevel@tonic-gate} disp_t;
837c478bdstevel@tonic-gate
84b819ceaGordon Ross#if defined(_KERNEL) || defined(_FAKE_KERNEL)
857c478bdstevel@tonic-gate
867c478bdstevel@tonic-gate#define	MAXCLSYSPRI	99
877c478bdstevel@tonic-gate#define	MINCLSYSPRI	60
887c478bdstevel@tonic-gate
897c478bdstevel@tonic-gate
907c478bdstevel@tonic-gate/*
917c478bdstevel@tonic-gate * Global scheduling variables.
927c478bdstevel@tonic-gate *	- See sys/cpuvar.h for CPU-local variables.
937c478bdstevel@tonic-gate */
947c478bdstevel@tonic-gateextern int	nswapped;	/* number of swapped threads */
957c478bdstevel@tonic-gate				/* nswapped protected by swap_lock */
967c478bdstevel@tonic-gate
977c478bdstevel@tonic-gateextern	pri_t	minclsyspri;	/* minimum level of any system class */
987c478bdstevel@tonic-gateextern	pri_t	maxclsyspri;	/* maximum level of any system class */
997c478bdstevel@tonic-gateextern	pri_t	intr_pri;	/* interrupt thread priority base level */
1007c478bdstevel@tonic-gate
101b819ceaGordon Ross#endif	/* _KERNEL || _FAKE_KERNEL */
102b819ceaGordon Ross#if defined(_KERNEL)
103b819ceaGordon Ross
1047c478bdstevel@tonic-gate/*
105685679fakolb * Minimum amount of time that a thread can remain runnable before it can
106685679fakolb * be stolen by another CPU (in nanoseconds).
107685679fakolb */
108685679fakolbextern hrtime_t nosteal_nsec;
109685679fakolb
110685679fakolb/*
1117c478bdstevel@tonic-gate * Kernel preemption occurs if a higher-priority thread is runnable with
1127c478bdstevel@tonic-gate * a priority at or above kpreemptpri.
1137c478bdstevel@tonic-gate *
1147c478bdstevel@tonic-gate * So that other processors can watch for such threads, a separate
1157c478bdstevel@tonic-gate * dispatch queue with unbound work above kpreemptpri is maintained.
1167c478bdstevel@tonic-gate * This is part of the CPU partition structure (cpupart_t).
1177c478bdstevel@tonic-gate */
1187c478bdstevel@tonic-gateextern	pri_t	kpreemptpri;	/* level above which preemption takes place */
1197c478bdstevel@tonic-gate
1207c478bdstevel@tonic-gateextern void		disp_kp_alloc(disp_t *, pri_t);	/* allocate kp queue */
1217c478bdstevel@tonic-gateextern void		disp_kp_free(disp_t *);		/* free kp queue */
1227c478bdstevel@tonic-gate
1237c478bdstevel@tonic-gate/*
1247c478bdstevel@tonic-gate * Macro for use by scheduling classes to decide whether the thread is about
1257c478bdstevel@tonic-gate * to be scheduled or not.  This returns the maximum run priority.
1267c478bdstevel@tonic-gate */
1277c478bdstevel@tonic-gate#define	DISP_MAXRUNPRI(t)	((t)->t_disp_queue->disp_maxrunpri)
1287c478bdstevel@tonic-gate
1297c478bdstevel@tonic-gate/*
1307c478bdstevel@tonic-gate * Platform callbacks for various dispatcher operations
1317c478bdstevel@tonic-gate *
1327c478bdstevel@tonic-gate * idle_cpu() is invoked when a cpu goes idle, and has nothing to do.
1337c478bdstevel@tonic-gate * disp_enq_thread() is invoked when a thread is placed on a run queue.
1347c478bdstevel@tonic-gate */
1357c478bdstevel@tonic-gateextern void	(*idle_cpu)();
1367c478bdstevel@tonic-gateextern void	(*disp_enq_thread)(struct cpu *, int);
1377c478bdstevel@tonic-gate
1387c478bdstevel@tonic-gate
1397c478bdstevel@tonic-gateextern int		dispdeq(kthread_t *);
1407c478bdstevel@tonic-gateextern void		dispinit(void);
1417c478bdstevel@tonic-gateextern void		disp_add(sclass_t *);
1427c478bdstevel@tonic-gateextern int		intr_active(struct cpu *, int);
1437c478bdstevel@tonic-gateextern int		servicing_interrupt(void);
1447c478bdstevel@tonic-gateextern void		preempt(void);
1457c478bdstevel@tonic-gateextern void		setbackdq(kthread_t *);
1467c478bdstevel@tonic-gateextern void		setfrontdq(kthread_t *);
1477c478bdstevel@tonic-gateextern void		swtch(void);
1487c478bdstevel@tonic-gateextern void		swtch_to(kthread_t *);
1497c478bdstevel@tonic-gateextern void		swtch_from_zombie(void)
1507c478bdstevel@tonic-gate				__NORETURN;
1517c478bdstevel@tonic-gateextern void		dq_sruninc(kthread_t *);
1527c478bdstevel@tonic-gateextern void		dq_srundec(kthread_t *);
1537c478bdstevel@tonic-gateextern void		cpu_rechoose(kthread_t *);
1547c478bdstevel@tonic-gateextern void		cpu_surrender(kthread_t *);
1557c478bdstevel@tonic-gateextern void		kpreempt(int);
156455e370John Levonextern struct cpu	*disp_lowpri_cpu(struct cpu *, kthread_t *, pri_t);
1577c478bdstevel@tonic-gateextern int		disp_bound_threads(struct cpu *, int);
1587c478bdstevel@tonic-gateextern int		disp_bound_anythreads(struct cpu *, int);
1597c478bdstevel@tonic-gateextern int		disp_bound_partition(struct cpu *, int);
1607c478bdstevel@tonic-gateextern void		disp_cpu_init(struct cpu *);
1617c478bdstevel@tonic-gateextern void		disp_cpu_fini(struct cpu *);
1627c478bdstevel@tonic-gateextern void		disp_cpu_inactive(struct cpu *);
1637c478bdstevel@tonic-gateextern void		disp_adjust_unbound_pri(kthread_t *);
1647c478bdstevel@tonic-gateextern void		resume(kthread_t *);
1657c478bdstevel@tonic-gateextern void		resume_from_intr(kthread_t *);
1667c478bdstevel@tonic-gateextern void		resume_from_zombie(kthread_t *)
1677c478bdstevel@tonic-gate				__NORETURN;
1687c478bdstevel@tonic-gateextern void		disp_swapped_enq(kthread_t *);
1697c478bdstevel@tonic-gateextern int		disp_anywork(void);
1707c478bdstevel@tonic-gate
171455e370John Levonextern struct cpu	*disp_choose_best_cpu(void);
172455e370John Levon
1737c478bdstevel@tonic-gate#define	KPREEMPT_SYNC		(-1)
1747c478bdstevel@tonic-gate#define	kpreempt_disable()				\
1757c478bdstevel@tonic-gate	{						\
1767c478bdstevel@tonic-gate		curthread->t_preempt++;			\
1777c478bdstevel@tonic-gate		ASSERT(curthread->t_preempt >= 1);	\
1787c478bdstevel@tonic-gate	}
1797c478bdstevel@tonic-gate#define	kpreempt_enable()				\
1807c478bdstevel@tonic-gate	{						\
1817c478bdstevel@tonic-gate		ASSERT(curthread->t_preempt >= 1);	\
1827c478bdstevel@tonic-gate		if (--curthread->t_preempt == 0 &&	\
1837c478bdstevel@tonic-gate		    CPU->cpu_kprunrun)			\
1847c478bdstevel@tonic-gate			kpreempt(KPREEMPT_SYNC);	\
1857c478bdstevel@tonic-gate	}
1867c478bdstevel@tonic-gate
1877c478bdstevel@tonic-gate#endif	/* _KERNEL */
1887c478bdstevel@tonic-gate
189455e370John Levon#define	CPU_IDLE_PRI (-1)
190455e370John Levon
1917c478bdstevel@tonic-gate#ifdef	__cplusplus
1927c478bdstevel@tonic-gate}
1937c478bdstevel@tonic-gate#endif
1947c478bdstevel@tonic-gate
1957c478bdstevel@tonic-gate#endif	/* _SYS_DISP_H */
196