1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21/*
22 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
23 * Use is subject to license terms.
24 */
25
26#pragma ident	"%Z%%M%	%I%	%E% SMI"
27
28#include <sys/types.h>
29#include <sys/param.h>
30#include <sys/sysmacros.h>
31#include <sys/systm.h>
32#include <sys/cmn_err.h>
33#include <sys/debug.h>
34#include <sys/inline.h>
35#include <sys/disp.h>
36#include <sys/kmem.h>
37#include <sys/cpuvar.h>
38#include <sys/vtrace.h>
39#include <sys/lockstat.h>
40#include <sys/spl.h>
41#include <sys/atomic.h>
42#include <sys/cpu.h>
43
44/*
45 * We check CPU_ON_INTR(CPU) when exiting a disp lock, rather than when
46 * entering it, for a purely pragmatic reason: when exiting a disp lock
47 * we know that we must be at PIL 10, and thus not preemptible; therefore
48 * we can safely load the CPU pointer without worrying about it changing.
49 */
50static void
51disp_onintr_panic(void)
52{
53	panic("dispatcher invoked from high-level interrupt handler");
54}
55
56/* ARGSUSED */
57void
58disp_lock_init(disp_lock_t *lp, char *name)
59{
60	DISP_LOCK_INIT(lp);
61}
62
63/* ARGSUSED */
64void
65disp_lock_destroy(disp_lock_t *lp)
66{
67	DISP_LOCK_DESTROY(lp);
68}
69
70void
71disp_lock_enter_high(disp_lock_t *lp)
72{
73	lock_set(lp);
74}
75
76void
77disp_lock_exit_high(disp_lock_t *lp)
78{
79	if (CPU_ON_INTR(CPU) != 0)
80		disp_onintr_panic();
81	ASSERT(DISP_LOCK_HELD(lp));
82	lock_clear(lp);
83}
84
85void
86disp_lock_enter(disp_lock_t *lp)
87{
88	lock_set_spl(lp, ipltospl(DISP_LEVEL), &curthread->t_oldspl);
89}
90
91void
92disp_lock_exit(disp_lock_t *lp)
93{
94	if (CPU_ON_INTR(CPU) != 0)
95		disp_onintr_panic();
96	ASSERT(DISP_LOCK_HELD(lp));
97	if (CPU->cpu_kprunrun) {
98		lock_clear_splx(lp, curthread->t_oldspl);
99		kpreempt(KPREEMPT_SYNC);
100	} else {
101		lock_clear_splx(lp, curthread->t_oldspl);
102	}
103}
104
105void
106disp_lock_exit_nopreempt(disp_lock_t *lp)
107{
108	if (CPU_ON_INTR(CPU) != 0)
109		disp_onintr_panic();
110	ASSERT(DISP_LOCK_HELD(lp));
111	lock_clear_splx(lp, curthread->t_oldspl);
112}
113
114/*
115 * Thread_lock() - get the correct dispatcher lock for the thread.
116 */
117void
118thread_lock(kthread_id_t t)
119{
120	int s = splhigh();
121
122	if (CPU_ON_INTR(CPU) != 0)
123		disp_onintr_panic();
124
125	for (;;) {
126		lock_t *volatile *tlpp = &t->t_lockp;
127		lock_t *lp = *tlpp;
128		if (lock_try(lp)) {
129			if (lp == *tlpp) {
130				curthread->t_oldspl = (ushort_t)s;
131				return;
132			}
133			lock_clear(lp);
134		} else {
135			hrtime_t spin_time =
136			    LOCKSTAT_START_TIME(LS_THREAD_LOCK_SPIN);
137			/*
138			 * Lower spl and spin on lock with non-atomic load
139			 * to avoid cache activity.  Spin until the lock
140			 * becomes available or spontaneously changes.
141			 */
142			splx(s);
143			while (lp == *tlpp && LOCK_HELD(lp)) {
144				if (panicstr) {
145					curthread->t_oldspl = splhigh();
146					return;
147				}
148				SMT_PAUSE();
149			}
150
151			LOCKSTAT_RECORD_TIME(LS_THREAD_LOCK_SPIN,
152			    lp, spin_time);
153			s = splhigh();
154		}
155	}
156}
157
158/*
159 * Thread_lock_high() - get the correct dispatcher lock for the thread.
160 *	This version is called when already at high spl.
161 */
162void
163thread_lock_high(kthread_id_t t)
164{
165	if (CPU_ON_INTR(CPU) != 0)
166		disp_onintr_panic();
167
168	for (;;) {
169		lock_t *volatile *tlpp = &t->t_lockp;
170		lock_t *lp = *tlpp;
171		if (lock_try(lp)) {
172			if (lp == *tlpp)
173				return;
174			lock_clear(lp);
175		} else {
176			hrtime_t spin_time =
177			    LOCKSTAT_START_TIME(LS_THREAD_LOCK_HIGH_SPIN);
178			while (lp == *tlpp && LOCK_HELD(lp)) {
179				if (panicstr)
180					return;
181				SMT_PAUSE();
182			}
183			LOCKSTAT_RECORD_TIME(LS_THREAD_LOCK_HIGH_SPIN,
184			    lp, spin_time);
185		}
186	}
187}
188
189/*
190 * Called by THREAD_TRANSITION macro to change the thread state to
191 * the intermediate state-in-transititon state.
192 */
193void
194thread_transition(kthread_id_t t)
195{
196	disp_lock_t	*lp;
197
198	ASSERT(THREAD_LOCK_HELD(t));
199	ASSERT(t->t_lockp != &transition_lock);
200
201	lp = t->t_lockp;
202	t->t_lockp = &transition_lock;
203	disp_lock_exit_high(lp);
204}
205
206/*
207 * Put thread in stop state, and set the lock pointer to the stop_lock.
208 * This effectively drops the lock on the thread, since the stop_lock
209 * isn't held.
210 * Eventually, stop_lock could be hashed if there is too much contention.
211 */
212void
213thread_stop(kthread_id_t t)
214{
215	disp_lock_t	*lp;
216
217	ASSERT(THREAD_LOCK_HELD(t));
218	ASSERT(t->t_lockp != &stop_lock);
219
220	lp = t->t_lockp;
221	t->t_state = TS_STOPPED;
222	/*
223	 * Ensure that t_state reaches global visibility before t_lockp
224	 */
225	membar_producer();
226	t->t_lockp = &stop_lock;
227	disp_lock_exit(lp);
228}
229