xref: /illumos-gate/usr/src/uts/common/disp/thread.c (revision 2570281c)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 1991, 2010, Oracle and/or its affiliates. All rights reserved.
24  * Copyright 2021 Joyent, Inc.
25  * Copyright 2021 Oxide Computer Company
26  */
27 
28 #include <sys/types.h>
29 #include <sys/param.h>
30 #include <sys/sysmacros.h>
31 #include <sys/signal.h>
32 #include <sys/stack.h>
33 #include <sys/pcb.h>
34 #include <sys/user.h>
35 #include <sys/systm.h>
36 #include <sys/sysinfo.h>
37 #include <sys/errno.h>
38 #include <sys/cmn_err.h>
39 #include <sys/cred.h>
40 #include <sys/resource.h>
41 #include <sys/task.h>
42 #include <sys/project.h>
43 #include <sys/proc.h>
44 #include <sys/debug.h>
45 #include <sys/disp.h>
46 #include <sys/class.h>
47 #include <vm/seg_kmem.h>
48 #include <vm/seg_kp.h>
49 #include <sys/machlock.h>
50 #include <sys/kmem.h>
51 #include <sys/varargs.h>
52 #include <sys/turnstile.h>
53 #include <sys/poll.h>
54 #include <sys/vtrace.h>
55 #include <sys/callb.h>
56 #include <c2/audit.h>
57 #include <sys/sobject.h>
58 #include <sys/cpupart.h>
59 #include <sys/pset.h>
60 #include <sys/door.h>
61 #include <sys/spl.h>
62 #include <sys/copyops.h>
63 #include <sys/rctl.h>
64 #include <sys/brand.h>
65 #include <sys/pool.h>
66 #include <sys/zone.h>
67 #include <sys/tsol/label.h>
68 #include <sys/tsol/tndb.h>
69 #include <sys/cpc_impl.h>
70 #include <sys/sdt.h>
71 #include <sys/reboot.h>
72 #include <sys/kdi.h>
73 #include <sys/schedctl.h>
74 #include <sys/waitq.h>
75 #include <sys/cpucaps.h>
76 #include <sys/kiconv.h>
77 #include <sys/ctype.h>
78 #include <sys/smt.h>
79 
80 struct kmem_cache *thread_cache;	/* cache of free threads */
81 struct kmem_cache *lwp_cache;		/* cache of free lwps */
82 struct kmem_cache *turnstile_cache;	/* cache of free turnstiles */
83 
84 /*
85  * allthreads is only for use by kmem_readers.  All kernel loops can use
86  * the current thread as a start/end point.
87  */
88 kthread_t *allthreads = &t0;	/* circular list of all threads */
89 
90 static kcondvar_t reaper_cv;		/* synchronization var */
91 kthread_t	*thread_deathrow;	/* circular list of reapable threads */
92 kthread_t	*lwp_deathrow;		/* circular list of reapable threads */
93 kmutex_t	reaplock;		/* protects lwp and thread deathrows */
94 int	thread_reapcnt = 0;		/* number of threads on deathrow */
95 int	lwp_reapcnt = 0;		/* number of lwps on deathrow */
96 int	reaplimit = 16;			/* delay reaping until reaplimit */
97 
98 thread_free_lock_t	*thread_free_lock;
99 					/* protects tick thread from reaper */
100 
101 extern int nthread;
102 
103 /* System Scheduling classes. */
104 id_t	syscid;				/* system scheduling class ID */
105 id_t	sysdccid = CLASS_UNUSED;	/* reset when SDC loads */
106 
107 void	*segkp_thread;			/* cookie for segkp pool */
108 
109 int lwp_cache_sz = 32;
110 int t_cache_sz = 8;
111 static kt_did_t next_t_id = 1;
112 
113 /* Default mode for thread binding to CPUs and processor sets */
114 int default_binding_mode = TB_ALLHARD;
115 
116 /*
117  * Min/Max stack sizes for stack size parameters
118  */
119 #define	MAX_STKSIZE	(32 * DEFAULTSTKSZ)
120 #define	MIN_STKSIZE	DEFAULTSTKSZ
121 
122 /*
123  * default_stksize overrides lwp_default_stksize if it is set.
124  */
125 int	default_stksize;
126 int	lwp_default_stksize;
127 
128 static zone_key_t zone_thread_key;
129 
130 unsigned int kmem_stackinfo;		/* stackinfo feature on-off */
131 kmem_stkinfo_t *kmem_stkinfo_log;	/* stackinfo circular log */
132 static kmutex_t kmem_stkinfo_lock;	/* protects kmem_stkinfo_log */
133 
134 /*
135  * forward declarations for internal thread specific data (tsd)
136  */
137 static void *tsd_realloc(void *, size_t, size_t);
138 
139 void thread_reaper(void);
140 
141 /* forward declarations for stackinfo feature */
142 static void stkinfo_begin(kthread_t *);
143 static void stkinfo_end(kthread_t *);
144 static size_t stkinfo_percent(caddr_t, caddr_t, caddr_t);
145 
146 /*ARGSUSED*/
147 static int
turnstile_constructor(void * buf,void * cdrarg,int kmflags)148 turnstile_constructor(void *buf, void *cdrarg, int kmflags)
149 {
150 	bzero(buf, sizeof (turnstile_t));
151 	return (0);
152 }
153 
154 /*ARGSUSED*/
155 static void
turnstile_destructor(void * buf,void * cdrarg)156 turnstile_destructor(void *buf, void *cdrarg)
157 {
158 	turnstile_t *ts = buf;
159 
160 	ASSERT(ts->ts_free == NULL);
161 	ASSERT(ts->ts_waiters == 0);
162 	ASSERT(ts->ts_inheritor == NULL);
163 	ASSERT(ts->ts_sleepq[0].sq_first == NULL);
164 	ASSERT(ts->ts_sleepq[1].sq_first == NULL);
165 }
166 
167 void
thread_init(void)168 thread_init(void)
169 {
170 	kthread_t *tp;
171 	extern char sys_name[];
172 	extern void idle();
173 	struct cpu *cpu = CPU;
174 	int i;
175 	kmutex_t *lp;
176 
177 	mutex_init(&reaplock, NULL, MUTEX_SPIN, (void *)ipltospl(DISP_LEVEL));
178 	thread_free_lock =
179 	    kmem_alloc(sizeof (thread_free_lock_t) * THREAD_FREE_NUM, KM_SLEEP);
180 	for (i = 0; i < THREAD_FREE_NUM; i++) {
181 		lp = &thread_free_lock[i].tf_lock;
182 		mutex_init(lp, NULL, MUTEX_DEFAULT, NULL);
183 	}
184 
185 #if defined(__x86)
186 	thread_cache = kmem_cache_create("thread_cache", sizeof (kthread_t),
187 	    PTR24_ALIGN, NULL, NULL, NULL, NULL, NULL, 0);
188 
189 	/*
190 	 * "struct _klwp" includes a "struct pcb", which includes a
191 	 * "struct fpu", which needs to be 64-byte aligned on amd64
192 	 * (and even on i386) for xsave/xrstor.
193 	 */
194 	lwp_cache = kmem_cache_create("lwp_cache", sizeof (klwp_t),
195 	    64, NULL, NULL, NULL, NULL, NULL, 0);
196 #else
197 	/*
198 	 * Allocate thread structures from static_arena.  This prevents
199 	 * issues where a thread tries to relocate its own thread
200 	 * structure and touches it after the mapping has been suspended.
201 	 */
202 	thread_cache = kmem_cache_create("thread_cache", sizeof (kthread_t),
203 	    PTR24_ALIGN, NULL, NULL, NULL, NULL, static_arena, 0);
204 
205 	lwp_stk_cache_init();
206 
207 	lwp_cache = kmem_cache_create("lwp_cache", sizeof (klwp_t),
208 	    0, NULL, NULL, NULL, NULL, NULL, 0);
209 #endif
210 
211 	turnstile_cache = kmem_cache_create("turnstile_cache",
212 	    sizeof (turnstile_t), 0,
213 	    turnstile_constructor, turnstile_destructor, NULL, NULL, NULL, 0);
214 
215 	label_init();
216 	cred_init();
217 
218 	/*
219 	 * Initialize various resource management facilities.
220 	 */
221 	rctl_init();
222 	cpucaps_init();
223 	/*
224 	 * Zone_init() should be called before project_init() so that project ID
225 	 * for the first project is initialized correctly.
226 	 */
227 	zone_init();
228 	project_init();
229 	brand_init();
230 	kiconv_init();
231 	task_init();
232 	tcache_init();
233 	pool_init();
234 
235 	curthread->t_ts = kmem_cache_alloc(turnstile_cache, KM_SLEEP);
236 
237 	/*
238 	 * Originally, we had two parameters to set default stack
239 	 * size: one for lwp's (lwp_default_stksize), and one for
240 	 * kernel-only threads (DEFAULTSTKSZ, a.k.a. _defaultstksz).
241 	 * Now we have a third parameter that overrides both if it is
242 	 * set to a legal stack size, called default_stksize.
243 	 */
244 
245 	if (default_stksize == 0) {
246 		default_stksize = DEFAULTSTKSZ;
247 	} else if (default_stksize % PAGESIZE != 0 ||
248 	    default_stksize > MAX_STKSIZE ||
249 	    default_stksize < MIN_STKSIZE) {
250 		cmn_err(CE_WARN, "Illegal stack size. Using %d",
251 		    (int)DEFAULTSTKSZ);
252 		default_stksize = DEFAULTSTKSZ;
253 	} else {
254 		lwp_default_stksize = default_stksize;
255 	}
256 
257 	if (lwp_default_stksize == 0) {
258 		lwp_default_stksize = default_stksize;
259 	} else if (lwp_default_stksize % PAGESIZE != 0 ||
260 	    lwp_default_stksize > MAX_STKSIZE ||
261 	    lwp_default_stksize < MIN_STKSIZE) {
262 		cmn_err(CE_WARN, "Illegal stack size. Using %d",
263 		    default_stksize);
264 		lwp_default_stksize = default_stksize;
265 	}
266 
267 	segkp_lwp = segkp_cache_init(segkp, lwp_cache_sz,
268 	    lwp_default_stksize,
269 	    (KPD_NOWAIT | KPD_HASREDZONE | KPD_LOCKED));
270 
271 	segkp_thread = segkp_cache_init(segkp, t_cache_sz,
272 	    default_stksize, KPD_HASREDZONE | KPD_LOCKED | KPD_NO_ANON);
273 
274 	(void) getcid(sys_name, &syscid);
275 	curthread->t_cid = syscid;	/* current thread is t0 */
276 
277 	/*
278 	 * Set up the first CPU's idle thread.
279 	 * It runs whenever the CPU has nothing worthwhile to do.
280 	 */
281 	tp = thread_create(NULL, 0, idle, NULL, 0, &p0, TS_STOPPED, -1);
282 	cpu->cpu_idle_thread = tp;
283 	tp->t_preempt = 1;
284 	tp->t_disp_queue = cpu->cpu_disp;
285 	ASSERT(tp->t_disp_queue != NULL);
286 	tp->t_bound_cpu = cpu;
287 	tp->t_affinitycnt = 1;
288 
289 	/*
290 	 * Registering a thread in the callback table is usually
291 	 * done in the initialization code of the thread. In this
292 	 * case, we do it right after thread creation to avoid
293 	 * blocking idle thread while registering itself. It also
294 	 * avoids the possibility of reregistration in case a CPU
295 	 * restarts its idle thread.
296 	 */
297 	CALLB_CPR_INIT_SAFE(tp, "idle");
298 
299 	/*
300 	 * Create the thread_reaper daemon. From this point on, exited
301 	 * threads will get reaped.
302 	 */
303 	(void) thread_create(NULL, 0, (void (*)())thread_reaper,
304 	    NULL, 0, &p0, TS_RUN, minclsyspri);
305 
306 	/*
307 	 * Finish initializing the kernel memory allocator now that
308 	 * thread_create() is available.
309 	 */
310 	kmem_thread_init();
311 
312 	if (boothowto & RB_DEBUG)
313 		kdi_dvec_thravail();
314 }
315 
316 /*
317  * Create a thread.
318  *
319  * thread_create() blocks for memory if necessary.  It never fails.
320  *
321  * If stk is NULL, the thread is created at the base of the stack
322  * and cannot be swapped.
323  */
324 kthread_t *
thread_create(caddr_t stk,size_t stksize,void (* proc)(),void * arg,size_t len,proc_t * pp,int state,pri_t pri)325 thread_create(
326 	caddr_t	stk,
327 	size_t	stksize,
328 	void	(*proc)(),
329 	void	*arg,
330 	size_t	len,
331 	proc_t	 *pp,
332 	int	state,
333 	pri_t	pri)
334 {
335 	kthread_t *t;
336 	extern struct classfuncs sys_classfuncs;
337 	turnstile_t *ts;
338 
339 	/*
340 	 * Every thread keeps a turnstile around in case it needs to block.
341 	 * The only reason the turnstile is not simply part of the thread
342 	 * structure is that we may have to break the association whenever
343 	 * more than one thread blocks on a given synchronization object.
344 	 * From a memory-management standpoint, turnstiles are like the
345 	 * "attached mblks" that hang off dblks in the streams allocator.
346 	 */
347 	ts = kmem_cache_alloc(turnstile_cache, KM_SLEEP);
348 
349 	if (stk == NULL) {
350 		/*
351 		 * alloc both thread and stack in segkp chunk
352 		 */
353 
354 		if (stksize < default_stksize)
355 			stksize = default_stksize;
356 
357 		if (stksize == default_stksize) {
358 			stk = (caddr_t)segkp_cache_get(segkp_thread);
359 		} else {
360 			stksize = roundup(stksize, PAGESIZE);
361 			stk = (caddr_t)segkp_get(segkp, stksize,
362 			    (KPD_HASREDZONE | KPD_NO_ANON | KPD_LOCKED));
363 		}
364 
365 		ASSERT(stk != NULL);
366 
367 		/*
368 		 * The machine-dependent mutex code may require that
369 		 * thread pointers (since they may be used for mutex owner
370 		 * fields) have certain alignment requirements.
371 		 * PTR24_ALIGN is the size of the alignment quanta.
372 		 * XXX - assumes stack grows toward low addresses.
373 		 */
374 		if (stksize <= sizeof (kthread_t) + PTR24_ALIGN)
375 			cmn_err(CE_PANIC, "thread_create: proposed stack size"
376 			    " too small to hold thread.");
377 #ifdef STACK_GROWTH_DOWN
378 		stksize -= SA(sizeof (kthread_t) + PTR24_ALIGN - 1);
379 		stksize &= -PTR24_ALIGN;	/* make thread aligned */
380 		t = (kthread_t *)(stk + stksize);
381 		bzero(t, sizeof (kthread_t));
382 		if (audit_active)
383 			audit_thread_create(t);
384 		t->t_stk = stk + stksize;
385 		t->t_stkbase = stk;
386 #else	/* stack grows to larger addresses */
387 		stksize -= SA(sizeof (kthread_t));
388 		t = (kthread_t *)(stk);
389 		bzero(t, sizeof (kthread_t));
390 		t->t_stk = stk + sizeof (kthread_t);
391 		t->t_stkbase = stk + stksize + sizeof (kthread_t);
392 #endif	/* STACK_GROWTH_DOWN */
393 		t->t_flag |= T_TALLOCSTK;
394 		t->t_swap = stk;
395 	} else {
396 		t = kmem_cache_alloc(thread_cache, KM_SLEEP);
397 		bzero(t, sizeof (kthread_t));
398 		ASSERT(((uintptr_t)t & (PTR24_ALIGN - 1)) == 0);
399 		if (audit_active)
400 			audit_thread_create(t);
401 		/*
402 		 * Initialize t_stk to the kernel stack pointer to use
403 		 * upon entry to the kernel
404 		 */
405 #ifdef STACK_GROWTH_DOWN
406 		t->t_stk = stk + stksize;
407 		t->t_stkbase = stk;
408 #else
409 		t->t_stk = stk;			/* 3b2-like */
410 		t->t_stkbase = stk + stksize;
411 #endif /* STACK_GROWTH_DOWN */
412 	}
413 
414 	if (kmem_stackinfo != 0) {
415 		stkinfo_begin(t);
416 	}
417 
418 	t->t_ts = ts;
419 
420 	/*
421 	 * p_cred could be NULL if it thread_create is called before cred_init
422 	 * is called in main.
423 	 */
424 	mutex_enter(&pp->p_crlock);
425 	if (pp->p_cred)
426 		crhold(t->t_cred = pp->p_cred);
427 	mutex_exit(&pp->p_crlock);
428 	t->t_start = gethrestime_sec();
429 	t->t_startpc = proc;
430 	t->t_procp = pp;
431 	t->t_clfuncs = &sys_classfuncs.thread;
432 	t->t_cid = syscid;
433 	t->t_pri = pri;
434 	t->t_stime = ddi_get_lbolt();
435 	t->t_schedflag = TS_LOAD | TS_DONT_SWAP;
436 	t->t_bind_cpu = PBIND_NONE;
437 	t->t_bindflag = (uchar_t)default_binding_mode;
438 	t->t_bind_pset = PS_NONE;
439 	t->t_plockp = &pp->p_lock;
440 	t->t_copyops = NULL;
441 	t->t_taskq = NULL;
442 	t->t_anttime = 0;
443 	t->t_hatdepth = 0;
444 
445 	t->t_dtrace_vtime = 1;	/* assure vtimestamp is always non-zero */
446 
447 	CPU_STATS_ADDQ(CPU, sys, nthreads, 1);
448 	LOCK_INIT_CLEAR(&t->t_lock);
449 
450 	/*
451 	 * Callers who give us a NULL proc must do their own
452 	 * stack initialization.  e.g. lwp_create()
453 	 */
454 	if (proc != NULL) {
455 		t->t_stk = thread_stk_init(t->t_stk);
456 		thread_load(t, proc, arg, len);
457 	}
458 
459 	/*
460 	 * Put a hold on project0. If this thread is actually in a
461 	 * different project, then t_proj will be changed later in
462 	 * lwp_create().  All kernel-only threads must be in project 0.
463 	 */
464 	t->t_proj = project_hold(proj0p);
465 
466 	lgrp_affinity_init(&t->t_lgrp_affinity);
467 
468 	mutex_enter(&pidlock);
469 	nthread++;
470 	t->t_did = next_t_id++;
471 	t->t_prev = curthread->t_prev;
472 	t->t_next = curthread;
473 
474 	/*
475 	 * Add the thread to the list of all threads, and initialize
476 	 * its t_cpu pointer.  We need to block preemption since
477 	 * cpu_offline walks the thread list looking for threads
478 	 * with t_cpu pointing to the CPU being offlined.  We want
479 	 * to make sure that the list is consistent and that if t_cpu
480 	 * is set, the thread is on the list.
481 	 */
482 	kpreempt_disable();
483 	curthread->t_prev->t_next = t;
484 	curthread->t_prev = t;
485 
486 	/*
487 	 * We'll always create in the default partition since that's where
488 	 * kernel threads go (we'll change this later if needed, in
489 	 * lwp_create()).
490 	 */
491 	t->t_cpupart = &cp_default;
492 
493 	/*
494 	 * For now, affiliate this thread with the root lgroup.
495 	 * Since the kernel does not (presently) allocate its memory
496 	 * in a locality aware fashion, the root is an appropriate home.
497 	 * If this thread is later associated with an lwp, it will have
498 	 * its lgroup re-assigned at that time.
499 	 */
500 	lgrp_move_thread(t, &cp_default.cp_lgrploads[LGRP_ROOTID], 1);
501 
502 	/*
503 	 * If the current CPU is in the default cpupart, use it.  Otherwise,
504 	 * pick one that is; before entering the dispatcher code, we'll
505 	 * make sure to keep the invariant that ->t_cpu is set.  (In fact, we
506 	 * rely on this, in ht_should_run(), in the call tree of
507 	 * disp_lowpri_cpu().)
508 	 */
509 	if (CPU->cpu_part == &cp_default) {
510 		t->t_cpu = CPU;
511 	} else {
512 		t->t_cpu = cp_default.cp_cpulist;
513 		t->t_cpu = disp_lowpri_cpu(t->t_cpu, t, t->t_pri);
514 	}
515 
516 	t->t_disp_queue = t->t_cpu->cpu_disp;
517 	kpreempt_enable();
518 
519 	/*
520 	 * Initialize thread state and the dispatcher lock pointer.
521 	 * Need to hold onto pidlock to block allthreads walkers until
522 	 * the state is set.
523 	 */
524 	switch (state) {
525 	case TS_RUN:
526 		curthread->t_oldspl = splhigh();	/* get dispatcher spl */
527 		THREAD_SET_STATE(t, TS_STOPPED, &transition_lock);
528 		CL_SETRUN(t);
529 		thread_unlock(t);
530 		break;
531 
532 	case TS_ONPROC:
533 		THREAD_ONPROC(t, t->t_cpu);
534 		break;
535 
536 	case TS_FREE:
537 		/*
538 		 * Free state will be used for intr threads.
539 		 * The interrupt routine must set the thread dispatcher
540 		 * lock pointer (t_lockp) if starting on a CPU
541 		 * other than the current one.
542 		 */
543 		THREAD_FREEINTR(t, CPU);
544 		break;
545 
546 	case TS_STOPPED:
547 		THREAD_SET_STATE(t, TS_STOPPED, &stop_lock);
548 		break;
549 
550 	default:			/* TS_SLEEP, TS_ZOMB or TS_TRANS */
551 		cmn_err(CE_PANIC, "thread_create: invalid state %d", state);
552 	}
553 	mutex_exit(&pidlock);
554 	return (t);
555 }
556 
557 /*
558  * Move thread to project0 and take care of project reference counters.
559  */
560 void
thread_rele(kthread_t * t)561 thread_rele(kthread_t *t)
562 {
563 	kproject_t *kpj;
564 
565 	thread_lock(t);
566 
567 	ASSERT(t == curthread || t->t_state == TS_FREE || t->t_procp == &p0);
568 	kpj = ttoproj(t);
569 	t->t_proj = proj0p;
570 
571 	thread_unlock(t);
572 
573 	if (kpj != proj0p) {
574 		project_rele(kpj);
575 		(void) project_hold(proj0p);
576 	}
577 }
578 
579 void
thread_exit(void)580 thread_exit(void)
581 {
582 	kthread_t *t = curthread;
583 
584 	if ((t->t_proc_flag & TP_ZTHREAD) != 0)
585 		cmn_err(CE_PANIC, "thread_exit: zthread_exit() not called");
586 
587 	tsd_exit();		/* Clean up this thread's TSD */
588 
589 	kcpc_passivate();	/* clean up performance counter state */
590 
591 	/*
592 	 * No kernel thread should have called poll() without arranging
593 	 * calling pollcleanup() here.
594 	 */
595 	ASSERT(t->t_pollstate == NULL);
596 	ASSERT(t->t_schedctl == NULL);
597 	if (t->t_door)
598 		door_slam();	/* in case thread did an upcall */
599 
600 	thread_rele(t);
601 	t->t_preempt++;
602 
603 	/*
604 	 * remove thread from the all threads list so that
605 	 * death-row can use the same pointers.
606 	 */
607 	mutex_enter(&pidlock);
608 	t->t_next->t_prev = t->t_prev;
609 	t->t_prev->t_next = t->t_next;
610 	ASSERT(allthreads != t);	/* t0 never exits */
611 	cv_broadcast(&t->t_joincv);	/* wake up anyone in thread_join */
612 	mutex_exit(&pidlock);
613 
614 	if (t->t_ctx != NULL)
615 		exitctx(t);
616 	if (t->t_procp->p_pctx != NULL)
617 		exitpctx(t->t_procp);
618 
619 	if (kmem_stackinfo != 0) {
620 		stkinfo_end(t);
621 	}
622 
623 	t->t_state = TS_ZOMB;	/* set zombie thread */
624 
625 	swtch_from_zombie();	/* give up the CPU */
626 	/* NOTREACHED */
627 }
628 
629 /*
630  * Check to see if the specified thread is active (defined as being on
631  * the thread list).  This is certainly a slow way to do this; if there's
632  * ever a reason to speed it up, we could maintain a hash table of active
633  * threads indexed by their t_did.
634  */
635 static kthread_t *
did_to_thread(kt_did_t tid)636 did_to_thread(kt_did_t tid)
637 {
638 	kthread_t *t;
639 
640 	ASSERT(MUTEX_HELD(&pidlock));
641 	for (t = curthread->t_next; t != curthread; t = t->t_next) {
642 		if (t->t_did == tid)
643 			break;
644 	}
645 	if (t->t_did == tid)
646 		return (t);
647 	else
648 		return (NULL);
649 }
650 
651 /*
652  * Wait for specified thread to exit.  Returns immediately if the thread
653  * could not be found, meaning that it has either already exited or never
654  * existed.
655  */
656 void
thread_join(kt_did_t tid)657 thread_join(kt_did_t tid)
658 {
659 	kthread_t *t;
660 
661 	ASSERT(tid != curthread->t_did);
662 	ASSERT(tid != t0.t_did);
663 
664 	mutex_enter(&pidlock);
665 	/*
666 	 * Make sure we check that the thread is on the thread list
667 	 * before blocking on it; otherwise we could end up blocking on
668 	 * a cv that's already been freed.  In other words, don't cache
669 	 * the thread pointer across calls to cv_wait.
670 	 *
671 	 * The choice of loop invariant means that whenever a thread
672 	 * is taken off the allthreads list, a cv_broadcast must be
673 	 * performed on that thread's t_joincv to wake up any waiters.
674 	 * The broadcast doesn't have to happen right away, but it
675 	 * shouldn't be postponed indefinitely (e.g., by doing it in
676 	 * thread_free which may only be executed when the deathrow
677 	 * queue is processed.
678 	 */
679 	while (t = did_to_thread(tid))
680 		cv_wait(&t->t_joincv, &pidlock);
681 	mutex_exit(&pidlock);
682 }
683 
684 void
thread_free_prevent(kthread_t * t)685 thread_free_prevent(kthread_t *t)
686 {
687 	kmutex_t *lp;
688 
689 	lp = &thread_free_lock[THREAD_FREE_HASH(t)].tf_lock;
690 	mutex_enter(lp);
691 }
692 
693 void
thread_free_allow(kthread_t * t)694 thread_free_allow(kthread_t *t)
695 {
696 	kmutex_t *lp;
697 
698 	lp = &thread_free_lock[THREAD_FREE_HASH(t)].tf_lock;
699 	mutex_exit(lp);
700 }
701 
702 static void
thread_free_barrier(kthread_t * t)703 thread_free_barrier(kthread_t *t)
704 {
705 	kmutex_t *lp;
706 
707 	lp = &thread_free_lock[THREAD_FREE_HASH(t)].tf_lock;
708 	mutex_enter(lp);
709 	mutex_exit(lp);
710 }
711 
712 void
thread_free(kthread_t * t)713 thread_free(kthread_t *t)
714 {
715 	boolean_t allocstk = (t->t_flag & T_TALLOCSTK);
716 	klwp_t *lwp = t->t_lwp;
717 	caddr_t swap = t->t_swap;
718 
719 	ASSERT(t != &t0 && t->t_state == TS_FREE);
720 	ASSERT(t->t_door == NULL);
721 	ASSERT(t->t_schedctl == NULL);
722 	ASSERT(t->t_pollstate == NULL);
723 
724 	t->t_pri = 0;
725 	t->t_pc = 0;
726 	t->t_sp = 0;
727 	t->t_wchan0 = NULL;
728 	t->t_wchan = NULL;
729 	if (t->t_cred != NULL) {
730 		crfree(t->t_cred);
731 		t->t_cred = 0;
732 	}
733 	if (t->t_pdmsg) {
734 		kmem_free(t->t_pdmsg, strlen(t->t_pdmsg) + 1);
735 		t->t_pdmsg = NULL;
736 	}
737 	if (audit_active)
738 		audit_thread_free(t);
739 	if (t->t_cldata) {
740 		CL_EXITCLASS(t->t_cid, (caddr_t *)t->t_cldata);
741 	}
742 	if (t->t_rprof != NULL) {
743 		kmem_free(t->t_rprof, sizeof (*t->t_rprof));
744 		t->t_rprof = NULL;
745 	}
746 	t->t_lockp = NULL;	/* nothing should try to lock this thread now */
747 	if (lwp)
748 		lwp_freeregs(lwp, 0);
749 	if (t->t_ctx)
750 		freectx(t, 0);
751 	t->t_stk = NULL;
752 	if (lwp)
753 		lwp_stk_fini(lwp);
754 	lock_clear(&t->t_lock);
755 
756 	if (t->t_ts->ts_waiters > 0)
757 		panic("thread_free: turnstile still active");
758 
759 	kmem_cache_free(turnstile_cache, t->t_ts);
760 
761 	free_afd(&t->t_activefd);
762 
763 	/*
764 	 * Barrier for the tick accounting code.  The tick accounting code
765 	 * holds this lock to keep the thread from going away while it's
766 	 * looking at it.
767 	 */
768 	thread_free_barrier(t);
769 
770 	ASSERT(ttoproj(t) == proj0p);
771 	project_rele(ttoproj(t));
772 
773 	lgrp_affinity_free(&t->t_lgrp_affinity);
774 
775 	mutex_enter(&pidlock);
776 	nthread--;
777 	mutex_exit(&pidlock);
778 
779 	if (t->t_name != NULL) {
780 		kmem_free(t->t_name, THREAD_NAME_MAX);
781 		t->t_name = NULL;
782 	}
783 
784 	/*
785 	 * Free thread, lwp and stack.  This needs to be done carefully, since
786 	 * if T_TALLOCSTK is set, the thread is part of the stack.
787 	 */
788 	t->t_lwp = NULL;
789 	t->t_swap = NULL;
790 
791 	if (swap) {
792 		segkp_release(segkp, swap);
793 	}
794 	if (lwp) {
795 		kmem_cache_free(lwp_cache, lwp);
796 	}
797 	if (!allocstk) {
798 		kmem_cache_free(thread_cache, t);
799 	}
800 }
801 
802 /*
803  * Removes threads associated with the given zone from a deathrow queue.
804  * tp is a pointer to the head of the deathrow queue, and countp is a
805  * pointer to the current deathrow count.  Returns a linked list of
806  * threads removed from the list.
807  */
808 static kthread_t *
thread_zone_cleanup(kthread_t ** tp,int * countp,zoneid_t zoneid)809 thread_zone_cleanup(kthread_t **tp, int *countp, zoneid_t zoneid)
810 {
811 	kthread_t *tmp, *list = NULL;
812 	cred_t *cr;
813 
814 	ASSERT(MUTEX_HELD(&reaplock));
815 	while (*tp != NULL) {
816 		if ((cr = (*tp)->t_cred) != NULL && crgetzoneid(cr) == zoneid) {
817 			tmp = *tp;
818 			*tp = tmp->t_forw;
819 			tmp->t_forw = list;
820 			list = tmp;
821 			(*countp)--;
822 		} else {
823 			tp = &(*tp)->t_forw;
824 		}
825 	}
826 	return (list);
827 }
828 
829 static void
thread_reap_list(kthread_t * t)830 thread_reap_list(kthread_t *t)
831 {
832 	kthread_t *next;
833 
834 	while (t != NULL) {
835 		next = t->t_forw;
836 		thread_free(t);
837 		t = next;
838 	}
839 }
840 
841 /* ARGSUSED */
842 static void
thread_zone_destroy(zoneid_t zoneid,void * unused)843 thread_zone_destroy(zoneid_t zoneid, void *unused)
844 {
845 	kthread_t *t, *l;
846 
847 	mutex_enter(&reaplock);
848 	/*
849 	 * Pull threads and lwps associated with zone off deathrow lists.
850 	 */
851 	t = thread_zone_cleanup(&thread_deathrow, &thread_reapcnt, zoneid);
852 	l = thread_zone_cleanup(&lwp_deathrow, &lwp_reapcnt, zoneid);
853 	mutex_exit(&reaplock);
854 
855 	/*
856 	 * Guard against race condition in mutex_owner_running:
857 	 *	thread=owner(mutex)
858 	 *	<interrupt>
859 	 *				thread exits mutex
860 	 *				thread exits
861 	 *				thread reaped
862 	 *				thread struct freed
863 	 * cpu = thread->t_cpu <- BAD POINTER DEREFERENCE.
864 	 * A cross call to all cpus will cause the interrupt handler
865 	 * to reset the PC if it is in mutex_owner_running, refreshing
866 	 * stale thread pointers.
867 	 */
868 	mutex_sync();   /* sync with mutex code */
869 
870 	/*
871 	 * Reap threads
872 	 */
873 	thread_reap_list(t);
874 
875 	/*
876 	 * Reap lwps
877 	 */
878 	thread_reap_list(l);
879 }
880 
881 /*
882  * cleanup zombie threads that are on deathrow.
883  */
884 void
thread_reaper()885 thread_reaper()
886 {
887 	kthread_t *t, *l;
888 	callb_cpr_t cprinfo;
889 
890 	/*
891 	 * Register callback to clean up threads when zone is destroyed.
892 	 */
893 	zone_key_create(&zone_thread_key, NULL, NULL, thread_zone_destroy);
894 
895 	CALLB_CPR_INIT(&cprinfo, &reaplock, callb_generic_cpr, "t_reaper");
896 	for (;;) {
897 		mutex_enter(&reaplock);
898 		while (thread_deathrow == NULL && lwp_deathrow == NULL) {
899 			CALLB_CPR_SAFE_BEGIN(&cprinfo);
900 			cv_wait(&reaper_cv, &reaplock);
901 			CALLB_CPR_SAFE_END(&cprinfo, &reaplock);
902 		}
903 		/*
904 		 * mutex_sync() needs to be called when reaping, but
905 		 * not too often.  We limit reaping rate to once
906 		 * per second.  Reaplimit is max rate at which threads can
907 		 * be freed. Does not impact thread destruction/creation.
908 		 */
909 		t = thread_deathrow;
910 		l = lwp_deathrow;
911 		thread_deathrow = NULL;
912 		lwp_deathrow = NULL;
913 		thread_reapcnt = 0;
914 		lwp_reapcnt = 0;
915 		mutex_exit(&reaplock);
916 
917 		/*
918 		 * Guard against race condition in mutex_owner_running:
919 		 *	thread=owner(mutex)
920 		 *	<interrupt>
921 		 *				thread exits mutex
922 		 *				thread exits
923 		 *				thread reaped
924 		 *				thread struct freed
925 		 * cpu = thread->t_cpu <- BAD POINTER DEREFERENCE.
926 		 * A cross call to all cpus will cause the interrupt handler
927 		 * to reset the PC if it is in mutex_owner_running, refreshing
928 		 * stale thread pointers.
929 		 */
930 		mutex_sync();   /* sync with mutex code */
931 		/*
932 		 * Reap threads
933 		 */
934 		thread_reap_list(t);
935 
936 		/*
937 		 * Reap lwps
938 		 */
939 		thread_reap_list(l);
940 		delay(hz);
941 	}
942 }
943 
944 /*
945  * This is called by lwpcreate, etc.() to put a lwp_deathrow thread onto
946  * thread_deathrow. The thread's state is changed already TS_FREE to indicate
947  * that is reapable. The thread already holds the reaplock, and was already
948  * freed.
949  */
950 void
reapq_move_lq_to_tq(kthread_t * t)951 reapq_move_lq_to_tq(kthread_t *t)
952 {
953 	ASSERT(t->t_state == TS_FREE);
954 	ASSERT(MUTEX_HELD(&reaplock));
955 	t->t_forw = thread_deathrow;
956 	thread_deathrow = t;
957 	thread_reapcnt++;
958 	if (lwp_reapcnt + thread_reapcnt > reaplimit)
959 		cv_signal(&reaper_cv);  /* wake the reaper */
960 }
961 
962 /*
963  * This is called by resume() to put a zombie thread onto deathrow.
964  * The thread's state is changed to TS_FREE to indicate that is reapable.
965  * This is called from the idle thread so it must not block - just spin.
966  */
967 void
reapq_add(kthread_t * t)968 reapq_add(kthread_t *t)
969 {
970 	mutex_enter(&reaplock);
971 
972 	/*
973 	 * lwp_deathrow contains threads with lwp linkage and
974 	 * swappable thread stacks which have the default stacksize.
975 	 * These threads' lwps and stacks may be reused by lwp_create().
976 	 *
977 	 * Anything else goes on thread_deathrow(), where it will eventually
978 	 * be thread_free()d.
979 	 */
980 	if (t->t_flag & T_LWPREUSE) {
981 		ASSERT(ttolwp(t) != NULL);
982 		t->t_forw = lwp_deathrow;
983 		lwp_deathrow = t;
984 		lwp_reapcnt++;
985 	} else {
986 		t->t_forw = thread_deathrow;
987 		thread_deathrow = t;
988 		thread_reapcnt++;
989 	}
990 	if (lwp_reapcnt + thread_reapcnt > reaplimit)
991 		cv_signal(&reaper_cv);	/* wake the reaper */
992 	t->t_state = TS_FREE;
993 	lock_clear(&t->t_lock);
994 
995 	/*
996 	 * Before we return, we need to grab and drop the thread lock for
997 	 * the dead thread.  At this point, the current thread is the idle
998 	 * thread, and the dead thread's CPU lock points to the current
999 	 * CPU -- and we must grab and drop the lock to synchronize with
1000 	 * a racing thread walking a blocking chain that the zombie thread
1001 	 * was recently in.  By this point, that blocking chain is (by
1002 	 * definition) stale:  the dead thread is not holding any locks, and
1003 	 * is therefore not in any blocking chains -- but if we do not regrab
1004 	 * our lock before freeing the dead thread's data structures, the
1005 	 * thread walking the (stale) blocking chain will die on memory
1006 	 * corruption when it attempts to drop the dead thread's lock.  We
1007 	 * only need do this once because there is no way for the dead thread
1008 	 * to ever again be on a blocking chain:  once we have grabbed and
1009 	 * dropped the thread lock, we are guaranteed that anyone that could
1010 	 * have seen this thread in a blocking chain can no longer see it.
1011 	 */
1012 	thread_lock(t);
1013 	thread_unlock(t);
1014 
1015 	mutex_exit(&reaplock);
1016 }
1017 
1018 static struct ctxop *
ctxop_find_by_tmpl(kthread_t * t,const struct ctxop_template * ct,void * arg)1019 ctxop_find_by_tmpl(kthread_t *t, const struct ctxop_template *ct, void *arg)
1020 {
1021 	struct ctxop *ctx, *head;
1022 
1023 	ASSERT(MUTEX_HELD(&t->t_ctx_lock));
1024 	ASSERT(curthread->t_preempt > 0);
1025 
1026 	if (t->t_ctx == NULL) {
1027 		return (NULL);
1028 	}
1029 
1030 	ctx = head = t->t_ctx;
1031 	do {
1032 		if (ctx->save_op == ct->ct_save &&
1033 		    ctx->restore_op == ct->ct_restore &&
1034 		    ctx->fork_op == ct->ct_fork &&
1035 		    ctx->lwp_create_op == ct->ct_lwp_create &&
1036 		    ctx->exit_op == ct->ct_exit &&
1037 		    ctx->free_op == ct->ct_free &&
1038 		    ctx->arg == arg) {
1039 			return (ctx);
1040 		}
1041 
1042 		ctx = ctx->next;
1043 	} while (ctx != head);
1044 
1045 	return (NULL);
1046 }
1047 
1048 static void
ctxop_detach_chain(kthread_t * t,struct ctxop * ctx)1049 ctxop_detach_chain(kthread_t *t, struct ctxop *ctx)
1050 {
1051 	ASSERT(t != NULL);
1052 	ASSERT(t->t_ctx != NULL);
1053 	ASSERT(ctx != NULL);
1054 	ASSERT(ctx->next != NULL && ctx->prev != NULL);
1055 
1056 	ctx->prev->next = ctx->next;
1057 	ctx->next->prev = ctx->prev;
1058 	if (ctx->next == ctx) {
1059 		/* last remaining item */
1060 		t->t_ctx = NULL;
1061 	} else if (ctx == t->t_ctx) {
1062 		/* fix up head of list */
1063 		t->t_ctx = ctx->next;
1064 	}
1065 	ctx->next = ctx->prev = NULL;
1066 }
1067 
1068 struct ctxop *
ctxop_allocate(const struct ctxop_template * ct,void * arg)1069 ctxop_allocate(const struct ctxop_template *ct, void *arg)
1070 {
1071 	struct ctxop *ctx;
1072 
1073 	/*
1074 	 * No changes have been made to the interface yet, so we expect all
1075 	 * callers to use the original revision.
1076 	 */
1077 	VERIFY3U(ct->ct_rev, ==, CTXOP_TPL_REV);
1078 
1079 	ctx = kmem_alloc(sizeof (struct ctxop), KM_SLEEP);
1080 	ctx->save_op = ct->ct_save;
1081 	ctx->restore_op = ct->ct_restore;
1082 	ctx->fork_op = ct->ct_fork;
1083 	ctx->lwp_create_op = ct->ct_lwp_create;
1084 	ctx->exit_op = ct->ct_exit;
1085 	ctx->free_op = ct->ct_free;
1086 	ctx->arg = arg;
1087 	ctx->save_ts = 0;
1088 	ctx->restore_ts = 0;
1089 	ctx->next = ctx->prev = NULL;
1090 
1091 	return (ctx);
1092 }
1093 
1094 void
ctxop_free(struct ctxop * ctx)1095 ctxop_free(struct ctxop *ctx)
1096 {
1097 	if (ctx->free_op != NULL)
1098 		(ctx->free_op)(ctx->arg, 0);
1099 
1100 	kmem_free(ctx, sizeof (struct ctxop));
1101 }
1102 
1103 void
ctxop_attach(kthread_t * t,struct ctxop * ctx)1104 ctxop_attach(kthread_t *t, struct ctxop *ctx)
1105 {
1106 	ASSERT(ctx->next == NULL && ctx->prev == NULL);
1107 
1108 	/*
1109 	 * Keep ctxops in a doubly-linked list to allow traversal in both
1110 	 * directions.  Using only the newest-to-oldest ordering was adequate
1111 	 * previously, but reversing the order for restore_op actions is
1112 	 * necessary if later-added ctxops depends on earlier ones.
1113 	 *
1114 	 * One example of such a dependency:  Hypervisor software handling the
1115 	 * guest FPU expects that it save FPU state prior to host FPU handling
1116 	 * and consequently handle the guest logic _after_ the host FPU has
1117 	 * been restored.
1118 	 *
1119 	 * The t_ctx member points to the most recently added ctxop or is NULL
1120 	 * if no ctxops are associated with the thread.  The 'next' pointers
1121 	 * form a loop of the ctxops in newest-to-oldest order.  The 'prev'
1122 	 * pointers form a loop in the reverse direction, where t_ctx->prev is
1123 	 * the oldest entry associated with the thread.
1124 	 *
1125 	 * The protection of kpreempt_disable is required to safely perform the
1126 	 * list insertion, since there are inconsistent states between some of
1127 	 * the pointer assignments.
1128 	 */
1129 	kpreempt_disable();
1130 	if (t->t_ctx == NULL) {
1131 		ctx->next = ctx;
1132 		ctx->prev = ctx;
1133 	} else {
1134 		struct ctxop *head = t->t_ctx, *tail = t->t_ctx->prev;
1135 
1136 		ctx->next = head;
1137 		ctx->prev = tail;
1138 		head->prev = ctx;
1139 		tail->next = ctx;
1140 	}
1141 	t->t_ctx = ctx;
1142 	kpreempt_enable();
1143 }
1144 
1145 void
ctxop_detach(kthread_t * t,struct ctxop * ctx)1146 ctxop_detach(kthread_t *t, struct ctxop *ctx)
1147 {
1148 	/*
1149 	 * The incoming kthread_t (which is the thread for which the
1150 	 * context ops will be detached) should be one of the following:
1151 	 *
1152 	 * a) the current thread,
1153 	 *
1154 	 * b) a thread of a process that's being forked (SIDL),
1155 	 *
1156 	 * c) a thread that belongs to the same process as the current
1157 	 *    thread and for which the current thread is the agent thread,
1158 	 *
1159 	 * d) a thread that is TS_STOPPED which is indicative of it
1160 	 *    being (if curthread is not an agent) a thread being created
1161 	 *    as part of an lwp creation.
1162 	 */
1163 	ASSERT(t == curthread || ttoproc(t)->p_stat == SIDL ||
1164 	    ttoproc(t)->p_agenttp == curthread || t->t_state == TS_STOPPED);
1165 
1166 	/*
1167 	 * Serialize modifications to t->t_ctx to prevent the agent thread
1168 	 * and the target thread from racing with each other during lwp exit.
1169 	 */
1170 	mutex_enter(&t->t_ctx_lock);
1171 	kpreempt_disable();
1172 
1173 	VERIFY(t->t_ctx != NULL);
1174 
1175 #ifdef	DEBUG
1176 	/* Check that provided `ctx` is actually present in the t_ctx chain */
1177 	struct ctxop *head, *cur;
1178 	head = cur = t->t_ctx;
1179 	for (;;) {
1180 		if (cur == ctx) {
1181 			break;
1182 		}
1183 		cur = cur->next;
1184 		/* If we wrap, having not found `ctx`, this assert will fail */
1185 		ASSERT3P(cur, !=, head);
1186 	}
1187 #endif /* DEBUG */
1188 
1189 	ctxop_detach_chain(t, ctx);
1190 
1191 	mutex_exit(&t->t_ctx_lock);
1192 	kpreempt_enable();
1193 }
1194 
1195 void
ctxop_install(kthread_t * t,const struct ctxop_template * ct,void * arg)1196 ctxop_install(kthread_t *t, const struct ctxop_template *ct, void *arg)
1197 {
1198 	ctxop_attach(t, ctxop_allocate(ct, arg));
1199 }
1200 
1201 int
ctxop_remove(kthread_t * t,const struct ctxop_template * ct,void * arg)1202 ctxop_remove(kthread_t *t, const struct ctxop_template *ct, void *arg)
1203 {
1204 	struct ctxop *ctx;
1205 
1206 	/*
1207 	 * ctxop_remove() shares the same requirements for the acted-upon thread
1208 	 * as ctxop_detach()
1209 	 */
1210 	ASSERT(t == curthread || ttoproc(t)->p_stat == SIDL ||
1211 	    ttoproc(t)->p_agenttp == curthread || t->t_state == TS_STOPPED);
1212 
1213 	/*
1214 	 * Serialize modifications to t->t_ctx to prevent the agent thread
1215 	 * and the target thread from racing with each other during lwp exit.
1216 	 */
1217 	mutex_enter(&t->t_ctx_lock);
1218 	kpreempt_disable();
1219 
1220 	ctx = ctxop_find_by_tmpl(t, ct, arg);
1221 	if (ctx != NULL) {
1222 		ctxop_detach_chain(t, ctx);
1223 		ctxop_free(ctx);
1224 	}
1225 
1226 	mutex_exit(&t->t_ctx_lock);
1227 	kpreempt_enable();
1228 
1229 	if (ctx != NULL) {
1230 		return (1);
1231 	}
1232 	return (0);
1233 }
1234 
1235 void
savectx(kthread_t * t)1236 savectx(kthread_t *t)
1237 {
1238 	ASSERT(t == curthread);
1239 
1240 	if (t->t_ctx != NULL) {
1241 		struct ctxop *ctx, *head;
1242 
1243 		/* Forward traversal */
1244 		ctx = head = t->t_ctx;
1245 		do {
1246 			if (ctx->save_op != NULL) {
1247 				ctx->save_ts = gethrtime_unscaled();
1248 				(ctx->save_op)(ctx->arg);
1249 			}
1250 			ctx = ctx->next;
1251 		} while (ctx != head);
1252 	}
1253 }
1254 
1255 void
restorectx(kthread_t * t)1256 restorectx(kthread_t *t)
1257 {
1258 	ASSERT(t == curthread);
1259 
1260 	if (t->t_ctx != NULL) {
1261 		struct ctxop *ctx, *tail;
1262 
1263 		/* Backward traversal (starting at the tail) */
1264 		ctx = tail = t->t_ctx->prev;
1265 		do {
1266 			if (ctx->restore_op != NULL) {
1267 				ctx->restore_ts = gethrtime_unscaled();
1268 				(ctx->restore_op)(ctx->arg);
1269 			}
1270 			ctx = ctx->prev;
1271 		} while (ctx != tail);
1272 	}
1273 }
1274 
1275 void
forkctx(kthread_t * t,kthread_t * ct)1276 forkctx(kthread_t *t, kthread_t *ct)
1277 {
1278 	if (t->t_ctx != NULL) {
1279 		struct ctxop *ctx, *head;
1280 
1281 		/* Forward traversal */
1282 		ctx = head = t->t_ctx;
1283 		do {
1284 			if (ctx->fork_op != NULL) {
1285 				(ctx->fork_op)(t, ct);
1286 			}
1287 			ctx = ctx->next;
1288 		} while (ctx != head);
1289 	}
1290 }
1291 
1292 /*
1293  * Note that this operator is only invoked via the _lwp_create
1294  * system call.  The system may have other reasons to create lwps
1295  * e.g. the agent lwp or the doors unreferenced lwp.
1296  */
1297 void
lwp_createctx(kthread_t * t,kthread_t * ct)1298 lwp_createctx(kthread_t *t, kthread_t *ct)
1299 {
1300 	if (t->t_ctx != NULL) {
1301 		struct ctxop *ctx, *head;
1302 
1303 		/* Forward traversal */
1304 		ctx = head = t->t_ctx;
1305 		do {
1306 			if (ctx->lwp_create_op != NULL) {
1307 				(ctx->lwp_create_op)(t, ct);
1308 			}
1309 			ctx = ctx->next;
1310 		} while (ctx != head);
1311 	}
1312 }
1313 
1314 /*
1315  * exitctx is called from thread_exit() and lwp_exit() to perform any actions
1316  * needed when the thread/LWP leaves the processor for the last time. This
1317  * routine is not intended to deal with freeing memory; freectx() is used for
1318  * that purpose during thread_free(). This routine is provided to allow for
1319  * clean-up that can't wait until thread_free().
1320  */
1321 void
exitctx(kthread_t * t)1322 exitctx(kthread_t *t)
1323 {
1324 	if (t->t_ctx != NULL) {
1325 		struct ctxop *ctx, *head;
1326 
1327 		/* Forward traversal */
1328 		ctx = head = t->t_ctx;
1329 		do {
1330 			if (ctx->exit_op != NULL) {
1331 				(ctx->exit_op)(t);
1332 			}
1333 			ctx = ctx->next;
1334 		} while (ctx != head);
1335 	}
1336 }
1337 
1338 /*
1339  * freectx is called from thread_free() and exec() to get
1340  * rid of old thread context ops.
1341  */
1342 void
freectx(kthread_t * t,int isexec)1343 freectx(kthread_t *t, int isexec)
1344 {
1345 	kpreempt_disable();
1346 	if (t->t_ctx != NULL) {
1347 		struct ctxop *ctx, *head;
1348 
1349 		ctx = head = t->t_ctx;
1350 		t->t_ctx = NULL;
1351 		do {
1352 			struct ctxop *next = ctx->next;
1353 
1354 			if (ctx->free_op != NULL) {
1355 				(ctx->free_op)(ctx->arg, isexec);
1356 			}
1357 			kmem_free(ctx, sizeof (struct ctxop));
1358 			ctx = next;
1359 		} while (ctx != head);
1360 	}
1361 	kpreempt_enable();
1362 }
1363 
1364 /*
1365  * freectx_ctx is called from lwp_create() when lwp is reused from
1366  * lwp_deathrow and its thread structure is added to thread_deathrow.
1367  * The thread structure to which this ctx was attached may be already
1368  * freed by the thread reaper so free_op implementations shouldn't rely
1369  * on thread structure to which this ctx was attached still being around.
1370  */
1371 void
freectx_ctx(struct ctxop * ctx)1372 freectx_ctx(struct ctxop *ctx)
1373 {
1374 	struct ctxop *head = ctx;
1375 
1376 	ASSERT(ctx != NULL);
1377 
1378 	kpreempt_disable();
1379 
1380 	head = ctx;
1381 	do {
1382 		struct ctxop *next = ctx->next;
1383 
1384 		if (ctx->free_op != NULL) {
1385 			(ctx->free_op)(ctx->arg, 0);
1386 		}
1387 		kmem_free(ctx, sizeof (struct ctxop));
1388 		ctx = next;
1389 	} while (ctx != head);
1390 	kpreempt_enable();
1391 }
1392 
1393 /*
1394  * Set the thread running; arrange for it to be swapped in if necessary.
1395  */
1396 void
setrun_locked(kthread_t * t)1397 setrun_locked(kthread_t *t)
1398 {
1399 	ASSERT(THREAD_LOCK_HELD(t));
1400 	if (t->t_state == TS_SLEEP) {
1401 		/*
1402 		 * Take off sleep queue.
1403 		 */
1404 		SOBJ_UNSLEEP(t->t_sobj_ops, t);
1405 	} else if (t->t_state & (TS_RUN | TS_ONPROC)) {
1406 		/*
1407 		 * Already on dispatcher queue.
1408 		 */
1409 		return;
1410 	} else if (t->t_state == TS_WAIT) {
1411 		waitq_setrun(t);
1412 	} else if (t->t_state == TS_STOPPED) {
1413 		/*
1414 		 * All of the sending of SIGCONT (TC_XSTART) and /proc
1415 		 * (TC_PSTART) and lwp_continue() (TC_CSTART) must have
1416 		 * requested that the thread be run.
1417 		 * Just calling setrun() is not sufficient to set a stopped
1418 		 * thread running.  TP_TXSTART is always set if the thread
1419 		 * is not stopped by a jobcontrol stop signal.
1420 		 * TP_TPSTART is always set if /proc is not controlling it.
1421 		 * TP_TCSTART is always set if lwp_suspend() didn't stop it.
1422 		 * The thread won't be stopped unless one of these
1423 		 * three mechanisms did it.
1424 		 *
1425 		 * These flags must be set before calling setrun_locked(t).
1426 		 * They can't be passed as arguments because the streams
1427 		 * code calls setrun() indirectly and the mechanism for
1428 		 * doing so admits only one argument.  Note that the
1429 		 * thread must be locked in order to change t_schedflags.
1430 		 */
1431 		if ((t->t_schedflag & TS_ALLSTART) != TS_ALLSTART)
1432 			return;
1433 		/*
1434 		 * Process is no longer stopped (a thread is running).
1435 		 */
1436 		t->t_whystop = 0;
1437 		t->t_whatstop = 0;
1438 		/*
1439 		 * Strictly speaking, we do not have to clear these
1440 		 * flags here; they are cleared on entry to stop().
1441 		 * However, they are confusing when doing kernel
1442 		 * debugging or when they are revealed by ps(1).
1443 		 */
1444 		t->t_schedflag &= ~TS_ALLSTART;
1445 		THREAD_TRANSITION(t);	/* drop stopped-thread lock */
1446 		ASSERT(t->t_lockp == &transition_lock);
1447 		ASSERT(t->t_wchan0 == NULL && t->t_wchan == NULL);
1448 		/*
1449 		 * Let the class put the process on the dispatcher queue.
1450 		 */
1451 		CL_SETRUN(t);
1452 	}
1453 }
1454 
1455 void
setrun(kthread_t * t)1456 setrun(kthread_t *t)
1457 {
1458 	thread_lock(t);
1459 	setrun_locked(t);
1460 	thread_unlock(t);
1461 }
1462 
1463 /*
1464  * Unpin an interrupted thread.
1465  *	When an interrupt occurs, the interrupt is handled on the stack
1466  *	of an interrupt thread, taken from a pool linked to the CPU structure.
1467  *
1468  *	When swtch() is switching away from an interrupt thread because it
1469  *	blocked or was preempted, this routine is called to complete the
1470  *	saving of the interrupted thread state, and returns the interrupted
1471  *	thread pointer so it may be resumed.
1472  *
1473  *	Called by swtch() only at high spl.
1474  */
1475 kthread_t *
thread_unpin()1476 thread_unpin()
1477 {
1478 	kthread_t	*t = curthread;	/* current thread */
1479 	kthread_t	*itp;		/* interrupted thread */
1480 	int		i;		/* interrupt level */
1481 	extern int	intr_passivate();
1482 
1483 	ASSERT(t->t_intr != NULL);
1484 
1485 	itp = t->t_intr;		/* interrupted thread */
1486 	t->t_intr = NULL;		/* clear interrupt ptr */
1487 
1488 	smt_end_intr();
1489 
1490 	/*
1491 	 * Get state from interrupt thread for the one
1492 	 * it interrupted.
1493 	 */
1494 
1495 	i = intr_passivate(t, itp);
1496 
1497 	TRACE_5(TR_FAC_INTR, TR_INTR_PASSIVATE,
1498 	    "intr_passivate:level %d curthread %p (%T) ithread %p (%T)",
1499 	    i, t, t, itp, itp);
1500 
1501 	/*
1502 	 * Dissociate the current thread from the interrupted thread's LWP.
1503 	 */
1504 	t->t_lwp = NULL;
1505 
1506 	/*
1507 	 * Interrupt handlers above the level that spinlocks block must
1508 	 * not block.
1509 	 */
1510 #if DEBUG
1511 	if (i < 0 || i > LOCK_LEVEL)
1512 		cmn_err(CE_PANIC, "thread_unpin: ipl out of range %x", i);
1513 #endif
1514 
1515 	/*
1516 	 * Compute the CPU's base interrupt level based on the active
1517 	 * interrupts.
1518 	 */
1519 	ASSERT(CPU->cpu_intr_actv & (1 << i));
1520 	set_base_spl();
1521 
1522 	return (itp);
1523 }
1524 
1525 /*
1526  * Create and initialize an interrupt thread.
1527  *	Returns non-zero on error.
1528  *	Called at spl7() or better.
1529  */
1530 void
thread_create_intr(struct cpu * cp)1531 thread_create_intr(struct cpu *cp)
1532 {
1533 	kthread_t *tp;
1534 
1535 	tp = thread_create(NULL, 0,
1536 	    (void (*)())thread_create_intr, NULL, 0, &p0, TS_ONPROC, 0);
1537 
1538 	/*
1539 	 * Set the thread in the TS_FREE state.  The state will change
1540 	 * to TS_ONPROC only while the interrupt is active.  Think of these
1541 	 * as being on a private free list for the CPU.  Being TS_FREE keeps
1542 	 * inactive interrupt threads out of debugger thread lists.
1543 	 *
1544 	 * We cannot call thread_create with TS_FREE because of the current
1545 	 * checks there for ONPROC.  Fix this when thread_create takes flags.
1546 	 */
1547 	THREAD_FREEINTR(tp, cp);
1548 
1549 	/*
1550 	 * Nobody should ever reference the credentials of an interrupt
1551 	 * thread so make it NULL to catch any such references.
1552 	 */
1553 	tp->t_cred = NULL;
1554 	tp->t_flag |= T_INTR_THREAD;
1555 	tp->t_cpu = cp;
1556 	tp->t_bound_cpu = cp;
1557 	tp->t_disp_queue = cp->cpu_disp;
1558 	tp->t_affinitycnt = 1;
1559 	tp->t_preempt = 1;
1560 
1561 	/*
1562 	 * Don't make a user-requested binding on this thread so that
1563 	 * the processor can be offlined.
1564 	 */
1565 	tp->t_bind_cpu = PBIND_NONE;	/* no USER-requested binding */
1566 	tp->t_bind_pset = PS_NONE;
1567 
1568 #if defined(__x86)
1569 	tp->t_stk -= STACK_ALIGN;
1570 	*(tp->t_stk) = 0;		/* terminate intr thread stack */
1571 #endif
1572 
1573 	/*
1574 	 * Link onto CPU's interrupt pool.
1575 	 */
1576 	tp->t_link = cp->cpu_intr_thread;
1577 	cp->cpu_intr_thread = tp;
1578 }
1579 
1580 /*
1581  * TSD -- THREAD SPECIFIC DATA
1582  */
1583 static kmutex_t		tsd_mutex;	 /* linked list spin lock */
1584 static uint_t		tsd_nkeys;	 /* size of destructor array */
1585 /* per-key destructor funcs */
1586 static void		(**tsd_destructor)(void *);
1587 /* list of tsd_thread's */
1588 static struct tsd_thread	*tsd_list;
1589 
1590 /*
1591  * Default destructor
1592  *	Needed because NULL destructor means that the key is unused
1593  */
1594 /* ARGSUSED */
1595 void
tsd_defaultdestructor(void * value)1596 tsd_defaultdestructor(void *value)
1597 {}
1598 
1599 /*
1600  * Create a key (index into per thread array)
1601  *	Locks out tsd_create, tsd_destroy, and tsd_exit
1602  *	May allocate memory with lock held
1603  */
1604 void
tsd_create(uint_t * keyp,void (* destructor)(void *))1605 tsd_create(uint_t *keyp, void (*destructor)(void *))
1606 {
1607 	int	i;
1608 	uint_t	nkeys;
1609 
1610 	/*
1611 	 * if key is allocated, do nothing
1612 	 */
1613 	mutex_enter(&tsd_mutex);
1614 	if (*keyp) {
1615 		mutex_exit(&tsd_mutex);
1616 		return;
1617 	}
1618 	/*
1619 	 * find an unused key
1620 	 */
1621 	if (destructor == NULL)
1622 		destructor = tsd_defaultdestructor;
1623 
1624 	for (i = 0; i < tsd_nkeys; ++i)
1625 		if (tsd_destructor[i] == NULL)
1626 			break;
1627 
1628 	/*
1629 	 * if no unused keys, increase the size of the destructor array
1630 	 */
1631 	if (i == tsd_nkeys) {
1632 		if ((nkeys = (tsd_nkeys << 1)) == 0)
1633 			nkeys = 1;
1634 		tsd_destructor =
1635 		    (void (**)(void *))tsd_realloc((void *)tsd_destructor,
1636 		    (size_t)(tsd_nkeys * sizeof (void (*)(void *))),
1637 		    (size_t)(nkeys * sizeof (void (*)(void *))));
1638 		tsd_nkeys = nkeys;
1639 	}
1640 
1641 	/*
1642 	 * allocate the next available unused key
1643 	 */
1644 	tsd_destructor[i] = destructor;
1645 	*keyp = i + 1;
1646 	mutex_exit(&tsd_mutex);
1647 }
1648 
1649 /*
1650  * Destroy a key -- this is for unloadable modules
1651  *
1652  * Assumes that the caller is preventing tsd_set and tsd_get
1653  * Locks out tsd_create, tsd_destroy, and tsd_exit
1654  * May free memory with lock held
1655  */
1656 void
tsd_destroy(uint_t * keyp)1657 tsd_destroy(uint_t *keyp)
1658 {
1659 	uint_t key;
1660 	struct tsd_thread *tsd;
1661 
1662 	/*
1663 	 * protect the key namespace and our destructor lists
1664 	 */
1665 	mutex_enter(&tsd_mutex);
1666 	key = *keyp;
1667 	*keyp = 0;
1668 
1669 	ASSERT(key <= tsd_nkeys);
1670 
1671 	/*
1672 	 * if the key is valid
1673 	 */
1674 	if (key != 0) {
1675 		uint_t k = key - 1;
1676 		/*
1677 		 * for every thread with TSD, call key's destructor
1678 		 */
1679 		for (tsd = tsd_list; tsd; tsd = tsd->ts_next) {
1680 			/*
1681 			 * no TSD for key in this thread
1682 			 */
1683 			if (key > tsd->ts_nkeys)
1684 				continue;
1685 			/*
1686 			 * call destructor for key
1687 			 */
1688 			if (tsd->ts_value[k] && tsd_destructor[k])
1689 				(*tsd_destructor[k])(tsd->ts_value[k]);
1690 			/*
1691 			 * reset value for key
1692 			 */
1693 			tsd->ts_value[k] = NULL;
1694 		}
1695 		/*
1696 		 * actually free the key (NULL destructor == unused)
1697 		 */
1698 		tsd_destructor[k] = NULL;
1699 	}
1700 
1701 	mutex_exit(&tsd_mutex);
1702 }
1703 
1704 /*
1705  * Quickly return the per thread value that was stored with the specified key
1706  * Assumes the caller is protecting key from tsd_create and tsd_destroy
1707  */
1708 void *
tsd_get(uint_t key)1709 tsd_get(uint_t key)
1710 {
1711 	return (tsd_agent_get(curthread, key));
1712 }
1713 
1714 /*
1715  * Set a per thread value indexed with the specified key
1716  */
1717 int
tsd_set(uint_t key,void * value)1718 tsd_set(uint_t key, void *value)
1719 {
1720 	return (tsd_agent_set(curthread, key, value));
1721 }
1722 
1723 /*
1724  * Like tsd_get(), except that the agent lwp can get the tsd of
1725  * another thread in the same process (the agent thread only runs when the
1726  * process is completely stopped by /proc), or syslwp is creating a new lwp.
1727  */
1728 void *
tsd_agent_get(kthread_t * t,uint_t key)1729 tsd_agent_get(kthread_t *t, uint_t key)
1730 {
1731 	struct tsd_thread *tsd = t->t_tsd;
1732 
1733 	ASSERT(t == curthread ||
1734 	    ttoproc(t)->p_agenttp == curthread || t->t_state == TS_STOPPED);
1735 
1736 	if (key && tsd != NULL && key <= tsd->ts_nkeys)
1737 		return (tsd->ts_value[key - 1]);
1738 	return (NULL);
1739 }
1740 
1741 /*
1742  * Like tsd_set(), except that the agent lwp can set the tsd of
1743  * another thread in the same process, or syslwp can set the tsd
1744  * of a thread it's in the middle of creating.
1745  *
1746  * Assumes the caller is protecting key from tsd_create and tsd_destroy
1747  * May lock out tsd_destroy (and tsd_create), may allocate memory with
1748  * lock held
1749  */
1750 int
tsd_agent_set(kthread_t * t,uint_t key,void * value)1751 tsd_agent_set(kthread_t *t, uint_t key, void *value)
1752 {
1753 	struct tsd_thread *tsd = t->t_tsd;
1754 
1755 	ASSERT(t == curthread ||
1756 	    ttoproc(t)->p_agenttp == curthread || t->t_state == TS_STOPPED);
1757 
1758 	if (key == 0)
1759 		return (EINVAL);
1760 	if (tsd == NULL)
1761 		tsd = t->t_tsd = kmem_zalloc(sizeof (*tsd), KM_SLEEP);
1762 	if (key <= tsd->ts_nkeys) {
1763 		tsd->ts_value[key - 1] = value;
1764 		return (0);
1765 	}
1766 
1767 	ASSERT(key <= tsd_nkeys);
1768 
1769 	/*
1770 	 * lock out tsd_destroy()
1771 	 */
1772 	mutex_enter(&tsd_mutex);
1773 	if (tsd->ts_nkeys == 0) {
1774 		/*
1775 		 * Link onto list of threads with TSD
1776 		 */
1777 		if ((tsd->ts_next = tsd_list) != NULL)
1778 			tsd_list->ts_prev = tsd;
1779 		tsd_list = tsd;
1780 	}
1781 
1782 	/*
1783 	 * Allocate thread local storage and set the value for key
1784 	 */
1785 	tsd->ts_value = tsd_realloc(tsd->ts_value,
1786 	    tsd->ts_nkeys * sizeof (void *),
1787 	    key * sizeof (void *));
1788 	tsd->ts_nkeys = key;
1789 	tsd->ts_value[key - 1] = value;
1790 	mutex_exit(&tsd_mutex);
1791 
1792 	return (0);
1793 }
1794 
1795 
1796 /*
1797  * Return the per thread value that was stored with the specified key
1798  *	If necessary, create the key and the value
1799  *	Assumes the caller is protecting *keyp from tsd_destroy
1800  */
1801 void *
tsd_getcreate(uint_t * keyp,void (* destroy)(void *),void * (* allocate)(void))1802 tsd_getcreate(uint_t *keyp, void (*destroy)(void *), void *(*allocate)(void))
1803 {
1804 	void *value;
1805 	uint_t key = *keyp;
1806 	struct tsd_thread *tsd = curthread->t_tsd;
1807 
1808 	if (tsd == NULL)
1809 		tsd = curthread->t_tsd = kmem_zalloc(sizeof (*tsd), KM_SLEEP);
1810 	if (key && key <= tsd->ts_nkeys && (value = tsd->ts_value[key - 1]))
1811 		return (value);
1812 	if (key == 0)
1813 		tsd_create(keyp, destroy);
1814 	(void) tsd_set(*keyp, value = (*allocate)());
1815 
1816 	return (value);
1817 }
1818 
1819 /*
1820  * Called from thread_exit() to run the destructor function for each tsd
1821  *	Locks out tsd_create and tsd_destroy
1822  *	Assumes that the destructor *DOES NOT* use tsd
1823  */
1824 void
tsd_exit(void)1825 tsd_exit(void)
1826 {
1827 	int i;
1828 	struct tsd_thread *tsd = curthread->t_tsd;
1829 
1830 	if (tsd == NULL)
1831 		return;
1832 
1833 	if (tsd->ts_nkeys == 0) {
1834 		kmem_free(tsd, sizeof (*tsd));
1835 		curthread->t_tsd = NULL;
1836 		return;
1837 	}
1838 
1839 	/*
1840 	 * lock out tsd_create and tsd_destroy, call
1841 	 * the destructor, and mark the value as destroyed.
1842 	 */
1843 	mutex_enter(&tsd_mutex);
1844 
1845 	for (i = 0; i < tsd->ts_nkeys; i++) {
1846 		if (tsd->ts_value[i] && tsd_destructor[i])
1847 			(*tsd_destructor[i])(tsd->ts_value[i]);
1848 		tsd->ts_value[i] = NULL;
1849 	}
1850 
1851 	/*
1852 	 * remove from linked list of threads with TSD
1853 	 */
1854 	if (tsd->ts_next)
1855 		tsd->ts_next->ts_prev = tsd->ts_prev;
1856 	if (tsd->ts_prev)
1857 		tsd->ts_prev->ts_next = tsd->ts_next;
1858 	if (tsd_list == tsd)
1859 		tsd_list = tsd->ts_next;
1860 
1861 	mutex_exit(&tsd_mutex);
1862 
1863 	/*
1864 	 * free up the TSD
1865 	 */
1866 	kmem_free(tsd->ts_value, tsd->ts_nkeys * sizeof (void *));
1867 	kmem_free(tsd, sizeof (struct tsd_thread));
1868 	curthread->t_tsd = NULL;
1869 }
1870 
1871 /*
1872  * realloc
1873  */
1874 static void *
tsd_realloc(void * old,size_t osize,size_t nsize)1875 tsd_realloc(void *old, size_t osize, size_t nsize)
1876 {
1877 	void *new;
1878 
1879 	new = kmem_zalloc(nsize, KM_SLEEP);
1880 	if (old) {
1881 		bcopy(old, new, osize);
1882 		kmem_free(old, osize);
1883 	}
1884 	return (new);
1885 }
1886 
1887 /*
1888  * Return non-zero if an interrupt is being serviced.
1889  */
1890 int
servicing_interrupt()1891 servicing_interrupt()
1892 {
1893 	int onintr = 0;
1894 
1895 	/* Are we an interrupt thread */
1896 	if (curthread->t_flag & T_INTR_THREAD)
1897 		return (1);
1898 	/* Are we servicing a high level interrupt? */
1899 	if (CPU_ON_INTR(CPU)) {
1900 		kpreempt_disable();
1901 		onintr = CPU_ON_INTR(CPU);
1902 		kpreempt_enable();
1903 	}
1904 	return (onintr);
1905 }
1906 
1907 
1908 /*
1909  * Change the dispatch priority of a thread in the system.
1910  * Used when raising or lowering a thread's priority.
1911  * (E.g., priority inheritance)
1912  *
1913  * Since threads are queued according to their priority, we
1914  * we must check the thread's state to determine whether it
1915  * is on a queue somewhere. If it is, we've got to:
1916  *
1917  *	o Dequeue the thread.
1918  *	o Change its effective priority.
1919  *	o Enqueue the thread.
1920  *
1921  * Assumptions: The thread whose priority we wish to change
1922  * must be locked before we call thread_change_(e)pri().
1923  * The thread_change(e)pri() function doesn't drop the thread
1924  * lock--that must be done by its caller.
1925  */
1926 void
thread_change_epri(kthread_t * t,pri_t disp_pri)1927 thread_change_epri(kthread_t *t, pri_t disp_pri)
1928 {
1929 	uint_t	state;
1930 
1931 	ASSERT(THREAD_LOCK_HELD(t));
1932 
1933 	/*
1934 	 * If the inherited priority hasn't actually changed,
1935 	 * just return.
1936 	 */
1937 	if (t->t_epri == disp_pri)
1938 		return;
1939 
1940 	state = t->t_state;
1941 
1942 	/*
1943 	 * If it's not on a queue, change the priority with impunity.
1944 	 */
1945 	if ((state & (TS_SLEEP | TS_RUN | TS_WAIT)) == 0) {
1946 		t->t_epri = disp_pri;
1947 		if (state == TS_ONPROC) {
1948 			cpu_t *cp = t->t_disp_queue->disp_cpu;
1949 
1950 			if (t == cp->cpu_dispthread)
1951 				cp->cpu_dispatch_pri = DISP_PRIO(t);
1952 		}
1953 	} else if (state == TS_SLEEP) {
1954 		/*
1955 		 * Take the thread out of its sleep queue.
1956 		 * Change the inherited priority.
1957 		 * Re-enqueue the thread.
1958 		 * Each synchronization object exports a function
1959 		 * to do this in an appropriate manner.
1960 		 */
1961 		SOBJ_CHANGE_EPRI(t->t_sobj_ops, t, disp_pri);
1962 	} else if (state == TS_WAIT) {
1963 		/*
1964 		 * Re-enqueue a thread on the wait queue if its
1965 		 * effective priority needs to change.
1966 		 */
1967 		if (disp_pri != t->t_epri)
1968 			waitq_change_pri(t, disp_pri);
1969 	} else {
1970 		/*
1971 		 * The thread is on a run queue.
1972 		 * Note: setbackdq() may not put the thread
1973 		 * back on the same run queue where it originally
1974 		 * resided.
1975 		 */
1976 		(void) dispdeq(t);
1977 		t->t_epri = disp_pri;
1978 		setbackdq(t);
1979 	}
1980 	schedctl_set_cidpri(t);
1981 }
1982 
1983 /*
1984  * Function: Change the t_pri field of a thread.
1985  * Side Effects: Adjust the thread ordering on a run queue
1986  *		 or sleep queue, if necessary.
1987  * Returns: 1 if the thread was on a run queue, else 0.
1988  */
1989 int
thread_change_pri(kthread_t * t,pri_t disp_pri,int front)1990 thread_change_pri(kthread_t *t, pri_t disp_pri, int front)
1991 {
1992 	uint_t	state;
1993 	int	on_rq = 0;
1994 
1995 	ASSERT(THREAD_LOCK_HELD(t));
1996 
1997 	state = t->t_state;
1998 	THREAD_WILLCHANGE_PRI(t, disp_pri);
1999 
2000 	/*
2001 	 * If it's not on a queue, change the priority with impunity.
2002 	 */
2003 	if ((state & (TS_SLEEP | TS_RUN | TS_WAIT)) == 0) {
2004 		t->t_pri = disp_pri;
2005 
2006 		if (state == TS_ONPROC) {
2007 			cpu_t *cp = t->t_disp_queue->disp_cpu;
2008 
2009 			if (t == cp->cpu_dispthread)
2010 				cp->cpu_dispatch_pri = DISP_PRIO(t);
2011 		}
2012 	} else if (state == TS_SLEEP) {
2013 		/*
2014 		 * If the priority has changed, take the thread out of
2015 		 * its sleep queue and change the priority.
2016 		 * Re-enqueue the thread.
2017 		 * Each synchronization object exports a function
2018 		 * to do this in an appropriate manner.
2019 		 */
2020 		if (disp_pri != t->t_pri)
2021 			SOBJ_CHANGE_PRI(t->t_sobj_ops, t, disp_pri);
2022 	} else if (state == TS_WAIT) {
2023 		/*
2024 		 * Re-enqueue a thread on the wait queue if its
2025 		 * priority needs to change.
2026 		 */
2027 		if (disp_pri != t->t_pri)
2028 			waitq_change_pri(t, disp_pri);
2029 	} else {
2030 		/*
2031 		 * The thread is on a run queue.
2032 		 * Note: setbackdq() may not put the thread
2033 		 * back on the same run queue where it originally
2034 		 * resided.
2035 		 *
2036 		 * We still requeue the thread even if the priority
2037 		 * is unchanged to preserve round-robin (and other)
2038 		 * effects between threads of the same priority.
2039 		 */
2040 		on_rq = dispdeq(t);
2041 		ASSERT(on_rq);
2042 		t->t_pri = disp_pri;
2043 		if (front) {
2044 			setfrontdq(t);
2045 		} else {
2046 			setbackdq(t);
2047 		}
2048 	}
2049 	schedctl_set_cidpri(t);
2050 	return (on_rq);
2051 }
2052 
2053 /*
2054  * Tunable kmem_stackinfo is set, fill the kernel thread stack with a
2055  * specific pattern.
2056  */
2057 static void
stkinfo_begin(kthread_t * t)2058 stkinfo_begin(kthread_t *t)
2059 {
2060 	caddr_t	start;	/* stack start */
2061 	caddr_t	end;	/* stack end  */
2062 	uint64_t *ptr;	/* pattern pointer */
2063 
2064 	/*
2065 	 * Stack grows up or down, see thread_create(),
2066 	 * compute stack memory area start and end (start < end).
2067 	 */
2068 	if (t->t_stk > t->t_stkbase) {
2069 		/* stack grows down */
2070 		start = t->t_stkbase;
2071 		end = t->t_stk;
2072 	} else {
2073 		/* stack grows up */
2074 		start = t->t_stk;
2075 		end = t->t_stkbase;
2076 	}
2077 
2078 	/*
2079 	 * Stackinfo pattern size is 8 bytes. Ensure proper 8 bytes
2080 	 * alignement for start and end in stack area boundaries
2081 	 * (protection against corrupt t_stkbase/t_stk data).
2082 	 */
2083 	if ((((uintptr_t)start) & 0x7) != 0) {
2084 		start = (caddr_t)((((uintptr_t)start) & (~0x7)) + 8);
2085 	}
2086 	end = (caddr_t)(((uintptr_t)end) & (~0x7));
2087 
2088 	if ((end <= start) || (end - start) > (1024 * 1024)) {
2089 		/* negative or stack size > 1 meg, assume bogus */
2090 		return;
2091 	}
2092 
2093 	/* fill stack area with a pattern (instead of zeros) */
2094 	ptr = (uint64_t *)((void *)start);
2095 	while (ptr < (uint64_t *)((void *)end)) {
2096 		*ptr++ = KMEM_STKINFO_PATTERN;
2097 	}
2098 }
2099 
2100 
2101 /*
2102  * Tunable kmem_stackinfo is set, create stackinfo log if doesn't already exist,
2103  * compute the percentage of kernel stack really used, and set in the log
2104  * if it's the latest highest percentage.
2105  */
2106 static void
stkinfo_end(kthread_t * t)2107 stkinfo_end(kthread_t *t)
2108 {
2109 	caddr_t	start;	/* stack start */
2110 	caddr_t	end;	/* stack end  */
2111 	uint64_t *ptr;	/* pattern pointer */
2112 	size_t stksz;	/* stack size */
2113 	size_t smallest = 0;
2114 	size_t percent = 0;
2115 	uint_t index = 0;
2116 	uint_t i;
2117 	static size_t smallest_percent = (size_t)-1;
2118 	static uint_t full = 0;
2119 
2120 	/* create the stackinfo log, if doesn't already exist */
2121 	mutex_enter(&kmem_stkinfo_lock);
2122 	if (kmem_stkinfo_log == NULL) {
2123 		kmem_stkinfo_log = (kmem_stkinfo_t *)
2124 		    kmem_zalloc(KMEM_STKINFO_LOG_SIZE *
2125 		    (sizeof (kmem_stkinfo_t)), KM_NOSLEEP);
2126 		if (kmem_stkinfo_log == NULL) {
2127 			mutex_exit(&kmem_stkinfo_lock);
2128 			return;
2129 		}
2130 	}
2131 	mutex_exit(&kmem_stkinfo_lock);
2132 
2133 	/*
2134 	 * Stack grows up or down, see thread_create(),
2135 	 * compute stack memory area start and end (start < end).
2136 	 */
2137 	if (t->t_stk > t->t_stkbase) {
2138 		/* stack grows down */
2139 		start = t->t_stkbase;
2140 		end = t->t_stk;
2141 	} else {
2142 		/* stack grows up */
2143 		start = t->t_stk;
2144 		end = t->t_stkbase;
2145 	}
2146 
2147 	/* stack size as found in kthread_t */
2148 	stksz = end - start;
2149 
2150 	/*
2151 	 * Stackinfo pattern size is 8 bytes. Ensure proper 8 bytes
2152 	 * alignement for start and end in stack area boundaries
2153 	 * (protection against corrupt t_stkbase/t_stk data).
2154 	 */
2155 	if ((((uintptr_t)start) & 0x7) != 0) {
2156 		start = (caddr_t)((((uintptr_t)start) & (~0x7)) + 8);
2157 	}
2158 	end = (caddr_t)(((uintptr_t)end) & (~0x7));
2159 
2160 	if ((end <= start) || (end - start) > (1024 * 1024)) {
2161 		/* negative or stack size > 1 meg, assume bogus */
2162 		return;
2163 	}
2164 
2165 	/* search until no pattern in the stack */
2166 	if (t->t_stk > t->t_stkbase) {
2167 		/* stack grows down */
2168 #if defined(__x86)
2169 		/*
2170 		 * 6 longs are pushed on stack, see thread_load(). Skip
2171 		 * them, so if kthread has never run, percent is zero.
2172 		 * 8 bytes alignement is preserved for a 32 bit kernel,
2173 		 * 6 x 4 = 24, 24 is a multiple of 8.
2174 		 *
2175 		 */
2176 		end -= (6 * sizeof (long));
2177 #endif
2178 		ptr = (uint64_t *)((void *)start);
2179 		while (ptr < (uint64_t *)((void *)end)) {
2180 			if (*ptr != KMEM_STKINFO_PATTERN) {
2181 				percent = stkinfo_percent(end,
2182 				    start, (caddr_t)ptr);
2183 				break;
2184 			}
2185 			ptr++;
2186 		}
2187 	} else {
2188 		/* stack grows up */
2189 		ptr = (uint64_t *)((void *)end);
2190 		ptr--;
2191 		while (ptr >= (uint64_t *)((void *)start)) {
2192 			if (*ptr != KMEM_STKINFO_PATTERN) {
2193 				percent = stkinfo_percent(start,
2194 				    end, (caddr_t)ptr);
2195 				break;
2196 			}
2197 			ptr--;
2198 		}
2199 	}
2200 
2201 	DTRACE_PROBE3(stack__usage, kthread_t *, t,
2202 	    size_t, stksz, size_t, percent);
2203 
2204 	if (percent == 0) {
2205 		return;
2206 	}
2207 
2208 	mutex_enter(&kmem_stkinfo_lock);
2209 	if (full == KMEM_STKINFO_LOG_SIZE && percent < smallest_percent) {
2210 		/*
2211 		 * The log is full and already contains the highest values
2212 		 */
2213 		mutex_exit(&kmem_stkinfo_lock);
2214 		return;
2215 	}
2216 
2217 	/* keep a log of the highest used stack */
2218 	for (i = 0; i < KMEM_STKINFO_LOG_SIZE; i++) {
2219 		if (kmem_stkinfo_log[i].percent == 0) {
2220 			index = i;
2221 			full++;
2222 			break;
2223 		}
2224 		if (smallest == 0) {
2225 			smallest = kmem_stkinfo_log[i].percent;
2226 			index = i;
2227 			continue;
2228 		}
2229 		if (kmem_stkinfo_log[i].percent < smallest) {
2230 			smallest = kmem_stkinfo_log[i].percent;
2231 			index = i;
2232 		}
2233 	}
2234 
2235 	if (percent >= kmem_stkinfo_log[index].percent) {
2236 		kmem_stkinfo_log[index].kthread = (caddr_t)t;
2237 		kmem_stkinfo_log[index].t_startpc = (caddr_t)t->t_startpc;
2238 		kmem_stkinfo_log[index].start = start;
2239 		kmem_stkinfo_log[index].stksz = stksz;
2240 		kmem_stkinfo_log[index].percent = percent;
2241 		kmem_stkinfo_log[index].t_tid = t->t_tid;
2242 		kmem_stkinfo_log[index].cmd[0] = '\0';
2243 		if (t->t_tid != 0) {
2244 			stksz = strlen((t->t_procp)->p_user.u_comm);
2245 			if (stksz >= KMEM_STKINFO_STR_SIZE) {
2246 				stksz = KMEM_STKINFO_STR_SIZE - 1;
2247 				kmem_stkinfo_log[index].cmd[stksz] = '\0';
2248 			} else {
2249 				stksz += 1;
2250 			}
2251 			(void) memcpy(kmem_stkinfo_log[index].cmd,
2252 			    (t->t_procp)->p_user.u_comm, stksz);
2253 		}
2254 		if (percent < smallest_percent) {
2255 			smallest_percent = percent;
2256 		}
2257 	}
2258 	mutex_exit(&kmem_stkinfo_lock);
2259 }
2260 
2261 /*
2262  * Tunable kmem_stackinfo is set, compute stack utilization percentage.
2263  */
2264 static size_t
stkinfo_percent(caddr_t t_stk,caddr_t t_stkbase,caddr_t sp)2265 stkinfo_percent(caddr_t t_stk, caddr_t t_stkbase, caddr_t sp)
2266 {
2267 	size_t percent;
2268 	size_t s;
2269 
2270 	if (t_stk > t_stkbase) {
2271 		/* stack grows down */
2272 		if (sp > t_stk) {
2273 			return (0);
2274 		}
2275 		if (sp < t_stkbase) {
2276 			return (100);
2277 		}
2278 		percent = t_stk - sp + 1;
2279 		s = t_stk - t_stkbase + 1;
2280 	} else {
2281 		/* stack grows up */
2282 		if (sp < t_stk) {
2283 			return (0);
2284 		}
2285 		if (sp > t_stkbase) {
2286 			return (100);
2287 		}
2288 		percent = sp - t_stk + 1;
2289 		s = t_stkbase - t_stk + 1;
2290 	}
2291 	percent = ((100 * percent) / s) + 1;
2292 	if (percent > 100) {
2293 		percent = 100;
2294 	}
2295 	return (percent);
2296 }
2297 
2298 /*
2299  * NOTE: This will silently truncate a name > THREAD_NAME_MAX - 1 characters
2300  * long.  It is expected that callers (acting on behalf of userland clients)
2301  * will perform any required checks to return the correct error semantics.
2302  * It is also expected callers on behalf of userland clients have done
2303  * any necessary permission checks.
2304  */
2305 int
thread_setname(kthread_t * t,const char * name)2306 thread_setname(kthread_t *t, const char *name)
2307 {
2308 	char *buf = NULL;
2309 
2310 	/*
2311 	 * We optimistically assume that a thread's name will only be set
2312 	 * once and so allocate memory in preparation of setting t_name.
2313 	 * If it turns out a name has already been set, we just discard (free)
2314 	 * the buffer we just allocated and reuse the current buffer
2315 	 * (as all should be THREAD_NAME_MAX large).
2316 	 *
2317 	 * Such an arrangement means over the lifetime of a kthread_t, t_name
2318 	 * is either NULL or has one value (the address of the buffer holding
2319 	 * the current thread name).   The assumption is that most kthread_t
2320 	 * instances will not have a name assigned, so dynamically allocating
2321 	 * the memory should minimize the footprint of this feature, but by
2322 	 * having the buffer persist for the life of the thread, it simplifies
2323 	 * usage in highly constrained situations (e.g. dtrace).
2324 	 */
2325 	if (name != NULL && name[0] != '\0') {
2326 		for (size_t i = 0; name[i] != '\0'; i++) {
2327 			if (!isprint(name[i]))
2328 				return (EINVAL);
2329 		}
2330 
2331 		buf = kmem_zalloc(THREAD_NAME_MAX, KM_SLEEP);
2332 		(void) strlcpy(buf, name, THREAD_NAME_MAX);
2333 	}
2334 
2335 	mutex_enter(&ttoproc(t)->p_lock);
2336 	if (t->t_name == NULL) {
2337 		t->t_name = buf;
2338 	} else {
2339 		if (buf != NULL) {
2340 			(void) strlcpy(t->t_name, name, THREAD_NAME_MAX);
2341 			kmem_free(buf, THREAD_NAME_MAX);
2342 		} else {
2343 			bzero(t->t_name, THREAD_NAME_MAX);
2344 		}
2345 	}
2346 	mutex_exit(&ttoproc(t)->p_lock);
2347 	return (0);
2348 }
2349 
2350 int
thread_vsetname(kthread_t * t,const char * fmt,...)2351 thread_vsetname(kthread_t *t, const char *fmt, ...)
2352 {
2353 	char name[THREAD_NAME_MAX];
2354 	va_list va;
2355 	int rc;
2356 
2357 	va_start(va, fmt);
2358 	rc = vsnprintf(name, sizeof (name), fmt, va);
2359 	va_end(va);
2360 
2361 	if (rc < 0)
2362 		return (EINVAL);
2363 
2364 	if (rc >= sizeof (name))
2365 		return (ENAMETOOLONG);
2366 
2367 	return (thread_setname(t, name));
2368 }
2369