1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22/*
23 * Copyright (c) 1991, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright 2019 Joyent, Inc.
25 */
26
27#include <sys/types.h>
28#include <sys/param.h>
29#include <sys/sysmacros.h>
30#include <sys/signal.h>
31#include <sys/stack.h>
32#include <sys/pcb.h>
33#include <sys/user.h>
34#include <sys/systm.h>
35#include <sys/sysinfo.h>
36#include <sys/errno.h>
37#include <sys/cmn_err.h>
38#include <sys/cred.h>
39#include <sys/resource.h>
40#include <sys/task.h>
41#include <sys/project.h>
42#include <sys/proc.h>
43#include <sys/debug.h>
44#include <sys/disp.h>
45#include <sys/class.h>
46#include <vm/seg_kmem.h>
47#include <vm/seg_kp.h>
48#include <sys/machlock.h>
49#include <sys/kmem.h>
50#include <sys/varargs.h>
51#include <sys/turnstile.h>
52#include <sys/poll.h>
53#include <sys/vtrace.h>
54#include <sys/callb.h>
55#include <c2/audit.h>
56#include <sys/tnf.h>
57#include <sys/sobject.h>
58#include <sys/cpupart.h>
59#include <sys/pset.h>
60#include <sys/door.h>
61#include <sys/spl.h>
62#include <sys/copyops.h>
63#include <sys/rctl.h>
64#include <sys/brand.h>
65#include <sys/pool.h>
66#include <sys/zone.h>
67#include <sys/tsol/label.h>
68#include <sys/tsol/tndb.h>
69#include <sys/cpc_impl.h>
70#include <sys/sdt.h>
71#include <sys/reboot.h>
72#include <sys/kdi.h>
73#include <sys/schedctl.h>
74#include <sys/waitq.h>
75#include <sys/cpucaps.h>
76#include <sys/kiconv.h>
77#include <sys/ctype.h>
78#include <sys/smt.h>
79
80struct kmem_cache *thread_cache;	/* cache of free threads */
81struct kmem_cache *lwp_cache;		/* cache of free lwps */
82struct kmem_cache *turnstile_cache;	/* cache of free turnstiles */
83
84/*
85 * allthreads is only for use by kmem_readers.  All kernel loops can use
86 * the current thread as a start/end point.
87 */
88kthread_t *allthreads = &t0;	/* circular list of all threads */
89
90static kcondvar_t reaper_cv;		/* synchronization var */
91kthread_t	*thread_deathrow;	/* circular list of reapable threads */
92kthread_t	*lwp_deathrow;		/* circular list of reapable threads */
93kmutex_t	reaplock;		/* protects lwp and thread deathrows */
94int	thread_reapcnt = 0;		/* number of threads on deathrow */
95int	lwp_reapcnt = 0;		/* number of lwps on deathrow */
96int	reaplimit = 16;			/* delay reaping until reaplimit */
97
98thread_free_lock_t	*thread_free_lock;
99					/* protects tick thread from reaper */
100
101extern int nthread;
102
103/* System Scheduling classes. */
104id_t	syscid;				/* system scheduling class ID */
105id_t	sysdccid = CLASS_UNUSED;	/* reset when SDC loads */
106
107void	*segkp_thread;			/* cookie for segkp pool */
108
109int lwp_cache_sz = 32;
110int t_cache_sz = 8;
111static kt_did_t next_t_id = 1;
112
113/* Default mode for thread binding to CPUs and processor sets */
114int default_binding_mode = TB_ALLHARD;
115
116/*
117 * Min/Max stack sizes for stack size parameters
118 */
119#define	MAX_STKSIZE	(32 * DEFAULTSTKSZ)
120#define	MIN_STKSIZE	DEFAULTSTKSZ
121
122/*
123 * default_stksize overrides lwp_default_stksize if it is set.
124 */
125int	default_stksize;
126int	lwp_default_stksize;
127
128static zone_key_t zone_thread_key;
129
130unsigned int kmem_stackinfo;		/* stackinfo feature on-off */
131kmem_stkinfo_t *kmem_stkinfo_log;	/* stackinfo circular log */
132static kmutex_t kmem_stkinfo_lock;	/* protects kmem_stkinfo_log */
133
134/*
135 * forward declarations for internal thread specific data (tsd)
136 */
137static void *tsd_realloc(void *, size_t, size_t);
138
139void thread_reaper(void);
140
141/* forward declarations for stackinfo feature */
142static void stkinfo_begin(kthread_t *);
143static void stkinfo_end(kthread_t *);
144static size_t stkinfo_percent(caddr_t, caddr_t, caddr_t);
145
146/*ARGSUSED*/
147static int
148turnstile_constructor(void *buf, void *cdrarg, int kmflags)
149{
150	bzero(buf, sizeof (turnstile_t));
151	return (0);
152}
153
154/*ARGSUSED*/
155static void
156turnstile_destructor(void *buf, void *cdrarg)
157{
158	turnstile_t *ts = buf;
159
160	ASSERT(ts->ts_free == NULL);
161	ASSERT(ts->ts_waiters == 0);
162	ASSERT(ts->ts_inheritor == NULL);
163	ASSERT(ts->ts_sleepq[0].sq_first == NULL);
164	ASSERT(ts->ts_sleepq[1].sq_first == NULL);
165}
166
167void
168thread_init(void)
169{
170	kthread_t *tp;
171	extern char sys_name[];
172	extern void idle();
173	struct cpu *cpu = CPU;
174	int i;
175	kmutex_t *lp;
176
177	mutex_init(&reaplock, NULL, MUTEX_SPIN, (void *)ipltospl(DISP_LEVEL));
178	thread_free_lock =
179	    kmem_alloc(sizeof (thread_free_lock_t) * THREAD_FREE_NUM, KM_SLEEP);
180	for (i = 0; i < THREAD_FREE_NUM; i++) {
181		lp = &thread_free_lock[i].tf_lock;
182		mutex_init(lp, NULL, MUTEX_DEFAULT, NULL);
183	}
184
185#if defined(__i386) || defined(__amd64)
186	thread_cache = kmem_cache_create("thread_cache", sizeof (kthread_t),
187	    PTR24_ALIGN, NULL, NULL, NULL, NULL, NULL, 0);
188
189	/*
190	 * "struct _klwp" includes a "struct pcb", which includes a
191	 * "struct fpu", which needs to be 64-byte aligned on amd64
192	 * (and even on i386) for xsave/xrstor.
193	 */
194	lwp_cache = kmem_cache_create("lwp_cache", sizeof (klwp_t),
195	    64, NULL, NULL, NULL, NULL, NULL, 0);
196#else
197	/*
198	 * Allocate thread structures from static_arena.  This prevents
199	 * issues where a thread tries to relocate its own thread
200	 * structure and touches it after the mapping has been suspended.
201	 */
202	thread_cache = kmem_cache_create("thread_cache", sizeof (kthread_t),
203	    PTR24_ALIGN, NULL, NULL, NULL, NULL, static_arena, 0);
204
205	lwp_stk_cache_init();
206
207	lwp_cache = kmem_cache_create("lwp_cache", sizeof (klwp_t),
208	    0, NULL, NULL, NULL, NULL, NULL, 0);
209#endif
210
211	turnstile_cache = kmem_cache_create("turnstile_cache",
212	    sizeof (turnstile_t), 0,
213	    turnstile_constructor, turnstile_destructor, NULL, NULL, NULL, 0);
214
215	label_init();
216	cred_init();
217
218	/*
219	 * Initialize various resource management facilities.
220	 */
221	rctl_init();
222	cpucaps_init();
223	/*
224	 * Zone_init() should be called before project_init() so that project ID
225	 * for the first project is initialized correctly.
226	 */
227	zone_init();
228	project_init();
229	brand_init();
230	kiconv_init();
231	task_init();
232	tcache_init();
233	pool_init();
234
235	curthread->t_ts = kmem_cache_alloc(turnstile_cache, KM_SLEEP);
236
237	/*
238	 * Originally, we had two parameters to set default stack
239	 * size: one for lwp's (lwp_default_stksize), and one for
240	 * kernel-only threads (DEFAULTSTKSZ, a.k.a. _defaultstksz).
241	 * Now we have a third parameter that overrides both if it is
242	 * set to a legal stack size, called default_stksize.
243	 */
244
245	if (default_stksize == 0) {
246		default_stksize = DEFAULTSTKSZ;
247	} else if (default_stksize % PAGESIZE != 0 ||
248	    default_stksize > MAX_STKSIZE ||
249	    default_stksize < MIN_STKSIZE) {
250		cmn_err(CE_WARN, "Illegal stack size. Using %d",
251		    (int)DEFAULTSTKSZ);
252		default_stksize = DEFAULTSTKSZ;
253	} else {
254		lwp_default_stksize = default_stksize;
255	}
256
257	if (lwp_default_stksize == 0) {
258		lwp_default_stksize = default_stksize;
259	} else if (lwp_default_stksize % PAGESIZE != 0 ||
260	    lwp_default_stksize > MAX_STKSIZE ||
261	    lwp_default_stksize < MIN_STKSIZE) {
262		cmn_err(CE_WARN, "Illegal stack size. Using %d",
263		    default_stksize);
264		lwp_default_stksize = default_stksize;
265	}
266
267	segkp_lwp = segkp_cache_init(segkp, lwp_cache_sz,
268	    lwp_default_stksize,
269	    (KPD_NOWAIT | KPD_HASREDZONE | KPD_LOCKED));
270
271	segkp_thread = segkp_cache_init(segkp, t_cache_sz,
272	    default_stksize, KPD_HASREDZONE | KPD_LOCKED | KPD_NO_ANON);
273
274	(void) getcid(sys_name, &syscid);
275	curthread->t_cid = syscid;	/* current thread is t0 */
276
277	/*
278	 * Set up the first CPU's idle thread.
279	 * It runs whenever the CPU has nothing worthwhile to do.
280	 */
281	tp = thread_create(NULL, 0, idle, NULL, 0, &p0, TS_STOPPED, -1);
282	cpu->cpu_idle_thread = tp;
283	tp->t_preempt = 1;
284	tp->t_disp_queue = cpu->cpu_disp;
285	ASSERT(tp->t_disp_queue != NULL);
286	tp->t_bound_cpu = cpu;
287	tp->t_affinitycnt = 1;
288
289	/*
290	 * Registering a thread in the callback table is usually
291	 * done in the initialization code of the thread. In this
292	 * case, we do it right after thread creation to avoid
293	 * blocking idle thread while registering itself. It also
294	 * avoids the possibility of reregistration in case a CPU
295	 * restarts its idle thread.
296	 */
297	CALLB_CPR_INIT_SAFE(tp, "idle");
298
299	/*
300	 * Create the thread_reaper daemon. From this point on, exited
301	 * threads will get reaped.
302	 */
303	(void) thread_create(NULL, 0, (void (*)())thread_reaper,
304	    NULL, 0, &p0, TS_RUN, minclsyspri);
305
306	/*
307	 * Finish initializing the kernel memory allocator now that
308	 * thread_create() is available.
309	 */
310	kmem_thread_init();
311
312	if (boothowto & RB_DEBUG)
313		kdi_dvec_thravail();
314}
315
316/*
317 * Create a thread.
318 *
319 * thread_create() blocks for memory if necessary.  It never fails.
320 *
321 * If stk is NULL, the thread is created at the base of the stack
322 * and cannot be swapped.
323 */
324kthread_t *
325thread_create(
326	caddr_t	stk,
327	size_t	stksize,
328	void	(*proc)(),
329	void	*arg,
330	size_t	len,
331	proc_t	 *pp,
332	int	state,
333	pri_t	pri)
334{
335	kthread_t *t;
336	extern struct classfuncs sys_classfuncs;
337	turnstile_t *ts;
338
339	/*
340	 * Every thread keeps a turnstile around in case it needs to block.
341	 * The only reason the turnstile is not simply part of the thread
342	 * structure is that we may have to break the association whenever
343	 * more than one thread blocks on a given synchronization object.
344	 * From a memory-management standpoint, turnstiles are like the
345	 * "attached mblks" that hang off dblks in the streams allocator.
346	 */
347	ts = kmem_cache_alloc(turnstile_cache, KM_SLEEP);
348
349	if (stk == NULL) {
350		/*
351		 * alloc both thread and stack in segkp chunk
352		 */
353
354		if (stksize < default_stksize)
355			stksize = default_stksize;
356
357		if (stksize == default_stksize) {
358			stk = (caddr_t)segkp_cache_get(segkp_thread);
359		} else {
360			stksize = roundup(stksize, PAGESIZE);
361			stk = (caddr_t)segkp_get(segkp, stksize,
362			    (KPD_HASREDZONE | KPD_NO_ANON | KPD_LOCKED));
363		}
364
365		ASSERT(stk != NULL);
366
367		/*
368		 * The machine-dependent mutex code may require that
369		 * thread pointers (since they may be used for mutex owner
370		 * fields) have certain alignment requirements.
371		 * PTR24_ALIGN is the size of the alignment quanta.
372		 * XXX - assumes stack grows toward low addresses.
373		 */
374		if (stksize <= sizeof (kthread_t) + PTR24_ALIGN)
375			cmn_err(CE_PANIC, "thread_create: proposed stack size"
376			    " too small to hold thread.");
377#ifdef STACK_GROWTH_DOWN
378		stksize -= SA(sizeof (kthread_t) + PTR24_ALIGN - 1);
379		stksize &= -PTR24_ALIGN;	/* make thread aligned */
380		t = (kthread_t *)(stk + stksize);
381		bzero(t, sizeof (kthread_t));
382		if (audit_active)
383			audit_thread_create(t);
384		t->t_stk = stk + stksize;
385		t->t_stkbase = stk;
386#else	/* stack grows to larger addresses */
387		stksize -= SA(sizeof (kthread_t));
388		t = (kthread_t *)(stk);
389		bzero(t, sizeof (kthread_t));
390		t->t_stk = stk + sizeof (kthread_t);
391		t->t_stkbase = stk + stksize + sizeof (kthread_t);
392#endif	/* STACK_GROWTH_DOWN */
393		t->t_flag |= T_TALLOCSTK;
394		t->t_swap = stk;
395	} else {
396		t = kmem_cache_alloc(thread_cache, KM_SLEEP);
397		bzero(t, sizeof (kthread_t));
398		ASSERT(((uintptr_t)t & (PTR24_ALIGN - 1)) == 0);
399		if (audit_active)
400			audit_thread_create(t);
401		/*
402		 * Initialize t_stk to the kernel stack pointer to use
403		 * upon entry to the kernel
404		 */
405#ifdef STACK_GROWTH_DOWN
406		t->t_stk = stk + stksize;
407		t->t_stkbase = stk;
408#else
409		t->t_stk = stk;			/* 3b2-like */
410		t->t_stkbase = stk + stksize;
411#endif /* STACK_GROWTH_DOWN */
412	}
413
414	if (kmem_stackinfo != 0) {
415		stkinfo_begin(t);
416	}
417
418	t->t_ts = ts;
419
420	/*
421	 * p_cred could be NULL if it thread_create is called before cred_init
422	 * is called in main.
423	 */
424	mutex_enter(&pp->p_crlock);
425	if (pp->p_cred)
426		crhold(t->t_cred = pp->p_cred);
427	mutex_exit(&pp->p_crlock);
428	t->t_start = gethrestime_sec();
429	t->t_startpc = proc;
430	t->t_procp = pp;
431	t->t_clfuncs = &sys_classfuncs.thread;
432	t->t_cid = syscid;
433	t->t_pri = pri;
434	t->t_stime = ddi_get_lbolt();
435	t->t_schedflag = TS_LOAD | TS_DONT_SWAP;
436	t->t_bind_cpu = PBIND_NONE;
437	t->t_bindflag = (uchar_t)default_binding_mode;
438	t->t_bind_pset = PS_NONE;
439	t->t_plockp = &pp->p_lock;
440	t->t_copyops = NULL;
441	t->t_taskq = NULL;
442	t->t_anttime = 0;
443	t->t_hatdepth = 0;
444
445	t->t_dtrace_vtime = 1;	/* assure vtimestamp is always non-zero */
446
447	CPU_STATS_ADDQ(CPU, sys, nthreads, 1);
448#ifndef NPROBE
449	/* Kernel probe */
450	tnf_thread_create(t);
451#endif /* NPROBE */
452	LOCK_INIT_CLEAR(&t->t_lock);
453
454	/*
455	 * Callers who give us a NULL proc must do their own
456	 * stack initialization.  e.g. lwp_create()
457	 */
458	if (proc != NULL) {
459		t->t_stk = thread_stk_init(t->t_stk);
460		thread_load(t, proc, arg, len);
461	}
462
463	/*
464	 * Put a hold on project0. If this thread is actually in a
465	 * different project, then t_proj will be changed later in
466	 * lwp_create().  All kernel-only threads must be in project 0.
467	 */
468	t->t_proj = project_hold(proj0p);
469
470	lgrp_affinity_init(&t->t_lgrp_affinity);
471
472	mutex_enter(&pidlock);
473	nthread++;
474	t->t_did = next_t_id++;
475	t->t_prev = curthread->t_prev;
476	t->t_next = curthread;
477
478	/*
479	 * Add the thread to the list of all threads, and initialize
480	 * its t_cpu pointer.  We need to block preemption since
481	 * cpu_offline walks the thread list looking for threads
482	 * with t_cpu pointing to the CPU being offlined.  We want
483	 * to make sure that the list is consistent and that if t_cpu
484	 * is set, the thread is on the list.
485	 */
486	kpreempt_disable();
487	curthread->t_prev->t_next = t;
488	curthread->t_prev = t;
489
490	/*
491	 * We'll always create in the default partition since that's where
492	 * kernel threads go (we'll change this later if needed, in
493	 * lwp_create()).
494	 */
495	t->t_cpupart = &cp_default;
496
497	/*
498	 * For now, affiliate this thread with the root lgroup.
499	 * Since the kernel does not (presently) allocate its memory
500	 * in a locality aware fashion, the root is an appropriate home.
501	 * If this thread is later associated with an lwp, it will have
502	 * its lgroup re-assigned at that time.
503	 */
504	lgrp_move_thread(t, &cp_default.cp_lgrploads[LGRP_ROOTID], 1);
505
506	/*
507	 * If the current CPU is in the default cpupart, use it.  Otherwise,
508	 * pick one that is; before entering the dispatcher code, we'll
509	 * make sure to keep the invariant that ->t_cpu is set.  (In fact, we
510	 * rely on this, in ht_should_run(), in the call tree of
511	 * disp_lowpri_cpu().)
512	 */
513	if (CPU->cpu_part == &cp_default) {
514		t->t_cpu = CPU;
515	} else {
516		t->t_cpu = cp_default.cp_cpulist;
517		t->t_cpu = disp_lowpri_cpu(t->t_cpu, t, t->t_pri);
518	}
519
520	t->t_disp_queue = t->t_cpu->cpu_disp;
521	kpreempt_enable();
522
523	/*
524	 * Initialize thread state and the dispatcher lock pointer.
525	 * Need to hold onto pidlock to block allthreads walkers until
526	 * the state is set.
527	 */
528	switch (state) {
529	case TS_RUN:
530		curthread->t_oldspl = splhigh();	/* get dispatcher spl */
531		THREAD_SET_STATE(t, TS_STOPPED, &transition_lock);
532		CL_SETRUN(t);
533		thread_unlock(t);
534		break;
535
536	case TS_ONPROC:
537		THREAD_ONPROC(t, t->t_cpu);
538		break;
539
540	case TS_FREE:
541		/*
542		 * Free state will be used for intr threads.
543		 * The interrupt routine must set the thread dispatcher
544		 * lock pointer (t_lockp) if starting on a CPU
545		 * other than the current one.
546		 */
547		THREAD_FREEINTR(t, CPU);
548		break;
549
550	case TS_STOPPED:
551		THREAD_SET_STATE(t, TS_STOPPED, &stop_lock);
552		break;
553
554	default:			/* TS_SLEEP, TS_ZOMB or TS_TRANS */
555		cmn_err(CE_PANIC, "thread_create: invalid state %d", state);
556	}
557	mutex_exit(&pidlock);
558	return (t);
559}
560
561/*
562 * Move thread to project0 and take care of project reference counters.
563 */
564void
565thread_rele(kthread_t *t)
566{
567	kproject_t *kpj;
568
569	thread_lock(t);
570
571	ASSERT(t == curthread || t->t_state == TS_FREE || t->t_procp == &p0);
572	kpj = ttoproj(t);
573	t->t_proj = proj0p;
574
575	thread_unlock(t);
576
577	if (kpj != proj0p) {
578		project_rele(kpj);
579		(void) project_hold(proj0p);
580	}
581}
582
583void
584thread_exit(void)
585{
586	kthread_t *t = curthread;
587
588	if ((t->t_proc_flag & TP_ZTHREAD) != 0)
589		cmn_err(CE_PANIC, "thread_exit: zthread_exit() not called");
590
591	tsd_exit();		/* Clean up this thread's TSD */
592
593	kcpc_passivate();	/* clean up performance counter state */
594
595	/*
596	 * No kernel thread should have called poll() without arranging
597	 * calling pollcleanup() here.
598	 */
599	ASSERT(t->t_pollstate == NULL);
600	ASSERT(t->t_schedctl == NULL);
601	if (t->t_door)
602		door_slam();	/* in case thread did an upcall */
603
604#ifndef NPROBE
605	/* Kernel probe */
606	if (t->t_tnf_tpdp)
607		tnf_thread_exit();
608#endif /* NPROBE */
609
610	thread_rele(t);
611	t->t_preempt++;
612
613	/*
614	 * remove thread from the all threads list so that
615	 * death-row can use the same pointers.
616	 */
617	mutex_enter(&pidlock);
618	t->t_next->t_prev = t->t_prev;
619	t->t_prev->t_next = t->t_next;
620	ASSERT(allthreads != t);	/* t0 never exits */
621	cv_broadcast(&t->t_joincv);	/* wake up anyone in thread_join */
622	mutex_exit(&pidlock);
623
624	if (t->t_ctx != NULL)
625		exitctx(t);
626	if (t->t_procp->p_pctx != NULL)
627		exitpctx(t->t_procp);
628
629	if (kmem_stackinfo != 0) {
630		stkinfo_end(t);
631	}
632
633	t->t_state = TS_ZOMB;	/* set zombie thread */
634
635	swtch_from_zombie();	/* give up the CPU */
636	/* NOTREACHED */
637}
638
639/*
640 * Check to see if the specified thread is active (defined as being on
641 * the thread list).  This is certainly a slow way to do this; if there's
642 * ever a reason to speed it up, we could maintain a hash table of active
643 * threads indexed by their t_did.
644 */
645static kthread_t *
646did_to_thread(kt_did_t tid)
647{
648	kthread_t *t;
649
650	ASSERT(MUTEX_HELD(&pidlock));
651	for (t = curthread->t_next; t != curthread; t = t->t_next) {
652		if (t->t_did == tid)
653			break;
654	}
655	if (t->t_did == tid)
656		return (t);
657	else
658		return (NULL);
659}
660
661/*
662 * Wait for specified thread to exit.  Returns immediately if the thread
663 * could not be found, meaning that it has either already exited or never
664 * existed.
665 */
666void
667thread_join(kt_did_t tid)
668{
669	kthread_t *t;
670
671	ASSERT(tid != curthread->t_did);
672	ASSERT(tid != t0.t_did);
673
674	mutex_enter(&pidlock);
675	/*
676	 * Make sure we check that the thread is on the thread list
677	 * before blocking on it; otherwise we could end up blocking on
678	 * a cv that's already been freed.  In other words, don't cache
679	 * the thread pointer across calls to cv_wait.
680	 *
681	 * The choice of loop invariant means that whenever a thread
682	 * is taken off the allthreads list, a cv_broadcast must be
683	 * performed on that thread's t_joincv to wake up any waiters.
684	 * The broadcast doesn't have to happen right away, but it
685	 * shouldn't be postponed indefinitely (e.g., by doing it in
686	 * thread_free which may only be executed when the deathrow
687	 * queue is processed.
688	 */
689	while (t = did_to_thread(tid))
690		cv_wait(&t->t_joincv, &pidlock);
691	mutex_exit(&pidlock);
692}
693
694void
695thread_free_prevent(kthread_t *t)
696{
697	kmutex_t *lp;
698
699	lp = &thread_free_lock[THREAD_FREE_HASH(t)].tf_lock;
700	mutex_enter(lp);
701}
702
703void
704thread_free_allow(kthread_t *t)
705{
706	kmutex_t *lp;
707
708	lp = &thread_free_lock[THREAD_FREE_HASH(t)].tf_lock;
709	mutex_exit(lp);
710}
711
712static void
713thread_free_barrier(kthread_t *t)
714{
715	kmutex_t *lp;
716
717	lp = &thread_free_lock[THREAD_FREE_HASH(t)].tf_lock;
718	mutex_enter(lp);
719	mutex_exit(lp);
720}
721
722void
723thread_free(kthread_t *t)
724{
725	boolean_t allocstk = (t->t_flag & T_TALLOCSTK);
726	klwp_t *lwp = t->t_lwp;
727	caddr_t swap = t->t_swap;
728
729	ASSERT(t != &t0 && t->t_state == TS_FREE);
730	ASSERT(t->t_door == NULL);
731	ASSERT(t->t_schedctl == NULL);
732	ASSERT(t->t_pollstate == NULL);
733
734	t->t_pri = 0;
735	t->t_pc = 0;
736	t->t_sp = 0;
737	t->t_wchan0 = NULL;
738	t->t_wchan = NULL;
739	if (t->t_cred != NULL) {
740		crfree(t->t_cred);
741		t->t_cred = 0;
742	}
743	if (t->t_pdmsg) {
744		kmem_free(t->t_pdmsg, strlen(t->t_pdmsg) + 1);
745		t->t_pdmsg = NULL;
746	}
747	if (audit_active)
748		audit_thread_free(t);
749#ifndef NPROBE
750	if (t->t_tnf_tpdp)
751		tnf_thread_free(t);
752#endif /* NPROBE */
753	if (t->t_cldata) {
754		CL_EXITCLASS(t->t_cid, (caddr_t *)t->t_cldata);
755	}
756	if (t->t_rprof != NULL) {
757		kmem_free(t->t_rprof, sizeof (*t->t_rprof));
758		t->t_rprof = NULL;
759	}
760	t->t_lockp = NULL;	/* nothing should try to lock this thread now */
761	if (lwp)
762		lwp_freeregs(lwp, 0);
763	if (t->t_ctx)
764		freectx(t, 0);
765	t->t_stk = NULL;
766	if (lwp)
767		lwp_stk_fini(lwp);
768	lock_clear(&t->t_lock);
769
770	if (t->t_ts->ts_waiters > 0)
771		panic("thread_free: turnstile still active");
772
773	kmem_cache_free(turnstile_cache, t->t_ts);
774
775	free_afd(&t->t_activefd);
776
777	/*
778	 * Barrier for the tick accounting code.  The tick accounting code
779	 * holds this lock to keep the thread from going away while it's
780	 * looking at it.
781	 */
782	thread_free_barrier(t);
783
784	ASSERT(ttoproj(t) == proj0p);
785	project_rele(ttoproj(t));
786
787	lgrp_affinity_free(&t->t_lgrp_affinity);
788
789	mutex_enter(&pidlock);
790	nthread--;
791	mutex_exit(&pidlock);
792
793	if (t->t_name != NULL) {
794		kmem_free(t->t_name, THREAD_NAME_MAX);
795		t->t_name = NULL;
796	}
797
798	/*
799	 * Free thread, lwp and stack.  This needs to be done carefully, since
800	 * if T_TALLOCSTK is set, the thread is part of the stack.
801	 */
802	t->t_lwp = NULL;
803	t->t_swap = NULL;
804
805	if (swap) {
806		segkp_release(segkp, swap);
807	}
808	if (lwp) {
809		kmem_cache_free(lwp_cache, lwp);
810	}
811	if (!allocstk) {
812		kmem_cache_free(thread_cache, t);
813	}
814}
815
816/*
817 * Removes threads associated with the given zone from a deathrow queue.
818 * tp is a pointer to the head of the deathrow queue, and countp is a
819 * pointer to the current deathrow count.  Returns a linked list of
820 * threads removed from the list.
821 */
822static kthread_t *
823thread_zone_cleanup(kthread_t **tp, int *countp, zoneid_t zoneid)
824{
825	kthread_t *tmp, *list = NULL;
826	cred_t *cr;
827
828	ASSERT(MUTEX_HELD(&reaplock));
829	while (*tp != NULL) {
830		if ((cr = (*tp)->t_cred) != NULL && crgetzoneid(cr) == zoneid) {
831			tmp = *tp;
832			*tp = tmp->t_forw;
833			tmp->t_forw = list;
834			list = tmp;
835			(*countp)--;
836		} else {
837			tp = &(*tp)->t_forw;
838		}
839	}
840	return (list);
841}
842
843static void
844thread_reap_list(kthread_t *t)
845{
846	kthread_t *next;
847
848	while (t != NULL) {
849		next = t->t_forw;
850		thread_free(t);
851		t = next;
852	}
853}
854
855/* ARGSUSED */
856static void
857thread_zone_destroy(zoneid_t zoneid, void *unused)
858{
859	kthread_t *t, *l;
860
861	mutex_enter(&reaplock);
862	/*
863	 * Pull threads and lwps associated with zone off deathrow lists.
864	 */
865	t = thread_zone_cleanup(&thread_deathrow, &thread_reapcnt, zoneid);
866	l = thread_zone_cleanup(&lwp_deathrow, &lwp_reapcnt, zoneid);
867	mutex_exit(&reaplock);
868
869	/*
870	 * Guard against race condition in mutex_owner_running:
871	 *	thread=owner(mutex)
872	 *	<interrupt>
873	 *				thread exits mutex
874	 *				thread exits
875	 *				thread reaped
876	 *				thread struct freed
877	 * cpu = thread->t_cpu <- BAD POINTER DEREFERENCE.
878	 * A cross call to all cpus will cause the interrupt handler
879	 * to reset the PC if it is in mutex_owner_running, refreshing
880	 * stale thread pointers.
881	 */
882	mutex_sync();   /* sync with mutex code */
883
884	/*
885	 * Reap threads
886	 */
887	thread_reap_list(t);
888
889	/*
890	 * Reap lwps
891	 */
892	thread_reap_list(l);
893}
894
895/*
896 * cleanup zombie threads that are on deathrow.
897 */
898void
899thread_reaper()
900{
901	kthread_t *t, *l;
902	callb_cpr_t cprinfo;
903
904	/*
905	 * Register callback to clean up threads when zone is destroyed.
906	 */
907	zone_key_create(&zone_thread_key, NULL, NULL, thread_zone_destroy);
908
909	CALLB_CPR_INIT(&cprinfo, &reaplock, callb_generic_cpr, "t_reaper");
910	for (;;) {
911		mutex_enter(&reaplock);
912		while (thread_deathrow == NULL && lwp_deathrow == NULL) {
913			CALLB_CPR_SAFE_BEGIN(&cprinfo);
914			cv_wait(&reaper_cv, &reaplock);
915			CALLB_CPR_SAFE_END(&cprinfo, &reaplock);
916		}
917		/*
918		 * mutex_sync() needs to be called when reaping, but
919		 * not too often.  We limit reaping rate to once
920		 * per second.  Reaplimit is max rate at which threads can
921		 * be freed. Does not impact thread destruction/creation.
922		 */
923		t = thread_deathrow;
924		l = lwp_deathrow;
925		thread_deathrow = NULL;
926		lwp_deathrow = NULL;
927		thread_reapcnt = 0;
928		lwp_reapcnt = 0;
929		mutex_exit(&reaplock);
930
931		/*
932		 * Guard against race condition in mutex_owner_running:
933		 *	thread=owner(mutex)
934		 *	<interrupt>
935		 *				thread exits mutex
936		 *				thread exits
937		 *				thread reaped
938		 *				thread struct freed
939		 * cpu = thread->t_cpu <- BAD POINTER DEREFERENCE.
940		 * A cross call to all cpus will cause the interrupt handler
941		 * to reset the PC if it is in mutex_owner_running, refreshing
942		 * stale thread pointers.
943		 */
944		mutex_sync();   /* sync with mutex code */
945		/*
946		 * Reap threads
947		 */
948		thread_reap_list(t);
949
950		/*
951		 * Reap lwps
952		 */
953		thread_reap_list(l);
954		delay(hz);
955	}
956}
957
958/*
959 * This is called by lwpcreate, etc.() to put a lwp_deathrow thread onto
960 * thread_deathrow. The thread's state is changed already TS_FREE to indicate
961 * that is reapable. The thread already holds the reaplock, and was already
962 * freed.
963 */
964void
965reapq_move_lq_to_tq(kthread_t *t)
966{
967	ASSERT(t->t_state == TS_FREE);
968	ASSERT(MUTEX_HELD(&reaplock));
969	t->t_forw = thread_deathrow;
970	thread_deathrow = t;
971	thread_reapcnt++;
972	if (lwp_reapcnt + thread_reapcnt > reaplimit)
973		cv_signal(&reaper_cv);  /* wake the reaper */
974}
975
976/*
977 * This is called by resume() to put a zombie thread onto deathrow.
978 * The thread's state is changed to TS_FREE to indicate that is reapable.
979 * This is called from the idle thread so it must not block - just spin.
980 */
981void
982reapq_add(kthread_t *t)
983{
984	mutex_enter(&reaplock);
985
986	/*
987	 * lwp_deathrow contains threads with lwp linkage and
988	 * swappable thread stacks which have the default stacksize.
989	 * These threads' lwps and stacks may be reused by lwp_create().
990	 *
991	 * Anything else goes on thread_deathrow(), where it will eventually
992	 * be thread_free()d.
993	 */
994	if (t->t_flag & T_LWPREUSE) {
995		ASSERT(ttolwp(t) != NULL);
996		t->t_forw = lwp_deathrow;
997		lwp_deathrow = t;
998		lwp_reapcnt++;
999	} else {
1000		t->t_forw = thread_deathrow;
1001		thread_deathrow = t;
1002		thread_reapcnt++;
1003	}
1004	if (lwp_reapcnt + thread_reapcnt > reaplimit)
1005		cv_signal(&reaper_cv);	/* wake the reaper */
1006	t->t_state = TS_FREE;
1007	lock_clear(&t->t_lock);
1008
1009	/*
1010	 * Before we return, we need to grab and drop the thread lock for
1011	 * the dead thread.  At this point, the current thread is the idle
1012	 * thread, and the dead thread's CPU lock points to the current
1013	 * CPU -- and we must grab and drop the lock to synchronize with
1014	 * a racing thread walking a blocking chain that the zombie thread
1015	 * was recently in.  By this point, that blocking chain is (by
1016	 * definition) stale:  the dead thread is not holding any locks, and
1017	 * is therefore not in any blocking chains -- but if we do not regrab
1018	 * our lock before freeing the dead thread's data structures, the
1019	 * thread walking the (stale) blocking chain will die on memory
1020	 * corruption when it attempts to drop the dead thread's lock.  We
1021	 * only need do this once because there is no way for the dead thread
1022	 * to ever again be on a blocking chain:  once we have grabbed and
1023	 * dropped the thread lock, we are guaranteed that anyone that could
1024	 * have seen this thread in a blocking chain can no longer see it.
1025	 */
1026	thread_lock(t);
1027	thread_unlock(t);
1028
1029	mutex_exit(&reaplock);
1030}
1031
1032/*
1033 * Install thread context ops for the current thread.
1034 */
1035void
1036installctx(
1037	kthread_t *t,
1038	void	*arg,
1039	void	(*save)(void *),
1040	void	(*restore)(void *),
1041	void	(*fork)(void *, void *),
1042	void	(*lwp_create)(void *, void *),
1043	void	(*exit)(void *),
1044	void	(*free)(void *, int))
1045{
1046	struct ctxop *ctx;
1047
1048	ctx = kmem_alloc(sizeof (struct ctxop), KM_SLEEP);
1049	ctx->save_op = save;
1050	ctx->restore_op = restore;
1051	ctx->fork_op = fork;
1052	ctx->lwp_create_op = lwp_create;
1053	ctx->exit_op = exit;
1054	ctx->free_op = free;
1055	ctx->arg = arg;
1056	ctx->next = t->t_ctx;
1057	t->t_ctx = ctx;
1058}
1059
1060/*
1061 * Remove the thread context ops from a thread.
1062 */
1063int
1064removectx(
1065	kthread_t *t,
1066	void	*arg,
1067	void	(*save)(void *),
1068	void	(*restore)(void *),
1069	void	(*fork)(void *, void *),
1070	void	(*lwp_create)(void *, void *),
1071	void	(*exit)(void *),
1072	void	(*free)(void *, int))
1073{
1074	struct ctxop *ctx, *prev_ctx;
1075
1076	/*
1077	 * The incoming kthread_t (which is the thread for which the
1078	 * context ops will be removed) should be one of the following:
1079	 *
1080	 * a) the current thread,
1081	 *
1082	 * b) a thread of a process that's being forked (SIDL),
1083	 *
1084	 * c) a thread that belongs to the same process as the current
1085	 *    thread and for which the current thread is the agent thread,
1086	 *
1087	 * d) a thread that is TS_STOPPED which is indicative of it
1088	 *    being (if curthread is not an agent) a thread being created
1089	 *    as part of an lwp creation.
1090	 */
1091	ASSERT(t == curthread || ttoproc(t)->p_stat == SIDL ||
1092	    ttoproc(t)->p_agenttp == curthread || t->t_state == TS_STOPPED);
1093
1094	/*
1095	 * Serialize modifications to t->t_ctx to prevent the agent thread
1096	 * and the target thread from racing with each other during lwp exit.
1097	 */
1098	mutex_enter(&t->t_ctx_lock);
1099	prev_ctx = NULL;
1100	kpreempt_disable();
1101	for (ctx = t->t_ctx; ctx != NULL; ctx = ctx->next) {
1102		if (ctx->save_op == save && ctx->restore_op == restore &&
1103		    ctx->fork_op == fork && ctx->lwp_create_op == lwp_create &&
1104		    ctx->exit_op == exit && ctx->free_op == free &&
1105		    ctx->arg == arg) {
1106			if (prev_ctx)
1107				prev_ctx->next = ctx->next;
1108			else
1109				t->t_ctx = ctx->next;
1110			mutex_exit(&t->t_ctx_lock);
1111			if (ctx->free_op != NULL)
1112				(ctx->free_op)(ctx->arg, 0);
1113			kmem_free(ctx, sizeof (struct ctxop));
1114			kpreempt_enable();
1115			return (1);
1116		}
1117		prev_ctx = ctx;
1118	}
1119	mutex_exit(&t->t_ctx_lock);
1120	kpreempt_enable();
1121
1122	return (0);
1123}
1124
1125void
1126savectx(kthread_t *t)
1127{
1128	struct ctxop *ctx;
1129
1130	ASSERT(t == curthread);
1131	for (ctx = t->t_ctx; ctx != 0; ctx = ctx->next)
1132		if (ctx->save_op != NULL)
1133			(ctx->save_op)(ctx->arg);
1134}
1135
1136void
1137restorectx(kthread_t *t)
1138{
1139	struct ctxop *ctx;
1140
1141	ASSERT(t == curthread);
1142	for (ctx = t->t_ctx; ctx != 0; ctx = ctx->next)
1143		if (ctx->restore_op != NULL)
1144			(ctx->restore_op)(ctx->arg);
1145}
1146
1147void
1148forkctx(kthread_t *t, kthread_t *ct)
1149{
1150	struct ctxop *ctx;
1151
1152	for (ctx = t->t_ctx; ctx != NULL; ctx = ctx->next)
1153		if (ctx->fork_op != NULL)
1154			(ctx->fork_op)(t, ct);
1155}
1156
1157/*
1158 * Note that this operator is only invoked via the _lwp_create
1159 * system call.  The system may have other reasons to create lwps
1160 * e.g. the agent lwp or the doors unreferenced lwp.
1161 */
1162void
1163lwp_createctx(kthread_t *t, kthread_t *ct)
1164{
1165	struct ctxop *ctx;
1166
1167	for (ctx = t->t_ctx; ctx != NULL; ctx = ctx->next)
1168		if (ctx->lwp_create_op != NULL)
1169			(ctx->lwp_create_op)(t, ct);
1170}
1171
1172/*
1173 * exitctx is called from thread_exit() and lwp_exit() to perform any actions
1174 * needed when the thread/LWP leaves the processor for the last time. This
1175 * routine is not intended to deal with freeing memory; freectx() is used for
1176 * that purpose during thread_free(). This routine is provided to allow for
1177 * clean-up that can't wait until thread_free().
1178 */
1179void
1180exitctx(kthread_t *t)
1181{
1182	struct ctxop *ctx;
1183
1184	for (ctx = t->t_ctx; ctx != NULL; ctx = ctx->next)
1185		if (ctx->exit_op != NULL)
1186			(ctx->exit_op)(t);
1187}
1188
1189/*
1190 * freectx is called from thread_free() and exec() to get
1191 * rid of old thread context ops.
1192 */
1193void
1194freectx(kthread_t *t, int isexec)
1195{
1196	struct ctxop *ctx;
1197
1198	kpreempt_disable();
1199	while ((ctx = t->t_ctx) != NULL) {
1200		t->t_ctx = ctx->next;
1201		if (ctx->free_op != NULL)
1202			(ctx->free_op)(ctx->arg, isexec);
1203		kmem_free(ctx, sizeof (struct ctxop));
1204	}
1205	kpreempt_enable();
1206}
1207
1208/*
1209 * freectx_ctx is called from lwp_create() when lwp is reused from
1210 * lwp_deathrow and its thread structure is added to thread_deathrow.
1211 * The thread structure to which this ctx was attached may be already
1212 * freed by the thread reaper so free_op implementations shouldn't rely
1213 * on thread structure to which this ctx was attached still being around.
1214 */
1215void
1216freectx_ctx(struct ctxop *ctx)
1217{
1218	struct ctxop *nctx;
1219
1220	ASSERT(ctx != NULL);
1221
1222	kpreempt_disable();
1223	do {
1224		nctx = ctx->next;
1225		if (ctx->free_op != NULL)
1226			(ctx->free_op)(ctx->arg, 0);
1227		kmem_free(ctx, sizeof (struct ctxop));
1228	} while ((ctx = nctx) != NULL);
1229	kpreempt_enable();
1230}
1231
1232/*
1233 * Set the thread running; arrange for it to be swapped in if necessary.
1234 */
1235void
1236setrun_locked(kthread_t *t)
1237{
1238	ASSERT(THREAD_LOCK_HELD(t));
1239	if (t->t_state == TS_SLEEP) {
1240		/*
1241		 * Take off sleep queue.
1242		 */
1243		SOBJ_UNSLEEP(t->t_sobj_ops, t);
1244	} else if (t->t_state & (TS_RUN | TS_ONPROC)) {
1245		/*
1246		 * Already on dispatcher queue.
1247		 */
1248		return;
1249	} else if (t->t_state == TS_WAIT) {
1250		waitq_setrun(t);
1251	} else if (t->t_state == TS_STOPPED) {
1252		/*
1253		 * All of the sending of SIGCONT (TC_XSTART) and /proc
1254		 * (TC_PSTART) and lwp_continue() (TC_CSTART) must have
1255		 * requested that the thread be run.
1256		 * Just calling setrun() is not sufficient to set a stopped
1257		 * thread running.  TP_TXSTART is always set if the thread
1258		 * is not stopped by a jobcontrol stop signal.
1259		 * TP_TPSTART is always set if /proc is not controlling it.
1260		 * TP_TCSTART is always set if lwp_suspend() didn't stop it.
1261		 * The thread won't be stopped unless one of these
1262		 * three mechanisms did it.
1263		 *
1264		 * These flags must be set before calling setrun_locked(t).
1265		 * They can't be passed as arguments because the streams
1266		 * code calls setrun() indirectly and the mechanism for
1267		 * doing so admits only one argument.  Note that the
1268		 * thread must be locked in order to change t_schedflags.
1269		 */
1270		if ((t->t_schedflag & TS_ALLSTART) != TS_ALLSTART)
1271			return;
1272		/*
1273		 * Process is no longer stopped (a thread is running).
1274		 */
1275		t->t_whystop = 0;
1276		t->t_whatstop = 0;
1277		/*
1278		 * Strictly speaking, we do not have to clear these
1279		 * flags here; they are cleared on entry to stop().
1280		 * However, they are confusing when doing kernel
1281		 * debugging or when they are revealed by ps(1).
1282		 */
1283		t->t_schedflag &= ~TS_ALLSTART;
1284		THREAD_TRANSITION(t);	/* drop stopped-thread lock */
1285		ASSERT(t->t_lockp == &transition_lock);
1286		ASSERT(t->t_wchan0 == NULL && t->t_wchan == NULL);
1287		/*
1288		 * Let the class put the process on the dispatcher queue.
1289		 */
1290		CL_SETRUN(t);
1291	}
1292}
1293
1294void
1295setrun(kthread_t *t)
1296{
1297	thread_lock(t);
1298	setrun_locked(t);
1299	thread_unlock(t);
1300}
1301
1302/*
1303 * Unpin an interrupted thread.
1304 *	When an interrupt occurs, the interrupt is handled on the stack
1305 *	of an interrupt thread, taken from a pool linked to the CPU structure.
1306 *
1307 *	When swtch() is switching away from an interrupt thread because it
1308 *	blocked or was preempted, this routine is called to complete the
1309 *	saving of the interrupted thread state, and returns the interrupted
1310 *	thread pointer so it may be resumed.
1311 *
1312 *	Called by swtch() only at high spl.
1313 */
1314kthread_t *
1315thread_unpin()
1316{
1317	kthread_t	*t = curthread;	/* current thread */
1318	kthread_t	*itp;		/* interrupted thread */
1319	int		i;		/* interrupt level */
1320	extern int	intr_passivate();
1321
1322	ASSERT(t->t_intr != NULL);
1323
1324	itp = t->t_intr;		/* interrupted thread */
1325	t->t_intr = NULL;		/* clear interrupt ptr */
1326
1327	smt_end_intr();
1328
1329	/*
1330	 * Get state from interrupt thread for the one
1331	 * it interrupted.
1332	 */
1333
1334	i = intr_passivate(t, itp);
1335
1336	TRACE_5(TR_FAC_INTR, TR_INTR_PASSIVATE,
1337	    "intr_passivate:level %d curthread %p (%T) ithread %p (%T)",
1338	    i, t, t, itp, itp);
1339
1340	/*
1341	 * Dissociate the current thread from the interrupted thread's LWP.
1342	 */
1343	t->t_lwp = NULL;
1344
1345	/*
1346	 * Interrupt handlers above the level that spinlocks block must
1347	 * not block.
1348	 */
1349#if DEBUG
1350	if (i < 0 || i > LOCK_LEVEL)
1351		cmn_err(CE_PANIC, "thread_unpin: ipl out of range %x", i);
1352#endif
1353
1354	/*
1355	 * Compute the CPU's base interrupt level based on the active
1356	 * interrupts.
1357	 */
1358	ASSERT(CPU->cpu_intr_actv & (1 << i));
1359	set_base_spl();
1360
1361	return (itp);
1362}
1363
1364/*
1365 * Create and initialize an interrupt thread.
1366 *	Returns non-zero on error.
1367 *	Called at spl7() or better.
1368 */
1369void
1370thread_create_intr(struct cpu *cp)
1371{
1372	kthread_t *tp;
1373
1374	tp = thread_create(NULL, 0,
1375	    (void (*)())thread_create_intr, NULL, 0, &p0, TS_ONPROC, 0);
1376
1377	/*
1378	 * Set the thread in the TS_FREE state.  The state will change
1379	 * to TS_ONPROC only while the interrupt is active.  Think of these
1380	 * as being on a private free list for the CPU.  Being TS_FREE keeps
1381	 * inactive interrupt threads out of debugger thread lists.
1382	 *
1383	 * We cannot call thread_create with TS_FREE because of the current
1384	 * checks there for ONPROC.  Fix this when thread_create takes flags.
1385	 */
1386	THREAD_FREEINTR(tp, cp);
1387
1388	/*
1389	 * Nobody should ever reference the credentials of an interrupt
1390	 * thread so make it NULL to catch any such references.
1391	 */
1392	tp->t_cred = NULL;
1393	tp->t_flag |= T_INTR_THREAD;
1394	tp->t_cpu = cp;
1395	tp->t_bound_cpu = cp;
1396	tp->t_disp_queue = cp->cpu_disp;
1397	tp->t_affinitycnt = 1;
1398	tp->t_preempt = 1;
1399
1400	/*
1401	 * Don't make a user-requested binding on this thread so that
1402	 * the processor can be offlined.
1403	 */
1404	tp->t_bind_cpu = PBIND_NONE;	/* no USER-requested binding */
1405	tp->t_bind_pset = PS_NONE;
1406
1407#if defined(__i386) || defined(__amd64)
1408	tp->t_stk -= STACK_ALIGN;
1409	*(tp->t_stk) = 0;		/* terminate intr thread stack */
1410#endif
1411
1412	/*
1413	 * Link onto CPU's interrupt pool.
1414	 */
1415	tp->t_link = cp->cpu_intr_thread;
1416	cp->cpu_intr_thread = tp;
1417}
1418
1419/*
1420 * TSD -- THREAD SPECIFIC DATA
1421 */
1422static kmutex_t		tsd_mutex;	 /* linked list spin lock */
1423static uint_t		tsd_nkeys;	 /* size of destructor array */
1424/* per-key destructor funcs */
1425static void		(**tsd_destructor)(void *);
1426/* list of tsd_thread's */
1427static struct tsd_thread	*tsd_list;
1428
1429/*
1430 * Default destructor
1431 *	Needed because NULL destructor means that the key is unused
1432 */
1433/* ARGSUSED */
1434void
1435tsd_defaultdestructor(void *value)
1436{}
1437
1438/*
1439 * Create a key (index into per thread array)
1440 *	Locks out tsd_create, tsd_destroy, and tsd_exit
1441 *	May allocate memory with lock held
1442 */
1443void
1444tsd_create(uint_t *keyp, void (*destructor)(void *))
1445{
1446	int	i;
1447	uint_t	nkeys;
1448
1449	/*
1450	 * if key is allocated, do nothing
1451	 */
1452	mutex_enter(&tsd_mutex);
1453	if (*keyp) {
1454		mutex_exit(&tsd_mutex);
1455		return;
1456	}
1457	/*
1458	 * find an unused key
1459	 */
1460	if (destructor == NULL)
1461		destructor = tsd_defaultdestructor;
1462
1463	for (i = 0; i < tsd_nkeys; ++i)
1464		if (tsd_destructor[i] == NULL)
1465			break;
1466
1467	/*
1468	 * if no unused keys, increase the size of the destructor array
1469	 */
1470	if (i == tsd_nkeys) {
1471		if ((nkeys = (tsd_nkeys << 1)) == 0)
1472			nkeys = 1;
1473		tsd_destructor =
1474		    (void (**)(void *))tsd_realloc((void *)tsd_destructor,
1475		    (size_t)(tsd_nkeys * sizeof (void (*)(void *))),
1476		    (size_t)(nkeys * sizeof (void (*)(void *))));
1477		tsd_nkeys = nkeys;
1478	}
1479
1480	/*
1481	 * allocate the next available unused key
1482	 */
1483	tsd_destructor[i] = destructor;
1484	*keyp = i + 1;
1485	mutex_exit(&tsd_mutex);
1486}
1487
1488/*
1489 * Destroy a key -- this is for unloadable modules
1490 *
1491 * Assumes that the caller is preventing tsd_set and tsd_get
1492 * Locks out tsd_create, tsd_destroy, and tsd_exit
1493 * May free memory with lock held
1494 */
1495void
1496tsd_destroy(uint_t *keyp)
1497{
1498	uint_t key;
1499	struct tsd_thread *tsd;
1500
1501	/*
1502	 * protect the key namespace and our destructor lists
1503	 */
1504	mutex_enter(&tsd_mutex);
1505	key = *keyp;
1506	*keyp = 0;
1507
1508	ASSERT(key <= tsd_nkeys);
1509
1510	/*
1511	 * if the key is valid
1512	 */
1513	if (key != 0) {
1514		uint_t k = key - 1;
1515		/*
1516		 * for every thread with TSD, call key's destructor
1517		 */
1518		for (tsd = tsd_list; tsd; tsd = tsd->ts_next) {
1519			/*
1520			 * no TSD for key in this thread
1521			 */
1522			if (key > tsd->ts_nkeys)
1523				continue;
1524			/*
1525			 * call destructor for key
1526			 */
1527			if (tsd->ts_value[k] && tsd_destructor[k])
1528				(*tsd_destructor[k])(tsd->ts_value[k]);
1529			/*
1530			 * reset value for key
1531			 */
1532			tsd->ts_value[k] = NULL;
1533		}
1534		/*
1535		 * actually free the key (NULL destructor == unused)
1536		 */
1537		tsd_destructor[k] = NULL;
1538	}
1539
1540	mutex_exit(&tsd_mutex);
1541}
1542
1543/*
1544 * Quickly return the per thread value that was stored with the specified key
1545 * Assumes the caller is protecting key from tsd_create and tsd_destroy
1546 */
1547void *
1548tsd_get(uint_t key)
1549{
1550	return (tsd_agent_get(curthread, key));
1551}
1552
1553/*
1554 * Set a per thread value indexed with the specified key
1555 */
1556int
1557tsd_set(uint_t key, void *value)
1558{
1559	return (tsd_agent_set(curthread, key, value));
1560}
1561
1562/*
1563 * Like tsd_get(), except that the agent lwp can get the tsd of
1564 * another thread in the same process (the agent thread only runs when the
1565 * process is completely stopped by /proc), or syslwp is creating a new lwp.
1566 */
1567void *
1568tsd_agent_get(kthread_t *t, uint_t key)
1569{
1570	struct tsd_thread *tsd = t->t_tsd;
1571
1572	ASSERT(t == curthread ||
1573	    ttoproc(t)->p_agenttp == curthread || t->t_state == TS_STOPPED);
1574
1575	if (key && tsd != NULL && key <= tsd->ts_nkeys)
1576		return (tsd->ts_value[key - 1]);
1577	return (NULL);
1578}
1579
1580/*
1581 * Like tsd_set(), except that the agent lwp can set the tsd of
1582 * another thread in the same process, or syslwp can set the tsd
1583 * of a thread it's in the middle of creating.
1584 *
1585 * Assumes the caller is protecting key from tsd_create and tsd_destroy
1586 * May lock out tsd_destroy (and tsd_create), may allocate memory with
1587 * lock held
1588 */
1589int
1590tsd_agent_set(kthread_t *t, uint_t key, void *value)
1591{
1592	struct tsd_thread *tsd = t->t_tsd;
1593
1594	ASSERT(t == curthread ||
1595	    ttoproc(t)->p_agenttp == curthread || t->t_state == TS_STOPPED);
1596
1597	if (key == 0)
1598		return (EINVAL);
1599	if (tsd == NULL)
1600		tsd = t->t_tsd = kmem_zalloc(sizeof (*tsd), KM_SLEEP);
1601	if (key <= tsd->ts_nkeys) {
1602		tsd->ts_value[key - 1] = value;
1603		return (0);
1604	}
1605
1606	ASSERT(key <= tsd_nkeys);
1607
1608	/*
1609	 * lock out tsd_destroy()
1610	 */
1611	mutex_enter(&tsd_mutex);
1612	if (tsd->ts_nkeys == 0) {
1613		/*
1614		 * Link onto list of threads with TSD
1615		 */
1616		if ((tsd->ts_next = tsd_list) != NULL)
1617			tsd_list->ts_prev = tsd;
1618		tsd_list = tsd;
1619	}
1620
1621	/*
1622	 * Allocate thread local storage and set the value for key
1623	 */
1624	tsd->ts_value = tsd_realloc(tsd->ts_value,
1625	    tsd->ts_nkeys * sizeof (void *),
1626	    key * sizeof (void *));
1627	tsd->ts_nkeys = key;
1628	tsd->ts_value[key - 1] = value;
1629	mutex_exit(&tsd_mutex);
1630
1631	return (0);
1632}
1633
1634
1635/*
1636 * Return the per thread value that was stored with the specified key
1637 *	If necessary, create the key and the value
1638 *	Assumes the caller is protecting *keyp from tsd_destroy
1639 */
1640void *
1641tsd_getcreate(uint_t *keyp, void (*destroy)(void *), void *(*allocate)(void))
1642{
1643	void *value;
1644	uint_t key = *keyp;
1645	struct tsd_thread *tsd = curthread->t_tsd;
1646
1647	if (tsd == NULL)
1648		tsd = curthread->t_tsd = kmem_zalloc(sizeof (*tsd), KM_SLEEP);
1649	if (key && key <= tsd->ts_nkeys && (value = tsd->ts_value[key - 1]))
1650		return (value);
1651	if (key == 0)
1652		tsd_create(keyp, destroy);
1653	(void) tsd_set(*keyp, value = (*allocate)());
1654
1655	return (value);
1656}
1657
1658/*
1659 * Called from thread_exit() to run the destructor function for each tsd
1660 *	Locks out tsd_create and tsd_destroy
1661 *	Assumes that the destructor *DOES NOT* use tsd
1662 */
1663void
1664tsd_exit(void)
1665{
1666	int i;
1667	struct tsd_thread *tsd = curthread->t_tsd;
1668
1669	if (tsd == NULL)
1670		return;
1671
1672	if (tsd->ts_nkeys == 0) {
1673		kmem_free(tsd, sizeof (*tsd));
1674		curthread->t_tsd = NULL;
1675		return;
1676	}
1677
1678	/*
1679	 * lock out tsd_create and tsd_destroy, call
1680	 * the destructor, and mark the value as destroyed.
1681	 */
1682	mutex_enter(&tsd_mutex);
1683
1684	for (i = 0; i < tsd->ts_nkeys; i++) {
1685		if (tsd->ts_value[i] && tsd_destructor[i])
1686			(*tsd_destructor[i])(tsd->ts_value[i]);
1687		tsd->ts_value[i] = NULL;
1688	}
1689
1690	/*
1691	 * remove from linked list of threads with TSD
1692	 */
1693	if (tsd->ts_next)
1694		tsd->ts_next->ts_prev = tsd->ts_prev;
1695	if (tsd->ts_prev)
1696		tsd->ts_prev->ts_next = tsd->ts_next;
1697	if (tsd_list == tsd)
1698		tsd_list = tsd->ts_next;
1699
1700	mutex_exit(&tsd_mutex);
1701
1702	/*
1703	 * free up the TSD
1704	 */
1705	kmem_free(tsd->ts_value, tsd->ts_nkeys * sizeof (void *));
1706	kmem_free(tsd, sizeof (struct tsd_thread));
1707	curthread->t_tsd = NULL;
1708}
1709
1710/*
1711 * realloc
1712 */
1713static void *
1714tsd_realloc(void *old, size_t osize, size_t nsize)
1715{
1716	void *new;
1717
1718	new = kmem_zalloc(nsize, KM_SLEEP);
1719	if (old) {
1720		bcopy(old, new, osize);
1721		kmem_free(old, osize);
1722	}
1723	return (new);
1724}
1725
1726/*
1727 * Return non-zero if an interrupt is being serviced.
1728 */
1729int
1730servicing_interrupt()
1731{
1732	int onintr = 0;
1733
1734	/* Are we an interrupt thread */
1735	if (curthread->t_flag & T_INTR_THREAD)
1736		return (1);
1737	/* Are we servicing a high level interrupt? */
1738	if (CPU_ON_INTR(CPU)) {
1739		kpreempt_disable();
1740		onintr = CPU_ON_INTR(CPU);
1741		kpreempt_enable();
1742	}
1743	return (onintr);
1744}
1745
1746
1747/*
1748 * Change the dispatch priority of a thread in the system.
1749 * Used when raising or lowering a thread's priority.
1750 * (E.g., priority inheritance)
1751 *
1752 * Since threads are queued according to their priority, we
1753 * we must check the thread's state to determine whether it
1754 * is on a queue somewhere. If it is, we've got to:
1755 *
1756 *	o Dequeue the thread.
1757 *	o Change its effective priority.
1758 *	o Enqueue the thread.
1759 *
1760 * Assumptions: The thread whose priority we wish to change
1761 * must be locked before we call thread_change_(e)pri().
1762 * The thread_change(e)pri() function doesn't drop the thread
1763 * lock--that must be done by its caller.
1764 */
1765void
1766thread_change_epri(kthread_t *t, pri_t disp_pri)
1767{
1768	uint_t	state;
1769
1770	ASSERT(THREAD_LOCK_HELD(t));
1771
1772	/*
1773	 * If the inherited priority hasn't actually changed,
1774	 * just return.
1775	 */
1776	if (t->t_epri == disp_pri)
1777		return;
1778
1779	state = t->t_state;
1780
1781	/*
1782	 * If it's not on a queue, change the priority with impunity.
1783	 */
1784	if ((state & (TS_SLEEP | TS_RUN | TS_WAIT)) == 0) {
1785		t->t_epri = disp_pri;
1786		if (state == TS_ONPROC) {
1787			cpu_t *cp = t->t_disp_queue->disp_cpu;
1788
1789			if (t == cp->cpu_dispthread)
1790				cp->cpu_dispatch_pri = DISP_PRIO(t);
1791		}
1792	} else if (state == TS_SLEEP) {
1793		/*
1794		 * Take the thread out of its sleep queue.
1795		 * Change the inherited priority.
1796		 * Re-enqueue the thread.
1797		 * Each synchronization object exports a function
1798		 * to do this in an appropriate manner.
1799		 */
1800		SOBJ_CHANGE_EPRI(t->t_sobj_ops, t, disp_pri);
1801	} else if (state == TS_WAIT) {
1802		/*
1803		 * Re-enqueue a thread on the wait queue if its
1804		 * effective priority needs to change.
1805		 */
1806		if (disp_pri != t->t_epri)
1807			waitq_change_pri(t, disp_pri);
1808	} else {
1809		/*
1810		 * The thread is on a run queue.
1811		 * Note: setbackdq() may not put the thread
1812		 * back on the same run queue where it originally
1813		 * resided.
1814		 */
1815		(void) dispdeq(t);
1816		t->t_epri = disp_pri;
1817		setbackdq(t);
1818	}
1819	schedctl_set_cidpri(t);
1820}
1821
1822/*
1823 * Function: Change the t_pri field of a thread.
1824 * Side Effects: Adjust the thread ordering on a run queue
1825 *		 or sleep queue, if necessary.
1826 * Returns: 1 if the thread was on a run queue, else 0.
1827 */
1828int
1829thread_change_pri(kthread_t *t, pri_t disp_pri, int front)
1830{
1831	uint_t	state;
1832	int	on_rq = 0;
1833
1834	ASSERT(THREAD_LOCK_HELD(t));
1835
1836	state = t->t_state;
1837	THREAD_WILLCHANGE_PRI(t, disp_pri);
1838
1839	/*
1840	 * If it's not on a queue, change the priority with impunity.
1841	 */
1842	if ((state & (TS_SLEEP | TS_RUN | TS_WAIT)) == 0) {
1843		t->t_pri = disp_pri;
1844
1845		if (state == TS_ONPROC) {
1846			cpu_t *cp = t->t_disp_queue->disp_cpu;
1847
1848			if (t == cp->cpu_dispthread)
1849				cp->cpu_dispatch_pri = DISP_PRIO(t);
1850		}
1851	} else if (state == TS_SLEEP) {
1852		/*
1853		 * If the priority has changed, take the thread out of
1854		 * its sleep queue and change the priority.
1855		 * Re-enqueue the thread.
1856		 * Each synchronization object exports a function
1857		 * to do this in an appropriate manner.
1858		 */
1859		if (disp_pri != t->t_pri)
1860			SOBJ_CHANGE_PRI(t->t_sobj_ops, t, disp_pri);
1861	} else if (state == TS_WAIT) {
1862		/*
1863		 * Re-enqueue a thread on the wait queue if its
1864		 * priority needs to change.
1865		 */
1866		if (disp_pri != t->t_pri)
1867			waitq_change_pri(t, disp_pri);
1868	} else {
1869		/*
1870		 * The thread is on a run queue.
1871		 * Note: setbackdq() may not put the thread
1872		 * back on the same run queue where it originally
1873		 * resided.
1874		 *
1875		 * We still requeue the thread even if the priority
1876		 * is unchanged to preserve round-robin (and other)
1877		 * effects between threads of the same priority.
1878		 */
1879		on_rq = dispdeq(t);
1880		ASSERT(on_rq);
1881		t->t_pri = disp_pri;
1882		if (front) {
1883			setfrontdq(t);
1884		} else {
1885			setbackdq(t);
1886		}
1887	}
1888	schedctl_set_cidpri(t);
1889	return (on_rq);
1890}
1891
1892/*
1893 * Tunable kmem_stackinfo is set, fill the kernel thread stack with a
1894 * specific pattern.
1895 */
1896static void
1897stkinfo_begin(kthread_t *t)
1898{
1899	caddr_t	start;	/* stack start */
1900	caddr_t	end;	/* stack end  */
1901	uint64_t *ptr;	/* pattern pointer */
1902
1903	/*
1904	 * Stack grows up or down, see thread_create(),
1905	 * compute stack memory area start and end (start < end).
1906	 */
1907	if (t->t_stk > t->t_stkbase) {
1908		/* stack grows down */
1909		start = t->t_stkbase;
1910		end = t->t_stk;
1911	} else {
1912		/* stack grows up */
1913		start = t->t_stk;
1914		end = t->t_stkbase;
1915	}
1916
1917	/*
1918	 * Stackinfo pattern size is 8 bytes. Ensure proper 8 bytes
1919	 * alignement for start and end in stack area boundaries
1920	 * (protection against corrupt t_stkbase/t_stk data).
1921	 */
1922	if ((((uintptr_t)start) & 0x7) != 0) {
1923		start = (caddr_t)((((uintptr_t)start) & (~0x7)) + 8);
1924	}
1925	end = (caddr_t)(((uintptr_t)end) & (~0x7));
1926
1927	if ((end <= start) || (end - start) > (1024 * 1024)) {
1928		/* negative or stack size > 1 meg, assume bogus */
1929		return;
1930	}
1931
1932	/* fill stack area with a pattern (instead of zeros) */
1933	ptr = (uint64_t *)((void *)start);
1934	while (ptr < (uint64_t *)((void *)end)) {
1935		*ptr++ = KMEM_STKINFO_PATTERN;
1936	}
1937}
1938
1939
1940/*
1941 * Tunable kmem_stackinfo is set, create stackinfo log if doesn't already exist,
1942 * compute the percentage of kernel stack really used, and set in the log
1943 * if it's the latest highest percentage.
1944 */
1945static void
1946stkinfo_end(kthread_t *t)
1947{
1948	caddr_t	start;	/* stack start */
1949	caddr_t	end;	/* stack end  */
1950	uint64_t *ptr;	/* pattern pointer */
1951	size_t stksz;	/* stack size */
1952	size_t smallest = 0;
1953	size_t percent = 0;
1954	uint_t index = 0;
1955	uint_t i;
1956	static size_t smallest_percent = (size_t)-1;
1957	static uint_t full = 0;
1958
1959	/* create the stackinfo log, if doesn't already exist */
1960	mutex_enter(&kmem_stkinfo_lock);
1961	if (kmem_stkinfo_log == NULL) {
1962		kmem_stkinfo_log = (kmem_stkinfo_t *)
1963		    kmem_zalloc(KMEM_STKINFO_LOG_SIZE *
1964		    (sizeof (kmem_stkinfo_t)), KM_NOSLEEP);
1965		if (kmem_stkinfo_log == NULL) {
1966			mutex_exit(&kmem_stkinfo_lock);
1967			return;
1968		}
1969	}
1970	mutex_exit(&kmem_stkinfo_lock);
1971
1972	/*
1973	 * Stack grows up or down, see thread_create(),
1974	 * compute stack memory area start and end (start < end).
1975	 */
1976	if (t->t_stk > t->t_stkbase) {
1977		/* stack grows down */
1978		start = t->t_stkbase;
1979		end = t->t_stk;
1980	} else {
1981		/* stack grows up */
1982		start = t->t_stk;
1983		end = t->t_stkbase;
1984	}
1985
1986	/* stack size as found in kthread_t */
1987	stksz = end - start;
1988
1989	/*
1990	 * Stackinfo pattern size is 8 bytes. Ensure proper 8 bytes
1991	 * alignement for start and end in stack area boundaries
1992	 * (protection against corrupt t_stkbase/t_stk data).
1993	 */
1994	if ((((uintptr_t)start) & 0x7) != 0) {
1995		start = (caddr_t)((((uintptr_t)start) & (~0x7)) + 8);
1996	}
1997	end = (caddr_t)(((uintptr_t)end) & (~0x7));
1998
1999	if ((end <= start) || (end - start) > (1024 * 1024)) {
2000		/* negative or stack size > 1 meg, assume bogus */
2001		return;
2002	}
2003
2004	/* search until no pattern in the stack */
2005	if (t->t_stk > t->t_stkbase) {
2006		/* stack grows down */
2007#if defined(__i386) || defined(__amd64)
2008		/*
2009		 * 6 longs are pushed on stack, see thread_load(). Skip
2010		 * them, so if kthread has never run, percent is zero.
2011		 * 8 bytes alignement is preserved for a 32 bit kernel,
2012		 * 6 x 4 = 24, 24 is a multiple of 8.
2013		 *
2014		 */
2015		end -= (6 * sizeof (long));
2016#endif
2017		ptr = (uint64_t *)((void *)start);
2018		while (ptr < (uint64_t *)((void *)end)) {
2019			if (*ptr != KMEM_STKINFO_PATTERN) {
2020				percent = stkinfo_percent(end,
2021				    start, (caddr_t)ptr);
2022				break;
2023			}
2024			ptr++;
2025		}
2026	} else {
2027		/* stack grows up */
2028		ptr = (uint64_t *)((void *)end);
2029		ptr--;
2030		while (ptr >= (uint64_t *)((void *)start)) {
2031			if (*ptr != KMEM_STKINFO_PATTERN) {
2032				percent = stkinfo_percent(start,
2033				    end, (caddr_t)ptr);
2034				break;
2035			}
2036			ptr--;
2037		}
2038	}
2039
2040	DTRACE_PROBE3(stack__usage, kthread_t *, t,
2041	    size_t, stksz, size_t, percent);
2042
2043	if (percent == 0) {
2044		return;
2045	}
2046
2047	mutex_enter(&kmem_stkinfo_lock);
2048	if (full == KMEM_STKINFO_LOG_SIZE && percent < smallest_percent) {
2049		/*
2050		 * The log is full and already contains the highest values
2051		 */
2052		mutex_exit(&kmem_stkinfo_lock);
2053		return;
2054	}
2055
2056	/* keep a log of the highest used stack */
2057	for (i = 0; i < KMEM_STKINFO_LOG_SIZE; i++) {
2058		if (kmem_stkinfo_log[i].percent == 0) {
2059			index = i;
2060			full++;
2061			break;
2062		}
2063		if (smallest == 0) {
2064			smallest = kmem_stkinfo_log[i].percent;
2065			index = i;
2066			continue;
2067		}
2068		if (kmem_stkinfo_log[i].percent < smallest) {
2069			smallest = kmem_stkinfo_log[i].percent;
2070			index = i;
2071		}
2072	}
2073
2074	if (percent >= kmem_stkinfo_log[index].percent) {
2075		kmem_stkinfo_log[index].kthread = (caddr_t)t;
2076		kmem_stkinfo_log[index].t_startpc = (caddr_t)t->t_startpc;
2077		kmem_stkinfo_log[index].start = start;
2078		kmem_stkinfo_log[index].stksz = stksz;
2079		kmem_stkinfo_log[index].percent = percent;
2080		kmem_stkinfo_log[index].t_tid = t->t_tid;
2081		kmem_stkinfo_log[index].cmd[0] = '\0';
2082		if (t->t_tid != 0) {
2083			stksz = strlen((t->t_procp)->p_user.u_comm);
2084			if (stksz >= KMEM_STKINFO_STR_SIZE) {
2085				stksz = KMEM_STKINFO_STR_SIZE - 1;
2086				kmem_stkinfo_log[index].cmd[stksz] = '\0';
2087			} else {
2088				stksz += 1;
2089			}
2090			(void) memcpy(kmem_stkinfo_log[index].cmd,
2091			    (t->t_procp)->p_user.u_comm, stksz);
2092		}
2093		if (percent < smallest_percent) {
2094			smallest_percent = percent;
2095		}
2096	}
2097	mutex_exit(&kmem_stkinfo_lock);
2098}
2099
2100/*
2101 * Tunable kmem_stackinfo is set, compute stack utilization percentage.
2102 */
2103static size_t
2104stkinfo_percent(caddr_t t_stk, caddr_t t_stkbase, caddr_t sp)
2105{
2106	size_t percent;
2107	size_t s;
2108
2109	if (t_stk > t_stkbase) {
2110		/* stack grows down */
2111		if (sp > t_stk) {
2112			return (0);
2113		}
2114		if (sp < t_stkbase) {
2115			return (100);
2116		}
2117		percent = t_stk - sp + 1;
2118		s = t_stk - t_stkbase + 1;
2119	} else {
2120		/* stack grows up */
2121		if (sp < t_stk) {
2122			return (0);
2123		}
2124		if (sp > t_stkbase) {
2125			return (100);
2126		}
2127		percent = sp - t_stk + 1;
2128		s = t_stkbase - t_stk + 1;
2129	}
2130	percent = ((100 * percent) / s) + 1;
2131	if (percent > 100) {
2132		percent = 100;
2133	}
2134	return (percent);
2135}
2136
2137/*
2138 * NOTE: This will silently truncate a name > THREAD_NAME_MAX - 1 characters
2139 * long.  It is expected that callers (acting on behalf of userland clients)
2140 * will perform any required checks to return the correct error semantics.
2141 * It is also expected callers on behalf of userland clients have done
2142 * any necessary permission checks.
2143 */
2144int
2145thread_setname(kthread_t *t, const char *name)
2146{
2147	char *buf = NULL;
2148
2149	/*
2150	 * We optimistically assume that a thread's name will only be set
2151	 * once and so allocate memory in preparation of setting t_name.
2152	 * If it turns out a name has already been set, we just discard (free)
2153	 * the buffer we just allocated and reuse the current buffer
2154	 * (as all should be THREAD_NAME_MAX large).
2155	 *
2156	 * Such an arrangement means over the lifetime of a kthread_t, t_name
2157	 * is either NULL or has one value (the address of the buffer holding
2158	 * the current thread name).   The assumption is that most kthread_t
2159	 * instances will not have a name assigned, so dynamically allocating
2160	 * the memory should minimize the footprint of this feature, but by
2161	 * having the buffer persist for the life of the thread, it simplifies
2162	 * usage in highly constrained situations (e.g. dtrace).
2163	 */
2164	if (name != NULL && name[0] != '\0') {
2165		for (size_t i = 0; name[i] != '\0'; i++) {
2166			if (!isprint(name[i]))
2167				return (EINVAL);
2168		}
2169
2170		buf = kmem_zalloc(THREAD_NAME_MAX, KM_SLEEP);
2171		(void) strlcpy(buf, name, THREAD_NAME_MAX);
2172	}
2173
2174	mutex_enter(&ttoproc(t)->p_lock);
2175	if (t->t_name == NULL) {
2176		t->t_name = buf;
2177	} else {
2178		if (buf != NULL) {
2179			(void) strlcpy(t->t_name, name, THREAD_NAME_MAX);
2180			kmem_free(buf, THREAD_NAME_MAX);
2181		} else {
2182			bzero(t->t_name, THREAD_NAME_MAX);
2183		}
2184	}
2185	mutex_exit(&ttoproc(t)->p_lock);
2186	return (0);
2187}
2188
2189int
2190thread_vsetname(kthread_t *t, const char *fmt, ...)
2191{
2192	char name[THREAD_NAME_MAX];
2193	va_list va;
2194	int rc;
2195
2196	va_start(va, fmt);
2197	rc = vsnprintf(name, sizeof (name), fmt, va);
2198	va_end(va);
2199
2200	if (rc < 0)
2201		return (EINVAL);
2202
2203	if (rc >= sizeof (name))
2204		return (ENAMETOOLONG);
2205
2206	return (thread_setname(t, name));
2207}
2208