xref: /illumos-gate/usr/src/uts/common/os/lwp.c (revision d32efdad)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 #include <sys/param.h>
28 #include <sys/types.h>
29 #include <sys/sysmacros.h>
30 #include <sys/systm.h>
31 #include <sys/thread.h>
32 #include <sys/proc.h>
33 #include <sys/task.h>
34 #include <sys/project.h>
35 #include <sys/signal.h>
36 #include <sys/errno.h>
37 #include <sys/vmparam.h>
38 #include <sys/stack.h>
39 #include <sys/procfs.h>
40 #include <sys/prsystm.h>
41 #include <sys/cpuvar.h>
42 #include <sys/kmem.h>
43 #include <sys/vtrace.h>
44 #include <sys/door.h>
45 #include <vm/seg_kp.h>
46 #include <sys/debug.h>
47 #include <sys/tnf.h>
48 #include <sys/schedctl.h>
49 #include <sys/poll.h>
50 #include <sys/copyops.h>
51 #include <sys/lwp_upimutex_impl.h>
52 #include <sys/cpupart.h>
53 #include <sys/lgrp.h>
54 #include <sys/rctl.h>
55 #include <sys/contract_impl.h>
56 #include <sys/cpc_impl.h>
57 #include <sys/sdt.h>
58 #include <sys/cmn_err.h>
59 #include <sys/brand.h>
60 #include <sys/cyclic.h>
61 #include <sys/pool.h>
62 
63 /* hash function for the lwpid hash table, p->p_tidhash[] */
64 #define	TIDHASH(tid, hash_sz)	((tid) & ((hash_sz) - 1))
65 
66 void *segkp_lwp;		/* cookie for pool of segkp resources */
67 extern void reapq_move_lq_to_tq(kthread_t *);
68 extern void freectx_ctx(struct ctxop *);
69 
70 /*
71  * Create a kernel thread associated with a particular system process.  Give
72  * it an LWP so that microstate accounting will be available for it.
73  */
74 kthread_t *
75 lwp_kernel_create(proc_t *p, void (*proc)(), void *arg, int state, pri_t pri)
76 {
77 	klwp_t *lwp;
78 
79 	VERIFY((p->p_flag & SSYS) != 0);
80 
81 	lwp = lwp_create(proc, arg, 0, p, state, pri, &t0.t_hold, syscid, 0);
82 
83 	VERIFY(lwp != NULL);
84 
85 	return (lwptot(lwp));
86 }
87 
88 /*
89  * Create a thread that appears to be stopped at sys_rtt.
90  */
91 klwp_t *
92 lwp_create(void (*proc)(), caddr_t arg, size_t len, proc_t *p,
93     int state, int pri, const k_sigset_t *smask, int cid, id_t lwpid)
94 {
95 	klwp_t *lwp = NULL;
96 	kthread_t *t;
97 	kthread_t *tx;
98 	cpupart_t *oldpart = NULL;
99 	size_t	stksize;
100 	caddr_t lwpdata = NULL;
101 	processorid_t	binding;
102 	int err = 0;
103 	kproject_t *oldkpj, *newkpj;
104 	void *bufp = NULL;
105 	klwp_t *curlwp;
106 	lwpent_t *lep;
107 	lwpdir_t *old_dir = NULL;
108 	uint_t old_dirsz = 0;
109 	tidhash_t *old_hash = NULL;
110 	uint_t old_hashsz = 0;
111 	ret_tidhash_t *ret_tidhash = NULL;
112 	int i;
113 	int rctlfail = 0;
114 	boolean_t branded = 0;
115 	struct ctxop *ctx = NULL;
116 
117 	ASSERT(cid != sysdccid);	/* system threads must start in SYS */
118 
119 	ASSERT(p != &p0);		/* No new LWPs in p0. */
120 
121 	mutex_enter(&p->p_lock);
122 	mutex_enter(&p->p_zone->zone_nlwps_lock);
123 	/*
124 	 * don't enforce rctl limits on system processes
125 	 */
126 	if (!CLASS_KERNEL(cid)) {
127 		if (p->p_task->tk_nlwps >= p->p_task->tk_nlwps_ctl)
128 			if (rctl_test(rc_task_lwps, p->p_task->tk_rctls, p,
129 			    1, 0) & RCT_DENY)
130 				rctlfail = 1;
131 		if (p->p_task->tk_proj->kpj_nlwps >=
132 		    p->p_task->tk_proj->kpj_nlwps_ctl)
133 			if (rctl_test(rc_project_nlwps,
134 			    p->p_task->tk_proj->kpj_rctls, p, 1, 0)
135 			    & RCT_DENY)
136 				rctlfail = 1;
137 		if (p->p_zone->zone_nlwps >= p->p_zone->zone_nlwps_ctl)
138 			if (rctl_test(rc_zone_nlwps, p->p_zone->zone_rctls, p,
139 			    1, 0) & RCT_DENY)
140 				rctlfail = 1;
141 	}
142 	if (rctlfail) {
143 		mutex_exit(&p->p_zone->zone_nlwps_lock);
144 		mutex_exit(&p->p_lock);
145 		return (NULL);
146 	}
147 	p->p_task->tk_nlwps++;
148 	p->p_task->tk_proj->kpj_nlwps++;
149 	p->p_zone->zone_nlwps++;
150 	mutex_exit(&p->p_zone->zone_nlwps_lock);
151 	mutex_exit(&p->p_lock);
152 
153 	curlwp = ttolwp(curthread);
154 	if (curlwp == NULL || (stksize = curlwp->lwp_childstksz) == 0)
155 		stksize = lwp_default_stksize;
156 
157 	if (CLASS_KERNEL(cid)) {
158 		/*
159 		 * Since we are creating an LWP in an SSYS process, we do not
160 		 * inherit anything from the current thread's LWP.  We set
161 		 * stksize and lwpdata to 0 in order to let thread_create()
162 		 * allocate a regular kernel thread stack for this thread.
163 		 */
164 		curlwp = NULL;
165 		stksize = 0;
166 		lwpdata = NULL;
167 
168 	} else if (stksize == lwp_default_stksize) {
169 		/*
170 		 * Try to reuse an <lwp,stack> from the LWP deathrow.
171 		 */
172 		if (lwp_reapcnt > 0) {
173 			mutex_enter(&reaplock);
174 			if ((t = lwp_deathrow) != NULL) {
175 				ASSERT(t->t_swap);
176 				lwp_deathrow = t->t_forw;
177 				lwp_reapcnt--;
178 				lwpdata = t->t_swap;
179 				lwp = t->t_lwp;
180 				ctx = t->t_ctx;
181 				t->t_swap = NULL;
182 				t->t_lwp = NULL;
183 				t->t_ctx = NULL;
184 				reapq_move_lq_to_tq(t);
185 			}
186 			mutex_exit(&reaplock);
187 			if (lwp != NULL) {
188 				lwp_stk_fini(lwp);
189 			}
190 			if (ctx != NULL) {
191 				freectx_ctx(ctx);
192 			}
193 		}
194 		if (lwpdata == NULL &&
195 		    (lwpdata = (caddr_t)segkp_cache_get(segkp_lwp)) == NULL) {
196 			mutex_enter(&p->p_lock);
197 			mutex_enter(&p->p_zone->zone_nlwps_lock);
198 			p->p_task->tk_nlwps--;
199 			p->p_task->tk_proj->kpj_nlwps--;
200 			p->p_zone->zone_nlwps--;
201 			mutex_exit(&p->p_zone->zone_nlwps_lock);
202 			mutex_exit(&p->p_lock);
203 			return (NULL);
204 		}
205 	} else {
206 		stksize = roundup(stksize, PAGESIZE);
207 		if ((lwpdata = (caddr_t)segkp_get(segkp, stksize,
208 		    (KPD_NOWAIT | KPD_HASREDZONE | KPD_LOCKED))) == NULL) {
209 			mutex_enter(&p->p_lock);
210 			mutex_enter(&p->p_zone->zone_nlwps_lock);
211 			p->p_task->tk_nlwps--;
212 			p->p_task->tk_proj->kpj_nlwps--;
213 			p->p_zone->zone_nlwps--;
214 			mutex_exit(&p->p_zone->zone_nlwps_lock);
215 			mutex_exit(&p->p_lock);
216 			return (NULL);
217 		}
218 	}
219 
220 	/*
221 	 * Create a thread, initializing the stack pointer
222 	 */
223 	t = thread_create(lwpdata, stksize, NULL, NULL, 0, p, TS_STOPPED, pri);
224 
225 	/*
226 	 * If a non-NULL stack base is passed in, thread_create() assumes
227 	 * that the stack might be statically allocated (as opposed to being
228 	 * allocated from segkp), and so it does not set t_swap.  Since
229 	 * the lwpdata was allocated from segkp, we must set t_swap to point
230 	 * to it ourselves.
231 	 *
232 	 * This would be less confusing if t_swap had a better name; it really
233 	 * indicates that the stack is allocated from segkp, regardless of
234 	 * whether or not it is swappable.
235 	 */
236 	if (lwpdata != NULL) {
237 		ASSERT(!CLASS_KERNEL(cid));
238 		ASSERT(t->t_swap == NULL);
239 		t->t_swap = lwpdata;	/* Start of page-able data */
240 	}
241 
242 	/*
243 	 * If the stack and lwp can be reused, mark the thread as such.
244 	 * When we get to reapq_add() from resume_from_zombie(), these
245 	 * threads will go onto lwp_deathrow instead of thread_deathrow.
246 	 */
247 	if (!CLASS_KERNEL(cid) && stksize == lwp_default_stksize)
248 		t->t_flag |= T_LWPREUSE;
249 
250 	if (lwp == NULL)
251 		lwp = kmem_cache_alloc(lwp_cache, KM_SLEEP);
252 	bzero(lwp, sizeof (*lwp));
253 	t->t_lwp = lwp;
254 
255 	t->t_hold = *smask;
256 	lwp->lwp_thread = t;
257 	lwp->lwp_procp = p;
258 	lwp->lwp_sigaltstack.ss_flags = SS_DISABLE;
259 	if (curlwp != NULL && curlwp->lwp_childstksz != 0)
260 		lwp->lwp_childstksz = curlwp->lwp_childstksz;
261 
262 	t->t_stk = lwp_stk_init(lwp, t->t_stk);
263 	thread_load(t, proc, arg, len);
264 
265 	/*
266 	 * Allocate the SIGPROF buffer if ITIMER_REALPROF is in effect.
267 	 */
268 	if (p->p_rprof_cyclic != CYCLIC_NONE)
269 		t->t_rprof = kmem_zalloc(sizeof (struct rprof), KM_SLEEP);
270 
271 	if (cid != NOCLASS)
272 		(void) CL_ALLOC(&bufp, cid, KM_SLEEP);
273 
274 	/*
275 	 * Allocate an lwp directory entry for the new lwp.
276 	 */
277 	lep = kmem_zalloc(sizeof (*lep), KM_SLEEP);
278 
279 	mutex_enter(&p->p_lock);
280 grow:
281 	/*
282 	 * Grow the lwp (thread) directory and lwpid hash table if necessary.
283 	 * A note on the growth algorithm:
284 	 *	The new lwp directory size is computed as:
285 	 *		new = 2 * old + 2
286 	 *	Starting with an initial size of 2 (see exec_common()),
287 	 *	this yields numbers that are a power of two minus 2:
288 	 *		2, 6, 14, 30, 62, 126, 254, 510, 1022, ...
289 	 *	The size of the lwpid hash table must be a power of two
290 	 *	and must be commensurate in size with the lwp directory
291 	 *	so that hash bucket chains remain short.  Therefore,
292 	 *	the lwpid hash table size is computed as:
293 	 *		hashsz = (dirsz + 2) / 2
294 	 *	which leads to these hash table sizes corresponding to
295 	 *	the above directory sizes:
296 	 *		2, 4, 8, 16, 32, 64, 128, 256, 512, ...
297 	 * A note on growing the hash table:
298 	 *	For performance reasons, code in lwp_unpark() does not
299 	 *	acquire curproc->p_lock when searching the hash table.
300 	 *	Rather, it calls lwp_hash_lookup_and_lock() which
301 	 *	acquires only the individual hash bucket lock, taking
302 	 *	care to deal with reallocation of the hash table
303 	 *	during the time it takes to acquire the lock.
304 	 *
305 	 *	This is sufficient to protect the integrity of the
306 	 *	hash table, but it requires us to acquire all of the
307 	 *	old hash bucket locks before growing the hash table
308 	 *	and to release them afterwards.  It also requires us
309 	 *	not to free the old hash table because some thread
310 	 *	in lwp_hash_lookup_and_lock() might still be trying
311 	 *	to acquire the old bucket lock.
312 	 *
313 	 *	So we adopt the tactic of keeping all of the retired
314 	 *	hash tables on a linked list, so they can be safely
315 	 *	freed when the process exits or execs.
316 	 *
317 	 *	Because the hash table grows in powers of two, the
318 	 *	total size of all of the hash tables will be slightly
319 	 *	less than twice the size of the largest hash table.
320 	 */
321 	while (p->p_lwpfree == NULL) {
322 		uint_t dirsz = p->p_lwpdir_sz;
323 		lwpdir_t *new_dir;
324 		uint_t new_dirsz;
325 		lwpdir_t *ldp;
326 		tidhash_t *new_hash;
327 		uint_t new_hashsz;
328 
329 		mutex_exit(&p->p_lock);
330 
331 		/*
332 		 * Prepare to remember the old p_tidhash for later
333 		 * kmem_free()ing when the process exits or execs.
334 		 */
335 		if (ret_tidhash == NULL)
336 			ret_tidhash = kmem_zalloc(sizeof (ret_tidhash_t),
337 			    KM_SLEEP);
338 		if (old_dir != NULL)
339 			kmem_free(old_dir, old_dirsz * sizeof (*old_dir));
340 		if (old_hash != NULL)
341 			kmem_free(old_hash, old_hashsz * sizeof (*old_hash));
342 
343 		new_dirsz = 2 * dirsz + 2;
344 		new_dir = kmem_zalloc(new_dirsz * sizeof (lwpdir_t), KM_SLEEP);
345 		for (ldp = new_dir, i = 1; i < new_dirsz; i++, ldp++)
346 			ldp->ld_next = ldp + 1;
347 		new_hashsz = (new_dirsz + 2) / 2;
348 		new_hash = kmem_zalloc(new_hashsz * sizeof (tidhash_t),
349 		    KM_SLEEP);
350 
351 		mutex_enter(&p->p_lock);
352 		if (p == curproc)
353 			prbarrier(p);
354 
355 		if (dirsz != p->p_lwpdir_sz || p->p_lwpfree != NULL) {
356 			/*
357 			 * Someone else beat us to it or some lwp exited.
358 			 * Set up to free our memory and take a lap.
359 			 */
360 			old_dir = new_dir;
361 			old_dirsz = new_dirsz;
362 			old_hash = new_hash;
363 			old_hashsz = new_hashsz;
364 		} else {
365 			/*
366 			 * For the benefit of lwp_hash_lookup_and_lock(),
367 			 * called from lwp_unpark(), which searches the
368 			 * tid hash table without acquiring p->p_lock,
369 			 * we must acquire all of the tid hash table
370 			 * locks before replacing p->p_tidhash.
371 			 */
372 			old_hash = p->p_tidhash;
373 			old_hashsz = p->p_tidhash_sz;
374 			for (i = 0; i < old_hashsz; i++) {
375 				mutex_enter(&old_hash[i].th_lock);
376 				mutex_enter(&new_hash[i].th_lock);
377 			}
378 
379 			/*
380 			 * We simply hash in all of the old directory entries.
381 			 * This works because the old directory has no empty
382 			 * slots and the new hash table starts out empty.
383 			 * This reproduces the original directory ordering
384 			 * (required for /proc directory semantics).
385 			 */
386 			old_dir = p->p_lwpdir;
387 			old_dirsz = p->p_lwpdir_sz;
388 			p->p_lwpdir = new_dir;
389 			p->p_lwpfree = new_dir;
390 			p->p_lwpdir_sz = new_dirsz;
391 			for (ldp = old_dir, i = 0; i < old_dirsz; i++, ldp++)
392 				lwp_hash_in(p, ldp->ld_entry,
393 				    new_hash, new_hashsz, 0);
394 
395 			/*
396 			 * Remember the old hash table along with all
397 			 * of the previously-remembered hash tables.
398 			 * We will free them at process exit or exec.
399 			 */
400 			ret_tidhash->rth_tidhash = old_hash;
401 			ret_tidhash->rth_tidhash_sz = old_hashsz;
402 			ret_tidhash->rth_next = p->p_ret_tidhash;
403 			p->p_ret_tidhash = ret_tidhash;
404 
405 			/*
406 			 * Now establish the new tid hash table.
407 			 * As soon as we assign p->p_tidhash,
408 			 * code in lwp_unpark() can start using it.
409 			 */
410 			membar_producer();
411 			p->p_tidhash = new_hash;
412 
413 			/*
414 			 * It is necessary that p_tidhash reach global
415 			 * visibility before p_tidhash_sz.  Otherwise,
416 			 * code in lwp_hash_lookup_and_lock() could
417 			 * index into the old p_tidhash using the new
418 			 * p_tidhash_sz and thereby access invalid data.
419 			 */
420 			membar_producer();
421 			p->p_tidhash_sz = new_hashsz;
422 
423 			/*
424 			 * Release the locks; allow lwp_unpark() to carry on.
425 			 */
426 			for (i = 0; i < old_hashsz; i++) {
427 				mutex_exit(&old_hash[i].th_lock);
428 				mutex_exit(&new_hash[i].th_lock);
429 			}
430 
431 			/*
432 			 * Avoid freeing these objects below.
433 			 */
434 			ret_tidhash = NULL;
435 			old_hash = NULL;
436 			old_hashsz = 0;
437 		}
438 	}
439 
440 	/*
441 	 * Block the process against /proc while we manipulate p->p_tlist,
442 	 * unless lwp_create() was called by /proc for the PCAGENT operation.
443 	 * We want to do this early enough so that we don't drop p->p_lock
444 	 * until the thread is put on the p->p_tlist.
445 	 */
446 	if (p == curproc) {
447 		prbarrier(p);
448 		/*
449 		 * If the current lwp has been requested to stop, do so now.
450 		 * Otherwise we have a race condition between /proc attempting
451 		 * to stop the process and this thread creating a new lwp
452 		 * that was not seen when the /proc PCSTOP request was issued.
453 		 * We rely on stop() to call prbarrier(p) before returning.
454 		 */
455 		while ((curthread->t_proc_flag & TP_PRSTOP) &&
456 		    !ttolwp(curthread)->lwp_nostop) {
457 			/*
458 			 * We called pool_barrier_enter() before calling
459 			 * here to lwp_create(). We have to call
460 			 * pool_barrier_exit() before stopping.
461 			 */
462 			pool_barrier_exit();
463 			prbarrier(p);
464 			stop(PR_REQUESTED, 0);
465 			/*
466 			 * And we have to repeat the call to
467 			 * pool_barrier_enter after stopping.
468 			 */
469 			pool_barrier_enter();
470 			prbarrier(p);
471 		}
472 
473 		/*
474 		 * If process is exiting, there could be a race between
475 		 * the agent lwp creation and the new lwp currently being
476 		 * created. So to prevent this race lwp creation is failed
477 		 * if the process is exiting.
478 		 */
479 		if (p->p_flag & (SEXITLWPS|SKILLED)) {
480 			err = 1;
481 			goto error;
482 		}
483 
484 		/*
485 		 * Since we might have dropped p->p_lock, the
486 		 * lwp directory free list might have changed.
487 		 */
488 		if (p->p_lwpfree == NULL)
489 			goto grow;
490 	}
491 
492 	kpreempt_disable();	/* can't grab cpu_lock here */
493 
494 	/*
495 	 * Inherit processor and processor set bindings from curthread.
496 	 *
497 	 * For kernel LWPs, we do not inherit processor set bindings at
498 	 * process creation time (i.e. when p != curproc).  After the
499 	 * kernel process is created, any subsequent LWPs must be created
500 	 * by threads in the kernel process, at which point we *will*
501 	 * inherit processor set bindings.
502 	 */
503 	if (CLASS_KERNEL(cid) && p != curproc) {
504 		t->t_bind_cpu = binding = PBIND_NONE;
505 		t->t_cpupart = oldpart = &cp_default;
506 		t->t_bind_pset = PS_NONE;
507 		t->t_bindflag = (uchar_t)default_binding_mode;
508 	} else {
509 		binding = curthread->t_bind_cpu;
510 		t->t_bind_cpu = binding;
511 		oldpart = t->t_cpupart;
512 		t->t_cpupart = curthread->t_cpupart;
513 		t->t_bind_pset = curthread->t_bind_pset;
514 		t->t_bindflag = curthread->t_bindflag |
515 		    (uchar_t)default_binding_mode;
516 	}
517 
518 	/*
519 	 * thread_create() initializes this thread's home lgroup to the root.
520 	 * Choose a more suitable lgroup, since this thread is associated
521 	 * with an lwp.
522 	 */
523 	ASSERT(oldpart != NULL);
524 	if (binding != PBIND_NONE && t->t_affinitycnt == 0) {
525 		t->t_bound_cpu = cpu[binding];
526 		if (t->t_lpl != t->t_bound_cpu->cpu_lpl)
527 			lgrp_move_thread(t, t->t_bound_cpu->cpu_lpl, 1);
528 	} else if (CLASS_KERNEL(cid)) {
529 		/*
530 		 * For kernel threads, assign ourselves to the root lgrp.
531 		 */
532 		lgrp_move_thread(t,
533 		    &curthread->t_cpupart->cp_lgrploads[LGRP_ROOTID], 1);
534 	} else {
535 		lgrp_move_thread(t, lgrp_choose(t, t->t_cpupart), 1);
536 	}
537 
538 	kpreempt_enable();
539 
540 	/*
541 	 * make sure lpl points to our own partition
542 	 */
543 	ASSERT(t->t_lpl >= t->t_cpupart->cp_lgrploads);
544 	ASSERT(t->t_lpl < t->t_cpupart->cp_lgrploads +
545 	    t->t_cpupart->cp_nlgrploads);
546 
547 	/*
548 	 * If we're creating a new process, then inherit the project from our
549 	 * parent. If we're only creating an additional lwp then use the
550 	 * project pointer of the target process.
551 	 */
552 	if (p->p_task == NULL)
553 		newkpj = ttoproj(curthread);
554 	else
555 		newkpj = p->p_task->tk_proj;
556 
557 	/*
558 	 * It is safe to point the thread to the new project without holding it
559 	 * since we're holding the target process' p_lock here and therefore
560 	 * we're guaranteed that it will not move to another project.
561 	 */
562 	oldkpj = ttoproj(t);
563 	if (newkpj != oldkpj) {
564 		t->t_proj = newkpj;
565 		(void) project_hold(newkpj);
566 		project_rele(oldkpj);
567 	}
568 
569 	if (cid != NOCLASS) {
570 		/*
571 		 * If the lwp is being created in the current process
572 		 * and matches the current thread's scheduling class,
573 		 * we should propagate the current thread's scheduling
574 		 * parameters by calling CL_FORK.  Otherwise just use
575 		 * the defaults by calling CL_ENTERCLASS.
576 		 */
577 		if (p != curproc || curthread->t_cid != cid) {
578 			err = CL_ENTERCLASS(t, cid, NULL, NULL, bufp);
579 			t->t_pri = pri;	/* CL_ENTERCLASS may have changed it */
580 			/*
581 			 * We don't call schedctl_set_cidpri(t) here
582 			 * because the schedctl data is not yet set
583 			 * up for the newly-created lwp.
584 			 */
585 		} else {
586 			t->t_clfuncs = &(sclass[cid].cl_funcs->thread);
587 			err = CL_FORK(curthread, t, bufp);
588 			t->t_cid = cid;
589 		}
590 		if (err)
591 			goto error;
592 		else
593 			bufp = NULL;
594 	}
595 
596 	/*
597 	 * If we were given an lwpid then use it, else allocate one.
598 	 */
599 	if (lwpid != 0)
600 		t->t_tid = lwpid;
601 	else {
602 		/*
603 		 * lwp/thread id 0 is never valid; reserved for special checks.
604 		 * lwp/thread id 1 is reserved for the main thread.
605 		 * Start again at 2 when INT_MAX has been reached
606 		 * (id_t is a signed 32-bit integer).
607 		 */
608 		id_t prev_id = p->p_lwpid;	/* last allocated tid */
609 
610 		do {			/* avoid lwpid duplication */
611 			if (p->p_lwpid == INT_MAX) {
612 				p->p_flag |= SLWPWRAP;
613 				p->p_lwpid = 1;
614 			}
615 			if ((t->t_tid = ++p->p_lwpid) == prev_id) {
616 				/*
617 				 * All lwpids are allocated; fail the request.
618 				 */
619 				err = 1;
620 				goto error;
621 			}
622 			/*
623 			 * We only need to worry about colliding with an id
624 			 * that's already in use if this process has
625 			 * cycled through all available lwp ids.
626 			 */
627 			if ((p->p_flag & SLWPWRAP) == 0)
628 				break;
629 		} while (lwp_hash_lookup(p, t->t_tid) != NULL);
630 	}
631 
632 	/*
633 	 * If this is a branded process, let the brand do any necessary lwp
634 	 * initialization.
635 	 */
636 	if (PROC_IS_BRANDED(p)) {
637 		if (BROP(p)->b_initlwp(lwp)) {
638 			err = 1;
639 			goto error;
640 		}
641 		branded = 1;
642 	}
643 
644 	if (t->t_tid == 1) {
645 		kpreempt_disable();
646 		ASSERT(t->t_lpl != NULL);
647 		p->p_t1_lgrpid = t->t_lpl->lpl_lgrpid;
648 		kpreempt_enable();
649 		if (p->p_tr_lgrpid != LGRP_NONE &&
650 		    p->p_tr_lgrpid != p->p_t1_lgrpid) {
651 			lgrp_update_trthr_migrations(1);
652 		}
653 	}
654 
655 	p->p_lwpcnt++;
656 	t->t_waitfor = -1;
657 
658 	/*
659 	 * Turn microstate accounting on for thread if on for process.
660 	 */
661 	if (p->p_flag & SMSACCT)
662 		t->t_proc_flag |= TP_MSACCT;
663 
664 	/*
665 	 * If the process has watchpoints, mark the new thread as such.
666 	 */
667 	if (pr_watch_active(p))
668 		watch_enable(t);
669 
670 	/*
671 	 * The lwp is being created in the stopped state.
672 	 * We set all the necessary flags to indicate that fact here.
673 	 * We omit the TS_CREATE flag from t_schedflag so that the lwp
674 	 * cannot be set running until the caller is finished with it,
675 	 * even if lwp_continue() is called on it after we drop p->p_lock.
676 	 * When the caller is finished with the newly-created lwp,
677 	 * the caller must call lwp_create_done() to allow the lwp
678 	 * to be set running.  If the TP_HOLDLWP is left set, the
679 	 * lwp will suspend itself after reaching system call exit.
680 	 */
681 	init_mstate(t, LMS_STOPPED);
682 	t->t_proc_flag |= TP_HOLDLWP;
683 	t->t_schedflag |= (TS_ALLSTART & ~(TS_CSTART | TS_CREATE));
684 	t->t_whystop = PR_SUSPENDED;
685 	t->t_whatstop = SUSPEND_NORMAL;
686 	t->t_sig_check = 1;	/* ensure that TP_HOLDLWP is honored */
687 
688 	/*
689 	 * Set system call processing flags in case tracing or profiling
690 	 * is set.  The first system call will evaluate these and turn
691 	 * them off if they aren't needed.
692 	 */
693 	t->t_pre_sys = 1;
694 	t->t_post_sys = 1;
695 
696 	/*
697 	 * Insert the new thread into the list of all threads.
698 	 */
699 	if ((tx = p->p_tlist) == NULL) {
700 		t->t_back = t;
701 		t->t_forw = t;
702 		p->p_tlist = t;
703 	} else {
704 		t->t_forw = tx;
705 		t->t_back = tx->t_back;
706 		tx->t_back->t_forw = t;
707 		tx->t_back = t;
708 	}
709 
710 	/*
711 	 * Insert the new lwp into an lwp directory slot position
712 	 * and into the lwpid hash table.
713 	 */
714 	lep->le_thread = t;
715 	lep->le_lwpid = t->t_tid;
716 	lep->le_start = t->t_start;
717 	lwp_hash_in(p, lep, p->p_tidhash, p->p_tidhash_sz, 1);
718 
719 	if (state == TS_RUN) {
720 		/*
721 		 * We set the new lwp running immediately.
722 		 */
723 		t->t_proc_flag &= ~TP_HOLDLWP;
724 		lwp_create_done(t);
725 	}
726 
727 error:
728 	if (err) {
729 		if (CLASS_KERNEL(cid)) {
730 			/*
731 			 * This should only happen if a system process runs
732 			 * out of lwpids, which shouldn't occur.
733 			 */
734 			panic("Failed to create a system LWP");
735 		}
736 		/*
737 		 * We have failed to create an lwp, so decrement the number
738 		 * of lwps in the task and let the lgroup load averages know
739 		 * that this thread isn't going to show up.
740 		 */
741 		kpreempt_disable();
742 		lgrp_move_thread(t, NULL, 1);
743 		kpreempt_enable();
744 
745 		ASSERT(MUTEX_HELD(&p->p_lock));
746 		mutex_enter(&p->p_zone->zone_nlwps_lock);
747 		p->p_task->tk_nlwps--;
748 		p->p_task->tk_proj->kpj_nlwps--;
749 		p->p_zone->zone_nlwps--;
750 		mutex_exit(&p->p_zone->zone_nlwps_lock);
751 		if (cid != NOCLASS && bufp != NULL)
752 			CL_FREE(cid, bufp);
753 
754 		if (branded)
755 			BROP(p)->b_freelwp(lwp);
756 
757 		mutex_exit(&p->p_lock);
758 		t->t_state = TS_FREE;
759 		thread_rele(t);
760 
761 		/*
762 		 * We need to remove t from the list of all threads
763 		 * because thread_exit()/lwp_exit() isn't called on t.
764 		 */
765 		mutex_enter(&pidlock);
766 		ASSERT(t != t->t_next);		/* t0 never exits */
767 		t->t_next->t_prev = t->t_prev;
768 		t->t_prev->t_next = t->t_next;
769 		mutex_exit(&pidlock);
770 
771 		thread_free(t);
772 		kmem_free(lep, sizeof (*lep));
773 		lwp = NULL;
774 	} else {
775 		mutex_exit(&p->p_lock);
776 	}
777 
778 	if (old_dir != NULL)
779 		kmem_free(old_dir, old_dirsz * sizeof (*old_dir));
780 	if (old_hash != NULL)
781 		kmem_free(old_hash, old_hashsz * sizeof (*old_hash));
782 	if (ret_tidhash != NULL)
783 		kmem_free(ret_tidhash, sizeof (ret_tidhash_t));
784 
785 	DTRACE_PROC1(lwp__create, kthread_t *, t);
786 	return (lwp);
787 }
788 
789 /*
790  * lwp_create_done() is called by the caller of lwp_create() to set the
791  * newly-created lwp running after the caller has finished manipulating it.
792  */
793 void
794 lwp_create_done(kthread_t *t)
795 {
796 	proc_t *p = ttoproc(t);
797 
798 	ASSERT(MUTEX_HELD(&p->p_lock));
799 
800 	/*
801 	 * We set the TS_CREATE and TS_CSTART flags and call setrun_locked().
802 	 * (The absence of the TS_CREATE flag prevents the lwp from running
803 	 * until we are finished with it, even if lwp_continue() is called on
804 	 * it by some other lwp in the process or elsewhere in the kernel.)
805 	 */
806 	thread_lock(t);
807 	ASSERT(t->t_state == TS_STOPPED && !(t->t_schedflag & TS_CREATE));
808 	/*
809 	 * If TS_CSTART is set, lwp_continue(t) has been called and
810 	 * has already incremented p_lwprcnt; avoid doing this twice.
811 	 */
812 	if (!(t->t_schedflag & TS_CSTART))
813 		p->p_lwprcnt++;
814 	t->t_schedflag |= (TS_CSTART | TS_CREATE);
815 	setrun_locked(t);
816 	thread_unlock(t);
817 }
818 
819 /*
820  * Copy an LWP's active templates, and clear the latest contracts.
821  */
822 void
823 lwp_ctmpl_copy(klwp_t *dst, klwp_t *src)
824 {
825 	int i;
826 
827 	for (i = 0; i < ct_ntypes; i++) {
828 		dst->lwp_ct_active[i] = ctmpl_dup(src->lwp_ct_active[i]);
829 		dst->lwp_ct_latest[i] = NULL;
830 	}
831 }
832 
833 /*
834  * Clear an LWP's contract template state.
835  */
836 void
837 lwp_ctmpl_clear(klwp_t *lwp)
838 {
839 	ct_template_t *tmpl;
840 	int i;
841 
842 	for (i = 0; i < ct_ntypes; i++) {
843 		if ((tmpl = lwp->lwp_ct_active[i]) != NULL) {
844 			ctmpl_free(tmpl);
845 			lwp->lwp_ct_active[i] = NULL;
846 		}
847 
848 		if (lwp->lwp_ct_latest[i] != NULL) {
849 			contract_rele(lwp->lwp_ct_latest[i]);
850 			lwp->lwp_ct_latest[i] = NULL;
851 		}
852 	}
853 }
854 
855 /*
856  * Individual lwp exit.
857  * If this is the last lwp, exit the whole process.
858  */
859 void
860 lwp_exit(void)
861 {
862 	kthread_t *t = curthread;
863 	klwp_t *lwp = ttolwp(t);
864 	proc_t *p = ttoproc(t);
865 
866 	ASSERT(MUTEX_HELD(&p->p_lock));
867 
868 	mutex_exit(&p->p_lock);
869 
870 #if defined(__sparc)
871 	/*
872 	 * Ensure that the user stack is fully abandoned..
873 	 */
874 	trash_user_windows();
875 #endif
876 
877 	tsd_exit();			/* free thread specific data */
878 
879 	kcpc_passivate();		/* Clean up performance counter state */
880 
881 	pollcleanup();
882 
883 	if (t->t_door)
884 		door_slam();
885 
886 	if (t->t_schedctl != NULL)
887 		schedctl_lwp_cleanup(t);
888 
889 	if (t->t_upimutex != NULL)
890 		upimutex_cleanup();
891 
892 	/*
893 	 * Perform any brand specific exit processing, then release any
894 	 * brand data associated with the lwp
895 	 */
896 	if (PROC_IS_BRANDED(p))
897 		BROP(p)->b_lwpexit(lwp);
898 
899 	mutex_enter(&p->p_lock);
900 	lwp_cleanup();
901 
902 	/*
903 	 * When this process is dumping core, its lwps are held here
904 	 * until the core dump is finished. Then exitlwps() is called
905 	 * again to release these lwps so that they can finish exiting.
906 	 */
907 	if (p->p_flag & SCOREDUMP)
908 		stop(PR_SUSPENDED, SUSPEND_NORMAL);
909 
910 	/*
911 	 * Block the process against /proc now that we have really acquired
912 	 * p->p_lock (to decrement p_lwpcnt and manipulate p_tlist at least).
913 	 */
914 	prbarrier(p);
915 
916 	/*
917 	 * Call proc_exit() if this is the last non-daemon lwp in the process.
918 	 */
919 	if (!(t->t_proc_flag & TP_DAEMON) &&
920 	    p->p_lwpcnt == p->p_lwpdaemon + 1) {
921 		mutex_exit(&p->p_lock);
922 		if (proc_exit(CLD_EXITED, 0) == 0) {
923 			/* Restarting init. */
924 			return;
925 		}
926 
927 		/*
928 		 * proc_exit() returns a non-zero value when some other
929 		 * lwp got there first.  We just have to continue in
930 		 * lwp_exit().
931 		 */
932 		mutex_enter(&p->p_lock);
933 		ASSERT(curproc->p_flag & SEXITLWPS);
934 		prbarrier(p);
935 	}
936 
937 	DTRACE_PROC(lwp__exit);
938 
939 	/*
940 	 * If the lwp is a detached lwp or if the process is exiting,
941 	 * remove (lwp_hash_out()) the lwp from the lwp directory.
942 	 * Otherwise null out the lwp's le_thread pointer in the lwp
943 	 * directory so that other threads will see it as a zombie lwp.
944 	 */
945 	prlwpexit(t);		/* notify /proc */
946 	if (!(t->t_proc_flag & TP_TWAIT) || (p->p_flag & SEXITLWPS))
947 		lwp_hash_out(p, t->t_tid);
948 	else {
949 		ASSERT(!(t->t_proc_flag & TP_DAEMON));
950 		p->p_lwpdir[t->t_dslot].ld_entry->le_thread = NULL;
951 		p->p_zombcnt++;
952 		cv_broadcast(&p->p_lwpexit);
953 	}
954 	if (t->t_proc_flag & TP_DAEMON) {
955 		p->p_lwpdaemon--;
956 		t->t_proc_flag &= ~TP_DAEMON;
957 	}
958 	t->t_proc_flag &= ~TP_TWAIT;
959 
960 	/*
961 	 * Maintain accurate lwp count for task.max-lwps resource control.
962 	 */
963 	mutex_enter(&p->p_zone->zone_nlwps_lock);
964 	p->p_task->tk_nlwps--;
965 	p->p_task->tk_proj->kpj_nlwps--;
966 	p->p_zone->zone_nlwps--;
967 	mutex_exit(&p->p_zone->zone_nlwps_lock);
968 
969 	CL_EXIT(t);		/* tell the scheduler that t is exiting */
970 	ASSERT(p->p_lwpcnt != 0);
971 	p->p_lwpcnt--;
972 
973 	/*
974 	 * If all remaining non-daemon lwps are waiting in lwp_wait(),
975 	 * wake them up so someone can return EDEADLK.
976 	 * (See the block comment preceeding lwp_wait().)
977 	 */
978 	if (p->p_lwpcnt == p->p_lwpdaemon + (p->p_lwpwait - p->p_lwpdwait))
979 		cv_broadcast(&p->p_lwpexit);
980 
981 	t->t_proc_flag |= TP_LWPEXIT;
982 	term_mstate(t);
983 
984 #ifndef NPROBE
985 	/* Kernel probe */
986 	if (t->t_tnf_tpdp)
987 		tnf_thread_exit();
988 #endif /* NPROBE */
989 
990 	t->t_forw->t_back = t->t_back;
991 	t->t_back->t_forw = t->t_forw;
992 	if (t == p->p_tlist)
993 		p->p_tlist = t->t_forw;
994 
995 	/*
996 	 * Clean up the signal state.
997 	 */
998 	if (t->t_sigqueue != NULL)
999 		sigdelq(p, t, 0);
1000 	if (lwp->lwp_curinfo != NULL) {
1001 		siginfofree(lwp->lwp_curinfo);
1002 		lwp->lwp_curinfo = NULL;
1003 	}
1004 
1005 	thread_rele(t);
1006 
1007 	/*
1008 	 * Terminated lwps are associated with process zero and are put onto
1009 	 * death-row by resume().  Avoid preemption after resetting t->t_procp.
1010 	 */
1011 	t->t_preempt++;
1012 
1013 	if (t->t_ctx != NULL)
1014 		exitctx(t);
1015 	if (p->p_pctx != NULL)
1016 		exitpctx(p);
1017 
1018 	t->t_procp = &p0;
1019 
1020 	/*
1021 	 * Notify the HAT about the change of address space
1022 	 */
1023 	hat_thread_exit(t);
1024 	/*
1025 	 * When this is the last running lwp in this process and some lwp is
1026 	 * waiting for this condition to become true, or this thread was being
1027 	 * suspended, then the waiting lwp is awakened.
1028 	 *
1029 	 * Also, if the process is exiting, we may have a thread waiting in
1030 	 * exitlwps() that needs to be notified.
1031 	 */
1032 	if (--p->p_lwprcnt == 0 || (t->t_proc_flag & TP_HOLDLWP) ||
1033 	    (p->p_flag & SEXITLWPS))
1034 		cv_broadcast(&p->p_holdlwps);
1035 
1036 	/*
1037 	 * Need to drop p_lock so we can reacquire pidlock.
1038 	 */
1039 	mutex_exit(&p->p_lock);
1040 	mutex_enter(&pidlock);
1041 
1042 	ASSERT(t != t->t_next);		/* t0 never exits */
1043 	t->t_next->t_prev = t->t_prev;
1044 	t->t_prev->t_next = t->t_next;
1045 	cv_broadcast(&t->t_joincv);	/* wake up anyone in thread_join */
1046 	mutex_exit(&pidlock);
1047 
1048 	lwp_pcb_exit();
1049 
1050 	t->t_state = TS_ZOMB;
1051 	swtch_from_zombie();
1052 	/* never returns */
1053 }
1054 
1055 
1056 /*
1057  * Cleanup function for an exiting lwp.
1058  * Called both from lwp_exit() and from proc_exit().
1059  * p->p_lock is repeatedly released and grabbed in this function.
1060  */
1061 void
1062 lwp_cleanup(void)
1063 {
1064 	kthread_t *t = curthread;
1065 	proc_t *p = ttoproc(t);
1066 
1067 	ASSERT(MUTEX_HELD(&p->p_lock));
1068 
1069 	/* untimeout any lwp-bound realtime timers */
1070 	if (p->p_itimer != NULL)
1071 		timer_lwpexit();
1072 
1073 	/*
1074 	 * If this is the /proc agent lwp that is exiting, readjust p_lwpid
1075 	 * so it appears that the agent never existed, and clear p_agenttp.
1076 	 */
1077 	if (t == p->p_agenttp) {
1078 		ASSERT(t->t_tid == p->p_lwpid);
1079 		p->p_lwpid--;
1080 		p->p_agenttp = NULL;
1081 	}
1082 
1083 	/*
1084 	 * Do lgroup bookkeeping to account for thread exiting.
1085 	 */
1086 	kpreempt_disable();
1087 	lgrp_move_thread(t, NULL, 1);
1088 	if (t->t_tid == 1) {
1089 		p->p_t1_lgrpid = LGRP_NONE;
1090 	}
1091 	kpreempt_enable();
1092 
1093 	lwp_ctmpl_clear(ttolwp(t));
1094 }
1095 
1096 int
1097 lwp_suspend(kthread_t *t)
1098 {
1099 	int tid;
1100 	proc_t *p = ttoproc(t);
1101 
1102 	ASSERT(MUTEX_HELD(&p->p_lock));
1103 
1104 	/*
1105 	 * Set the thread's TP_HOLDLWP flag so it will stop in holdlwp().
1106 	 * If an lwp is stopping itself, there is no need to wait.
1107 	 */
1108 top:
1109 	t->t_proc_flag |= TP_HOLDLWP;
1110 	if (t == curthread) {
1111 		t->t_sig_check = 1;
1112 	} else {
1113 		/*
1114 		 * Make sure the lwp stops promptly.
1115 		 */
1116 		thread_lock(t);
1117 		t->t_sig_check = 1;
1118 		/*
1119 		 * XXX Should use virtual stop like /proc does instead of
1120 		 * XXX waking the thread to get it to stop.
1121 		 */
1122 		if (ISWAKEABLE(t) || ISWAITING(t)) {
1123 			setrun_locked(t);
1124 		} else if (t->t_state == TS_ONPROC && t->t_cpu != CPU) {
1125 			poke_cpu(t->t_cpu->cpu_id);
1126 		}
1127 
1128 		tid = t->t_tid;	 /* remember thread ID */
1129 		/*
1130 		 * Wait for lwp to stop
1131 		 */
1132 		while (!SUSPENDED(t)) {
1133 			/*
1134 			 * Drop the thread lock before waiting and reacquire it
1135 			 * afterwards, so the thread can change its t_state
1136 			 * field.
1137 			 */
1138 			thread_unlock(t);
1139 
1140 			/*
1141 			 * Check if aborted by exitlwps().
1142 			 */
1143 			if (p->p_flag & SEXITLWPS)
1144 				lwp_exit();
1145 
1146 			/*
1147 			 * Cooperate with jobcontrol signals and /proc stopping
1148 			 * by calling cv_wait_sig() to wait for the target
1149 			 * lwp to stop.  Just using cv_wait() can lead to
1150 			 * deadlock because, if some other lwp has stopped
1151 			 * by either of these mechanisms, then p_lwprcnt will
1152 			 * never become zero if we do a cv_wait().
1153 			 */
1154 			if (!cv_wait_sig(&p->p_holdlwps, &p->p_lock))
1155 				return (EINTR);
1156 
1157 			/*
1158 			 * Check to see if thread died while we were
1159 			 * waiting for it to suspend.
1160 			 */
1161 			if (idtot(p, tid) == NULL)
1162 				return (ESRCH);
1163 
1164 			thread_lock(t);
1165 			/*
1166 			 * If the TP_HOLDLWP flag went away, lwp_continue()
1167 			 * or vfork() must have been called while we were
1168 			 * waiting, so start over again.
1169 			 */
1170 			if ((t->t_proc_flag & TP_HOLDLWP) == 0) {
1171 				thread_unlock(t);
1172 				goto top;
1173 			}
1174 		}
1175 		thread_unlock(t);
1176 	}
1177 	return (0);
1178 }
1179 
1180 /*
1181  * continue a lwp that's been stopped by lwp_suspend().
1182  */
1183 void
1184 lwp_continue(kthread_t *t)
1185 {
1186 	proc_t *p = ttoproc(t);
1187 	int was_suspended = t->t_proc_flag & TP_HOLDLWP;
1188 
1189 	ASSERT(MUTEX_HELD(&p->p_lock));
1190 
1191 	t->t_proc_flag &= ~TP_HOLDLWP;
1192 	thread_lock(t);
1193 	if (SUSPENDED(t) &&
1194 	    !(p->p_flag & (SHOLDFORK | SHOLDFORK1 | SHOLDWATCH))) {
1195 		p->p_lwprcnt++;
1196 		t->t_schedflag |= TS_CSTART;
1197 		setrun_locked(t);
1198 	}
1199 	thread_unlock(t);
1200 	/*
1201 	 * Wakeup anyone waiting for this thread to be suspended
1202 	 */
1203 	if (was_suspended)
1204 		cv_broadcast(&p->p_holdlwps);
1205 }
1206 
1207 /*
1208  * ********************************
1209  *  Miscellaneous lwp routines	  *
1210  * ********************************
1211  */
1212 /*
1213  * When a process is undergoing a forkall(), its p_flag is set to SHOLDFORK.
1214  * This will cause the process's lwps to stop at a hold point.  A hold
1215  * point is where a kernel thread has a flat stack.  This is at the
1216  * return from a system call and at the return from a user level trap.
1217  *
1218  * When a process is undergoing a fork1() or vfork(), its p_flag is set to
1219  * SHOLDFORK1.  This will cause the process's lwps to stop at a modified
1220  * hold point.  The lwps in the process are not being cloned, so they
1221  * are held at the usual hold points and also within issig_forreal().
1222  * This has the side-effect that their system calls do not return
1223  * showing EINTR.
1224  *
1225  * An lwp can also be held.  This is identified by the TP_HOLDLWP flag on
1226  * the thread.  The TP_HOLDLWP flag is set in lwp_suspend(), where the active
1227  * lwp is waiting for the target lwp to be stopped.
1228  */
1229 void
1230 holdlwp(void)
1231 {
1232 	proc_t *p = curproc;
1233 	kthread_t *t = curthread;
1234 
1235 	mutex_enter(&p->p_lock);
1236 	/*
1237 	 * Don't terminate immediately if the process is dumping core.
1238 	 * Once the process has dumped core, all lwps are terminated.
1239 	 */
1240 	if (!(p->p_flag & SCOREDUMP)) {
1241 		if ((p->p_flag & SEXITLWPS) || (t->t_proc_flag & TP_EXITLWP))
1242 			lwp_exit();
1243 	}
1244 	if (!(ISHOLD(p)) && !(p->p_flag & (SHOLDFORK1 | SHOLDWATCH))) {
1245 		mutex_exit(&p->p_lock);
1246 		return;
1247 	}
1248 	/*
1249 	 * stop() decrements p->p_lwprcnt and cv_signal()s &p->p_holdlwps
1250 	 * when p->p_lwprcnt becomes zero.
1251 	 */
1252 	stop(PR_SUSPENDED, SUSPEND_NORMAL);
1253 	if (p->p_flag & SEXITLWPS)
1254 		lwp_exit();
1255 	mutex_exit(&p->p_lock);
1256 }
1257 
1258 /*
1259  * Have all lwps within the process hold at a point where they are
1260  * cloneable (SHOLDFORK) or just safe w.r.t. fork1 (SHOLDFORK1).
1261  */
1262 int
1263 holdlwps(int holdflag)
1264 {
1265 	proc_t *p = curproc;
1266 
1267 	ASSERT(holdflag == SHOLDFORK || holdflag == SHOLDFORK1);
1268 	mutex_enter(&p->p_lock);
1269 	schedctl_finish_sigblock(curthread);
1270 again:
1271 	while (p->p_flag & (SEXITLWPS | SHOLDFORK | SHOLDFORK1 | SHOLDWATCH)) {
1272 		/*
1273 		 * If another lwp is doing a forkall() or proc_exit(), bail out.
1274 		 */
1275 		if (p->p_flag & (SEXITLWPS | SHOLDFORK)) {
1276 			mutex_exit(&p->p_lock);
1277 			return (0);
1278 		}
1279 		/*
1280 		 * Another lwp is doing a fork1() or is undergoing
1281 		 * watchpoint activity.  We hold here for it to complete.
1282 		 */
1283 		stop(PR_SUSPENDED, SUSPEND_NORMAL);
1284 	}
1285 	p->p_flag |= holdflag;
1286 	pokelwps(p);
1287 	--p->p_lwprcnt;
1288 	/*
1289 	 * Wait for the process to become quiescent (p->p_lwprcnt == 0).
1290 	 */
1291 	while (p->p_lwprcnt > 0) {
1292 		/*
1293 		 * Check if aborted by exitlwps().
1294 		 * Also check if SHOLDWATCH is set; it takes precedence.
1295 		 */
1296 		if (p->p_flag & (SEXITLWPS | SHOLDWATCH)) {
1297 			p->p_lwprcnt++;
1298 			p->p_flag &= ~holdflag;
1299 			cv_broadcast(&p->p_holdlwps);
1300 			goto again;
1301 		}
1302 		/*
1303 		 * Cooperate with jobcontrol signals and /proc stopping.
1304 		 * If some other lwp has stopped by either of these
1305 		 * mechanisms, then p_lwprcnt will never become zero
1306 		 * and the process will appear deadlocked unless we
1307 		 * stop here in sympathy with the other lwp before
1308 		 * doing the cv_wait() below.
1309 		 *
1310 		 * If the other lwp stops after we do the cv_wait(), it
1311 		 * will wake us up to loop around and do the sympathy stop.
1312 		 *
1313 		 * Since stop() drops p->p_lock, we must start from
1314 		 * the top again on returning from stop().
1315 		 */
1316 		if (p->p_stopsig | (curthread->t_proc_flag & TP_PRSTOP)) {
1317 			int whystop = p->p_stopsig? PR_JOBCONTROL :
1318 			    PR_REQUESTED;
1319 			p->p_lwprcnt++;
1320 			p->p_flag &= ~holdflag;
1321 			stop(whystop, p->p_stopsig);
1322 			goto again;
1323 		}
1324 		cv_wait(&p->p_holdlwps, &p->p_lock);
1325 	}
1326 	p->p_lwprcnt++;
1327 	p->p_flag &= ~holdflag;
1328 	mutex_exit(&p->p_lock);
1329 	return (1);
1330 }
1331 
1332 /*
1333  * See comments for holdwatch(), below.
1334  */
1335 static int
1336 holdcheck(int clearflags)
1337 {
1338 	proc_t *p = curproc;
1339 
1340 	/*
1341 	 * If we are trying to exit, that takes precedence over anything else.
1342 	 */
1343 	if (p->p_flag & SEXITLWPS) {
1344 		p->p_lwprcnt++;
1345 		p->p_flag &= ~clearflags;
1346 		lwp_exit();
1347 	}
1348 
1349 	/*
1350 	 * If another thread is calling fork1(), stop the current thread so the
1351 	 * other can complete.
1352 	 */
1353 	if (p->p_flag & SHOLDFORK1) {
1354 		p->p_lwprcnt++;
1355 		stop(PR_SUSPENDED, SUSPEND_NORMAL);
1356 		if (p->p_flag & SEXITLWPS) {
1357 			p->p_flag &= ~clearflags;
1358 			lwp_exit();
1359 		}
1360 		return (-1);
1361 	}
1362 
1363 	/*
1364 	 * If another thread is calling fork(), then indicate we are doing
1365 	 * watchpoint activity.  This will cause holdlwps() above to stop the
1366 	 * forking thread, at which point we can continue with watchpoint
1367 	 * activity.
1368 	 */
1369 	if (p->p_flag & SHOLDFORK) {
1370 		p->p_lwprcnt++;
1371 		while (p->p_flag & SHOLDFORK) {
1372 			p->p_flag |= SHOLDWATCH;
1373 			cv_broadcast(&p->p_holdlwps);
1374 			cv_wait(&p->p_holdlwps, &p->p_lock);
1375 			p->p_flag &= ~SHOLDWATCH;
1376 		}
1377 		return (-1);
1378 	}
1379 
1380 	return (0);
1381 }
1382 
1383 /*
1384  * Stop all lwps within the process, holding themselves in the kernel while the
1385  * active lwp undergoes watchpoint activity.  This is more complicated than
1386  * expected because stop() relies on calling holdwatch() in order to copyin data
1387  * from the user's address space.  A double barrier is used to prevent an
1388  * infinite loop.
1389  *
1390  * 	o The first thread into holdwatch() is the 'master' thread and does
1391  *        the following:
1392  *
1393  *              - Sets SHOLDWATCH on the current process
1394  *              - Sets TP_WATCHSTOP on the current thread
1395  *              - Waits for all threads to be either stopped or have
1396  *                TP_WATCHSTOP set.
1397  *              - Sets the SWATCHOK flag on the process
1398  *              - Unsets TP_WATCHSTOP
1399  *              - Waits for the other threads to completely stop
1400  *              - Unsets SWATCHOK
1401  *
1402  * 	o If SHOLDWATCH is already set when we enter this function, then another
1403  *        thread is already trying to stop this thread.  This 'slave' thread
1404  *        does the following:
1405  *
1406  *              - Sets TP_WATCHSTOP on the current thread
1407  *              - Waits for SWATCHOK flag to be set
1408  *              - Calls stop()
1409  *
1410  * 	o If SWATCHOK is set on the process, then this function immediately
1411  *        returns, as we must have been called via stop().
1412  *
1413  * In addition, there are other flags that take precedence over SHOLDWATCH:
1414  *
1415  * 	o If SEXITLWPS is set, exit immediately.
1416  *
1417  * 	o If SHOLDFORK1 is set, wait for fork1() to complete.
1418  *
1419  * 	o If SHOLDFORK is set, then watchpoint activity takes precedence In this
1420  *        case, set SHOLDWATCH, signalling the forking thread to stop first.
1421  *
1422  * 	o If the process is being stopped via /proc (TP_PRSTOP is set), then we
1423  *        stop the current thread.
1424  *
1425  * Returns 0 if all threads have been quiesced.  Returns non-zero if not all
1426  * threads were stopped, or the list of watched pages has changed.
1427  */
1428 int
1429 holdwatch(void)
1430 {
1431 	proc_t *p = curproc;
1432 	kthread_t *t = curthread;
1433 	int ret = 0;
1434 
1435 	mutex_enter(&p->p_lock);
1436 
1437 	p->p_lwprcnt--;
1438 
1439 	/*
1440 	 * Check for bail-out conditions as outlined above.
1441 	 */
1442 	if (holdcheck(0) != 0) {
1443 		mutex_exit(&p->p_lock);
1444 		return (-1);
1445 	}
1446 
1447 	if (!(p->p_flag & SHOLDWATCH)) {
1448 		/*
1449 		 * We are the master watchpoint thread.  Set SHOLDWATCH and poke
1450 		 * the other threads.
1451 		 */
1452 		p->p_flag |= SHOLDWATCH;
1453 		pokelwps(p);
1454 
1455 		/*
1456 		 * Wait for all threads to be stopped or have TP_WATCHSTOP set.
1457 		 */
1458 		while (pr_allstopped(p, 1) > 0) {
1459 			if (holdcheck(SHOLDWATCH) != 0) {
1460 				p->p_flag &= ~SHOLDWATCH;
1461 				mutex_exit(&p->p_lock);
1462 				return (-1);
1463 			}
1464 
1465 			cv_wait(&p->p_holdlwps, &p->p_lock);
1466 		}
1467 
1468 		/*
1469 		 * All threads are now stopped or in the process of stopping.
1470 		 * Set SWATCHOK and let them stop completely.
1471 		 */
1472 		p->p_flag |= SWATCHOK;
1473 		t->t_proc_flag &= ~TP_WATCHSTOP;
1474 		cv_broadcast(&p->p_holdlwps);
1475 
1476 		while (pr_allstopped(p, 0) > 0) {
1477 			/*
1478 			 * At first glance, it may appear that we don't need a
1479 			 * call to holdcheck() here.  But if the process gets a
1480 			 * SIGKILL signal, one of our stopped threads may have
1481 			 * been awakened and is waiting in exitlwps(), which
1482 			 * takes precedence over watchpoints.
1483 			 */
1484 			if (holdcheck(SHOLDWATCH | SWATCHOK) != 0) {
1485 				p->p_flag &= ~(SHOLDWATCH | SWATCHOK);
1486 				mutex_exit(&p->p_lock);
1487 				return (-1);
1488 			}
1489 
1490 			cv_wait(&p->p_holdlwps, &p->p_lock);
1491 		}
1492 
1493 		/*
1494 		 * All threads are now completely stopped.
1495 		 */
1496 		p->p_flag &= ~SWATCHOK;
1497 		p->p_flag &= ~SHOLDWATCH;
1498 		p->p_lwprcnt++;
1499 
1500 	} else if (!(p->p_flag & SWATCHOK)) {
1501 
1502 		/*
1503 		 * SHOLDWATCH is set, so another thread is trying to do
1504 		 * watchpoint activity.  Indicate this thread is stopping, and
1505 		 * wait for the OK from the master thread.
1506 		 */
1507 		t->t_proc_flag |= TP_WATCHSTOP;
1508 		cv_broadcast(&p->p_holdlwps);
1509 
1510 		while (!(p->p_flag & SWATCHOK)) {
1511 			if (holdcheck(0) != 0) {
1512 				t->t_proc_flag &= ~TP_WATCHSTOP;
1513 				mutex_exit(&p->p_lock);
1514 				return (-1);
1515 			}
1516 
1517 			cv_wait(&p->p_holdlwps, &p->p_lock);
1518 		}
1519 
1520 		/*
1521 		 * Once the master thread has given the OK, this thread can
1522 		 * actually call stop().
1523 		 */
1524 		t->t_proc_flag &= ~TP_WATCHSTOP;
1525 		p->p_lwprcnt++;
1526 
1527 		stop(PR_SUSPENDED, SUSPEND_NORMAL);
1528 
1529 		/*
1530 		 * It's not OK to do watchpoint activity, notify caller to
1531 		 * retry.
1532 		 */
1533 		ret = -1;
1534 
1535 	} else {
1536 
1537 		/*
1538 		 * The only way we can hit the case where SHOLDWATCH is set and
1539 		 * SWATCHOK is set is if we are triggering this from within a
1540 		 * stop() call.  Assert that this is the case.
1541 		 */
1542 
1543 		ASSERT(t->t_proc_flag & TP_STOPPING);
1544 		p->p_lwprcnt++;
1545 	}
1546 
1547 	mutex_exit(&p->p_lock);
1548 
1549 	return (ret);
1550 }
1551 
1552 /*
1553  * force all interruptible lwps to trap into the kernel.
1554  */
1555 void
1556 pokelwps(proc_t *p)
1557 {
1558 	kthread_t *t;
1559 
1560 	ASSERT(MUTEX_HELD(&p->p_lock));
1561 
1562 	t = p->p_tlist;
1563 	do {
1564 		if (t == curthread)
1565 			continue;
1566 		thread_lock(t);
1567 		aston(t);	/* make thread trap or do post_syscall */
1568 		if (ISWAKEABLE(t) || ISWAITING(t)) {
1569 			setrun_locked(t);
1570 		} else if (t->t_state == TS_STOPPED) {
1571 			/*
1572 			 * Ensure that proc_exit() is not blocked by lwps
1573 			 * that were stopped via jobcontrol or /proc.
1574 			 */
1575 			if (p->p_flag & SEXITLWPS) {
1576 				p->p_stopsig = 0;
1577 				t->t_schedflag |= (TS_XSTART | TS_PSTART);
1578 				setrun_locked(t);
1579 			}
1580 			/*
1581 			 * If we are holding lwps for a forkall(),
1582 			 * force lwps that have been suspended via
1583 			 * lwp_suspend() and are suspended inside
1584 			 * of a system call to proceed to their
1585 			 * holdlwp() points where they are clonable.
1586 			 */
1587 			if ((p->p_flag & SHOLDFORK) && SUSPENDED(t)) {
1588 				if ((t->t_schedflag & TS_CSTART) == 0) {
1589 					p->p_lwprcnt++;
1590 					t->t_schedflag |= TS_CSTART;
1591 					setrun_locked(t);
1592 				}
1593 			}
1594 		} else if (t->t_state == TS_ONPROC) {
1595 			if (t->t_cpu != CPU)
1596 				poke_cpu(t->t_cpu->cpu_id);
1597 		}
1598 		thread_unlock(t);
1599 	} while ((t = t->t_forw) != p->p_tlist);
1600 }
1601 
1602 /*
1603  * undo the effects of holdlwps() or holdwatch().
1604  */
1605 void
1606 continuelwps(proc_t *p)
1607 {
1608 	kthread_t *t;
1609 
1610 	/*
1611 	 * If this flag is set, then the original holdwatch() didn't actually
1612 	 * stop the process.  See comments for holdwatch().
1613 	 */
1614 	if (p->p_flag & SWATCHOK) {
1615 		ASSERT(curthread->t_proc_flag & TP_STOPPING);
1616 		return;
1617 	}
1618 
1619 	ASSERT(MUTEX_HELD(&p->p_lock));
1620 	ASSERT((p->p_flag & (SHOLDFORK | SHOLDFORK1 | SHOLDWATCH)) == 0);
1621 
1622 	t = p->p_tlist;
1623 	do {
1624 		thread_lock(t);		/* SUSPENDED looks at t_schedflag */
1625 		if (SUSPENDED(t) && !(t->t_proc_flag & TP_HOLDLWP)) {
1626 			p->p_lwprcnt++;
1627 			t->t_schedflag |= TS_CSTART;
1628 			setrun_locked(t);
1629 		}
1630 		thread_unlock(t);
1631 	} while ((t = t->t_forw) != p->p_tlist);
1632 }
1633 
1634 /*
1635  * Force all other LWPs in the current process other than the caller to exit,
1636  * and then cv_wait() on p_holdlwps for them to exit.  The exitlwps() function
1637  * is typically used in these situations:
1638  *
1639  *   (a) prior to an exec() system call
1640  *   (b) prior to dumping a core file
1641  *   (c) prior to a uadmin() shutdown
1642  *
1643  * If the 'coredump' flag is set, other LWPs are quiesced but not destroyed.
1644  * Multiple threads in the process can call this function at one time by
1645  * triggering execs or core dumps simultaneously, so the SEXITLWPS bit is used
1646  * to declare one particular thread the winner who gets to kill the others.
1647  * If a thread wins the exitlwps() dance, zero is returned; otherwise an
1648  * appropriate errno value is returned to caller for its system call to return.
1649  */
1650 int
1651 exitlwps(int coredump)
1652 {
1653 	proc_t *p = curproc;
1654 	int heldcnt;
1655 
1656 	if (curthread->t_door)
1657 		door_slam();
1658 	if (p->p_door_list)
1659 		door_revoke_all();
1660 	if (curthread->t_schedctl != NULL)
1661 		schedctl_lwp_cleanup(curthread);
1662 
1663 	/*
1664 	 * Ensure that before starting to wait for other lwps to exit,
1665 	 * cleanup all upimutexes held by curthread. Otherwise, some other
1666 	 * lwp could be waiting (uninterruptibly) for a upimutex held by
1667 	 * curthread, and the call to pokelwps() below would deadlock.
1668 	 * Even if a blocked upimutex_lock is made interruptible,
1669 	 * curthread's upimutexes need to be unlocked: do it here.
1670 	 */
1671 	if (curthread->t_upimutex != NULL)
1672 		upimutex_cleanup();
1673 
1674 	/*
1675 	 * Grab p_lock in order to check and set SEXITLWPS to declare a winner.
1676 	 * We must also block any further /proc access from this point forward.
1677 	 */
1678 	mutex_enter(&p->p_lock);
1679 	prbarrier(p);
1680 
1681 	if (p->p_flag & SEXITLWPS) {
1682 		mutex_exit(&p->p_lock);
1683 		aston(curthread);	/* force a trip through post_syscall */
1684 		return (set_errno(EINTR));
1685 	}
1686 
1687 	p->p_flag |= SEXITLWPS;
1688 	if (coredump)		/* tell other lwps to stop, not exit */
1689 		p->p_flag |= SCOREDUMP;
1690 
1691 	/*
1692 	 * Give precedence to exitlwps() if a holdlwps() is
1693 	 * in progress. The lwp doing the holdlwps() operation
1694 	 * is aborted when it is awakened.
1695 	 */
1696 	while (p->p_flag & (SHOLDFORK | SHOLDFORK1 | SHOLDWATCH)) {
1697 		cv_broadcast(&p->p_holdlwps);
1698 		cv_wait(&p->p_holdlwps, &p->p_lock);
1699 		prbarrier(p);
1700 	}
1701 	p->p_flag |= SHOLDFORK;
1702 	pokelwps(p);
1703 
1704 	/*
1705 	 * Wait for process to become quiescent.
1706 	 */
1707 	--p->p_lwprcnt;
1708 	while (p->p_lwprcnt > 0) {
1709 		cv_wait(&p->p_holdlwps, &p->p_lock);
1710 		prbarrier(p);
1711 	}
1712 	p->p_lwprcnt++;
1713 	ASSERT(p->p_lwprcnt == 1);
1714 
1715 	/*
1716 	 * The SCOREDUMP flag puts the process into a quiescent
1717 	 * state.  The process's lwps remain attached to this
1718 	 * process until exitlwps() is called again without the
1719 	 * 'coredump' flag set, then the lwps are terminated
1720 	 * and the process can exit.
1721 	 */
1722 	if (coredump) {
1723 		p->p_flag &= ~(SCOREDUMP | SHOLDFORK | SEXITLWPS);
1724 		goto out;
1725 	}
1726 
1727 	/*
1728 	 * Determine if there are any lwps left dangling in
1729 	 * the stopped state.  This happens when exitlwps()
1730 	 * aborts a holdlwps() operation.
1731 	 */
1732 	p->p_flag &= ~SHOLDFORK;
1733 	if ((heldcnt = p->p_lwpcnt) > 1) {
1734 		kthread_t *t;
1735 		for (t = curthread->t_forw; --heldcnt > 0; t = t->t_forw) {
1736 			t->t_proc_flag &= ~TP_TWAIT;
1737 			lwp_continue(t);
1738 		}
1739 	}
1740 
1741 	/*
1742 	 * Wait for all other lwps to exit.
1743 	 */
1744 	--p->p_lwprcnt;
1745 	while (p->p_lwpcnt > 1) {
1746 		cv_wait(&p->p_holdlwps, &p->p_lock);
1747 		prbarrier(p);
1748 	}
1749 	++p->p_lwprcnt;
1750 	ASSERT(p->p_lwpcnt == 1 && p->p_lwprcnt == 1);
1751 
1752 	p->p_flag &= ~SEXITLWPS;
1753 	curthread->t_proc_flag &= ~TP_TWAIT;
1754 
1755 out:
1756 	if (!coredump && p->p_zombcnt) {	/* cleanup the zombie lwps */
1757 		lwpdir_t *ldp;
1758 		lwpent_t *lep;
1759 		int i;
1760 
1761 		for (ldp = p->p_lwpdir, i = 0; i < p->p_lwpdir_sz; i++, ldp++) {
1762 			lep = ldp->ld_entry;
1763 			if (lep != NULL && lep->le_thread != curthread) {
1764 				ASSERT(lep->le_thread == NULL);
1765 				p->p_zombcnt--;
1766 				lwp_hash_out(p, lep->le_lwpid);
1767 			}
1768 		}
1769 		ASSERT(p->p_zombcnt == 0);
1770 	}
1771 
1772 	/*
1773 	 * If some other LWP in the process wanted us to suspend ourself,
1774 	 * then we will not do it.  The other LWP is now terminated and
1775 	 * no one will ever continue us again if we suspend ourself.
1776 	 */
1777 	curthread->t_proc_flag &= ~TP_HOLDLWP;
1778 	p->p_flag &= ~(SHOLDFORK | SHOLDFORK1 | SHOLDWATCH | SLWPWRAP);
1779 	mutex_exit(&p->p_lock);
1780 	return (0);
1781 }
1782 
1783 /*
1784  * duplicate a lwp.
1785  */
1786 klwp_t *
1787 forklwp(klwp_t *lwp, proc_t *cp, id_t lwpid)
1788 {
1789 	klwp_t *clwp;
1790 	void *tregs, *tfpu;
1791 	kthread_t *t = lwptot(lwp);
1792 	kthread_t *ct;
1793 	proc_t *p = lwptoproc(lwp);
1794 	int cid;
1795 	void *bufp;
1796 	void *brand_data;
1797 	int val;
1798 
1799 	ASSERT(p == curproc);
1800 	ASSERT(t == curthread || (SUSPENDED(t) && lwp->lwp_asleep == 0));
1801 
1802 #if defined(__sparc)
1803 	if (t == curthread)
1804 		(void) flush_user_windows_to_stack(NULL);
1805 #endif
1806 
1807 	if (t == curthread)
1808 		/* copy args out of registers first */
1809 		(void) save_syscall_args();
1810 
1811 	clwp = lwp_create(cp->p_lwpcnt == 0 ? lwp_rtt_initial : lwp_rtt,
1812 	    NULL, 0, cp, TS_STOPPED, t->t_pri, &t->t_hold, NOCLASS, lwpid);
1813 	if (clwp == NULL)
1814 		return (NULL);
1815 
1816 	/*
1817 	 * most of the parent's lwp can be copied to its duplicate,
1818 	 * except for the fields that are unique to each lwp, like
1819 	 * lwp_thread, lwp_procp, lwp_regs, and lwp_ap.
1820 	 */
1821 	ct = clwp->lwp_thread;
1822 	tregs = clwp->lwp_regs;
1823 	tfpu = clwp->lwp_fpu;
1824 	brand_data = clwp->lwp_brand;
1825 
1826 	/*
1827 	 * Copy parent lwp to child lwp.  Hold child's p_lock to prevent
1828 	 * mstate_aggr_state() from reading stale mstate entries copied
1829 	 * from lwp to clwp.
1830 	 */
1831 	mutex_enter(&cp->p_lock);
1832 	*clwp = *lwp;
1833 
1834 	/* clear microstate and resource usage data in new lwp */
1835 	init_mstate(ct, LMS_STOPPED);
1836 	bzero(&clwp->lwp_ru, sizeof (clwp->lwp_ru));
1837 	mutex_exit(&cp->p_lock);
1838 
1839 	/* fix up child's lwp */
1840 
1841 	clwp->lwp_pcb.pcb_flags = 0;
1842 #if defined(__sparc)
1843 	clwp->lwp_pcb.pcb_step = STEP_NONE;
1844 #endif
1845 	clwp->lwp_cursig = 0;
1846 	clwp->lwp_extsig = 0;
1847 	clwp->lwp_curinfo = (struct sigqueue *)0;
1848 	clwp->lwp_thread = ct;
1849 	ct->t_sysnum = t->t_sysnum;
1850 	clwp->lwp_regs = tregs;
1851 	clwp->lwp_fpu = tfpu;
1852 	clwp->lwp_brand = brand_data;
1853 	clwp->lwp_ap = clwp->lwp_arg;
1854 	clwp->lwp_procp = cp;
1855 	bzero(clwp->lwp_timer, sizeof (clwp->lwp_timer));
1856 	clwp->lwp_lastfault = 0;
1857 	clwp->lwp_lastfaddr = 0;
1858 
1859 	/* copy parent's struct regs to child. */
1860 	lwp_forkregs(lwp, clwp);
1861 
1862 	/*
1863 	 * Fork thread context ops, if any.
1864 	 */
1865 	if (t->t_ctx)
1866 		forkctx(t, ct);
1867 
1868 	/* fix door state in the child */
1869 	if (t->t_door)
1870 		door_fork(t, ct);
1871 
1872 	/* copy current contract templates, clear latest contracts */
1873 	lwp_ctmpl_copy(clwp, lwp);
1874 
1875 	mutex_enter(&cp->p_lock);
1876 	/* lwp_create() set the TP_HOLDLWP flag */
1877 	if (!(t->t_proc_flag & TP_HOLDLWP))
1878 		ct->t_proc_flag &= ~TP_HOLDLWP;
1879 	if (cp->p_flag & SMSACCT)
1880 		ct->t_proc_flag |= TP_MSACCT;
1881 	mutex_exit(&cp->p_lock);
1882 
1883 	/* Allow brand to propagate brand-specific state */
1884 	if (PROC_IS_BRANDED(p))
1885 		BROP(p)->b_forklwp(lwp, clwp);
1886 
1887 retry:
1888 	cid = t->t_cid;
1889 
1890 	val = CL_ALLOC(&bufp, cid, KM_SLEEP);
1891 	ASSERT(val == 0);
1892 
1893 	mutex_enter(&p->p_lock);
1894 	if (cid != t->t_cid) {
1895 		/*
1896 		 * Someone just changed this thread's scheduling class,
1897 		 * so try pre-allocating the buffer again.  Hopefully we
1898 		 * don't hit this often.
1899 		 */
1900 		mutex_exit(&p->p_lock);
1901 		CL_FREE(cid, bufp);
1902 		goto retry;
1903 	}
1904 
1905 	ct->t_unpark = t->t_unpark;
1906 	ct->t_clfuncs = t->t_clfuncs;
1907 	CL_FORK(t, ct, bufp);
1908 	ct->t_cid = t->t_cid;	/* after data allocated so prgetpsinfo works */
1909 	mutex_exit(&p->p_lock);
1910 
1911 	return (clwp);
1912 }
1913 
1914 /*
1915  * Add a new lwp entry to the lwp directory and to the lwpid hash table.
1916  */
1917 void
1918 lwp_hash_in(proc_t *p, lwpent_t *lep, tidhash_t *tidhash, uint_t tidhash_sz,
1919     int do_lock)
1920 {
1921 	tidhash_t *thp = &tidhash[TIDHASH(lep->le_lwpid, tidhash_sz)];
1922 	lwpdir_t **ldpp;
1923 	lwpdir_t *ldp;
1924 	kthread_t *t;
1925 
1926 	/*
1927 	 * Allocate a directory element from the free list.
1928 	 * Code elsewhere guarantees a free slot.
1929 	 */
1930 	ldp = p->p_lwpfree;
1931 	p->p_lwpfree = ldp->ld_next;
1932 	ASSERT(ldp->ld_entry == NULL);
1933 	ldp->ld_entry = lep;
1934 
1935 	if (do_lock)
1936 		mutex_enter(&thp->th_lock);
1937 
1938 	/*
1939 	 * Insert it into the lwpid hash table.
1940 	 */
1941 	ldpp = &thp->th_list;
1942 	ldp->ld_next = *ldpp;
1943 	*ldpp = ldp;
1944 
1945 	/*
1946 	 * Set the active thread's directory slot entry.
1947 	 */
1948 	if ((t = lep->le_thread) != NULL) {
1949 		ASSERT(lep->le_lwpid == t->t_tid);
1950 		t->t_dslot = (int)(ldp - p->p_lwpdir);
1951 	}
1952 
1953 	if (do_lock)
1954 		mutex_exit(&thp->th_lock);
1955 }
1956 
1957 /*
1958  * Remove an lwp from the lwpid hash table and free its directory entry.
1959  * This is done when a detached lwp exits in lwp_exit() or
1960  * when a non-detached lwp is waited for in lwp_wait() or
1961  * when a zombie lwp is detached in lwp_detach().
1962  */
1963 void
1964 lwp_hash_out(proc_t *p, id_t lwpid)
1965 {
1966 	tidhash_t *thp = &p->p_tidhash[TIDHASH(lwpid, p->p_tidhash_sz)];
1967 	lwpdir_t **ldpp;
1968 	lwpdir_t *ldp;
1969 	lwpent_t *lep;
1970 
1971 	mutex_enter(&thp->th_lock);
1972 	for (ldpp = &thp->th_list;
1973 	    (ldp = *ldpp) != NULL; ldpp = &ldp->ld_next) {
1974 		lep = ldp->ld_entry;
1975 		if (lep->le_lwpid == lwpid) {
1976 			prlwpfree(p, lep);	/* /proc deals with le_trace */
1977 			*ldpp = ldp->ld_next;
1978 			ldp->ld_entry = NULL;
1979 			ldp->ld_next = p->p_lwpfree;
1980 			p->p_lwpfree = ldp;
1981 			kmem_free(lep, sizeof (*lep));
1982 			break;
1983 		}
1984 	}
1985 	mutex_exit(&thp->th_lock);
1986 }
1987 
1988 /*
1989  * Lookup an lwp in the lwpid hash table by lwpid.
1990  */
1991 lwpdir_t *
1992 lwp_hash_lookup(proc_t *p, id_t lwpid)
1993 {
1994 	tidhash_t *thp;
1995 	lwpdir_t *ldp;
1996 
1997 	/*
1998 	 * The process may be exiting, after p_tidhash has been set to NULL in
1999 	 * proc_exit() but before prfee() has been called.  Return failure in
2000 	 * this case.
2001 	 */
2002 	if (p->p_tidhash == NULL)
2003 		return (NULL);
2004 
2005 	thp = &p->p_tidhash[TIDHASH(lwpid, p->p_tidhash_sz)];
2006 	for (ldp = thp->th_list; ldp != NULL; ldp = ldp->ld_next) {
2007 		if (ldp->ld_entry->le_lwpid == lwpid)
2008 			return (ldp);
2009 	}
2010 
2011 	return (NULL);
2012 }
2013 
2014 /*
2015  * Same as lwp_hash_lookup(), but acquire and return
2016  * the tid hash table entry lock on success.
2017  */
2018 lwpdir_t *
2019 lwp_hash_lookup_and_lock(proc_t *p, id_t lwpid, kmutex_t **mpp)
2020 {
2021 	tidhash_t *tidhash;
2022 	uint_t tidhash_sz;
2023 	tidhash_t *thp;
2024 	lwpdir_t *ldp;
2025 
2026 top:
2027 	tidhash_sz = p->p_tidhash_sz;
2028 	membar_consumer();
2029 	if ((tidhash = p->p_tidhash) == NULL)
2030 		return (NULL);
2031 
2032 	thp = &tidhash[TIDHASH(lwpid, tidhash_sz)];
2033 	mutex_enter(&thp->th_lock);
2034 
2035 	/*
2036 	 * Since we are not holding p->p_lock, the tid hash table
2037 	 * may have changed.  If so, start over.  If not, then
2038 	 * it cannot change until after we drop &thp->th_lock;
2039 	 */
2040 	if (tidhash != p->p_tidhash || tidhash_sz != p->p_tidhash_sz) {
2041 		mutex_exit(&thp->th_lock);
2042 		goto top;
2043 	}
2044 
2045 	for (ldp = thp->th_list; ldp != NULL; ldp = ldp->ld_next) {
2046 		if (ldp->ld_entry->le_lwpid == lwpid) {
2047 			*mpp = &thp->th_lock;
2048 			return (ldp);
2049 		}
2050 	}
2051 
2052 	mutex_exit(&thp->th_lock);
2053 	return (NULL);
2054 }
2055 
2056 /*
2057  * Update the indicated LWP usage statistic for the current LWP.
2058  */
2059 void
2060 lwp_stat_update(lwp_stat_id_t lwp_stat_id, long inc)
2061 {
2062 	klwp_t *lwp = ttolwp(curthread);
2063 
2064 	if (lwp == NULL)
2065 		return;
2066 
2067 	switch (lwp_stat_id) {
2068 	case LWP_STAT_INBLK:
2069 		lwp->lwp_ru.inblock += inc;
2070 		break;
2071 	case LWP_STAT_OUBLK:
2072 		lwp->lwp_ru.oublock += inc;
2073 		break;
2074 	case LWP_STAT_MSGRCV:
2075 		lwp->lwp_ru.msgrcv += inc;
2076 		break;
2077 	case LWP_STAT_MSGSND:
2078 		lwp->lwp_ru.msgsnd += inc;
2079 		break;
2080 	default:
2081 		panic("lwp_stat_update: invalid lwp_stat_id 0x%x", lwp_stat_id);
2082 	}
2083 }
2084