xref: /illumos-gate/usr/src/uts/common/os/lwp.c (revision 2cb27123)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 #pragma ident	"%Z%%M%	%I%	%E% SMI"
28 
29 #include <sys/param.h>
30 #include <sys/types.h>
31 #include <sys/sysmacros.h>
32 #include <sys/systm.h>
33 #include <sys/thread.h>
34 #include <sys/proc.h>
35 #include <sys/task.h>
36 #include <sys/project.h>
37 #include <sys/signal.h>
38 #include <sys/errno.h>
39 #include <sys/vmparam.h>
40 #include <sys/stack.h>
41 #include <sys/procfs.h>
42 #include <sys/prsystm.h>
43 #include <sys/cpuvar.h>
44 #include <sys/kmem.h>
45 #include <sys/vtrace.h>
46 #include <sys/door.h>
47 #include <vm/seg_kp.h>
48 #include <sys/debug.h>
49 #include <sys/tnf.h>
50 #include <sys/schedctl.h>
51 #include <sys/poll.h>
52 #include <sys/copyops.h>
53 #include <sys/lwp_upimutex_impl.h>
54 #include <sys/cpupart.h>
55 #include <sys/lgrp.h>
56 #include <sys/rctl.h>
57 #include <sys/contract_impl.h>
58 #include <sys/cpc_impl.h>
59 #include <sys/sdt.h>
60 #include <sys/cmn_err.h>
61 #include <sys/brand.h>
62 
63 void *segkp_lwp;		/* cookie for pool of segkp resources */
64 
65 /*
66  * Create a thread that appears to be stopped at sys_rtt.
67  */
68 klwp_t *
69 lwp_create(void (*proc)(), caddr_t arg, size_t len, proc_t *p,
70     int state, int pri, const k_sigset_t *smask, int cid, id_t lwpid)
71 {
72 	klwp_t *lwp = NULL;
73 	kthread_t *t;
74 	kthread_t *tx;
75 	cpupart_t *oldpart = NULL;
76 	size_t	stksize;
77 	caddr_t lwpdata = NULL;
78 	processorid_t	binding;
79 	int err = 0;
80 	kproject_t *oldkpj, *newkpj;
81 	void *bufp = NULL;
82 	klwp_t *curlwp = ttolwp(curthread);
83 	lwpent_t *lep;
84 	lwpdir_t *old_dir = NULL;
85 	uint_t old_dirsz = 0;
86 	lwpdir_t **old_hash = NULL;
87 	uint_t old_hashsz = 0;
88 	int i;
89 	int rctlfail = 0;
90 	boolean_t branded = 0;
91 
92 	mutex_enter(&p->p_lock);
93 	mutex_enter(&p->p_zone->zone_nlwps_lock);
94 	/*
95 	 * don't enforce rctl limits on system processes
96 	 */
97 	if (cid != syscid) {
98 		if (p->p_task->tk_nlwps >= p->p_task->tk_nlwps_ctl)
99 			if (rctl_test(rc_task_lwps, p->p_task->tk_rctls, p,
100 			    1, 0) & RCT_DENY)
101 				rctlfail = 1;
102 		if (p->p_task->tk_proj->kpj_nlwps >=
103 		    p->p_task->tk_proj->kpj_nlwps_ctl)
104 			if (rctl_test(rc_project_nlwps,
105 			    p->p_task->tk_proj->kpj_rctls, p, 1, 0)
106 			    & RCT_DENY)
107 				rctlfail = 1;
108 		if (p->p_zone->zone_nlwps >= p->p_zone->zone_nlwps_ctl)
109 			if (rctl_test(rc_zone_nlwps, p->p_zone->zone_rctls, p,
110 			    1, 0) & RCT_DENY)
111 				rctlfail = 1;
112 	}
113 	if (rctlfail) {
114 		mutex_exit(&p->p_zone->zone_nlwps_lock);
115 		mutex_exit(&p->p_lock);
116 		return (NULL);
117 	}
118 	p->p_task->tk_nlwps++;
119 	p->p_task->tk_proj->kpj_nlwps++;
120 	p->p_zone->zone_nlwps++;
121 	mutex_exit(&p->p_zone->zone_nlwps_lock);
122 	mutex_exit(&p->p_lock);
123 
124 	if (curlwp == NULL || (stksize = curlwp->lwp_childstksz) == 0)
125 		stksize = lwp_default_stksize;
126 
127 	/*
128 	 * Try to reclaim a <lwp,stack> from 'deathrow'
129 	 */
130 	if (stksize == lwp_default_stksize) {
131 		if (lwp_reapcnt > 0) {
132 			mutex_enter(&reaplock);
133 			if ((t = lwp_deathrow) != NULL) {
134 				ASSERT(t->t_swap);
135 				lwp_deathrow = t->t_forw;
136 				lwp_reapcnt--;
137 				lwpdata = t->t_swap;
138 				lwp = t->t_lwp;
139 			}
140 			mutex_exit(&reaplock);
141 			if (t) {
142 				t->t_swap = NULL;
143 				lwp_stk_fini(t->t_lwp);
144 				t->t_lwp = NULL;
145 				t->t_forw = NULL;
146 				thread_free(t);
147 			}
148 		}
149 		if (lwpdata == NULL &&
150 		    (lwpdata = (caddr_t)segkp_cache_get(segkp_lwp)) == NULL) {
151 			mutex_enter(&p->p_lock);
152 			mutex_enter(&p->p_zone->zone_nlwps_lock);
153 			p->p_task->tk_nlwps--;
154 			p->p_task->tk_proj->kpj_nlwps--;
155 			p->p_zone->zone_nlwps--;
156 			mutex_exit(&p->p_zone->zone_nlwps_lock);
157 			mutex_exit(&p->p_lock);
158 			return (NULL);
159 		}
160 	} else {
161 		stksize = roundup(stksize, PAGESIZE);
162 		if ((lwpdata = (caddr_t)segkp_get(segkp, stksize,
163 		    (KPD_NOWAIT | KPD_HASREDZONE | KPD_LOCKED))) == NULL) {
164 			mutex_enter(&p->p_lock);
165 			mutex_enter(&p->p_zone->zone_nlwps_lock);
166 			p->p_task->tk_nlwps--;
167 			p->p_task->tk_proj->kpj_nlwps--;
168 			p->p_zone->zone_nlwps--;
169 			mutex_exit(&p->p_zone->zone_nlwps_lock);
170 			mutex_exit(&p->p_lock);
171 			return (NULL);
172 		}
173 	}
174 
175 	/*
176 	 * Create a thread, initializing the stack pointer
177 	 */
178 	t = thread_create(lwpdata, stksize, NULL, NULL, 0, p, TS_STOPPED, pri);
179 
180 	t->t_swap = lwpdata;	/* Start of page-able data */
181 	if (lwp == NULL)
182 		lwp = kmem_cache_alloc(lwp_cache, KM_SLEEP);
183 	bzero(lwp, sizeof (*lwp));
184 	t->t_lwp = lwp;
185 
186 	t->t_hold = *smask;
187 	lwp->lwp_thread = t;
188 	lwp->lwp_procp = p;
189 	lwp->lwp_sigaltstack.ss_flags = SS_DISABLE;
190 	if (curlwp != NULL && curlwp->lwp_childstksz != 0)
191 		lwp->lwp_childstksz = curlwp->lwp_childstksz;
192 
193 	t->t_stk = lwp_stk_init(lwp, t->t_stk);
194 	thread_load(t, proc, arg, len);
195 
196 	/*
197 	 * Allocate the SIGPROF buffer if ITIMER_REALPROF is in effect.
198 	 */
199 	if (timerisset(&p->p_rprof_timer.it_value))
200 		t->t_rprof = kmem_zalloc(sizeof (struct rprof), KM_SLEEP);
201 
202 	if (cid != NOCLASS)
203 		(void) CL_ALLOC(&bufp, cid, KM_SLEEP);
204 
205 	/*
206 	 * Allocate an lwp directory entry for the new lwp.
207 	 */
208 	lep = kmem_zalloc(sizeof (*lep), KM_SLEEP);
209 
210 	mutex_enter(&p->p_lock);
211 grow:
212 	/*
213 	 * Grow the lwp (thread) directory and lwpid hash table if necessary.
214 	 * A note on the growth algorithm:
215 	 *	The new lwp directory size is computed as:
216 	 *		new = 2 * old + 2
217 	 *	Starting with an initial size of 2 (see exec_common()),
218 	 *	this yields numbers that are a power of two minus 2:
219 	 *		2, 6, 14, 30, 62, 126, 254, 510, 1022, ...
220 	 *	The size of the lwpid hash table must be a power of two
221 	 *	and must be commensurate in size with the lwp directory
222 	 *	so that hash bucket chains remain short.  Therefore,
223 	 *	the lwpid hash table size is computed as:
224 	 *		hashsz = (dirsz + 2) / 2
225 	 *	which leads to these hash table sizes corresponding to
226 	 *	the above directory sizes:
227 	 *		2, 4, 8, 16, 32, 64, 128, 256, 512, ...
228 	 */
229 	while (p->p_lwpfree == NULL) {
230 		uint_t dirsz = p->p_lwpdir_sz;
231 		uint_t new_dirsz;
232 		uint_t new_hashsz;
233 		lwpdir_t *new_dir;
234 		lwpdir_t *ldp;
235 		lwpdir_t **new_hash;
236 
237 		mutex_exit(&p->p_lock);
238 
239 		if (old_dir != NULL) {
240 			kmem_free(old_dir, old_dirsz * sizeof (*old_dir));
241 			kmem_free(old_hash, old_hashsz * sizeof (*old_hash));
242 			old_dir = NULL;
243 			old_dirsz = 0;
244 			old_hash = NULL;
245 			old_hashsz = 0;
246 		}
247 		new_dirsz = 2 * dirsz + 2;
248 		new_dir = kmem_zalloc(new_dirsz * sizeof (lwpdir_t), KM_SLEEP);
249 		for (ldp = new_dir, i = 1; i < new_dirsz; i++, ldp++)
250 			ldp->ld_next = ldp + 1;
251 		new_hashsz = (new_dirsz + 2) / 2;
252 		new_hash = kmem_zalloc(new_hashsz * sizeof (lwpdir_t *),
253 			KM_SLEEP);
254 
255 		mutex_enter(&p->p_lock);
256 		if (p == curproc)
257 			prbarrier(p);
258 
259 		if (dirsz != p->p_lwpdir_sz || p->p_lwpfree != NULL) {
260 			/*
261 			 * Someone else beat us to it or some lwp exited.
262 			 * Set up to free our memory and take a lap.
263 			 */
264 			old_dir = new_dir;
265 			old_dirsz = new_dirsz;
266 			old_hash = new_hash;
267 			old_hashsz = new_hashsz;
268 		} else {
269 			old_dir = p->p_lwpdir;
270 			old_dirsz = p->p_lwpdir_sz;
271 			old_hash = p->p_tidhash;
272 			old_hashsz = p->p_tidhash_sz;
273 			p->p_lwpdir = new_dir;
274 			p->p_lwpfree = new_dir;
275 			p->p_lwpdir_sz = new_dirsz;
276 			p->p_tidhash = new_hash;
277 			p->p_tidhash_sz = new_hashsz;
278 			/*
279 			 * We simply hash in all of the old directory entries.
280 			 * This works because the old directory has no empty
281 			 * slots and the new hash table starts out empty.
282 			 * This reproduces the original directory ordering
283 			 * (required for /proc directory semantics).
284 			 */
285 			for (ldp = old_dir, i = 0; i < dirsz; i++, ldp++)
286 				lwp_hash_in(p, ldp->ld_entry);
287 			/*
288 			 * Defer freeing memory until we drop p->p_lock,
289 			 */
290 		}
291 	}
292 
293 	/*
294 	 * Block the process against /proc while we manipulate p->p_tlist,
295 	 * unless lwp_create() was called by /proc for the PCAGENT operation.
296 	 * We want to do this early enough so that we don't drop p->p_lock
297 	 * until the thread is put on the p->p_tlist.
298 	 */
299 	if (p == curproc) {
300 		prbarrier(p);
301 		/*
302 		 * If the current lwp has been requested to stop, do so now.
303 		 * Otherwise we have a race condition between /proc attempting
304 		 * to stop the process and this thread creating a new lwp
305 		 * that was not seen when the /proc PCSTOP request was issued.
306 		 * We rely on stop() to call prbarrier(p) before returning.
307 		 */
308 		while ((curthread->t_proc_flag & TP_PRSTOP) &&
309 		    !ttolwp(curthread)->lwp_nostop)
310 			stop(PR_REQUESTED, 0);
311 
312 		/*
313 		 * If process is exiting, there could be a race between
314 		 * the agent lwp creation and the new lwp currently being
315 		 * created. So to prevent this race lwp creation is failed
316 		 * if the process is exiting.
317 		 */
318 		if (p->p_flag & (SEXITLWPS|SKILLED)) {
319 			err = 1;
320 			goto error;
321 		}
322 
323 		/*
324 		 * Since we might have dropped p->p_lock, the
325 		 * lwp directory free list might have changed.
326 		 */
327 		if (p->p_lwpfree == NULL)
328 			goto grow;
329 	}
330 
331 	kpreempt_disable();	/* can't grab cpu_lock here */
332 
333 	/*
334 	 * Inherit processor and processor set bindings from curthread,
335 	 * unless we're creating a new kernel process, in which case
336 	 * clear all bindings.
337 	 */
338 	if (cid == syscid) {
339 		t->t_bind_cpu = binding = PBIND_NONE;
340 		t->t_cpupart = oldpart = &cp_default;
341 		t->t_bind_pset = PS_NONE;
342 	} else {
343 		binding = curthread->t_bind_cpu;
344 		t->t_bind_cpu = binding;
345 		oldpart = t->t_cpupart;
346 		t->t_cpupart = curthread->t_cpupart;
347 		t->t_bind_pset = curthread->t_bind_pset;
348 	}
349 
350 	/*
351 	 * thread_create() initializes this thread's home lgroup to the root.
352 	 * Choose a more suitable lgroup, since this thread is associated
353 	 * with an lwp.
354 	 */
355 	ASSERT(oldpart != NULL);
356 	if (binding != PBIND_NONE && t->t_affinitycnt == 0) {
357 		t->t_bound_cpu = cpu[binding];
358 		if (t->t_lpl != t->t_bound_cpu->cpu_lpl)
359 			lgrp_move_thread(t, t->t_bound_cpu->cpu_lpl, 1);
360 	} else {
361 		lgrp_move_thread(t, lgrp_choose(t, t->t_cpupart), 1);
362 	}
363 
364 	kpreempt_enable();
365 
366 	/*
367 	 * make sure lpl points to our own partition
368 	 */
369 	ASSERT(t->t_lpl >= t->t_cpupart->cp_lgrploads);
370 	ASSERT(t->t_lpl < t->t_cpupart->cp_lgrploads +
371 	    t->t_cpupart->cp_nlgrploads);
372 
373 	/*
374 	 * If we're creating a new process, then inherit the project from our
375 	 * parent. If we're only creating an additional lwp then use the
376 	 * project pointer of the target process.
377 	 */
378 	if (p->p_task == NULL)
379 		newkpj = ttoproj(curthread);
380 	else
381 		newkpj = p->p_task->tk_proj;
382 
383 	/*
384 	 * It is safe to point the thread to the new project without holding it
385 	 * since we're holding the target process' p_lock here and therefore
386 	 * we're guaranteed that it will not move to another project.
387 	 */
388 	oldkpj = ttoproj(t);
389 	if (newkpj != oldkpj) {
390 		t->t_proj = newkpj;
391 		(void) project_hold(newkpj);
392 		project_rele(oldkpj);
393 	}
394 
395 	if (cid != NOCLASS) {
396 		/*
397 		 * If the lwp is being created in the current process
398 		 * and matches the current thread's scheduling class,
399 		 * we should propagate the current thread's scheduling
400 		 * parameters by calling CL_FORK.  Otherwise just use
401 		 * the defaults by calling CL_ENTERCLASS.
402 		 */
403 		if (p != curproc || curthread->t_cid != cid) {
404 			err = CL_ENTERCLASS(t, cid, NULL, NULL, bufp);
405 			t->t_pri = pri;	/* CL_ENTERCLASS may have changed it */
406 		} else {
407 			t->t_clfuncs = &(sclass[cid].cl_funcs->thread);
408 			err = CL_FORK(curthread, t, bufp);
409 			t->t_cid = cid;
410 		}
411 		if (err)
412 			goto error;
413 		else
414 			bufp = NULL;
415 	}
416 
417 	/*
418 	 * If we were given an lwpid then use it, else allocate one.
419 	 */
420 	if (lwpid != 0)
421 		t->t_tid = lwpid;
422 	else {
423 		/*
424 		 * lwp/thread id 0 is never valid; reserved for special checks.
425 		 * lwp/thread id 1 is reserved for the main thread.
426 		 * Start again at 2 when INT_MAX has been reached
427 		 * (id_t is a signed 32-bit integer).
428 		 */
429 		id_t prev_id = p->p_lwpid;	/* last allocated tid */
430 
431 		do {			/* avoid lwpid duplication */
432 			if (p->p_lwpid == INT_MAX) {
433 				p->p_flag |= SLWPWRAP;
434 				p->p_lwpid = 1;
435 			}
436 			if ((t->t_tid = ++p->p_lwpid) == prev_id) {
437 				/*
438 				 * All lwpids are allocated; fail the request.
439 				 */
440 				err = 1;
441 				goto error;
442 			}
443 			/*
444 			 * We only need to worry about colliding with an id
445 			 * that's already in use if this process has
446 			 * cycled through all available lwp ids.
447 			 */
448 			if ((p->p_flag & SLWPWRAP) == 0)
449 				break;
450 		} while (lwp_hash_lookup(p, t->t_tid) != NULL);
451 	}
452 
453 	/*
454 	 * If this is a branded process, let the brand do any necessary lwp
455 	 * initialization.
456 	 */
457 	if (PROC_IS_BRANDED(p)) {
458 		if (BROP(p)->b_initlwp(lwp)) {
459 			err = 1;
460 			goto error;
461 		}
462 		branded = 1;
463 	}
464 
465 	if (t->t_tid == 1) {
466 		kpreempt_disable();
467 		ASSERT(t->t_lpl != NULL);
468 		p->p_t1_lgrpid = t->t_lpl->lpl_lgrpid;
469 		kpreempt_enable();
470 		if (p->p_tr_lgrpid != LGRP_NONE &&
471 		    p->p_tr_lgrpid != p->p_t1_lgrpid) {
472 			lgrp_update_trthr_migrations(1);
473 		}
474 	}
475 
476 	p->p_lwpcnt++;
477 	t->t_waitfor = -1;
478 
479 	/*
480 	 * Turn microstate accounting on for thread if on for process.
481 	 */
482 	if (p->p_flag & SMSACCT)
483 		t->t_proc_flag |= TP_MSACCT;
484 
485 	/*
486 	 * If the process has watchpoints, mark the new thread as such.
487 	 */
488 	if (pr_watch_active(p))
489 		watch_enable(t);
490 
491 	/*
492 	 * The lwp is being created in the stopped state.
493 	 * We set all the necessary flags to indicate that fact here.
494 	 * We omit the TS_CREATE flag from t_schedflag so that the lwp
495 	 * cannot be set running until the caller is finished with it,
496 	 * even if lwp_continue() is called on it after we drop p->p_lock.
497 	 * When the caller is finished with the newly-created lwp,
498 	 * the caller must call lwp_create_done() to allow the lwp
499 	 * to be set running.  If the TP_HOLDLWP is left set, the
500 	 * lwp will suspend itself after reaching system call exit.
501 	 */
502 	init_mstate(t, LMS_STOPPED);
503 	t->t_proc_flag |= TP_HOLDLWP;
504 	t->t_schedflag |= (TS_ALLSTART & ~(TS_CSTART | TS_CREATE));
505 	t->t_whystop = PR_SUSPENDED;
506 	t->t_whatstop = SUSPEND_NORMAL;
507 	t->t_sig_check = 1;	/* ensure that TP_HOLDLWP is honored */
508 
509 	/*
510 	 * Set system call processing flags in case tracing or profiling
511 	 * is set.  The first system call will evaluate these and turn
512 	 * them off if they aren't needed.
513 	 */
514 	t->t_pre_sys = 1;
515 	t->t_post_sys = 1;
516 
517 	/*
518 	 * Insert the new thread into the list of all threads.
519 	 */
520 	if ((tx = p->p_tlist) == NULL) {
521 		t->t_back = t;
522 		t->t_forw = t;
523 		p->p_tlist = t;
524 	} else {
525 		t->t_forw = tx;
526 		t->t_back = tx->t_back;
527 		tx->t_back->t_forw = t;
528 		tx->t_back = t;
529 	}
530 
531 	/*
532 	 * Insert the new lwp into an lwp directory slot position
533 	 * and into the lwpid hash table.
534 	 */
535 	lep->le_thread = t;
536 	lep->le_lwpid = t->t_tid;
537 	lep->le_start = t->t_start;
538 	lwp_hash_in(p, lep);
539 
540 	if (state == TS_RUN) {
541 		/*
542 		 * We set the new lwp running immediately.
543 		 */
544 		t->t_proc_flag &= ~TP_HOLDLWP;
545 		lwp_create_done(t);
546 	}
547 
548 error:
549 	if (err) {
550 		/*
551 		 * We have failed to create an lwp, so decrement the number
552 		 * of lwps in the task and let the lgroup load averages know
553 		 * that this thread isn't going to show up.
554 		 */
555 		kpreempt_disable();
556 		lgrp_move_thread(t, NULL, 1);
557 		kpreempt_enable();
558 
559 		ASSERT(MUTEX_HELD(&p->p_lock));
560 		mutex_enter(&p->p_zone->zone_nlwps_lock);
561 		p->p_task->tk_nlwps--;
562 		p->p_task->tk_proj->kpj_nlwps--;
563 		p->p_zone->zone_nlwps--;
564 		mutex_exit(&p->p_zone->zone_nlwps_lock);
565 		if (cid != NOCLASS && bufp != NULL)
566 			CL_FREE(cid, bufp);
567 
568 		if (branded)
569 			BROP(p)->b_freelwp(lwp);
570 
571 		mutex_exit(&p->p_lock);
572 		t->t_state = TS_FREE;
573 		thread_rele(t);
574 
575 		/*
576 		 * We need to remove t from the list of all threads
577 		 * because thread_exit()/lwp_exit() isn't called on t.
578 		 */
579 		mutex_enter(&pidlock);
580 		ASSERT(t != t->t_next);		/* t0 never exits */
581 		t->t_next->t_prev = t->t_prev;
582 		t->t_prev->t_next = t->t_next;
583 		mutex_exit(&pidlock);
584 
585 		thread_free(t);
586 		kmem_free(lep, sizeof (*lep));
587 		lwp = NULL;
588 	} else {
589 		mutex_exit(&p->p_lock);
590 	}
591 
592 	if (old_dir != NULL) {
593 		kmem_free(old_dir, old_dirsz * sizeof (*old_dir));
594 		kmem_free(old_hash, old_hashsz * sizeof (*old_hash));
595 	}
596 
597 	DTRACE_PROC1(lwp__create, kthread_t *, t);
598 	return (lwp);
599 }
600 
601 /*
602  * lwp_create_done() is called by the caller of lwp_create() to set the
603  * newly-created lwp running after the caller has finished manipulating it.
604  */
605 void
606 lwp_create_done(kthread_t *t)
607 {
608 	proc_t *p = ttoproc(t);
609 
610 	ASSERT(MUTEX_HELD(&p->p_lock));
611 
612 	/*
613 	 * We set the TS_CREATE and TS_CSTART flags and call setrun_locked().
614 	 * (The absence of the TS_CREATE flag prevents the lwp from running
615 	 * until we are finished with it, even if lwp_continue() is called on
616 	 * it by some other lwp in the process or elsewhere in the kernel.)
617 	 */
618 	thread_lock(t);
619 	ASSERT(t->t_state == TS_STOPPED && !(t->t_schedflag & TS_CREATE));
620 	/*
621 	 * If TS_CSTART is set, lwp_continue(t) has been called and
622 	 * has already incremented p_lwprcnt; avoid doing this twice.
623 	 */
624 	if (!(t->t_schedflag & TS_CSTART))
625 		p->p_lwprcnt++;
626 	t->t_schedflag |= (TS_CSTART | TS_CREATE);
627 	setrun_locked(t);
628 	thread_unlock(t);
629 }
630 
631 /*
632  * Copy an LWP's active templates, and clear the latest contracts.
633  */
634 void
635 lwp_ctmpl_copy(klwp_t *dst, klwp_t *src)
636 {
637 	int i;
638 
639 	for (i = 0; i < ct_ntypes; i++) {
640 		dst->lwp_ct_active[i] = ctmpl_dup(src->lwp_ct_active[i]);
641 		dst->lwp_ct_latest[i] = NULL;
642 	}
643 }
644 
645 /*
646  * Clear an LWP's contract template state.
647  */
648 void
649 lwp_ctmpl_clear(klwp_t *lwp)
650 {
651 	ct_template_t *tmpl;
652 	int i;
653 
654 	for (i = 0; i < ct_ntypes; i++) {
655 		if ((tmpl = lwp->lwp_ct_active[i]) != NULL) {
656 			ctmpl_free(tmpl);
657 			lwp->lwp_ct_active[i] = NULL;
658 		}
659 
660 		if (lwp->lwp_ct_latest[i] != NULL) {
661 			contract_rele(lwp->lwp_ct_latest[i]);
662 			lwp->lwp_ct_latest[i] = NULL;
663 		}
664 	}
665 }
666 
667 /*
668  * Individual lwp exit.
669  * If this is the last lwp, exit the whole process.
670  */
671 void
672 lwp_exit(void)
673 {
674 	kthread_t *t = curthread;
675 	klwp_t *lwp = ttolwp(t);
676 	proc_t *p = ttoproc(t);
677 
678 	ASSERT(MUTEX_HELD(&p->p_lock));
679 
680 	mutex_exit(&p->p_lock);
681 
682 #if defined(__sparc)
683 	/*
684 	 * Ensure that the user stack is fully abandoned..
685 	 */
686 	trash_user_windows();
687 #endif
688 
689 	tsd_exit();			/* free thread specific data */
690 
691 	kcpc_passivate();		/* Clean up performance counter state */
692 
693 	pollcleanup();
694 
695 	if (t->t_door)
696 		door_slam();
697 
698 	if (t->t_schedctl != NULL)
699 		schedctl_lwp_cleanup(t);
700 
701 	if (t->t_upimutex != NULL)
702 		upimutex_cleanup();
703 
704 	/*
705 	 * Perform any brand specific exit processing, then release any
706 	 * brand data associated with the lwp
707 	 */
708 	if (PROC_IS_BRANDED(p))
709 		BROP(p)->b_lwpexit(lwp);
710 
711 	mutex_enter(&p->p_lock);
712 	lwp_cleanup();
713 
714 	/*
715 	 * When this process is dumping core, its lwps are held here
716 	 * until the core dump is finished. Then exitlwps() is called
717 	 * again to release these lwps so that they can finish exiting.
718 	 */
719 	if (p->p_flag & SCOREDUMP)
720 		stop(PR_SUSPENDED, SUSPEND_NORMAL);
721 
722 	/*
723 	 * Block the process against /proc now that we have really acquired
724 	 * p->p_lock (to decrement p_lwpcnt and manipulate p_tlist at least).
725 	 */
726 	prbarrier(p);
727 
728 	/*
729 	 * Call proc_exit() if this is the last non-daemon lwp in the process.
730 	 */
731 	if (!(t->t_proc_flag & TP_DAEMON) &&
732 	    p->p_lwpcnt == p->p_lwpdaemon + 1) {
733 		mutex_exit(&p->p_lock);
734 		if (proc_exit(CLD_EXITED, 0) == 0) {
735 			/* Restarting init. */
736 			return;
737 		}
738 
739 		/*
740 		 * proc_exit() returns a non-zero value when some other
741 		 * lwp got there first.  We just have to continue in
742 		 * lwp_exit().
743 		 */
744 		mutex_enter(&p->p_lock);
745 		ASSERT(curproc->p_flag & SEXITLWPS);
746 		prbarrier(p);
747 	}
748 
749 	DTRACE_PROC(lwp__exit);
750 
751 	/*
752 	 * If the lwp is a detached lwp or if the process is exiting,
753 	 * remove (lwp_hash_out()) the lwp from the lwp directory.
754 	 * Otherwise null out the lwp's le_thread pointer in the lwp
755 	 * directory so that other threads will see it as a zombie lwp.
756 	 */
757 	prlwpexit(t);		/* notify /proc */
758 	if (!(t->t_proc_flag & TP_TWAIT) || (p->p_flag & SEXITLWPS))
759 		lwp_hash_out(p, t->t_tid);
760 	else {
761 		ASSERT(!(t->t_proc_flag & TP_DAEMON));
762 		p->p_lwpdir[t->t_dslot].ld_entry->le_thread = NULL;
763 		p->p_zombcnt++;
764 		cv_broadcast(&p->p_lwpexit);
765 	}
766 	if (t->t_proc_flag & TP_DAEMON) {
767 		p->p_lwpdaemon--;
768 		t->t_proc_flag &= ~TP_DAEMON;
769 	}
770 	t->t_proc_flag &= ~TP_TWAIT;
771 
772 	/*
773 	 * Maintain accurate lwp count for task.max-lwps resource control.
774 	 */
775 	mutex_enter(&p->p_zone->zone_nlwps_lock);
776 	p->p_task->tk_nlwps--;
777 	p->p_task->tk_proj->kpj_nlwps--;
778 	p->p_zone->zone_nlwps--;
779 	mutex_exit(&p->p_zone->zone_nlwps_lock);
780 
781 	CL_EXIT(t);		/* tell the scheduler that t is exiting */
782 	ASSERT(p->p_lwpcnt != 0);
783 	p->p_lwpcnt--;
784 
785 	/*
786 	 * If all remaining non-daemon lwps are waiting in lwp_wait(),
787 	 * wake them up so someone can return EDEADLK.
788 	 * (See the block comment preceeding lwp_wait().)
789 	 */
790 	if (p->p_lwpcnt == p->p_lwpdaemon + (p->p_lwpwait - p->p_lwpdwait))
791 		cv_broadcast(&p->p_lwpexit);
792 
793 	t->t_proc_flag |= TP_LWPEXIT;
794 	term_mstate(t);
795 
796 #ifndef NPROBE
797 	/* Kernel probe */
798 	if (t->t_tnf_tpdp)
799 		tnf_thread_exit();
800 #endif /* NPROBE */
801 
802 	t->t_forw->t_back = t->t_back;
803 	t->t_back->t_forw = t->t_forw;
804 	if (t == p->p_tlist)
805 		p->p_tlist = t->t_forw;
806 
807 	/*
808 	 * Clean up the signal state.
809 	 */
810 	if (t->t_sigqueue != NULL)
811 		sigdelq(p, t, 0);
812 	if (lwp->lwp_curinfo != NULL) {
813 		siginfofree(lwp->lwp_curinfo);
814 		lwp->lwp_curinfo = NULL;
815 	}
816 
817 	thread_rele(t);
818 
819 	/*
820 	 * Terminated lwps are associated with process zero and are put onto
821 	 * death-row by resume().  Avoid preemption after resetting t->t_procp.
822 	 */
823 	t->t_preempt++;
824 
825 	if (t->t_ctx != NULL)
826 		exitctx(t);
827 	if (p->p_pctx != NULL)
828 		exitpctx(p);
829 
830 	t->t_procp = &p0;
831 
832 	/*
833 	 * Notify the HAT about the change of address space
834 	 */
835 	hat_thread_exit(t);
836 	/*
837 	 * When this is the last running lwp in this process and some lwp is
838 	 * waiting for this condition to become true, or this thread was being
839 	 * suspended, then the waiting lwp is awakened.
840 	 *
841 	 * Also, if the process is exiting, we may have a thread waiting in
842 	 * exitlwps() that needs to be notified.
843 	 */
844 	if (--p->p_lwprcnt == 0 || (t->t_proc_flag & TP_HOLDLWP) ||
845 	    (p->p_flag & SEXITLWPS))
846 		cv_broadcast(&p->p_holdlwps);
847 
848 	/*
849 	 * Need to drop p_lock so we can reacquire pidlock.
850 	 */
851 	mutex_exit(&p->p_lock);
852 	mutex_enter(&pidlock);
853 
854 	ASSERT(t != t->t_next);		/* t0 never exits */
855 	t->t_next->t_prev = t->t_prev;
856 	t->t_prev->t_next = t->t_next;
857 	cv_broadcast(&t->t_joincv);	/* wake up anyone in thread_join */
858 	mutex_exit(&pidlock);
859 
860 	lwp_pcb_exit();
861 
862 	t->t_state = TS_ZOMB;
863 	swtch_from_zombie();
864 	/* never returns */
865 }
866 
867 
868 /*
869  * Cleanup function for an exiting lwp.
870  * Called both from lwp_exit() and from proc_exit().
871  * p->p_lock is repeatedly released and grabbed in this function.
872  */
873 void
874 lwp_cleanup(void)
875 {
876 	kthread_t *t = curthread;
877 	proc_t *p = ttoproc(t);
878 
879 	ASSERT(MUTEX_HELD(&p->p_lock));
880 
881 	/* untimeout any lwp-bound realtime timers */
882 	if (p->p_itimer != NULL)
883 		timer_lwpexit();
884 
885 	/*
886 	 * If this is the /proc agent lwp that is exiting, readjust p_lwpid
887 	 * so it appears that the agent never existed, and clear p_agenttp.
888 	 */
889 	if (t == p->p_agenttp) {
890 		ASSERT(t->t_tid == p->p_lwpid);
891 		p->p_lwpid--;
892 		p->p_agenttp = NULL;
893 	}
894 
895 	/*
896 	 * Do lgroup bookkeeping to account for thread exiting.
897 	 */
898 	kpreempt_disable();
899 	lgrp_move_thread(t, NULL, 1);
900 	if (t->t_tid == 1) {
901 		p->p_t1_lgrpid = LGRP_NONE;
902 	}
903 	kpreempt_enable();
904 
905 	lwp_ctmpl_clear(ttolwp(t));
906 }
907 
908 int
909 lwp_suspend(kthread_t *t)
910 {
911 	int tid;
912 	proc_t *p = ttoproc(t);
913 
914 	ASSERT(MUTEX_HELD(&p->p_lock));
915 
916 	/*
917 	 * Set the thread's TP_HOLDLWP flag so it will stop in holdlwp().
918 	 * If an lwp is stopping itself, there is no need to wait.
919 	 */
920 top:
921 	t->t_proc_flag |= TP_HOLDLWP;
922 	if (t == curthread) {
923 		t->t_sig_check = 1;
924 	} else {
925 		/*
926 		 * Make sure the lwp stops promptly.
927 		 */
928 		thread_lock(t);
929 		t->t_sig_check = 1;
930 		/*
931 		 * XXX Should use virtual stop like /proc does instead of
932 		 * XXX waking the thread to get it to stop.
933 		 */
934 		if (ISWAKEABLE(t) || ISWAITING(t)) {
935 			setrun_locked(t);
936 		} else if (t->t_state == TS_ONPROC && t->t_cpu != CPU) {
937 			poke_cpu(t->t_cpu->cpu_id);
938 		}
939 
940 		tid = t->t_tid;	 /* remember thread ID */
941 		/*
942 		 * Wait for lwp to stop
943 		 */
944 		while (!SUSPENDED(t)) {
945 			/*
946 			 * Drop the thread lock before waiting and reacquire it
947 			 * afterwards, so the thread can change its t_state
948 			 * field.
949 			 */
950 			thread_unlock(t);
951 
952 			/*
953 			 * Check if aborted by exitlwps().
954 			 */
955 			if (p->p_flag & SEXITLWPS)
956 				lwp_exit();
957 
958 			/*
959 			 * Cooperate with jobcontrol signals and /proc stopping
960 			 * by calling cv_wait_sig() to wait for the target
961 			 * lwp to stop.  Just using cv_wait() can lead to
962 			 * deadlock because, if some other lwp has stopped
963 			 * by either of these mechanisms, then p_lwprcnt will
964 			 * never become zero if we do a cv_wait().
965 			 */
966 			if (!cv_wait_sig(&p->p_holdlwps, &p->p_lock))
967 				return (EINTR);
968 
969 			/*
970 			 * Check to see if thread died while we were
971 			 * waiting for it to suspend.
972 			 */
973 			if (idtot(p, tid) == NULL)
974 				return (ESRCH);
975 
976 			thread_lock(t);
977 			/*
978 			 * If the TP_HOLDLWP flag went away, lwp_continue()
979 			 * or vfork() must have been called while we were
980 			 * waiting, so start over again.
981 			 */
982 			if ((t->t_proc_flag & TP_HOLDLWP) == 0) {
983 				thread_unlock(t);
984 				goto top;
985 			}
986 		}
987 		thread_unlock(t);
988 	}
989 	return (0);
990 }
991 
992 /*
993  * continue a lwp that's been stopped by lwp_suspend().
994  */
995 void
996 lwp_continue(kthread_t *t)
997 {
998 	proc_t *p = ttoproc(t);
999 	int was_suspended = t->t_proc_flag & TP_HOLDLWP;
1000 
1001 	ASSERT(MUTEX_HELD(&p->p_lock));
1002 
1003 	t->t_proc_flag &= ~TP_HOLDLWP;
1004 	thread_lock(t);
1005 	if (SUSPENDED(t) &&
1006 	    !(p->p_flag & (SHOLDFORK | SHOLDFORK1 | SHOLDWATCH))) {
1007 		p->p_lwprcnt++;
1008 		t->t_schedflag |= TS_CSTART;
1009 		setrun_locked(t);
1010 	}
1011 	thread_unlock(t);
1012 	/*
1013 	 * Wakeup anyone waiting for this thread to be suspended
1014 	 */
1015 	if (was_suspended)
1016 		cv_broadcast(&p->p_holdlwps);
1017 }
1018 
1019 /*
1020  * ********************************
1021  *  Miscellaneous lwp routines	  *
1022  * ********************************
1023  */
1024 /*
1025  * When a process is undergoing a forkall(), its p_flag is set to SHOLDFORK.
1026  * This will cause the process's lwps to stop at a hold point.  A hold
1027  * point is where a kernel thread has a flat stack.  This is at the
1028  * return from a system call and at the return from a user level trap.
1029  *
1030  * When a process is undergoing a fork1() or vfork(), its p_flag is set to
1031  * SHOLDFORK1.  This will cause the process's lwps to stop at a modified
1032  * hold point.  The lwps in the process are not being cloned, so they
1033  * are held at the usual hold points and also within issig_forreal().
1034  * This has the side-effect that their system calls do not return
1035  * showing EINTR.
1036  *
1037  * An lwp can also be held.  This is identified by the TP_HOLDLWP flag on
1038  * the thread.  The TP_HOLDLWP flag is set in lwp_suspend(), where the active
1039  * lwp is waiting for the target lwp to be stopped.
1040  */
1041 void
1042 holdlwp(void)
1043 {
1044 	proc_t *p = curproc;
1045 	kthread_t *t = curthread;
1046 
1047 	mutex_enter(&p->p_lock);
1048 	/*
1049 	 * Don't terminate immediately if the process is dumping core.
1050 	 * Once the process has dumped core, all lwps are terminated.
1051 	 */
1052 	if (!(p->p_flag & SCOREDUMP)) {
1053 		if ((p->p_flag & SEXITLWPS) || (t->t_proc_flag & TP_EXITLWP))
1054 			lwp_exit();
1055 	}
1056 	if (!(ISHOLD(p)) && !(p->p_flag & (SHOLDFORK1 | SHOLDWATCH))) {
1057 		mutex_exit(&p->p_lock);
1058 		return;
1059 	}
1060 	/*
1061 	 * stop() decrements p->p_lwprcnt and cv_signal()s &p->p_holdlwps
1062 	 * when p->p_lwprcnt becomes zero.
1063 	 */
1064 	stop(PR_SUSPENDED, SUSPEND_NORMAL);
1065 	if (p->p_flag & SEXITLWPS)
1066 		lwp_exit();
1067 	mutex_exit(&p->p_lock);
1068 }
1069 
1070 /*
1071  * Have all lwps within the process hold at a point where they are
1072  * cloneable (SHOLDFORK) or just safe w.r.t. fork1 (SHOLDFORK1).
1073  */
1074 int
1075 holdlwps(int holdflag)
1076 {
1077 	proc_t *p = curproc;
1078 
1079 	ASSERT(holdflag == SHOLDFORK || holdflag == SHOLDFORK1);
1080 	mutex_enter(&p->p_lock);
1081 	schedctl_finish_sigblock(curthread);
1082 again:
1083 	while (p->p_flag & (SEXITLWPS | SHOLDFORK | SHOLDFORK1 | SHOLDWATCH)) {
1084 		/*
1085 		 * If another lwp is doing a forkall() or proc_exit(), bail out.
1086 		 */
1087 		if (p->p_flag & (SEXITLWPS | SHOLDFORK)) {
1088 			mutex_exit(&p->p_lock);
1089 			return (0);
1090 		}
1091 		/*
1092 		 * Another lwp is doing a fork1() or is undergoing
1093 		 * watchpoint activity.  We hold here for it to complete.
1094 		 */
1095 		stop(PR_SUSPENDED, SUSPEND_NORMAL);
1096 	}
1097 	p->p_flag |= holdflag;
1098 	pokelwps(p);
1099 	--p->p_lwprcnt;
1100 	/*
1101 	 * Wait for the process to become quiescent (p->p_lwprcnt == 0).
1102 	 */
1103 	while (p->p_lwprcnt > 0) {
1104 		/*
1105 		 * Check if aborted by exitlwps().
1106 		 * Also check if SHOLDWATCH is set; it takes precedence.
1107 		 */
1108 		if (p->p_flag & (SEXITLWPS | SHOLDWATCH)) {
1109 			p->p_lwprcnt++;
1110 			p->p_flag &= ~holdflag;
1111 			cv_broadcast(&p->p_holdlwps);
1112 			goto again;
1113 		}
1114 		/*
1115 		 * Cooperate with jobcontrol signals and /proc stopping.
1116 		 * If some other lwp has stopped by either of these
1117 		 * mechanisms, then p_lwprcnt will never become zero
1118 		 * and the process will appear deadlocked unless we
1119 		 * stop here in sympathy with the other lwp before
1120 		 * doing the cv_wait() below.
1121 		 *
1122 		 * If the other lwp stops after we do the cv_wait(), it
1123 		 * will wake us up to loop around and do the sympathy stop.
1124 		 *
1125 		 * Since stop() drops p->p_lock, we must start from
1126 		 * the top again on returning from stop().
1127 		 */
1128 		if (p->p_stopsig | (curthread->t_proc_flag & TP_PRSTOP)) {
1129 			int whystop = p->p_stopsig? PR_JOBCONTROL :
1130 			    PR_REQUESTED;
1131 			p->p_lwprcnt++;
1132 			p->p_flag &= ~holdflag;
1133 			stop(whystop, p->p_stopsig);
1134 			goto again;
1135 		}
1136 		cv_wait(&p->p_holdlwps, &p->p_lock);
1137 	}
1138 	p->p_lwprcnt++;
1139 	p->p_flag &= ~holdflag;
1140 	mutex_exit(&p->p_lock);
1141 	return (1);
1142 }
1143 
1144 /*
1145  * See comments for holdwatch(), below.
1146  */
1147 static int
1148 holdcheck(int clearflags)
1149 {
1150 	proc_t *p = curproc;
1151 
1152 	/*
1153 	 * If we are trying to exit, that takes precedence over anything else.
1154 	 */
1155 	if (p->p_flag & SEXITLWPS) {
1156 		p->p_lwprcnt++;
1157 		p->p_flag &= ~clearflags;
1158 		lwp_exit();
1159 	}
1160 
1161 	/*
1162 	 * If another thread is calling fork1(), stop the current thread so the
1163 	 * other can complete.
1164 	 */
1165 	if (p->p_flag & SHOLDFORK1) {
1166 		p->p_lwprcnt++;
1167 		stop(PR_SUSPENDED, SUSPEND_NORMAL);
1168 		if (p->p_flag & SEXITLWPS) {
1169 			p->p_flag &= ~clearflags;
1170 			lwp_exit();
1171 		}
1172 		return (-1);
1173 	}
1174 
1175 	/*
1176 	 * If another thread is calling fork(), then indicate we are doing
1177 	 * watchpoint activity.  This will cause holdlwps() above to stop the
1178 	 * forking thread, at which point we can continue with watchpoint
1179 	 * activity.
1180 	 */
1181 	if (p->p_flag & SHOLDFORK) {
1182 		p->p_lwprcnt++;
1183 		while (p->p_flag & SHOLDFORK) {
1184 			p->p_flag |= SHOLDWATCH;
1185 			cv_broadcast(&p->p_holdlwps);
1186 			cv_wait(&p->p_holdlwps, &p->p_lock);
1187 			p->p_flag &= ~SHOLDWATCH;
1188 		}
1189 		return (-1);
1190 	}
1191 
1192 	return (0);
1193 }
1194 
1195 /*
1196  * Stop all lwps within the process, holding themselves in the kernel while the
1197  * active lwp undergoes watchpoint activity.  This is more complicated than
1198  * expected because stop() relies on calling holdwatch() in order to copyin data
1199  * from the user's address space.  A double barrier is used to prevent an
1200  * infinite loop.
1201  *
1202  * 	o The first thread into holdwatch() is the 'master' thread and does
1203  *        the following:
1204  *
1205  *              - Sets SHOLDWATCH on the current process
1206  *              - Sets TP_WATCHSTOP on the current thread
1207  *              - Waits for all threads to be either stopped or have
1208  *                TP_WATCHSTOP set.
1209  *              - Sets the SWATCHOK flag on the process
1210  *              - Unsets TP_WATCHSTOP
1211  *              - Waits for the other threads to completely stop
1212  *              - Unsets SWATCHOK
1213  *
1214  * 	o If SHOLDWATCH is already set when we enter this function, then another
1215  *        thread is already trying to stop this thread.  This 'slave' thread
1216  *        does the following:
1217  *
1218  *              - Sets TP_WATCHSTOP on the current thread
1219  *              - Waits for SWATCHOK flag to be set
1220  *              - Calls stop()
1221  *
1222  * 	o If SWATCHOK is set on the process, then this function immediately
1223  *        returns, as we must have been called via stop().
1224  *
1225  * In addition, there are other flags that take precedence over SHOLDWATCH:
1226  *
1227  * 	o If SEXITLWPS is set, exit immediately.
1228  *
1229  * 	o If SHOLDFORK1 is set, wait for fork1() to complete.
1230  *
1231  * 	o If SHOLDFORK is set, then watchpoint activity takes precedence In this
1232  *        case, set SHOLDWATCH, signalling the forking thread to stop first.
1233  *
1234  * 	o If the process is being stopped via /proc (TP_PRSTOP is set), then we
1235  *        stop the current thread.
1236  *
1237  * Returns 0 if all threads have been quiesced.  Returns non-zero if not all
1238  * threads were stopped, or the list of watched pages has changed.
1239  */
1240 int
1241 holdwatch(void)
1242 {
1243 	proc_t *p = curproc;
1244 	kthread_t *t = curthread;
1245 	int ret = 0;
1246 
1247 	mutex_enter(&p->p_lock);
1248 
1249 	p->p_lwprcnt--;
1250 
1251 	/*
1252 	 * Check for bail-out conditions as outlined above.
1253 	 */
1254 	if (holdcheck(0) != 0) {
1255 		mutex_exit(&p->p_lock);
1256 		return (-1);
1257 	}
1258 
1259 	if (!(p->p_flag & SHOLDWATCH)) {
1260 		/*
1261 		 * We are the master watchpoint thread.  Set SHOLDWATCH and poke
1262 		 * the other threads.
1263 		 */
1264 		p->p_flag |= SHOLDWATCH;
1265 		pokelwps(p);
1266 
1267 		/*
1268 		 * Wait for all threads to be stopped or have TP_WATCHSTOP set.
1269 		 */
1270 		while (pr_allstopped(p, 1) > 0) {
1271 			if (holdcheck(SHOLDWATCH) != 0) {
1272 				p->p_flag &= ~SHOLDWATCH;
1273 				mutex_exit(&p->p_lock);
1274 				return (-1);
1275 			}
1276 
1277 			cv_wait(&p->p_holdlwps, &p->p_lock);
1278 		}
1279 
1280 		/*
1281 		 * All threads are now stopped or in the process of stopping.
1282 		 * Set SWATCHOK and let them stop completely.
1283 		 */
1284 		p->p_flag |= SWATCHOK;
1285 		t->t_proc_flag &= ~TP_WATCHSTOP;
1286 		cv_broadcast(&p->p_holdlwps);
1287 
1288 		while (pr_allstopped(p, 0) > 0) {
1289 			/*
1290 			 * At first glance, it may appear that we don't need a
1291 			 * call to holdcheck() here.  But if the process gets a
1292 			 * SIGKILL signal, one of our stopped threads may have
1293 			 * been awakened and is waiting in exitlwps(), which
1294 			 * takes precedence over watchpoints.
1295 			 */
1296 			if (holdcheck(SHOLDWATCH | SWATCHOK) != 0) {
1297 				p->p_flag &= ~(SHOLDWATCH | SWATCHOK);
1298 				mutex_exit(&p->p_lock);
1299 				return (-1);
1300 			}
1301 
1302 			cv_wait(&p->p_holdlwps, &p->p_lock);
1303 		}
1304 
1305 		/*
1306 		 * All threads are now completely stopped.
1307 		 */
1308 		p->p_flag &= ~SWATCHOK;
1309 		p->p_flag &= ~SHOLDWATCH;
1310 		p->p_lwprcnt++;
1311 
1312 	} else if (!(p->p_flag & SWATCHOK)) {
1313 
1314 		/*
1315 		 * SHOLDWATCH is set, so another thread is trying to do
1316 		 * watchpoint activity.  Indicate this thread is stopping, and
1317 		 * wait for the OK from the master thread.
1318 		 */
1319 		t->t_proc_flag |= TP_WATCHSTOP;
1320 		cv_broadcast(&p->p_holdlwps);
1321 
1322 		while (!(p->p_flag & SWATCHOK)) {
1323 			if (holdcheck(0) != 0) {
1324 				t->t_proc_flag &= ~TP_WATCHSTOP;
1325 				mutex_exit(&p->p_lock);
1326 				return (-1);
1327 			}
1328 
1329 			cv_wait(&p->p_holdlwps, &p->p_lock);
1330 		}
1331 
1332 		/*
1333 		 * Once the master thread has given the OK, this thread can
1334 		 * actually call stop().
1335 		 */
1336 		t->t_proc_flag &= ~TP_WATCHSTOP;
1337 		p->p_lwprcnt++;
1338 
1339 		stop(PR_SUSPENDED, SUSPEND_NORMAL);
1340 
1341 		/*
1342 		 * It's not OK to do watchpoint activity, notify caller to
1343 		 * retry.
1344 		 */
1345 		ret = -1;
1346 
1347 	} else {
1348 
1349 		/*
1350 		 * The only way we can hit the case where SHOLDWATCH is set and
1351 		 * SWATCHOK is set is if we are triggering this from within a
1352 		 * stop() call.  Assert that this is the case.
1353 		 */
1354 
1355 		ASSERT(t->t_proc_flag & TP_STOPPING);
1356 		p->p_lwprcnt++;
1357 	}
1358 
1359 	mutex_exit(&p->p_lock);
1360 
1361 	return (ret);
1362 }
1363 
1364 /*
1365  * force all interruptible lwps to trap into the kernel.
1366  */
1367 void
1368 pokelwps(proc_t *p)
1369 {
1370 	kthread_t *t;
1371 
1372 	ASSERT(MUTEX_HELD(&p->p_lock));
1373 
1374 	t = p->p_tlist;
1375 	do {
1376 		if (t == curthread)
1377 			continue;
1378 		thread_lock(t);
1379 		aston(t);	/* make thread trap or do post_syscall */
1380 		if (ISWAKEABLE(t) || ISWAITING(t)) {
1381 			setrun_locked(t);
1382 		} else if (t->t_state == TS_STOPPED) {
1383 			/*
1384 			 * Ensure that proc_exit() is not blocked by lwps
1385 			 * that were stopped via jobcontrol or /proc.
1386 			 */
1387 			if (p->p_flag & SEXITLWPS) {
1388 				p->p_stopsig = 0;
1389 				t->t_schedflag |= (TS_XSTART | TS_PSTART);
1390 				setrun_locked(t);
1391 			}
1392 			/*
1393 			 * If we are holding lwps for a forkall(),
1394 			 * force lwps that have been suspended via
1395 			 * lwp_suspend() and are suspended inside
1396 			 * of a system call to proceed to their
1397 			 * holdlwp() points where they are clonable.
1398 			 */
1399 			if ((p->p_flag & SHOLDFORK) && SUSPENDED(t)) {
1400 				if ((t->t_schedflag & TS_CSTART) == 0) {
1401 					p->p_lwprcnt++;
1402 					t->t_schedflag |= TS_CSTART;
1403 					setrun_locked(t);
1404 				}
1405 			}
1406 		} else if (t->t_state == TS_ONPROC) {
1407 			if (t->t_cpu != CPU)
1408 				poke_cpu(t->t_cpu->cpu_id);
1409 		}
1410 		thread_unlock(t);
1411 	} while ((t = t->t_forw) != p->p_tlist);
1412 }
1413 
1414 /*
1415  * undo the effects of holdlwps() or holdwatch().
1416  */
1417 void
1418 continuelwps(proc_t *p)
1419 {
1420 	kthread_t *t;
1421 
1422 	/*
1423 	 * If this flag is set, then the original holdwatch() didn't actually
1424 	 * stop the process.  See comments for holdwatch().
1425 	 */
1426 	if (p->p_flag & SWATCHOK) {
1427 		ASSERT(curthread->t_proc_flag & TP_STOPPING);
1428 		return;
1429 	}
1430 
1431 	ASSERT(MUTEX_HELD(&p->p_lock));
1432 	ASSERT((p->p_flag & (SHOLDFORK | SHOLDFORK1 | SHOLDWATCH)) == 0);
1433 
1434 	t = p->p_tlist;
1435 	do {
1436 		thread_lock(t);		/* SUSPENDED looks at t_schedflag */
1437 		if (SUSPENDED(t) && !(t->t_proc_flag & TP_HOLDLWP)) {
1438 			p->p_lwprcnt++;
1439 			t->t_schedflag |= TS_CSTART;
1440 			setrun_locked(t);
1441 		}
1442 		thread_unlock(t);
1443 	} while ((t = t->t_forw) != p->p_tlist);
1444 }
1445 
1446 /*
1447  * Force all other LWPs in the current process other than the caller to exit,
1448  * and then cv_wait() on p_holdlwps for them to exit.  The exitlwps() function
1449  * is typically used in these situations:
1450  *
1451  *   (a) prior to an exec() system call
1452  *   (b) prior to dumping a core file
1453  *   (c) prior to a uadmin() shutdown
1454  *
1455  * If the 'coredump' flag is set, other LWPs are quiesced but not destroyed.
1456  * Multiple threads in the process can call this function at one time by
1457  * triggering execs or core dumps simultaneously, so the SEXITLWPS bit is used
1458  * to declare one particular thread the winner who gets to kill the others.
1459  * If a thread wins the exitlwps() dance, zero is returned; otherwise an
1460  * appropriate errno value is returned to caller for its system call to return.
1461  */
1462 int
1463 exitlwps(int coredump)
1464 {
1465 	proc_t *p = curproc;
1466 	int heldcnt;
1467 
1468 	if (curthread->t_door)
1469 		door_slam();
1470 	if (p->p_door_list)
1471 		door_revoke_all();
1472 	if (curthread->t_schedctl != NULL)
1473 		schedctl_lwp_cleanup(curthread);
1474 
1475 	/*
1476 	 * Ensure that before starting to wait for other lwps to exit,
1477 	 * cleanup all upimutexes held by curthread. Otherwise, some other
1478 	 * lwp could be waiting (uninterruptibly) for a upimutex held by
1479 	 * curthread, and the call to pokelwps() below would deadlock.
1480 	 * Even if a blocked upimutex_lock is made interruptible,
1481 	 * curthread's upimutexes need to be unlocked: do it here.
1482 	 */
1483 	if (curthread->t_upimutex != NULL)
1484 		upimutex_cleanup();
1485 
1486 	/*
1487 	 * Grab p_lock in order to check and set SEXITLWPS to declare a winner.
1488 	 * We must also block any further /proc access from this point forward.
1489 	 */
1490 	mutex_enter(&p->p_lock);
1491 	prbarrier(p);
1492 
1493 	if (p->p_flag & SEXITLWPS) {
1494 		mutex_exit(&p->p_lock);
1495 		aston(curthread);	/* force a trip through post_syscall */
1496 		return (set_errno(EINTR));
1497 	}
1498 
1499 	p->p_flag |= SEXITLWPS;
1500 	if (coredump)		/* tell other lwps to stop, not exit */
1501 		p->p_flag |= SCOREDUMP;
1502 
1503 	/*
1504 	 * Give precedence to exitlwps() if a holdlwps() is
1505 	 * in progress. The lwp doing the holdlwps() operation
1506 	 * is aborted when it is awakened.
1507 	 */
1508 	while (p->p_flag & (SHOLDFORK | SHOLDFORK1 | SHOLDWATCH)) {
1509 		cv_broadcast(&p->p_holdlwps);
1510 		cv_wait(&p->p_holdlwps, &p->p_lock);
1511 		prbarrier(p);
1512 	}
1513 	p->p_flag |= SHOLDFORK;
1514 	pokelwps(p);
1515 
1516 	/*
1517 	 * Wait for process to become quiescent.
1518 	 */
1519 	--p->p_lwprcnt;
1520 	while (p->p_lwprcnt > 0) {
1521 		cv_wait(&p->p_holdlwps, &p->p_lock);
1522 		prbarrier(p);
1523 	}
1524 	p->p_lwprcnt++;
1525 	ASSERT(p->p_lwprcnt == 1);
1526 
1527 	/*
1528 	 * The SCOREDUMP flag puts the process into a quiescent
1529 	 * state.  The process's lwps remain attached to this
1530 	 * process until exitlwps() is called again without the
1531 	 * 'coredump' flag set, then the lwps are terminated
1532 	 * and the process can exit.
1533 	 */
1534 	if (coredump) {
1535 		p->p_flag &= ~(SCOREDUMP | SHOLDFORK | SEXITLWPS);
1536 		goto out;
1537 	}
1538 
1539 	/*
1540 	 * Determine if there are any lwps left dangling in
1541 	 * the stopped state.  This happens when exitlwps()
1542 	 * aborts a holdlwps() operation.
1543 	 */
1544 	p->p_flag &= ~SHOLDFORK;
1545 	if ((heldcnt = p->p_lwpcnt) > 1) {
1546 		kthread_t *t;
1547 		for (t = curthread->t_forw; --heldcnt > 0; t = t->t_forw) {
1548 			t->t_proc_flag &= ~TP_TWAIT;
1549 			lwp_continue(t);
1550 		}
1551 	}
1552 
1553 	/*
1554 	 * Wait for all other lwps to exit.
1555 	 */
1556 	--p->p_lwprcnt;
1557 	while (p->p_lwpcnt > 1) {
1558 		cv_wait(&p->p_holdlwps, &p->p_lock);
1559 		prbarrier(p);
1560 	}
1561 	++p->p_lwprcnt;
1562 	ASSERT(p->p_lwpcnt == 1 && p->p_lwprcnt == 1);
1563 
1564 	p->p_flag &= ~SEXITLWPS;
1565 	curthread->t_proc_flag &= ~TP_TWAIT;
1566 
1567 out:
1568 	if (!coredump && p->p_zombcnt) {	/* cleanup the zombie lwps */
1569 		lwpdir_t *ldp;
1570 		lwpent_t *lep;
1571 		int i;
1572 
1573 		for (ldp = p->p_lwpdir, i = 0; i < p->p_lwpdir_sz; i++, ldp++) {
1574 			lep = ldp->ld_entry;
1575 			if (lep != NULL && lep->le_thread != curthread) {
1576 				ASSERT(lep->le_thread == NULL);
1577 				p->p_zombcnt--;
1578 				lwp_hash_out(p, lep->le_lwpid);
1579 			}
1580 		}
1581 		ASSERT(p->p_zombcnt == 0);
1582 	}
1583 
1584 	/*
1585 	 * If some other LWP in the process wanted us to suspend ourself,
1586 	 * then we will not do it.  The other LWP is now terminated and
1587 	 * no one will ever continue us again if we suspend ourself.
1588 	 */
1589 	curthread->t_proc_flag &= ~TP_HOLDLWP;
1590 	p->p_flag &= ~(SHOLDFORK | SHOLDFORK1 | SHOLDWATCH | SLWPWRAP);
1591 	mutex_exit(&p->p_lock);
1592 	return (0);
1593 }
1594 
1595 /*
1596  * duplicate a lwp.
1597  */
1598 klwp_t *
1599 forklwp(klwp_t *lwp, proc_t *cp, id_t lwpid)
1600 {
1601 	klwp_t *clwp;
1602 	void *tregs, *tfpu;
1603 	kthread_t *t = lwptot(lwp);
1604 	kthread_t *ct;
1605 	proc_t *p = lwptoproc(lwp);
1606 	int cid;
1607 	void *bufp;
1608 	void *brand_data;
1609 	int val;
1610 
1611 	ASSERT(p == curproc);
1612 	ASSERT(t == curthread || (SUSPENDED(t) && lwp->lwp_asleep == 0));
1613 
1614 #if defined(__sparc)
1615 	if (t == curthread)
1616 		(void) flush_user_windows_to_stack(NULL);
1617 #endif
1618 
1619 	if (t == curthread)
1620 		/* copy args out of registers first */
1621 		(void) save_syscall_args();
1622 
1623 	clwp = lwp_create(cp->p_lwpcnt == 0 ? lwp_rtt_initial : lwp_rtt,
1624 	    NULL, 0, cp, TS_STOPPED, t->t_pri, &t->t_hold, NOCLASS, lwpid);
1625 	if (clwp == NULL)
1626 		return (NULL);
1627 
1628 	/*
1629 	 * most of the parent's lwp can be copied to its duplicate,
1630 	 * except for the fields that are unique to each lwp, like
1631 	 * lwp_thread, lwp_procp, lwp_regs, and lwp_ap.
1632 	 */
1633 	ct = clwp->lwp_thread;
1634 	tregs = clwp->lwp_regs;
1635 	tfpu = clwp->lwp_fpu;
1636 	brand_data = clwp->lwp_brand;
1637 
1638 	/*
1639 	 * Copy parent lwp to child lwp.  Hold child's p_lock to prevent
1640 	 * mstate_aggr_state() from reading stale mstate entries copied
1641 	 * from lwp to clwp.
1642 	 */
1643 	mutex_enter(&cp->p_lock);
1644 	*clwp = *lwp;
1645 
1646 	/* clear microstate and resource usage data in new lwp */
1647 	init_mstate(ct, LMS_STOPPED);
1648 	bzero(&clwp->lwp_ru, sizeof (clwp->lwp_ru));
1649 	mutex_exit(&cp->p_lock);
1650 
1651 	/* fix up child's lwp */
1652 
1653 #if defined(__i386) || defined(__amd64)
1654 	clwp->lwp_pcb.pcb_flags = clwp->lwp_pcb.pcb_flags & RUPDATE_PENDING;
1655 #elif defined(__sparc)
1656 	clwp->lwp_pcb.pcb_step = STEP_NONE;
1657 #endif
1658 	clwp->lwp_cursig = 0;
1659 	clwp->lwp_extsig = 0;
1660 	clwp->lwp_curinfo = (struct sigqueue *)0;
1661 	clwp->lwp_thread = ct;
1662 	ct->t_sysnum = t->t_sysnum;
1663 	clwp->lwp_regs = tregs;
1664 	clwp->lwp_fpu = tfpu;
1665 	clwp->lwp_brand = brand_data;
1666 	clwp->lwp_ap = clwp->lwp_arg;
1667 	clwp->lwp_procp = cp;
1668 	bzero(clwp->lwp_timer, sizeof (clwp->lwp_timer));
1669 	clwp->lwp_lastfault = 0;
1670 	clwp->lwp_lastfaddr = 0;
1671 
1672 	/* copy parent's struct regs to child. */
1673 	lwp_forkregs(lwp, clwp);
1674 
1675 	/*
1676 	 * Fork thread context ops, if any.
1677 	 */
1678 	if (t->t_ctx)
1679 		forkctx(t, ct);
1680 
1681 	/* fix door state in the child */
1682 	if (t->t_door)
1683 		door_fork(t, ct);
1684 
1685 	/* copy current contract templates, clear latest contracts */
1686 	lwp_ctmpl_copy(clwp, lwp);
1687 
1688 	mutex_enter(&cp->p_lock);
1689 	/* lwp_create() set the TP_HOLDLWP flag */
1690 	if (!(t->t_proc_flag & TP_HOLDLWP))
1691 		ct->t_proc_flag &= ~TP_HOLDLWP;
1692 	if (cp->p_flag & SMSACCT)
1693 		ct->t_proc_flag |= TP_MSACCT;
1694 	mutex_exit(&cp->p_lock);
1695 
1696 	/* Allow brand to propagate brand-specific state */
1697 	if (PROC_IS_BRANDED(p))
1698 		BROP(p)->b_forklwp(lwp, clwp);
1699 
1700 retry:
1701 	cid = t->t_cid;
1702 
1703 	val = CL_ALLOC(&bufp, cid, KM_SLEEP);
1704 	ASSERT(val == 0);
1705 
1706 	mutex_enter(&p->p_lock);
1707 	if (cid != t->t_cid) {
1708 		/*
1709 		 * Someone just changed this thread's scheduling class,
1710 		 * so try pre-allocating the buffer again.  Hopefully we
1711 		 * don't hit this often.
1712 		 */
1713 		mutex_exit(&p->p_lock);
1714 		CL_FREE(cid, bufp);
1715 		goto retry;
1716 	}
1717 
1718 	ct->t_unpark = t->t_unpark;
1719 	ct->t_clfuncs = t->t_clfuncs;
1720 	CL_FORK(t, ct, bufp);
1721 	ct->t_cid = t->t_cid;	/* after data allocated so prgetpsinfo works */
1722 	mutex_exit(&p->p_lock);
1723 
1724 	return (clwp);
1725 }
1726 
1727 /*
1728  * Add a new lwp entry to the lwp directory and to the lwpid hash table.
1729  */
1730 void
1731 lwp_hash_in(proc_t *p, lwpent_t *lep)
1732 {
1733 	lwpdir_t **ldpp;
1734 	lwpdir_t *ldp;
1735 	kthread_t *t;
1736 
1737 	/*
1738 	 * Allocate a directory element from the free list.
1739 	 * Code elsewhere guarantees a free slot.
1740 	 */
1741 	ldp = p->p_lwpfree;
1742 	p->p_lwpfree = ldp->ld_next;
1743 	ASSERT(ldp->ld_entry == NULL);
1744 	ldp->ld_entry = lep;
1745 
1746 	/*
1747 	 * Insert it into the lwpid hash table.
1748 	 */
1749 	ldpp = &p->p_tidhash[TIDHASH(p, lep->le_lwpid)];
1750 	ldp->ld_next = *ldpp;
1751 	*ldpp = ldp;
1752 
1753 	/*
1754 	 * Set the active thread's directory slot entry.
1755 	 */
1756 	if ((t = lep->le_thread) != NULL) {
1757 		ASSERT(lep->le_lwpid == t->t_tid);
1758 		t->t_dslot = (int)(ldp - p->p_lwpdir);
1759 	}
1760 }
1761 
1762 /*
1763  * Remove an lwp from the lwpid hash table and free its directory entry.
1764  * This is done when a detached lwp exits in lwp_exit() or
1765  * when a non-detached lwp is waited for in lwp_wait() or
1766  * when a zombie lwp is detached in lwp_detach().
1767  */
1768 void
1769 lwp_hash_out(proc_t *p, id_t lwpid)
1770 {
1771 	lwpdir_t **ldpp;
1772 	lwpdir_t *ldp;
1773 	lwpent_t *lep;
1774 
1775 	for (ldpp = &p->p_tidhash[TIDHASH(p, lwpid)];
1776 	    (ldp = *ldpp) != NULL; ldpp = &ldp->ld_next) {
1777 		lep = ldp->ld_entry;
1778 		if (lep->le_lwpid == lwpid) {
1779 			prlwpfree(p, lep);	/* /proc deals with le_trace */
1780 			*ldpp = ldp->ld_next;
1781 			ldp->ld_entry = NULL;
1782 			ldp->ld_next = p->p_lwpfree;
1783 			p->p_lwpfree = ldp;
1784 			kmem_free(lep, sizeof (*lep));
1785 			break;
1786 		}
1787 	}
1788 }
1789 
1790 /*
1791  * Lookup an lwp in the lwpid hash table by lwpid.
1792  */
1793 lwpdir_t *
1794 lwp_hash_lookup(proc_t *p, id_t lwpid)
1795 {
1796 	lwpdir_t *ldp;
1797 
1798 	/*
1799 	 * The process may be exiting, after p_tidhash has been set to NULL in
1800 	 * proc_exit() but before prfee() has been called.  Return failure in
1801 	 * this case.
1802 	 */
1803 	if (p->p_tidhash == NULL)
1804 		return (NULL);
1805 
1806 	for (ldp = p->p_tidhash[TIDHASH(p, lwpid)];
1807 	    ldp != NULL; ldp = ldp->ld_next) {
1808 		if (ldp->ld_entry->le_lwpid == lwpid)
1809 			return (ldp);
1810 	}
1811 
1812 	return (NULL);
1813 }
1814 
1815 /*
1816  * Update the indicated LWP usage statistic for the current LWP.
1817  */
1818 void
1819 lwp_stat_update(lwp_stat_id_t lwp_stat_id, long inc)
1820 {
1821 	klwp_t *lwp = ttolwp(curthread);
1822 
1823 	if (lwp == NULL)
1824 		return;
1825 
1826 	switch (lwp_stat_id) {
1827 	case LWP_STAT_INBLK:
1828 		lwp->lwp_ru.inblock += inc;
1829 		break;
1830 	case LWP_STAT_OUBLK:
1831 		lwp->lwp_ru.oublock += inc;
1832 		break;
1833 	case LWP_STAT_MSGRCV:
1834 		lwp->lwp_ru.msgrcv += inc;
1835 		break;
1836 	case LWP_STAT_MSGSND:
1837 		lwp->lwp_ru.msgsnd += inc;
1838 		break;
1839 	default:
1840 		panic("lwp_stat_update: invalid lwp_stat_id 0x%x", lwp_stat_id);
1841 	}
1842 }
1843