xref: /illumos-gate/usr/src/uts/common/os/lwp.c (revision 0b70c467)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 #pragma ident	"%Z%%M%	%I%	%E% SMI"
28 
29 #include <sys/param.h>
30 #include <sys/types.h>
31 #include <sys/sysmacros.h>
32 #include <sys/systm.h>
33 #include <sys/thread.h>
34 #include <sys/proc.h>
35 #include <sys/task.h>
36 #include <sys/project.h>
37 #include <sys/signal.h>
38 #include <sys/errno.h>
39 #include <sys/vmparam.h>
40 #include <sys/stack.h>
41 #include <sys/procfs.h>
42 #include <sys/prsystm.h>
43 #include <sys/cpuvar.h>
44 #include <sys/kmem.h>
45 #include <sys/vtrace.h>
46 #include <sys/door.h>
47 #include <vm/seg_kp.h>
48 #include <sys/debug.h>
49 #include <sys/tnf.h>
50 #include <sys/schedctl.h>
51 #include <sys/poll.h>
52 #include <sys/copyops.h>
53 #include <sys/lwp_upimutex_impl.h>
54 #include <sys/cpupart.h>
55 #include <sys/lgrp.h>
56 #include <sys/rctl.h>
57 #include <sys/contract_impl.h>
58 #include <sys/cpc_impl.h>
59 #include <sys/sdt.h>
60 #include <sys/cmn_err.h>
61 #include <sys/brand.h>
62 
63 void *segkp_lwp;		/* cookie for pool of segkp resources */
64 extern void reapq_move_lq_to_tq(kthread_t *);
65 extern void freectx_ctx(struct ctxop *);
66 
67 /*
68  * Create a thread that appears to be stopped at sys_rtt.
69  */
70 klwp_t *
71 lwp_create(void (*proc)(), caddr_t arg, size_t len, proc_t *p,
72     int state, int pri, const k_sigset_t *smask, int cid, id_t lwpid)
73 {
74 	klwp_t *lwp = NULL;
75 	kthread_t *t;
76 	kthread_t *tx;
77 	cpupart_t *oldpart = NULL;
78 	size_t	stksize;
79 	caddr_t lwpdata = NULL;
80 	processorid_t	binding;
81 	int err = 0;
82 	kproject_t *oldkpj, *newkpj;
83 	void *bufp = NULL;
84 	klwp_t *curlwp = ttolwp(curthread);
85 	lwpent_t *lep;
86 	lwpdir_t *old_dir = NULL;
87 	uint_t old_dirsz = 0;
88 	lwpdir_t **old_hash = NULL;
89 	uint_t old_hashsz = 0;
90 	int i;
91 	int rctlfail = 0;
92 	boolean_t branded = 0;
93 	struct ctxop *ctx = NULL;
94 
95 	mutex_enter(&p->p_lock);
96 	mutex_enter(&p->p_zone->zone_nlwps_lock);
97 	/*
98 	 * don't enforce rctl limits on system processes
99 	 */
100 	if (cid != syscid) {
101 		if (p->p_task->tk_nlwps >= p->p_task->tk_nlwps_ctl)
102 			if (rctl_test(rc_task_lwps, p->p_task->tk_rctls, p,
103 			    1, 0) & RCT_DENY)
104 				rctlfail = 1;
105 		if (p->p_task->tk_proj->kpj_nlwps >=
106 		    p->p_task->tk_proj->kpj_nlwps_ctl)
107 			if (rctl_test(rc_project_nlwps,
108 			    p->p_task->tk_proj->kpj_rctls, p, 1, 0)
109 			    & RCT_DENY)
110 				rctlfail = 1;
111 		if (p->p_zone->zone_nlwps >= p->p_zone->zone_nlwps_ctl)
112 			if (rctl_test(rc_zone_nlwps, p->p_zone->zone_rctls, p,
113 			    1, 0) & RCT_DENY)
114 				rctlfail = 1;
115 	}
116 	if (rctlfail) {
117 		mutex_exit(&p->p_zone->zone_nlwps_lock);
118 		mutex_exit(&p->p_lock);
119 		return (NULL);
120 	}
121 	p->p_task->tk_nlwps++;
122 	p->p_task->tk_proj->kpj_nlwps++;
123 	p->p_zone->zone_nlwps++;
124 	mutex_exit(&p->p_zone->zone_nlwps_lock);
125 	mutex_exit(&p->p_lock);
126 
127 	if (curlwp == NULL || (stksize = curlwp->lwp_childstksz) == 0)
128 		stksize = lwp_default_stksize;
129 
130 	/*
131 	 * Try to reclaim a <lwp,stack> from 'deathrow'
132 	 */
133 	if (stksize == lwp_default_stksize) {
134 		if (lwp_reapcnt > 0) {
135 			mutex_enter(&reaplock);
136 			if ((t = lwp_deathrow) != NULL) {
137 				ASSERT(t->t_swap);
138 				lwp_deathrow = t->t_forw;
139 				lwp_reapcnt--;
140 				lwpdata = t->t_swap;
141 				lwp = t->t_lwp;
142 				ctx = t->t_ctx;
143 				t->t_swap = NULL;
144 				t->t_lwp = NULL;
145 				t->t_ctx = NULL;
146 				reapq_move_lq_to_tq(t);
147 			}
148 			mutex_exit(&reaplock);
149 			if (lwp != NULL) {
150 				lwp_stk_fini(lwp);
151 			}
152 			if (ctx != NULL) {
153 				freectx_ctx(ctx);
154 			}
155 		}
156 		if (lwpdata == NULL &&
157 		    (lwpdata = (caddr_t)segkp_cache_get(segkp_lwp)) == NULL) {
158 			mutex_enter(&p->p_lock);
159 			mutex_enter(&p->p_zone->zone_nlwps_lock);
160 			p->p_task->tk_nlwps--;
161 			p->p_task->tk_proj->kpj_nlwps--;
162 			p->p_zone->zone_nlwps--;
163 			mutex_exit(&p->p_zone->zone_nlwps_lock);
164 			mutex_exit(&p->p_lock);
165 			return (NULL);
166 		}
167 	} else {
168 		stksize = roundup(stksize, PAGESIZE);
169 		if ((lwpdata = (caddr_t)segkp_get(segkp, stksize,
170 		    (KPD_NOWAIT | KPD_HASREDZONE | KPD_LOCKED))) == NULL) {
171 			mutex_enter(&p->p_lock);
172 			mutex_enter(&p->p_zone->zone_nlwps_lock);
173 			p->p_task->tk_nlwps--;
174 			p->p_task->tk_proj->kpj_nlwps--;
175 			p->p_zone->zone_nlwps--;
176 			mutex_exit(&p->p_zone->zone_nlwps_lock);
177 			mutex_exit(&p->p_lock);
178 			return (NULL);
179 		}
180 	}
181 
182 	/*
183 	 * Create a thread, initializing the stack pointer
184 	 */
185 	t = thread_create(lwpdata, stksize, NULL, NULL, 0, p, TS_STOPPED, pri);
186 
187 	t->t_swap = lwpdata;	/* Start of page-able data */
188 	if (lwp == NULL)
189 		lwp = kmem_cache_alloc(lwp_cache, KM_SLEEP);
190 	bzero(lwp, sizeof (*lwp));
191 	t->t_lwp = lwp;
192 
193 	t->t_hold = *smask;
194 	lwp->lwp_thread = t;
195 	lwp->lwp_procp = p;
196 	lwp->lwp_sigaltstack.ss_flags = SS_DISABLE;
197 	if (curlwp != NULL && curlwp->lwp_childstksz != 0)
198 		lwp->lwp_childstksz = curlwp->lwp_childstksz;
199 
200 	t->t_stk = lwp_stk_init(lwp, t->t_stk);
201 	thread_load(t, proc, arg, len);
202 
203 	/*
204 	 * Allocate the SIGPROF buffer if ITIMER_REALPROF is in effect.
205 	 */
206 	if (timerisset(&p->p_rprof_timer.it_value))
207 		t->t_rprof = kmem_zalloc(sizeof (struct rprof), KM_SLEEP);
208 
209 	if (cid != NOCLASS)
210 		(void) CL_ALLOC(&bufp, cid, KM_SLEEP);
211 
212 	/*
213 	 * Allocate an lwp directory entry for the new lwp.
214 	 */
215 	lep = kmem_zalloc(sizeof (*lep), KM_SLEEP);
216 
217 	mutex_enter(&p->p_lock);
218 grow:
219 	/*
220 	 * Grow the lwp (thread) directory and lwpid hash table if necessary.
221 	 * A note on the growth algorithm:
222 	 *	The new lwp directory size is computed as:
223 	 *		new = 2 * old + 2
224 	 *	Starting with an initial size of 2 (see exec_common()),
225 	 *	this yields numbers that are a power of two minus 2:
226 	 *		2, 6, 14, 30, 62, 126, 254, 510, 1022, ...
227 	 *	The size of the lwpid hash table must be a power of two
228 	 *	and must be commensurate in size with the lwp directory
229 	 *	so that hash bucket chains remain short.  Therefore,
230 	 *	the lwpid hash table size is computed as:
231 	 *		hashsz = (dirsz + 2) / 2
232 	 *	which leads to these hash table sizes corresponding to
233 	 *	the above directory sizes:
234 	 *		2, 4, 8, 16, 32, 64, 128, 256, 512, ...
235 	 */
236 	while (p->p_lwpfree == NULL) {
237 		uint_t dirsz = p->p_lwpdir_sz;
238 		uint_t new_dirsz;
239 		uint_t new_hashsz;
240 		lwpdir_t *new_dir;
241 		lwpdir_t *ldp;
242 		lwpdir_t **new_hash;
243 
244 		mutex_exit(&p->p_lock);
245 
246 		if (old_dir != NULL) {
247 			kmem_free(old_dir, old_dirsz * sizeof (*old_dir));
248 			kmem_free(old_hash, old_hashsz * sizeof (*old_hash));
249 			old_dir = NULL;
250 			old_dirsz = 0;
251 			old_hash = NULL;
252 			old_hashsz = 0;
253 		}
254 		new_dirsz = 2 * dirsz + 2;
255 		new_dir = kmem_zalloc(new_dirsz * sizeof (lwpdir_t), KM_SLEEP);
256 		for (ldp = new_dir, i = 1; i < new_dirsz; i++, ldp++)
257 			ldp->ld_next = ldp + 1;
258 		new_hashsz = (new_dirsz + 2) / 2;
259 		new_hash = kmem_zalloc(new_hashsz * sizeof (lwpdir_t *),
260 		    KM_SLEEP);
261 
262 		mutex_enter(&p->p_lock);
263 		if (p == curproc)
264 			prbarrier(p);
265 
266 		if (dirsz != p->p_lwpdir_sz || p->p_lwpfree != NULL) {
267 			/*
268 			 * Someone else beat us to it or some lwp exited.
269 			 * Set up to free our memory and take a lap.
270 			 */
271 			old_dir = new_dir;
272 			old_dirsz = new_dirsz;
273 			old_hash = new_hash;
274 			old_hashsz = new_hashsz;
275 		} else {
276 			old_dir = p->p_lwpdir;
277 			old_dirsz = p->p_lwpdir_sz;
278 			old_hash = p->p_tidhash;
279 			old_hashsz = p->p_tidhash_sz;
280 			p->p_lwpdir = new_dir;
281 			p->p_lwpfree = new_dir;
282 			p->p_lwpdir_sz = new_dirsz;
283 			p->p_tidhash = new_hash;
284 			p->p_tidhash_sz = new_hashsz;
285 			/*
286 			 * We simply hash in all of the old directory entries.
287 			 * This works because the old directory has no empty
288 			 * slots and the new hash table starts out empty.
289 			 * This reproduces the original directory ordering
290 			 * (required for /proc directory semantics).
291 			 */
292 			for (ldp = old_dir, i = 0; i < dirsz; i++, ldp++)
293 				lwp_hash_in(p, ldp->ld_entry);
294 			/*
295 			 * Defer freeing memory until we drop p->p_lock,
296 			 */
297 		}
298 	}
299 
300 	/*
301 	 * Block the process against /proc while we manipulate p->p_tlist,
302 	 * unless lwp_create() was called by /proc for the PCAGENT operation.
303 	 * We want to do this early enough so that we don't drop p->p_lock
304 	 * until the thread is put on the p->p_tlist.
305 	 */
306 	if (p == curproc) {
307 		prbarrier(p);
308 		/*
309 		 * If the current lwp has been requested to stop, do so now.
310 		 * Otherwise we have a race condition between /proc attempting
311 		 * to stop the process and this thread creating a new lwp
312 		 * that was not seen when the /proc PCSTOP request was issued.
313 		 * We rely on stop() to call prbarrier(p) before returning.
314 		 */
315 		while ((curthread->t_proc_flag & TP_PRSTOP) &&
316 		    !ttolwp(curthread)->lwp_nostop)
317 			stop(PR_REQUESTED, 0);
318 
319 		/*
320 		 * If process is exiting, there could be a race between
321 		 * the agent lwp creation and the new lwp currently being
322 		 * created. So to prevent this race lwp creation is failed
323 		 * if the process is exiting.
324 		 */
325 		if (p->p_flag & (SEXITLWPS|SKILLED)) {
326 			err = 1;
327 			goto error;
328 		}
329 
330 		/*
331 		 * Since we might have dropped p->p_lock, the
332 		 * lwp directory free list might have changed.
333 		 */
334 		if (p->p_lwpfree == NULL)
335 			goto grow;
336 	}
337 
338 	kpreempt_disable();	/* can't grab cpu_lock here */
339 
340 	/*
341 	 * Inherit processor and processor set bindings from curthread,
342 	 * unless we're creating a new kernel process, in which case
343 	 * clear all bindings.
344 	 */
345 	if (cid == syscid) {
346 		t->t_bind_cpu = binding = PBIND_NONE;
347 		t->t_cpupart = oldpart = &cp_default;
348 		t->t_bind_pset = PS_NONE;
349 		t->t_bindflag = (uchar_t)default_binding_mode;
350 	} else {
351 		binding = curthread->t_bind_cpu;
352 		t->t_bind_cpu = binding;
353 		oldpart = t->t_cpupart;
354 		t->t_cpupart = curthread->t_cpupart;
355 		t->t_bind_pset = curthread->t_bind_pset;
356 		t->t_bindflag = curthread->t_bindflag |
357 		    (uchar_t)default_binding_mode;
358 	}
359 
360 	/*
361 	 * thread_create() initializes this thread's home lgroup to the root.
362 	 * Choose a more suitable lgroup, since this thread is associated
363 	 * with an lwp.
364 	 */
365 	ASSERT(oldpart != NULL);
366 	if (binding != PBIND_NONE && t->t_affinitycnt == 0) {
367 		t->t_bound_cpu = cpu[binding];
368 		if (t->t_lpl != t->t_bound_cpu->cpu_lpl)
369 			lgrp_move_thread(t, t->t_bound_cpu->cpu_lpl, 1);
370 	} else {
371 		lgrp_move_thread(t, lgrp_choose(t, t->t_cpupart), 1);
372 	}
373 
374 	kpreempt_enable();
375 
376 	/*
377 	 * make sure lpl points to our own partition
378 	 */
379 	ASSERT(t->t_lpl >= t->t_cpupart->cp_lgrploads);
380 	ASSERT(t->t_lpl < t->t_cpupart->cp_lgrploads +
381 	    t->t_cpupart->cp_nlgrploads);
382 
383 	/*
384 	 * If we're creating a new process, then inherit the project from our
385 	 * parent. If we're only creating an additional lwp then use the
386 	 * project pointer of the target process.
387 	 */
388 	if (p->p_task == NULL)
389 		newkpj = ttoproj(curthread);
390 	else
391 		newkpj = p->p_task->tk_proj;
392 
393 	/*
394 	 * It is safe to point the thread to the new project without holding it
395 	 * since we're holding the target process' p_lock here and therefore
396 	 * we're guaranteed that it will not move to another project.
397 	 */
398 	oldkpj = ttoproj(t);
399 	if (newkpj != oldkpj) {
400 		t->t_proj = newkpj;
401 		(void) project_hold(newkpj);
402 		project_rele(oldkpj);
403 	}
404 
405 	if (cid != NOCLASS) {
406 		/*
407 		 * If the lwp is being created in the current process
408 		 * and matches the current thread's scheduling class,
409 		 * we should propagate the current thread's scheduling
410 		 * parameters by calling CL_FORK.  Otherwise just use
411 		 * the defaults by calling CL_ENTERCLASS.
412 		 */
413 		if (p != curproc || curthread->t_cid != cid) {
414 			err = CL_ENTERCLASS(t, cid, NULL, NULL, bufp);
415 			t->t_pri = pri;	/* CL_ENTERCLASS may have changed it */
416 			/*
417 			 * We don't call schedctl_set_cidpri(t) here
418 			 * because the schedctl data is not yet set
419 			 * up for the newly-created lwp.
420 			 */
421 		} else {
422 			t->t_clfuncs = &(sclass[cid].cl_funcs->thread);
423 			err = CL_FORK(curthread, t, bufp);
424 			t->t_cid = cid;
425 		}
426 		if (err)
427 			goto error;
428 		else
429 			bufp = NULL;
430 	}
431 
432 	/*
433 	 * If we were given an lwpid then use it, else allocate one.
434 	 */
435 	if (lwpid != 0)
436 		t->t_tid = lwpid;
437 	else {
438 		/*
439 		 * lwp/thread id 0 is never valid; reserved for special checks.
440 		 * lwp/thread id 1 is reserved for the main thread.
441 		 * Start again at 2 when INT_MAX has been reached
442 		 * (id_t is a signed 32-bit integer).
443 		 */
444 		id_t prev_id = p->p_lwpid;	/* last allocated tid */
445 
446 		do {			/* avoid lwpid duplication */
447 			if (p->p_lwpid == INT_MAX) {
448 				p->p_flag |= SLWPWRAP;
449 				p->p_lwpid = 1;
450 			}
451 			if ((t->t_tid = ++p->p_lwpid) == prev_id) {
452 				/*
453 				 * All lwpids are allocated; fail the request.
454 				 */
455 				err = 1;
456 				goto error;
457 			}
458 			/*
459 			 * We only need to worry about colliding with an id
460 			 * that's already in use if this process has
461 			 * cycled through all available lwp ids.
462 			 */
463 			if ((p->p_flag & SLWPWRAP) == 0)
464 				break;
465 		} while (lwp_hash_lookup(p, t->t_tid) != NULL);
466 	}
467 
468 	/*
469 	 * If this is a branded process, let the brand do any necessary lwp
470 	 * initialization.
471 	 */
472 	if (PROC_IS_BRANDED(p)) {
473 		if (BROP(p)->b_initlwp(lwp)) {
474 			err = 1;
475 			goto error;
476 		}
477 		branded = 1;
478 	}
479 
480 	if (t->t_tid == 1) {
481 		kpreempt_disable();
482 		ASSERT(t->t_lpl != NULL);
483 		p->p_t1_lgrpid = t->t_lpl->lpl_lgrpid;
484 		kpreempt_enable();
485 		if (p->p_tr_lgrpid != LGRP_NONE &&
486 		    p->p_tr_lgrpid != p->p_t1_lgrpid) {
487 			lgrp_update_trthr_migrations(1);
488 		}
489 	}
490 
491 	p->p_lwpcnt++;
492 	t->t_waitfor = -1;
493 
494 	/*
495 	 * Turn microstate accounting on for thread if on for process.
496 	 */
497 	if (p->p_flag & SMSACCT)
498 		t->t_proc_flag |= TP_MSACCT;
499 
500 	/*
501 	 * If the process has watchpoints, mark the new thread as such.
502 	 */
503 	if (pr_watch_active(p))
504 		watch_enable(t);
505 
506 	/*
507 	 * The lwp is being created in the stopped state.
508 	 * We set all the necessary flags to indicate that fact here.
509 	 * We omit the TS_CREATE flag from t_schedflag so that the lwp
510 	 * cannot be set running until the caller is finished with it,
511 	 * even if lwp_continue() is called on it after we drop p->p_lock.
512 	 * When the caller is finished with the newly-created lwp,
513 	 * the caller must call lwp_create_done() to allow the lwp
514 	 * to be set running.  If the TP_HOLDLWP is left set, the
515 	 * lwp will suspend itself after reaching system call exit.
516 	 */
517 	init_mstate(t, LMS_STOPPED);
518 	t->t_proc_flag |= TP_HOLDLWP;
519 	t->t_schedflag |= (TS_ALLSTART & ~(TS_CSTART | TS_CREATE));
520 	t->t_whystop = PR_SUSPENDED;
521 	t->t_whatstop = SUSPEND_NORMAL;
522 	t->t_sig_check = 1;	/* ensure that TP_HOLDLWP is honored */
523 
524 	/*
525 	 * Set system call processing flags in case tracing or profiling
526 	 * is set.  The first system call will evaluate these and turn
527 	 * them off if they aren't needed.
528 	 */
529 	t->t_pre_sys = 1;
530 	t->t_post_sys = 1;
531 
532 	/*
533 	 * Insert the new thread into the list of all threads.
534 	 */
535 	if ((tx = p->p_tlist) == NULL) {
536 		t->t_back = t;
537 		t->t_forw = t;
538 		p->p_tlist = t;
539 	} else {
540 		t->t_forw = tx;
541 		t->t_back = tx->t_back;
542 		tx->t_back->t_forw = t;
543 		tx->t_back = t;
544 	}
545 
546 	/*
547 	 * Insert the new lwp into an lwp directory slot position
548 	 * and into the lwpid hash table.
549 	 */
550 	lep->le_thread = t;
551 	lep->le_lwpid = t->t_tid;
552 	lep->le_start = t->t_start;
553 	lwp_hash_in(p, lep);
554 
555 	if (state == TS_RUN) {
556 		/*
557 		 * We set the new lwp running immediately.
558 		 */
559 		t->t_proc_flag &= ~TP_HOLDLWP;
560 		lwp_create_done(t);
561 	}
562 
563 error:
564 	if (err) {
565 		/*
566 		 * We have failed to create an lwp, so decrement the number
567 		 * of lwps in the task and let the lgroup load averages know
568 		 * that this thread isn't going to show up.
569 		 */
570 		kpreempt_disable();
571 		lgrp_move_thread(t, NULL, 1);
572 		kpreempt_enable();
573 
574 		ASSERT(MUTEX_HELD(&p->p_lock));
575 		mutex_enter(&p->p_zone->zone_nlwps_lock);
576 		p->p_task->tk_nlwps--;
577 		p->p_task->tk_proj->kpj_nlwps--;
578 		p->p_zone->zone_nlwps--;
579 		mutex_exit(&p->p_zone->zone_nlwps_lock);
580 		if (cid != NOCLASS && bufp != NULL)
581 			CL_FREE(cid, bufp);
582 
583 		if (branded)
584 			BROP(p)->b_freelwp(lwp);
585 
586 		mutex_exit(&p->p_lock);
587 		t->t_state = TS_FREE;
588 		thread_rele(t);
589 
590 		/*
591 		 * We need to remove t from the list of all threads
592 		 * because thread_exit()/lwp_exit() isn't called on t.
593 		 */
594 		mutex_enter(&pidlock);
595 		ASSERT(t != t->t_next);		/* t0 never exits */
596 		t->t_next->t_prev = t->t_prev;
597 		t->t_prev->t_next = t->t_next;
598 		mutex_exit(&pidlock);
599 
600 		thread_free(t);
601 		kmem_free(lep, sizeof (*lep));
602 		lwp = NULL;
603 	} else {
604 		mutex_exit(&p->p_lock);
605 	}
606 
607 	if (old_dir != NULL) {
608 		kmem_free(old_dir, old_dirsz * sizeof (*old_dir));
609 		kmem_free(old_hash, old_hashsz * sizeof (*old_hash));
610 	}
611 
612 	DTRACE_PROC1(lwp__create, kthread_t *, t);
613 	return (lwp);
614 }
615 
616 /*
617  * lwp_create_done() is called by the caller of lwp_create() to set the
618  * newly-created lwp running after the caller has finished manipulating it.
619  */
620 void
621 lwp_create_done(kthread_t *t)
622 {
623 	proc_t *p = ttoproc(t);
624 
625 	ASSERT(MUTEX_HELD(&p->p_lock));
626 
627 	/*
628 	 * We set the TS_CREATE and TS_CSTART flags and call setrun_locked().
629 	 * (The absence of the TS_CREATE flag prevents the lwp from running
630 	 * until we are finished with it, even if lwp_continue() is called on
631 	 * it by some other lwp in the process or elsewhere in the kernel.)
632 	 */
633 	thread_lock(t);
634 	ASSERT(t->t_state == TS_STOPPED && !(t->t_schedflag & TS_CREATE));
635 	/*
636 	 * If TS_CSTART is set, lwp_continue(t) has been called and
637 	 * has already incremented p_lwprcnt; avoid doing this twice.
638 	 */
639 	if (!(t->t_schedflag & TS_CSTART))
640 		p->p_lwprcnt++;
641 	t->t_schedflag |= (TS_CSTART | TS_CREATE);
642 	setrun_locked(t);
643 	thread_unlock(t);
644 }
645 
646 /*
647  * Copy an LWP's active templates, and clear the latest contracts.
648  */
649 void
650 lwp_ctmpl_copy(klwp_t *dst, klwp_t *src)
651 {
652 	int i;
653 
654 	for (i = 0; i < ct_ntypes; i++) {
655 		dst->lwp_ct_active[i] = ctmpl_dup(src->lwp_ct_active[i]);
656 		dst->lwp_ct_latest[i] = NULL;
657 	}
658 }
659 
660 /*
661  * Clear an LWP's contract template state.
662  */
663 void
664 lwp_ctmpl_clear(klwp_t *lwp)
665 {
666 	ct_template_t *tmpl;
667 	int i;
668 
669 	for (i = 0; i < ct_ntypes; i++) {
670 		if ((tmpl = lwp->lwp_ct_active[i]) != NULL) {
671 			ctmpl_free(tmpl);
672 			lwp->lwp_ct_active[i] = NULL;
673 		}
674 
675 		if (lwp->lwp_ct_latest[i] != NULL) {
676 			contract_rele(lwp->lwp_ct_latest[i]);
677 			lwp->lwp_ct_latest[i] = NULL;
678 		}
679 	}
680 }
681 
682 /*
683  * Individual lwp exit.
684  * If this is the last lwp, exit the whole process.
685  */
686 void
687 lwp_exit(void)
688 {
689 	kthread_t *t = curthread;
690 	klwp_t *lwp = ttolwp(t);
691 	proc_t *p = ttoproc(t);
692 
693 	ASSERT(MUTEX_HELD(&p->p_lock));
694 
695 	mutex_exit(&p->p_lock);
696 
697 #if defined(__sparc)
698 	/*
699 	 * Ensure that the user stack is fully abandoned..
700 	 */
701 	trash_user_windows();
702 #endif
703 
704 	tsd_exit();			/* free thread specific data */
705 
706 	kcpc_passivate();		/* Clean up performance counter state */
707 
708 	pollcleanup();
709 
710 	if (t->t_door)
711 		door_slam();
712 
713 	if (t->t_schedctl != NULL)
714 		schedctl_lwp_cleanup(t);
715 
716 	if (t->t_upimutex != NULL)
717 		upimutex_cleanup();
718 
719 	/*
720 	 * Perform any brand specific exit processing, then release any
721 	 * brand data associated with the lwp
722 	 */
723 	if (PROC_IS_BRANDED(p))
724 		BROP(p)->b_lwpexit(lwp);
725 
726 	mutex_enter(&p->p_lock);
727 	lwp_cleanup();
728 
729 	/*
730 	 * When this process is dumping core, its lwps are held here
731 	 * until the core dump is finished. Then exitlwps() is called
732 	 * again to release these lwps so that they can finish exiting.
733 	 */
734 	if (p->p_flag & SCOREDUMP)
735 		stop(PR_SUSPENDED, SUSPEND_NORMAL);
736 
737 	/*
738 	 * Block the process against /proc now that we have really acquired
739 	 * p->p_lock (to decrement p_lwpcnt and manipulate p_tlist at least).
740 	 */
741 	prbarrier(p);
742 
743 	/*
744 	 * Call proc_exit() if this is the last non-daemon lwp in the process.
745 	 */
746 	if (!(t->t_proc_flag & TP_DAEMON) &&
747 	    p->p_lwpcnt == p->p_lwpdaemon + 1) {
748 		mutex_exit(&p->p_lock);
749 		if (proc_exit(CLD_EXITED, 0) == 0) {
750 			/* Restarting init. */
751 			return;
752 		}
753 
754 		/*
755 		 * proc_exit() returns a non-zero value when some other
756 		 * lwp got there first.  We just have to continue in
757 		 * lwp_exit().
758 		 */
759 		mutex_enter(&p->p_lock);
760 		ASSERT(curproc->p_flag & SEXITLWPS);
761 		prbarrier(p);
762 	}
763 
764 	DTRACE_PROC(lwp__exit);
765 
766 	/*
767 	 * If the lwp is a detached lwp or if the process is exiting,
768 	 * remove (lwp_hash_out()) the lwp from the lwp directory.
769 	 * Otherwise null out the lwp's le_thread pointer in the lwp
770 	 * directory so that other threads will see it as a zombie lwp.
771 	 */
772 	prlwpexit(t);		/* notify /proc */
773 	if (!(t->t_proc_flag & TP_TWAIT) || (p->p_flag & SEXITLWPS))
774 		lwp_hash_out(p, t->t_tid);
775 	else {
776 		ASSERT(!(t->t_proc_flag & TP_DAEMON));
777 		p->p_lwpdir[t->t_dslot].ld_entry->le_thread = NULL;
778 		p->p_zombcnt++;
779 		cv_broadcast(&p->p_lwpexit);
780 	}
781 	if (t->t_proc_flag & TP_DAEMON) {
782 		p->p_lwpdaemon--;
783 		t->t_proc_flag &= ~TP_DAEMON;
784 	}
785 	t->t_proc_flag &= ~TP_TWAIT;
786 
787 	/*
788 	 * Maintain accurate lwp count for task.max-lwps resource control.
789 	 */
790 	mutex_enter(&p->p_zone->zone_nlwps_lock);
791 	p->p_task->tk_nlwps--;
792 	p->p_task->tk_proj->kpj_nlwps--;
793 	p->p_zone->zone_nlwps--;
794 	mutex_exit(&p->p_zone->zone_nlwps_lock);
795 
796 	CL_EXIT(t);		/* tell the scheduler that t is exiting */
797 	ASSERT(p->p_lwpcnt != 0);
798 	p->p_lwpcnt--;
799 
800 	/*
801 	 * If all remaining non-daemon lwps are waiting in lwp_wait(),
802 	 * wake them up so someone can return EDEADLK.
803 	 * (See the block comment preceeding lwp_wait().)
804 	 */
805 	if (p->p_lwpcnt == p->p_lwpdaemon + (p->p_lwpwait - p->p_lwpdwait))
806 		cv_broadcast(&p->p_lwpexit);
807 
808 	t->t_proc_flag |= TP_LWPEXIT;
809 	term_mstate(t);
810 
811 #ifndef NPROBE
812 	/* Kernel probe */
813 	if (t->t_tnf_tpdp)
814 		tnf_thread_exit();
815 #endif /* NPROBE */
816 
817 	t->t_forw->t_back = t->t_back;
818 	t->t_back->t_forw = t->t_forw;
819 	if (t == p->p_tlist)
820 		p->p_tlist = t->t_forw;
821 
822 	/*
823 	 * Clean up the signal state.
824 	 */
825 	if (t->t_sigqueue != NULL)
826 		sigdelq(p, t, 0);
827 	if (lwp->lwp_curinfo != NULL) {
828 		siginfofree(lwp->lwp_curinfo);
829 		lwp->lwp_curinfo = NULL;
830 	}
831 
832 	thread_rele(t);
833 
834 	/*
835 	 * Terminated lwps are associated with process zero and are put onto
836 	 * death-row by resume().  Avoid preemption after resetting t->t_procp.
837 	 */
838 	t->t_preempt++;
839 
840 	if (t->t_ctx != NULL)
841 		exitctx(t);
842 	if (p->p_pctx != NULL)
843 		exitpctx(p);
844 
845 	t->t_procp = &p0;
846 
847 	/*
848 	 * Notify the HAT about the change of address space
849 	 */
850 	hat_thread_exit(t);
851 	/*
852 	 * When this is the last running lwp in this process and some lwp is
853 	 * waiting for this condition to become true, or this thread was being
854 	 * suspended, then the waiting lwp is awakened.
855 	 *
856 	 * Also, if the process is exiting, we may have a thread waiting in
857 	 * exitlwps() that needs to be notified.
858 	 */
859 	if (--p->p_lwprcnt == 0 || (t->t_proc_flag & TP_HOLDLWP) ||
860 	    (p->p_flag & SEXITLWPS))
861 		cv_broadcast(&p->p_holdlwps);
862 
863 	/*
864 	 * Need to drop p_lock so we can reacquire pidlock.
865 	 */
866 	mutex_exit(&p->p_lock);
867 	mutex_enter(&pidlock);
868 
869 	ASSERT(t != t->t_next);		/* t0 never exits */
870 	t->t_next->t_prev = t->t_prev;
871 	t->t_prev->t_next = t->t_next;
872 	cv_broadcast(&t->t_joincv);	/* wake up anyone in thread_join */
873 	mutex_exit(&pidlock);
874 
875 	lwp_pcb_exit();
876 
877 	t->t_state = TS_ZOMB;
878 	swtch_from_zombie();
879 	/* never returns */
880 }
881 
882 
883 /*
884  * Cleanup function for an exiting lwp.
885  * Called both from lwp_exit() and from proc_exit().
886  * p->p_lock is repeatedly released and grabbed in this function.
887  */
888 void
889 lwp_cleanup(void)
890 {
891 	kthread_t *t = curthread;
892 	proc_t *p = ttoproc(t);
893 
894 	ASSERT(MUTEX_HELD(&p->p_lock));
895 
896 	/* untimeout any lwp-bound realtime timers */
897 	if (p->p_itimer != NULL)
898 		timer_lwpexit();
899 
900 	/*
901 	 * If this is the /proc agent lwp that is exiting, readjust p_lwpid
902 	 * so it appears that the agent never existed, and clear p_agenttp.
903 	 */
904 	if (t == p->p_agenttp) {
905 		ASSERT(t->t_tid == p->p_lwpid);
906 		p->p_lwpid--;
907 		p->p_agenttp = NULL;
908 	}
909 
910 	/*
911 	 * Do lgroup bookkeeping to account for thread exiting.
912 	 */
913 	kpreempt_disable();
914 	lgrp_move_thread(t, NULL, 1);
915 	if (t->t_tid == 1) {
916 		p->p_t1_lgrpid = LGRP_NONE;
917 	}
918 	kpreempt_enable();
919 
920 	lwp_ctmpl_clear(ttolwp(t));
921 }
922 
923 int
924 lwp_suspend(kthread_t *t)
925 {
926 	int tid;
927 	proc_t *p = ttoproc(t);
928 
929 	ASSERT(MUTEX_HELD(&p->p_lock));
930 
931 	/*
932 	 * Set the thread's TP_HOLDLWP flag so it will stop in holdlwp().
933 	 * If an lwp is stopping itself, there is no need to wait.
934 	 */
935 top:
936 	t->t_proc_flag |= TP_HOLDLWP;
937 	if (t == curthread) {
938 		t->t_sig_check = 1;
939 	} else {
940 		/*
941 		 * Make sure the lwp stops promptly.
942 		 */
943 		thread_lock(t);
944 		t->t_sig_check = 1;
945 		/*
946 		 * XXX Should use virtual stop like /proc does instead of
947 		 * XXX waking the thread to get it to stop.
948 		 */
949 		if (ISWAKEABLE(t) || ISWAITING(t)) {
950 			setrun_locked(t);
951 		} else if (t->t_state == TS_ONPROC && t->t_cpu != CPU) {
952 			poke_cpu(t->t_cpu->cpu_id);
953 		}
954 
955 		tid = t->t_tid;	 /* remember thread ID */
956 		/*
957 		 * Wait for lwp to stop
958 		 */
959 		while (!SUSPENDED(t)) {
960 			/*
961 			 * Drop the thread lock before waiting and reacquire it
962 			 * afterwards, so the thread can change its t_state
963 			 * field.
964 			 */
965 			thread_unlock(t);
966 
967 			/*
968 			 * Check if aborted by exitlwps().
969 			 */
970 			if (p->p_flag & SEXITLWPS)
971 				lwp_exit();
972 
973 			/*
974 			 * Cooperate with jobcontrol signals and /proc stopping
975 			 * by calling cv_wait_sig() to wait for the target
976 			 * lwp to stop.  Just using cv_wait() can lead to
977 			 * deadlock because, if some other lwp has stopped
978 			 * by either of these mechanisms, then p_lwprcnt will
979 			 * never become zero if we do a cv_wait().
980 			 */
981 			if (!cv_wait_sig(&p->p_holdlwps, &p->p_lock))
982 				return (EINTR);
983 
984 			/*
985 			 * Check to see if thread died while we were
986 			 * waiting for it to suspend.
987 			 */
988 			if (idtot(p, tid) == NULL)
989 				return (ESRCH);
990 
991 			thread_lock(t);
992 			/*
993 			 * If the TP_HOLDLWP flag went away, lwp_continue()
994 			 * or vfork() must have been called while we were
995 			 * waiting, so start over again.
996 			 */
997 			if ((t->t_proc_flag & TP_HOLDLWP) == 0) {
998 				thread_unlock(t);
999 				goto top;
1000 			}
1001 		}
1002 		thread_unlock(t);
1003 	}
1004 	return (0);
1005 }
1006 
1007 /*
1008  * continue a lwp that's been stopped by lwp_suspend().
1009  */
1010 void
1011 lwp_continue(kthread_t *t)
1012 {
1013 	proc_t *p = ttoproc(t);
1014 	int was_suspended = t->t_proc_flag & TP_HOLDLWP;
1015 
1016 	ASSERT(MUTEX_HELD(&p->p_lock));
1017 
1018 	t->t_proc_flag &= ~TP_HOLDLWP;
1019 	thread_lock(t);
1020 	if (SUSPENDED(t) &&
1021 	    !(p->p_flag & (SHOLDFORK | SHOLDFORK1 | SHOLDWATCH))) {
1022 		p->p_lwprcnt++;
1023 		t->t_schedflag |= TS_CSTART;
1024 		setrun_locked(t);
1025 	}
1026 	thread_unlock(t);
1027 	/*
1028 	 * Wakeup anyone waiting for this thread to be suspended
1029 	 */
1030 	if (was_suspended)
1031 		cv_broadcast(&p->p_holdlwps);
1032 }
1033 
1034 /*
1035  * ********************************
1036  *  Miscellaneous lwp routines	  *
1037  * ********************************
1038  */
1039 /*
1040  * When a process is undergoing a forkall(), its p_flag is set to SHOLDFORK.
1041  * This will cause the process's lwps to stop at a hold point.  A hold
1042  * point is where a kernel thread has a flat stack.  This is at the
1043  * return from a system call and at the return from a user level trap.
1044  *
1045  * When a process is undergoing a fork1() or vfork(), its p_flag is set to
1046  * SHOLDFORK1.  This will cause the process's lwps to stop at a modified
1047  * hold point.  The lwps in the process are not being cloned, so they
1048  * are held at the usual hold points and also within issig_forreal().
1049  * This has the side-effect that their system calls do not return
1050  * showing EINTR.
1051  *
1052  * An lwp can also be held.  This is identified by the TP_HOLDLWP flag on
1053  * the thread.  The TP_HOLDLWP flag is set in lwp_suspend(), where the active
1054  * lwp is waiting for the target lwp to be stopped.
1055  */
1056 void
1057 holdlwp(void)
1058 {
1059 	proc_t *p = curproc;
1060 	kthread_t *t = curthread;
1061 
1062 	mutex_enter(&p->p_lock);
1063 	/*
1064 	 * Don't terminate immediately if the process is dumping core.
1065 	 * Once the process has dumped core, all lwps are terminated.
1066 	 */
1067 	if (!(p->p_flag & SCOREDUMP)) {
1068 		if ((p->p_flag & SEXITLWPS) || (t->t_proc_flag & TP_EXITLWP))
1069 			lwp_exit();
1070 	}
1071 	if (!(ISHOLD(p)) && !(p->p_flag & (SHOLDFORK1 | SHOLDWATCH))) {
1072 		mutex_exit(&p->p_lock);
1073 		return;
1074 	}
1075 	/*
1076 	 * stop() decrements p->p_lwprcnt and cv_signal()s &p->p_holdlwps
1077 	 * when p->p_lwprcnt becomes zero.
1078 	 */
1079 	stop(PR_SUSPENDED, SUSPEND_NORMAL);
1080 	if (p->p_flag & SEXITLWPS)
1081 		lwp_exit();
1082 	mutex_exit(&p->p_lock);
1083 }
1084 
1085 /*
1086  * Have all lwps within the process hold at a point where they are
1087  * cloneable (SHOLDFORK) or just safe w.r.t. fork1 (SHOLDFORK1).
1088  */
1089 int
1090 holdlwps(int holdflag)
1091 {
1092 	proc_t *p = curproc;
1093 
1094 	ASSERT(holdflag == SHOLDFORK || holdflag == SHOLDFORK1);
1095 	mutex_enter(&p->p_lock);
1096 	schedctl_finish_sigblock(curthread);
1097 again:
1098 	while (p->p_flag & (SEXITLWPS | SHOLDFORK | SHOLDFORK1 | SHOLDWATCH)) {
1099 		/*
1100 		 * If another lwp is doing a forkall() or proc_exit(), bail out.
1101 		 */
1102 		if (p->p_flag & (SEXITLWPS | SHOLDFORK)) {
1103 			mutex_exit(&p->p_lock);
1104 			return (0);
1105 		}
1106 		/*
1107 		 * Another lwp is doing a fork1() or is undergoing
1108 		 * watchpoint activity.  We hold here for it to complete.
1109 		 */
1110 		stop(PR_SUSPENDED, SUSPEND_NORMAL);
1111 	}
1112 	p->p_flag |= holdflag;
1113 	pokelwps(p);
1114 	--p->p_lwprcnt;
1115 	/*
1116 	 * Wait for the process to become quiescent (p->p_lwprcnt == 0).
1117 	 */
1118 	while (p->p_lwprcnt > 0) {
1119 		/*
1120 		 * Check if aborted by exitlwps().
1121 		 * Also check if SHOLDWATCH is set; it takes precedence.
1122 		 */
1123 		if (p->p_flag & (SEXITLWPS | SHOLDWATCH)) {
1124 			p->p_lwprcnt++;
1125 			p->p_flag &= ~holdflag;
1126 			cv_broadcast(&p->p_holdlwps);
1127 			goto again;
1128 		}
1129 		/*
1130 		 * Cooperate with jobcontrol signals and /proc stopping.
1131 		 * If some other lwp has stopped by either of these
1132 		 * mechanisms, then p_lwprcnt will never become zero
1133 		 * and the process will appear deadlocked unless we
1134 		 * stop here in sympathy with the other lwp before
1135 		 * doing the cv_wait() below.
1136 		 *
1137 		 * If the other lwp stops after we do the cv_wait(), it
1138 		 * will wake us up to loop around and do the sympathy stop.
1139 		 *
1140 		 * Since stop() drops p->p_lock, we must start from
1141 		 * the top again on returning from stop().
1142 		 */
1143 		if (p->p_stopsig | (curthread->t_proc_flag & TP_PRSTOP)) {
1144 			int whystop = p->p_stopsig? PR_JOBCONTROL :
1145 			    PR_REQUESTED;
1146 			p->p_lwprcnt++;
1147 			p->p_flag &= ~holdflag;
1148 			stop(whystop, p->p_stopsig);
1149 			goto again;
1150 		}
1151 		cv_wait(&p->p_holdlwps, &p->p_lock);
1152 	}
1153 	p->p_lwprcnt++;
1154 	p->p_flag &= ~holdflag;
1155 	mutex_exit(&p->p_lock);
1156 	return (1);
1157 }
1158 
1159 /*
1160  * See comments for holdwatch(), below.
1161  */
1162 static int
1163 holdcheck(int clearflags)
1164 {
1165 	proc_t *p = curproc;
1166 
1167 	/*
1168 	 * If we are trying to exit, that takes precedence over anything else.
1169 	 */
1170 	if (p->p_flag & SEXITLWPS) {
1171 		p->p_lwprcnt++;
1172 		p->p_flag &= ~clearflags;
1173 		lwp_exit();
1174 	}
1175 
1176 	/*
1177 	 * If another thread is calling fork1(), stop the current thread so the
1178 	 * other can complete.
1179 	 */
1180 	if (p->p_flag & SHOLDFORK1) {
1181 		p->p_lwprcnt++;
1182 		stop(PR_SUSPENDED, SUSPEND_NORMAL);
1183 		if (p->p_flag & SEXITLWPS) {
1184 			p->p_flag &= ~clearflags;
1185 			lwp_exit();
1186 		}
1187 		return (-1);
1188 	}
1189 
1190 	/*
1191 	 * If another thread is calling fork(), then indicate we are doing
1192 	 * watchpoint activity.  This will cause holdlwps() above to stop the
1193 	 * forking thread, at which point we can continue with watchpoint
1194 	 * activity.
1195 	 */
1196 	if (p->p_flag & SHOLDFORK) {
1197 		p->p_lwprcnt++;
1198 		while (p->p_flag & SHOLDFORK) {
1199 			p->p_flag |= SHOLDWATCH;
1200 			cv_broadcast(&p->p_holdlwps);
1201 			cv_wait(&p->p_holdlwps, &p->p_lock);
1202 			p->p_flag &= ~SHOLDWATCH;
1203 		}
1204 		return (-1);
1205 	}
1206 
1207 	return (0);
1208 }
1209 
1210 /*
1211  * Stop all lwps within the process, holding themselves in the kernel while the
1212  * active lwp undergoes watchpoint activity.  This is more complicated than
1213  * expected because stop() relies on calling holdwatch() in order to copyin data
1214  * from the user's address space.  A double barrier is used to prevent an
1215  * infinite loop.
1216  *
1217  * 	o The first thread into holdwatch() is the 'master' thread and does
1218  *        the following:
1219  *
1220  *              - Sets SHOLDWATCH on the current process
1221  *              - Sets TP_WATCHSTOP on the current thread
1222  *              - Waits for all threads to be either stopped or have
1223  *                TP_WATCHSTOP set.
1224  *              - Sets the SWATCHOK flag on the process
1225  *              - Unsets TP_WATCHSTOP
1226  *              - Waits for the other threads to completely stop
1227  *              - Unsets SWATCHOK
1228  *
1229  * 	o If SHOLDWATCH is already set when we enter this function, then another
1230  *        thread is already trying to stop this thread.  This 'slave' thread
1231  *        does the following:
1232  *
1233  *              - Sets TP_WATCHSTOP on the current thread
1234  *              - Waits for SWATCHOK flag to be set
1235  *              - Calls stop()
1236  *
1237  * 	o If SWATCHOK is set on the process, then this function immediately
1238  *        returns, as we must have been called via stop().
1239  *
1240  * In addition, there are other flags that take precedence over SHOLDWATCH:
1241  *
1242  * 	o If SEXITLWPS is set, exit immediately.
1243  *
1244  * 	o If SHOLDFORK1 is set, wait for fork1() to complete.
1245  *
1246  * 	o If SHOLDFORK is set, then watchpoint activity takes precedence In this
1247  *        case, set SHOLDWATCH, signalling the forking thread to stop first.
1248  *
1249  * 	o If the process is being stopped via /proc (TP_PRSTOP is set), then we
1250  *        stop the current thread.
1251  *
1252  * Returns 0 if all threads have been quiesced.  Returns non-zero if not all
1253  * threads were stopped, or the list of watched pages has changed.
1254  */
1255 int
1256 holdwatch(void)
1257 {
1258 	proc_t *p = curproc;
1259 	kthread_t *t = curthread;
1260 	int ret = 0;
1261 
1262 	mutex_enter(&p->p_lock);
1263 
1264 	p->p_lwprcnt--;
1265 
1266 	/*
1267 	 * Check for bail-out conditions as outlined above.
1268 	 */
1269 	if (holdcheck(0) != 0) {
1270 		mutex_exit(&p->p_lock);
1271 		return (-1);
1272 	}
1273 
1274 	if (!(p->p_flag & SHOLDWATCH)) {
1275 		/*
1276 		 * We are the master watchpoint thread.  Set SHOLDWATCH and poke
1277 		 * the other threads.
1278 		 */
1279 		p->p_flag |= SHOLDWATCH;
1280 		pokelwps(p);
1281 
1282 		/*
1283 		 * Wait for all threads to be stopped or have TP_WATCHSTOP set.
1284 		 */
1285 		while (pr_allstopped(p, 1) > 0) {
1286 			if (holdcheck(SHOLDWATCH) != 0) {
1287 				p->p_flag &= ~SHOLDWATCH;
1288 				mutex_exit(&p->p_lock);
1289 				return (-1);
1290 			}
1291 
1292 			cv_wait(&p->p_holdlwps, &p->p_lock);
1293 		}
1294 
1295 		/*
1296 		 * All threads are now stopped or in the process of stopping.
1297 		 * Set SWATCHOK and let them stop completely.
1298 		 */
1299 		p->p_flag |= SWATCHOK;
1300 		t->t_proc_flag &= ~TP_WATCHSTOP;
1301 		cv_broadcast(&p->p_holdlwps);
1302 
1303 		while (pr_allstopped(p, 0) > 0) {
1304 			/*
1305 			 * At first glance, it may appear that we don't need a
1306 			 * call to holdcheck() here.  But if the process gets a
1307 			 * SIGKILL signal, one of our stopped threads may have
1308 			 * been awakened and is waiting in exitlwps(), which
1309 			 * takes precedence over watchpoints.
1310 			 */
1311 			if (holdcheck(SHOLDWATCH | SWATCHOK) != 0) {
1312 				p->p_flag &= ~(SHOLDWATCH | SWATCHOK);
1313 				mutex_exit(&p->p_lock);
1314 				return (-1);
1315 			}
1316 
1317 			cv_wait(&p->p_holdlwps, &p->p_lock);
1318 		}
1319 
1320 		/*
1321 		 * All threads are now completely stopped.
1322 		 */
1323 		p->p_flag &= ~SWATCHOK;
1324 		p->p_flag &= ~SHOLDWATCH;
1325 		p->p_lwprcnt++;
1326 
1327 	} else if (!(p->p_flag & SWATCHOK)) {
1328 
1329 		/*
1330 		 * SHOLDWATCH is set, so another thread is trying to do
1331 		 * watchpoint activity.  Indicate this thread is stopping, and
1332 		 * wait for the OK from the master thread.
1333 		 */
1334 		t->t_proc_flag |= TP_WATCHSTOP;
1335 		cv_broadcast(&p->p_holdlwps);
1336 
1337 		while (!(p->p_flag & SWATCHOK)) {
1338 			if (holdcheck(0) != 0) {
1339 				t->t_proc_flag &= ~TP_WATCHSTOP;
1340 				mutex_exit(&p->p_lock);
1341 				return (-1);
1342 			}
1343 
1344 			cv_wait(&p->p_holdlwps, &p->p_lock);
1345 		}
1346 
1347 		/*
1348 		 * Once the master thread has given the OK, this thread can
1349 		 * actually call stop().
1350 		 */
1351 		t->t_proc_flag &= ~TP_WATCHSTOP;
1352 		p->p_lwprcnt++;
1353 
1354 		stop(PR_SUSPENDED, SUSPEND_NORMAL);
1355 
1356 		/*
1357 		 * It's not OK to do watchpoint activity, notify caller to
1358 		 * retry.
1359 		 */
1360 		ret = -1;
1361 
1362 	} else {
1363 
1364 		/*
1365 		 * The only way we can hit the case where SHOLDWATCH is set and
1366 		 * SWATCHOK is set is if we are triggering this from within a
1367 		 * stop() call.  Assert that this is the case.
1368 		 */
1369 
1370 		ASSERT(t->t_proc_flag & TP_STOPPING);
1371 		p->p_lwprcnt++;
1372 	}
1373 
1374 	mutex_exit(&p->p_lock);
1375 
1376 	return (ret);
1377 }
1378 
1379 /*
1380  * force all interruptible lwps to trap into the kernel.
1381  */
1382 void
1383 pokelwps(proc_t *p)
1384 {
1385 	kthread_t *t;
1386 
1387 	ASSERT(MUTEX_HELD(&p->p_lock));
1388 
1389 	t = p->p_tlist;
1390 	do {
1391 		if (t == curthread)
1392 			continue;
1393 		thread_lock(t);
1394 		aston(t);	/* make thread trap or do post_syscall */
1395 		if (ISWAKEABLE(t) || ISWAITING(t)) {
1396 			setrun_locked(t);
1397 		} else if (t->t_state == TS_STOPPED) {
1398 			/*
1399 			 * Ensure that proc_exit() is not blocked by lwps
1400 			 * that were stopped via jobcontrol or /proc.
1401 			 */
1402 			if (p->p_flag & SEXITLWPS) {
1403 				p->p_stopsig = 0;
1404 				t->t_schedflag |= (TS_XSTART | TS_PSTART);
1405 				setrun_locked(t);
1406 			}
1407 			/*
1408 			 * If we are holding lwps for a forkall(),
1409 			 * force lwps that have been suspended via
1410 			 * lwp_suspend() and are suspended inside
1411 			 * of a system call to proceed to their
1412 			 * holdlwp() points where they are clonable.
1413 			 */
1414 			if ((p->p_flag & SHOLDFORK) && SUSPENDED(t)) {
1415 				if ((t->t_schedflag & TS_CSTART) == 0) {
1416 					p->p_lwprcnt++;
1417 					t->t_schedflag |= TS_CSTART;
1418 					setrun_locked(t);
1419 				}
1420 			}
1421 		} else if (t->t_state == TS_ONPROC) {
1422 			if (t->t_cpu != CPU)
1423 				poke_cpu(t->t_cpu->cpu_id);
1424 		}
1425 		thread_unlock(t);
1426 	} while ((t = t->t_forw) != p->p_tlist);
1427 }
1428 
1429 /*
1430  * undo the effects of holdlwps() or holdwatch().
1431  */
1432 void
1433 continuelwps(proc_t *p)
1434 {
1435 	kthread_t *t;
1436 
1437 	/*
1438 	 * If this flag is set, then the original holdwatch() didn't actually
1439 	 * stop the process.  See comments for holdwatch().
1440 	 */
1441 	if (p->p_flag & SWATCHOK) {
1442 		ASSERT(curthread->t_proc_flag & TP_STOPPING);
1443 		return;
1444 	}
1445 
1446 	ASSERT(MUTEX_HELD(&p->p_lock));
1447 	ASSERT((p->p_flag & (SHOLDFORK | SHOLDFORK1 | SHOLDWATCH)) == 0);
1448 
1449 	t = p->p_tlist;
1450 	do {
1451 		thread_lock(t);		/* SUSPENDED looks at t_schedflag */
1452 		if (SUSPENDED(t) && !(t->t_proc_flag & TP_HOLDLWP)) {
1453 			p->p_lwprcnt++;
1454 			t->t_schedflag |= TS_CSTART;
1455 			setrun_locked(t);
1456 		}
1457 		thread_unlock(t);
1458 	} while ((t = t->t_forw) != p->p_tlist);
1459 }
1460 
1461 /*
1462  * Force all other LWPs in the current process other than the caller to exit,
1463  * and then cv_wait() on p_holdlwps for them to exit.  The exitlwps() function
1464  * is typically used in these situations:
1465  *
1466  *   (a) prior to an exec() system call
1467  *   (b) prior to dumping a core file
1468  *   (c) prior to a uadmin() shutdown
1469  *
1470  * If the 'coredump' flag is set, other LWPs are quiesced but not destroyed.
1471  * Multiple threads in the process can call this function at one time by
1472  * triggering execs or core dumps simultaneously, so the SEXITLWPS bit is used
1473  * to declare one particular thread the winner who gets to kill the others.
1474  * If a thread wins the exitlwps() dance, zero is returned; otherwise an
1475  * appropriate errno value is returned to caller for its system call to return.
1476  */
1477 int
1478 exitlwps(int coredump)
1479 {
1480 	proc_t *p = curproc;
1481 	int heldcnt;
1482 
1483 	if (curthread->t_door)
1484 		door_slam();
1485 	if (p->p_door_list)
1486 		door_revoke_all();
1487 	if (curthread->t_schedctl != NULL)
1488 		schedctl_lwp_cleanup(curthread);
1489 
1490 	/*
1491 	 * Ensure that before starting to wait for other lwps to exit,
1492 	 * cleanup all upimutexes held by curthread. Otherwise, some other
1493 	 * lwp could be waiting (uninterruptibly) for a upimutex held by
1494 	 * curthread, and the call to pokelwps() below would deadlock.
1495 	 * Even if a blocked upimutex_lock is made interruptible,
1496 	 * curthread's upimutexes need to be unlocked: do it here.
1497 	 */
1498 	if (curthread->t_upimutex != NULL)
1499 		upimutex_cleanup();
1500 
1501 	/*
1502 	 * Grab p_lock in order to check and set SEXITLWPS to declare a winner.
1503 	 * We must also block any further /proc access from this point forward.
1504 	 */
1505 	mutex_enter(&p->p_lock);
1506 	prbarrier(p);
1507 
1508 	if (p->p_flag & SEXITLWPS) {
1509 		mutex_exit(&p->p_lock);
1510 		aston(curthread);	/* force a trip through post_syscall */
1511 		return (set_errno(EINTR));
1512 	}
1513 
1514 	p->p_flag |= SEXITLWPS;
1515 	if (coredump)		/* tell other lwps to stop, not exit */
1516 		p->p_flag |= SCOREDUMP;
1517 
1518 	/*
1519 	 * Give precedence to exitlwps() if a holdlwps() is
1520 	 * in progress. The lwp doing the holdlwps() operation
1521 	 * is aborted when it is awakened.
1522 	 */
1523 	while (p->p_flag & (SHOLDFORK | SHOLDFORK1 | SHOLDWATCH)) {
1524 		cv_broadcast(&p->p_holdlwps);
1525 		cv_wait(&p->p_holdlwps, &p->p_lock);
1526 		prbarrier(p);
1527 	}
1528 	p->p_flag |= SHOLDFORK;
1529 	pokelwps(p);
1530 
1531 	/*
1532 	 * Wait for process to become quiescent.
1533 	 */
1534 	--p->p_lwprcnt;
1535 	while (p->p_lwprcnt > 0) {
1536 		cv_wait(&p->p_holdlwps, &p->p_lock);
1537 		prbarrier(p);
1538 	}
1539 	p->p_lwprcnt++;
1540 	ASSERT(p->p_lwprcnt == 1);
1541 
1542 	/*
1543 	 * The SCOREDUMP flag puts the process into a quiescent
1544 	 * state.  The process's lwps remain attached to this
1545 	 * process until exitlwps() is called again without the
1546 	 * 'coredump' flag set, then the lwps are terminated
1547 	 * and the process can exit.
1548 	 */
1549 	if (coredump) {
1550 		p->p_flag &= ~(SCOREDUMP | SHOLDFORK | SEXITLWPS);
1551 		goto out;
1552 	}
1553 
1554 	/*
1555 	 * Determine if there are any lwps left dangling in
1556 	 * the stopped state.  This happens when exitlwps()
1557 	 * aborts a holdlwps() operation.
1558 	 */
1559 	p->p_flag &= ~SHOLDFORK;
1560 	if ((heldcnt = p->p_lwpcnt) > 1) {
1561 		kthread_t *t;
1562 		for (t = curthread->t_forw; --heldcnt > 0; t = t->t_forw) {
1563 			t->t_proc_flag &= ~TP_TWAIT;
1564 			lwp_continue(t);
1565 		}
1566 	}
1567 
1568 	/*
1569 	 * Wait for all other lwps to exit.
1570 	 */
1571 	--p->p_lwprcnt;
1572 	while (p->p_lwpcnt > 1) {
1573 		cv_wait(&p->p_holdlwps, &p->p_lock);
1574 		prbarrier(p);
1575 	}
1576 	++p->p_lwprcnt;
1577 	ASSERT(p->p_lwpcnt == 1 && p->p_lwprcnt == 1);
1578 
1579 	p->p_flag &= ~SEXITLWPS;
1580 	curthread->t_proc_flag &= ~TP_TWAIT;
1581 
1582 out:
1583 	if (!coredump && p->p_zombcnt) {	/* cleanup the zombie lwps */
1584 		lwpdir_t *ldp;
1585 		lwpent_t *lep;
1586 		int i;
1587 
1588 		for (ldp = p->p_lwpdir, i = 0; i < p->p_lwpdir_sz; i++, ldp++) {
1589 			lep = ldp->ld_entry;
1590 			if (lep != NULL && lep->le_thread != curthread) {
1591 				ASSERT(lep->le_thread == NULL);
1592 				p->p_zombcnt--;
1593 				lwp_hash_out(p, lep->le_lwpid);
1594 			}
1595 		}
1596 		ASSERT(p->p_zombcnt == 0);
1597 	}
1598 
1599 	/*
1600 	 * If some other LWP in the process wanted us to suspend ourself,
1601 	 * then we will not do it.  The other LWP is now terminated and
1602 	 * no one will ever continue us again if we suspend ourself.
1603 	 */
1604 	curthread->t_proc_flag &= ~TP_HOLDLWP;
1605 	p->p_flag &= ~(SHOLDFORK | SHOLDFORK1 | SHOLDWATCH | SLWPWRAP);
1606 	mutex_exit(&p->p_lock);
1607 	return (0);
1608 }
1609 
1610 /*
1611  * duplicate a lwp.
1612  */
1613 klwp_t *
1614 forklwp(klwp_t *lwp, proc_t *cp, id_t lwpid)
1615 {
1616 	klwp_t *clwp;
1617 	void *tregs, *tfpu;
1618 	kthread_t *t = lwptot(lwp);
1619 	kthread_t *ct;
1620 	proc_t *p = lwptoproc(lwp);
1621 	int cid;
1622 	void *bufp;
1623 	void *brand_data;
1624 	int val;
1625 
1626 	ASSERT(p == curproc);
1627 	ASSERT(t == curthread || (SUSPENDED(t) && lwp->lwp_asleep == 0));
1628 
1629 #if defined(__sparc)
1630 	if (t == curthread)
1631 		(void) flush_user_windows_to_stack(NULL);
1632 #endif
1633 
1634 	if (t == curthread)
1635 		/* copy args out of registers first */
1636 		(void) save_syscall_args();
1637 
1638 	clwp = lwp_create(cp->p_lwpcnt == 0 ? lwp_rtt_initial : lwp_rtt,
1639 	    NULL, 0, cp, TS_STOPPED, t->t_pri, &t->t_hold, NOCLASS, lwpid);
1640 	if (clwp == NULL)
1641 		return (NULL);
1642 
1643 	/*
1644 	 * most of the parent's lwp can be copied to its duplicate,
1645 	 * except for the fields that are unique to each lwp, like
1646 	 * lwp_thread, lwp_procp, lwp_regs, and lwp_ap.
1647 	 */
1648 	ct = clwp->lwp_thread;
1649 	tregs = clwp->lwp_regs;
1650 	tfpu = clwp->lwp_fpu;
1651 	brand_data = clwp->lwp_brand;
1652 
1653 	/*
1654 	 * Copy parent lwp to child lwp.  Hold child's p_lock to prevent
1655 	 * mstate_aggr_state() from reading stale mstate entries copied
1656 	 * from lwp to clwp.
1657 	 */
1658 	mutex_enter(&cp->p_lock);
1659 	*clwp = *lwp;
1660 
1661 	/* clear microstate and resource usage data in new lwp */
1662 	init_mstate(ct, LMS_STOPPED);
1663 	bzero(&clwp->lwp_ru, sizeof (clwp->lwp_ru));
1664 	mutex_exit(&cp->p_lock);
1665 
1666 	/* fix up child's lwp */
1667 
1668 	clwp->lwp_pcb.pcb_flags = 0;
1669 #if defined(__sparc)
1670 	clwp->lwp_pcb.pcb_step = STEP_NONE;
1671 #endif
1672 	clwp->lwp_cursig = 0;
1673 	clwp->lwp_extsig = 0;
1674 	clwp->lwp_curinfo = (struct sigqueue *)0;
1675 	clwp->lwp_thread = ct;
1676 	ct->t_sysnum = t->t_sysnum;
1677 	clwp->lwp_regs = tregs;
1678 	clwp->lwp_fpu = tfpu;
1679 	clwp->lwp_brand = brand_data;
1680 	clwp->lwp_ap = clwp->lwp_arg;
1681 	clwp->lwp_procp = cp;
1682 	bzero(clwp->lwp_timer, sizeof (clwp->lwp_timer));
1683 	clwp->lwp_lastfault = 0;
1684 	clwp->lwp_lastfaddr = 0;
1685 
1686 	/* copy parent's struct regs to child. */
1687 	lwp_forkregs(lwp, clwp);
1688 
1689 	/*
1690 	 * Fork thread context ops, if any.
1691 	 */
1692 	if (t->t_ctx)
1693 		forkctx(t, ct);
1694 
1695 	/* fix door state in the child */
1696 	if (t->t_door)
1697 		door_fork(t, ct);
1698 
1699 	/* copy current contract templates, clear latest contracts */
1700 	lwp_ctmpl_copy(clwp, lwp);
1701 
1702 	mutex_enter(&cp->p_lock);
1703 	/* lwp_create() set the TP_HOLDLWP flag */
1704 	if (!(t->t_proc_flag & TP_HOLDLWP))
1705 		ct->t_proc_flag &= ~TP_HOLDLWP;
1706 	if (cp->p_flag & SMSACCT)
1707 		ct->t_proc_flag |= TP_MSACCT;
1708 	mutex_exit(&cp->p_lock);
1709 
1710 	/* Allow brand to propagate brand-specific state */
1711 	if (PROC_IS_BRANDED(p))
1712 		BROP(p)->b_forklwp(lwp, clwp);
1713 
1714 retry:
1715 	cid = t->t_cid;
1716 
1717 	val = CL_ALLOC(&bufp, cid, KM_SLEEP);
1718 	ASSERT(val == 0);
1719 
1720 	mutex_enter(&p->p_lock);
1721 	if (cid != t->t_cid) {
1722 		/*
1723 		 * Someone just changed this thread's scheduling class,
1724 		 * so try pre-allocating the buffer again.  Hopefully we
1725 		 * don't hit this often.
1726 		 */
1727 		mutex_exit(&p->p_lock);
1728 		CL_FREE(cid, bufp);
1729 		goto retry;
1730 	}
1731 
1732 	ct->t_unpark = t->t_unpark;
1733 	ct->t_clfuncs = t->t_clfuncs;
1734 	CL_FORK(t, ct, bufp);
1735 	ct->t_cid = t->t_cid;	/* after data allocated so prgetpsinfo works */
1736 	mutex_exit(&p->p_lock);
1737 
1738 	return (clwp);
1739 }
1740 
1741 /*
1742  * Add a new lwp entry to the lwp directory and to the lwpid hash table.
1743  */
1744 void
1745 lwp_hash_in(proc_t *p, lwpent_t *lep)
1746 {
1747 	lwpdir_t **ldpp;
1748 	lwpdir_t *ldp;
1749 	kthread_t *t;
1750 
1751 	/*
1752 	 * Allocate a directory element from the free list.
1753 	 * Code elsewhere guarantees a free slot.
1754 	 */
1755 	ldp = p->p_lwpfree;
1756 	p->p_lwpfree = ldp->ld_next;
1757 	ASSERT(ldp->ld_entry == NULL);
1758 	ldp->ld_entry = lep;
1759 
1760 	/*
1761 	 * Insert it into the lwpid hash table.
1762 	 */
1763 	ldpp = &p->p_tidhash[TIDHASH(p, lep->le_lwpid)];
1764 	ldp->ld_next = *ldpp;
1765 	*ldpp = ldp;
1766 
1767 	/*
1768 	 * Set the active thread's directory slot entry.
1769 	 */
1770 	if ((t = lep->le_thread) != NULL) {
1771 		ASSERT(lep->le_lwpid == t->t_tid);
1772 		t->t_dslot = (int)(ldp - p->p_lwpdir);
1773 	}
1774 }
1775 
1776 /*
1777  * Remove an lwp from the lwpid hash table and free its directory entry.
1778  * This is done when a detached lwp exits in lwp_exit() or
1779  * when a non-detached lwp is waited for in lwp_wait() or
1780  * when a zombie lwp is detached in lwp_detach().
1781  */
1782 void
1783 lwp_hash_out(proc_t *p, id_t lwpid)
1784 {
1785 	lwpdir_t **ldpp;
1786 	lwpdir_t *ldp;
1787 	lwpent_t *lep;
1788 
1789 	for (ldpp = &p->p_tidhash[TIDHASH(p, lwpid)];
1790 	    (ldp = *ldpp) != NULL; ldpp = &ldp->ld_next) {
1791 		lep = ldp->ld_entry;
1792 		if (lep->le_lwpid == lwpid) {
1793 			prlwpfree(p, lep);	/* /proc deals with le_trace */
1794 			*ldpp = ldp->ld_next;
1795 			ldp->ld_entry = NULL;
1796 			ldp->ld_next = p->p_lwpfree;
1797 			p->p_lwpfree = ldp;
1798 			kmem_free(lep, sizeof (*lep));
1799 			break;
1800 		}
1801 	}
1802 }
1803 
1804 /*
1805  * Lookup an lwp in the lwpid hash table by lwpid.
1806  */
1807 lwpdir_t *
1808 lwp_hash_lookup(proc_t *p, id_t lwpid)
1809 {
1810 	lwpdir_t *ldp;
1811 
1812 	/*
1813 	 * The process may be exiting, after p_tidhash has been set to NULL in
1814 	 * proc_exit() but before prfee() has been called.  Return failure in
1815 	 * this case.
1816 	 */
1817 	if (p->p_tidhash == NULL)
1818 		return (NULL);
1819 
1820 	for (ldp = p->p_tidhash[TIDHASH(p, lwpid)];
1821 	    ldp != NULL; ldp = ldp->ld_next) {
1822 		if (ldp->ld_entry->le_lwpid == lwpid)
1823 			return (ldp);
1824 	}
1825 
1826 	return (NULL);
1827 }
1828 
1829 /*
1830  * Update the indicated LWP usage statistic for the current LWP.
1831  */
1832 void
1833 lwp_stat_update(lwp_stat_id_t lwp_stat_id, long inc)
1834 {
1835 	klwp_t *lwp = ttolwp(curthread);
1836 
1837 	if (lwp == NULL)
1838 		return;
1839 
1840 	switch (lwp_stat_id) {
1841 	case LWP_STAT_INBLK:
1842 		lwp->lwp_ru.inblock += inc;
1843 		break;
1844 	case LWP_STAT_OUBLK:
1845 		lwp->lwp_ru.oublock += inc;
1846 		break;
1847 	case LWP_STAT_MSGRCV:
1848 		lwp->lwp_ru.msgrcv += inc;
1849 		break;
1850 	case LWP_STAT_MSGSND:
1851 		lwp->lwp_ru.msgsnd += inc;
1852 		break;
1853 	default:
1854 		panic("lwp_stat_update: invalid lwp_stat_id 0x%x", lwp_stat_id);
1855 	}
1856 }
1857