xref: /illumos-gate/usr/src/uts/common/os/lwp.c (revision 8132eb48)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License, Version 1.0 only
6  * (the "License").  You may not use this file except in compliance
7  * with the License.
8  *
9  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10  * or http://www.opensolaris.org/os/licensing.
11  * See the License for the specific language governing permissions
12  * and limitations under the License.
13  *
14  * When distributing Covered Code, include this CDDL HEADER in each
15  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16  * If applicable, add the following below this CDDL HEADER, with the
17  * fields enclosed by brackets "[]" replaced with your own identifying
18  * information: Portions Copyright [yyyy] [name of copyright owner]
19  *
20  * CDDL HEADER END
21  */
22 
23 /*
24  * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
25  * Use is subject to license terms.
26  */
27 
28 #pragma ident	"%Z%%M%	%I%	%E% SMI"
29 
30 #include <sys/param.h>
31 #include <sys/types.h>
32 #include <sys/sysmacros.h>
33 #include <sys/systm.h>
34 #include <sys/thread.h>
35 #include <sys/proc.h>
36 #include <sys/task.h>
37 #include <sys/project.h>
38 #include <sys/signal.h>
39 #include <sys/errno.h>
40 #include <sys/vmparam.h>
41 #include <sys/stack.h>
42 #include <sys/procfs.h>
43 #include <sys/prsystm.h>
44 #include <sys/cpuvar.h>
45 #include <sys/kmem.h>
46 #include <sys/vtrace.h>
47 #include <sys/door.h>
48 #include <vm/seg_kp.h>
49 #include <sys/debug.h>
50 #include <sys/tnf.h>
51 #include <sys/schedctl.h>
52 #include <sys/poll.h>
53 #include <sys/copyops.h>
54 #include <sys/lwp_upimutex_impl.h>
55 #include <sys/cpupart.h>
56 #include <sys/lgrp.h>
57 #include <sys/rctl.h>
58 #include <sys/contract_impl.h>
59 #include <sys/cpc_impl.h>
60 #include <sys/sdt.h>
61 #include <sys/cmn_err.h>
62 
63 void *segkp_lwp;		/* cookie for pool of segkp resources */
64 
65 /*
66  * Create a thread that appears to be stopped at sys_rtt.
67  */
68 klwp_t *
69 lwp_create(void (*proc)(), caddr_t arg, size_t len, proc_t *p,
70     int state, int pri, const k_sigset_t *smask, int cid, id_t lwpid)
71 {
72 	klwp_t *lwp = NULL;
73 	kthread_t *t;
74 	kthread_t *tx;
75 	cpupart_t *oldpart = NULL;
76 	size_t	stksize;
77 	caddr_t lwpdata = NULL;
78 	processorid_t	binding;
79 	int err = 0;
80 	kproject_t *oldkpj, *newkpj;
81 	void *bufp = NULL;
82 	klwp_t *curlwp = ttolwp(curthread);
83 	lwpent_t *lep;
84 	lwpdir_t *old_dir = NULL;
85 	uint_t old_dirsz = 0;
86 	lwpdir_t **old_hash = NULL;
87 	uint_t old_hashsz = 0;
88 	int i;
89 	int rctlfail = 0;
90 
91 	mutex_enter(&p->p_lock);
92 	mutex_enter(&p->p_zone->zone_nlwps_lock);
93 	/*
94 	 * don't enforce rctl limits on system processes
95 	 */
96 	if (cid != syscid) {
97 		if (p->p_task->tk_nlwps >= p->p_task->tk_nlwps_ctl)
98 			if (rctl_test(rc_task_lwps, p->p_task->tk_rctls, p,
99 			    1, 0) & RCT_DENY)
100 				rctlfail = 1;
101 		if (p->p_task->tk_proj->kpj_nlwps >=
102 		    p->p_task->tk_proj->kpj_nlwps_ctl)
103 			if (rctl_test(rc_project_nlwps,
104 			    p->p_task->tk_proj->kpj_rctls, p, 1, 0)
105 			    & RCT_DENY)
106 				rctlfail = 1;
107 		if (p->p_zone->zone_nlwps >= p->p_zone->zone_nlwps_ctl)
108 			if (rctl_test(rc_zone_nlwps, p->p_zone->zone_rctls, p,
109 			    1, 0) & RCT_DENY)
110 				rctlfail = 1;
111 	}
112 	if (rctlfail) {
113 		mutex_exit(&p->p_zone->zone_nlwps_lock);
114 		mutex_exit(&p->p_lock);
115 		return (NULL);
116 	}
117 	p->p_task->tk_nlwps++;
118 	p->p_task->tk_proj->kpj_nlwps++;
119 	p->p_zone->zone_nlwps++;
120 	mutex_exit(&p->p_zone->zone_nlwps_lock);
121 	mutex_exit(&p->p_lock);
122 
123 	if (curlwp == NULL || (stksize = curlwp->lwp_childstksz) == 0)
124 		stksize = lwp_default_stksize;
125 
126 	/*
127 	 * Try to reclaim a <lwp,stack> from 'deathrow'
128 	 */
129 	if (stksize == lwp_default_stksize) {
130 		if (lwp_reapcnt > 0) {
131 			mutex_enter(&reaplock);
132 			if ((t = lwp_deathrow) != NULL) {
133 				ASSERT(t->t_swap);
134 				lwp_deathrow = t->t_forw;
135 				lwp_reapcnt--;
136 				lwpdata = t->t_swap;
137 				lwp = t->t_lwp;
138 			}
139 			mutex_exit(&reaplock);
140 			if (t) {
141 				t->t_swap = NULL;
142 				lwp_stk_fini(t->t_lwp);
143 				t->t_lwp = NULL;
144 				t->t_forw = NULL;
145 				thread_free(t);
146 			}
147 		}
148 		if (lwpdata == NULL &&
149 		    (lwpdata = (caddr_t)segkp_cache_get(segkp_lwp)) == NULL) {
150 			mutex_enter(&p->p_lock);
151 			mutex_enter(&p->p_zone->zone_nlwps_lock);
152 			p->p_task->tk_nlwps--;
153 			p->p_task->tk_proj->kpj_nlwps--;
154 			p->p_zone->zone_nlwps--;
155 			mutex_exit(&p->p_zone->zone_nlwps_lock);
156 			mutex_exit(&p->p_lock);
157 			return (NULL);
158 		}
159 	} else {
160 		stksize = roundup(stksize, PAGESIZE);
161 		if ((lwpdata = (caddr_t)segkp_get(segkp, stksize,
162 		    (KPD_NOWAIT | KPD_HASREDZONE | KPD_LOCKED))) == NULL) {
163 			mutex_enter(&p->p_lock);
164 			mutex_enter(&p->p_zone->zone_nlwps_lock);
165 			p->p_task->tk_nlwps--;
166 			p->p_task->tk_proj->kpj_nlwps--;
167 			p->p_zone->zone_nlwps--;
168 			mutex_exit(&p->p_zone->zone_nlwps_lock);
169 			mutex_exit(&p->p_lock);
170 			return (NULL);
171 		}
172 	}
173 
174 	/*
175 	 * Create a thread, initializing the stack pointer
176 	 */
177 	t = thread_create(lwpdata, stksize, NULL, NULL, 0, p, TS_STOPPED, pri);
178 
179 	t->t_swap = lwpdata;	/* Start of page-able data */
180 	if (lwp == NULL)
181 		lwp = kmem_cache_alloc(lwp_cache, KM_SLEEP);
182 	bzero(lwp, sizeof (*lwp));
183 	t->t_lwp = lwp;
184 
185 	t->t_hold = *smask;
186 	lwp->lwp_thread = t;
187 	lwp->lwp_procp = p;
188 	lwp->lwp_sigaltstack.ss_flags = SS_DISABLE;
189 	if (curlwp != NULL && curlwp->lwp_childstksz != 0)
190 		lwp->lwp_childstksz = curlwp->lwp_childstksz;
191 
192 	t->t_stk = lwp_stk_init(lwp, t->t_stk);
193 	thread_load(t, proc, arg, len);
194 
195 	/*
196 	 * Allocate the SIGPROF buffer if ITIMER_REALPROF is in effect.
197 	 */
198 	if (timerisset(&p->p_rprof_timer.it_value))
199 		t->t_rprof = kmem_zalloc(sizeof (struct rprof), KM_SLEEP);
200 
201 	if (cid != NOCLASS)
202 		(void) CL_ALLOC(&bufp, cid, KM_SLEEP);
203 
204 	/*
205 	 * Allocate an lwp directory entry for the new lwp.
206 	 */
207 	lep = kmem_zalloc(sizeof (*lep), KM_SLEEP);
208 
209 	mutex_enter(&p->p_lock);
210 grow:
211 	/*
212 	 * Grow the lwp (thread) directory and lwpid hash table if necessary.
213 	 * A note on the growth algorithm:
214 	 *	The new lwp directory size is computed as:
215 	 *		new = 2 * old + 2
216 	 *	Starting with an initial size of 2 (see exec_common()),
217 	 *	this yields numbers that are a power of two minus 2:
218 	 *		2, 6, 14, 30, 62, 126, 254, 510, 1022, ...
219 	 *	The size of the lwpid hash table must be a power of two
220 	 *	and must be commensurate in size with the lwp directory
221 	 *	so that hash bucket chains remain short.  Therefore,
222 	 *	the lwpid hash table size is computed as:
223 	 *		hashsz = (dirsz + 2) / 2
224 	 *	which leads to these hash table sizes corresponding to
225 	 *	the above directory sizes:
226 	 *		2, 4, 8, 16, 32, 64, 128, 256, 512, ...
227 	 */
228 	while (p->p_lwpfree == NULL) {
229 		uint_t dirsz = p->p_lwpdir_sz;
230 		uint_t new_dirsz;
231 		uint_t new_hashsz;
232 		lwpdir_t *new_dir;
233 		lwpdir_t *ldp;
234 		lwpdir_t **new_hash;
235 
236 		mutex_exit(&p->p_lock);
237 
238 		if (old_dir != NULL) {
239 			kmem_free(old_dir, old_dirsz * sizeof (*old_dir));
240 			kmem_free(old_hash, old_hashsz * sizeof (*old_hash));
241 			old_dir = NULL;
242 			old_dirsz = 0;
243 			old_hash = NULL;
244 			old_hashsz = 0;
245 		}
246 		new_dirsz = 2 * dirsz + 2;
247 		new_dir = kmem_zalloc(new_dirsz * sizeof (lwpdir_t), KM_SLEEP);
248 		for (ldp = new_dir, i = 1; i < new_dirsz; i++, ldp++)
249 			ldp->ld_next = ldp + 1;
250 		new_hashsz = (new_dirsz + 2) / 2;
251 		new_hash = kmem_zalloc(new_hashsz * sizeof (lwpdir_t *),
252 			KM_SLEEP);
253 
254 		mutex_enter(&p->p_lock);
255 		if (p == curproc)
256 			prbarrier(p);
257 
258 		if (dirsz != p->p_lwpdir_sz || p->p_lwpfree != NULL) {
259 			/*
260 			 * Someone else beat us to it or some lwp exited.
261 			 * Set up to free our memory and take a lap.
262 			 */
263 			old_dir = new_dir;
264 			old_dirsz = new_dirsz;
265 			old_hash = new_hash;
266 			old_hashsz = new_hashsz;
267 		} else {
268 			old_dir = p->p_lwpdir;
269 			old_dirsz = p->p_lwpdir_sz;
270 			old_hash = p->p_tidhash;
271 			old_hashsz = p->p_tidhash_sz;
272 			p->p_lwpdir = new_dir;
273 			p->p_lwpfree = new_dir;
274 			p->p_lwpdir_sz = new_dirsz;
275 			p->p_tidhash = new_hash;
276 			p->p_tidhash_sz = new_hashsz;
277 			/*
278 			 * We simply hash in all of the old directory entries.
279 			 * This works because the old directory has no empty
280 			 * slots and the new hash table starts out empty.
281 			 * This reproduces the original directory ordering
282 			 * (required for /proc directory semantics).
283 			 */
284 			for (ldp = old_dir, i = 0; i < dirsz; i++, ldp++)
285 				lwp_hash_in(p, ldp->ld_entry);
286 			/*
287 			 * Defer freeing memory until we drop p->p_lock,
288 			 */
289 		}
290 	}
291 
292 	/*
293 	 * Block the process against /proc while we manipulate p->p_tlist,
294 	 * unless lwp_create() was called by /proc for the PCAGENT operation.
295 	 * We want to do this early enough so that we don't drop p->p_lock
296 	 * until the thread is put on the p->p_tlist.
297 	 */
298 	if (p == curproc) {
299 		prbarrier(p);
300 		/*
301 		 * If the current lwp has been requested to stop, do so now.
302 		 * Otherwise we have a race condition between /proc attempting
303 		 * to stop the process and this thread creating a new lwp
304 		 * that was not seen when the /proc PCSTOP request was issued.
305 		 * We rely on stop() to call prbarrier(p) before returning.
306 		 */
307 		while ((curthread->t_proc_flag & TP_PRSTOP) &&
308 		    !ttolwp(curthread)->lwp_nostop)
309 			stop(PR_REQUESTED, 0);
310 
311 		/*
312 		 * If process is exiting, there could be a race between
313 		 * the agent lwp creation and the new lwp currently being
314 		 * created. So to prevent this race lwp creation is failed
315 		 * if the process is exiting.
316 		 */
317 		if (p->p_flag & (SEXITLWPS|SKILLED)) {
318 			err = 1;
319 			goto error;
320 		}
321 
322 		/*
323 		 * Since we might have dropped p->p_lock, the
324 		 * lwp directory free list might have changed.
325 		 */
326 		if (p->p_lwpfree == NULL)
327 			goto grow;
328 	}
329 
330 	kpreempt_disable();	/* can't grab cpu_lock here */
331 
332 	/*
333 	 * Inherit processor and processor set bindings from curthread,
334 	 * unless we're creating a new kernel process, in which case
335 	 * clear all bindings.
336 	 */
337 	if (cid == syscid) {
338 		t->t_bind_cpu = binding = PBIND_NONE;
339 		t->t_cpupart = oldpart = &cp_default;
340 		t->t_bind_pset = PS_NONE;
341 	} else {
342 		binding = curthread->t_bind_cpu;
343 		t->t_bind_cpu = binding;
344 		oldpart = t->t_cpupart;
345 		t->t_cpupart = curthread->t_cpupart;
346 		t->t_bind_pset = curthread->t_bind_pset;
347 	}
348 
349 	/*
350 	 * thread_create() initializes this thread's home lgroup to the root.
351 	 * Choose a more suitable lgroup, since this thread is associated
352 	 * with an lwp.
353 	 */
354 	ASSERT(oldpart != NULL);
355 	if (binding != PBIND_NONE && t->t_affinitycnt == 0) {
356 		t->t_bound_cpu = cpu[binding];
357 		if (t->t_lpl != t->t_bound_cpu->cpu_lpl)
358 			lgrp_move_thread(t, t->t_bound_cpu->cpu_lpl, 1);
359 	} else {
360 		lgrp_move_thread(t, lgrp_choose(t, t->t_cpupart), 1);
361 	}
362 
363 	kpreempt_enable();
364 
365 	/*
366 	 * make sure lpl points to our own partition
367 	 */
368 	ASSERT(t->t_lpl >= t->t_cpupart->cp_lgrploads);
369 	ASSERT(t->t_lpl < t->t_cpupart->cp_lgrploads +
370 	    t->t_cpupart->cp_nlgrploads);
371 
372 	/*
373 	 * If we're creating a new process, then inherit the project from our
374 	 * parent. If we're only creating an additional lwp then use the
375 	 * project pointer of the target process.
376 	 */
377 	if (p->p_task == NULL)
378 		newkpj = ttoproj(curthread);
379 	else
380 		newkpj = p->p_task->tk_proj;
381 
382 	/*
383 	 * It is safe to point the thread to the new project without holding it
384 	 * since we're holding the target process' p_lock here and therefore
385 	 * we're guaranteed that it will not move to another project.
386 	 */
387 	oldkpj = ttoproj(t);
388 	if (newkpj != oldkpj) {
389 		t->t_proj = newkpj;
390 		(void) project_hold(newkpj);
391 		project_rele(oldkpj);
392 	}
393 
394 	if (cid != NOCLASS) {
395 		/*
396 		 * If the lwp is being created in the current process
397 		 * and matches the current thread's scheduling class,
398 		 * we should propagate the current thread's scheduling
399 		 * parameters by calling CL_FORK.  Otherwise just use
400 		 * the defaults by calling CL_ENTERCLASS.
401 		 */
402 		if (p != curproc || curthread->t_cid != cid) {
403 			err = CL_ENTERCLASS(t, cid, NULL, NULL, bufp);
404 			t->t_pri = pri;	/* CL_ENTERCLASS may have changed it */
405 		} else {
406 			t->t_clfuncs = &(sclass[cid].cl_funcs->thread);
407 			err = CL_FORK(curthread, t, bufp);
408 			t->t_cid = cid;
409 		}
410 		if (err)
411 			goto error;
412 		else
413 			bufp = NULL;
414 	}
415 
416 	/*
417 	 * If we were given an lwpid then use it, else allocate one.
418 	 */
419 	if (lwpid != 0)
420 		t->t_tid = lwpid;
421 	else {
422 		/*
423 		 * lwp/thread id 0 is never valid; reserved for special checks.
424 		 * lwp/thread id 1 is reserved for the main thread.
425 		 * Start again at 2 when INT_MAX has been reached
426 		 * (id_t is a signed 32-bit integer).
427 		 */
428 		id_t prev_id = p->p_lwpid;	/* last allocated tid */
429 
430 		do {			/* avoid lwpid duplication */
431 			if (p->p_lwpid == INT_MAX) {
432 				p->p_flag |= SLWPWRAP;
433 				p->p_lwpid = 1;
434 			}
435 			if ((t->t_tid = ++p->p_lwpid) == prev_id) {
436 				/*
437 				 * All lwpids are allocated; fail the request.
438 				 */
439 				err = 1;
440 				goto error;
441 			}
442 			/*
443 			 * We only need to worry about colliding with an id
444 			 * that's already in use if this process has
445 			 * cycled through all available lwp ids.
446 			 */
447 			if ((p->p_flag & SLWPWRAP) == 0)
448 				break;
449 		} while (lwp_hash_lookup(p, t->t_tid) != NULL);
450 	}
451 	p->p_lwpcnt++;
452 	t->t_waitfor = -1;
453 
454 	/*
455 	 * Turn microstate accounting on for thread if on for process.
456 	 */
457 	if (p->p_flag & SMSACCT)
458 		t->t_proc_flag |= TP_MSACCT;
459 
460 	/*
461 	 * If the process has watchpoints, mark the new thread as such.
462 	 */
463 	if (pr_watch_active(p))
464 		watch_enable(t);
465 
466 	/*
467 	 * The lwp is being created in the stopped state.
468 	 * We set all the necessary flags to indicate that fact here.
469 	 * We omit the TS_CREATE flag from t_schedflag so that the lwp
470 	 * cannot be set running until the caller is finished with it,
471 	 * even if lwp_continue() is called on it after we drop p->p_lock.
472 	 * When the caller is finished with the newly-created lwp,
473 	 * the caller must call lwp_create_done() to allow the lwp
474 	 * to be set running.  If the TP_HOLDLWP is left set, the
475 	 * lwp will suspend itself after reaching system call exit.
476 	 */
477 	init_mstate(t, LMS_STOPPED);
478 	t->t_proc_flag |= TP_HOLDLWP;
479 	t->t_schedflag |= (TS_ALLSTART & ~(TS_CSTART | TS_CREATE));
480 	t->t_whystop = PR_SUSPENDED;
481 	t->t_whatstop = SUSPEND_NORMAL;
482 	t->t_sig_check = 1;	/* ensure that TP_HOLDLWP is honored */
483 
484 	/*
485 	 * Set system call processing flags in case tracing or profiling
486 	 * is set.  The first system call will evaluate these and turn
487 	 * them off if they aren't needed.
488 	 */
489 	t->t_pre_sys = 1;
490 	t->t_post_sys = 1;
491 
492 	/*
493 	 * Insert the new thread into the list of all threads.
494 	 */
495 	if ((tx = p->p_tlist) == NULL) {
496 		t->t_back = t;
497 		t->t_forw = t;
498 		p->p_tlist = t;
499 	} else {
500 		t->t_forw = tx;
501 		t->t_back = tx->t_back;
502 		tx->t_back->t_forw = t;
503 		tx->t_back = t;
504 	}
505 
506 	/*
507 	 * Insert the new lwp into an lwp directory slot position
508 	 * and into the lwpid hash table.
509 	 */
510 	lep->le_thread = t;
511 	lep->le_lwpid = t->t_tid;
512 	lep->le_start = t->t_start;
513 	lwp_hash_in(p, lep);
514 
515 	if (state == TS_RUN) {
516 		/*
517 		 * We set the new lwp running immediately.
518 		 */
519 		t->t_proc_flag &= ~TP_HOLDLWP;
520 		lwp_create_done(t);
521 	}
522 
523 error:
524 	if (err) {
525 		/*
526 		 * We have failed to create an lwp, so decrement the number
527 		 * of lwps in the task and let the lgroup load averages know
528 		 * that this thread isn't going to show up.
529 		 */
530 		kpreempt_disable();
531 		lgrp_move_thread(t, NULL, 1);
532 		kpreempt_enable();
533 
534 		ASSERT(MUTEX_HELD(&p->p_lock));
535 		mutex_enter(&p->p_zone->zone_nlwps_lock);
536 		p->p_task->tk_nlwps--;
537 		p->p_task->tk_proj->kpj_nlwps--;
538 		p->p_zone->zone_nlwps--;
539 		mutex_exit(&p->p_zone->zone_nlwps_lock);
540 		if (cid != NOCLASS && bufp != NULL)
541 			CL_FREE(cid, bufp);
542 
543 		mutex_exit(&p->p_lock);
544 		t->t_state = TS_FREE;
545 		thread_rele(t);
546 
547 		/*
548 		 * We need to remove t from the list of all threads
549 		 * because thread_exit()/lwp_exit() isn't called on t.
550 		 */
551 		mutex_enter(&pidlock);
552 		ASSERT(t != t->t_next);		/* t0 never exits */
553 		t->t_next->t_prev = t->t_prev;
554 		t->t_prev->t_next = t->t_next;
555 		mutex_exit(&pidlock);
556 
557 		thread_free(t);
558 		kmem_free(lep, sizeof (*lep));
559 		lwp = NULL;
560 	} else {
561 		mutex_exit(&p->p_lock);
562 	}
563 
564 	if (old_dir != NULL) {
565 		kmem_free(old_dir, old_dirsz * sizeof (*old_dir));
566 		kmem_free(old_hash, old_hashsz * sizeof (*old_hash));
567 	}
568 
569 	DTRACE_PROC1(lwp__create, kthread_t *, t);
570 	return (lwp);
571 }
572 
573 /*
574  * lwp_create_done() is called by the caller of lwp_create() to set the
575  * newly-created lwp running after the caller has finished manipulating it.
576  */
577 void
578 lwp_create_done(kthread_t *t)
579 {
580 	proc_t *p = ttoproc(t);
581 
582 	ASSERT(MUTEX_HELD(&p->p_lock));
583 
584 	/*
585 	 * We set the TS_CREATE and TS_CSTART flags and call setrun_locked().
586 	 * (The absence of the TS_CREATE flag prevents the lwp from running
587 	 * until we are finished with it, even if lwp_continue() is called on
588 	 * it by some other lwp in the process or elsewhere in the kernel.)
589 	 */
590 	thread_lock(t);
591 	ASSERT(t->t_state == TS_STOPPED && !(t->t_schedflag & TS_CREATE));
592 	/*
593 	 * If TS_CSTART is set, lwp_continue(t) has been called and
594 	 * has already incremented p_lwprcnt; avoid doing this twice.
595 	 */
596 	if (!(t->t_schedflag & TS_CSTART))
597 		p->p_lwprcnt++;
598 	t->t_schedflag |= (TS_CSTART | TS_CREATE);
599 	setrun_locked(t);
600 	thread_unlock(t);
601 }
602 
603 /*
604  * Copy an LWP's active templates, and clear the latest contracts.
605  */
606 void
607 lwp_ctmpl_copy(klwp_t *dst, klwp_t *src)
608 {
609 	int i;
610 
611 	for (i = 0; i < ct_ntypes; i++) {
612 		dst->lwp_ct_active[i] = ctmpl_dup(src->lwp_ct_active[i]);
613 		dst->lwp_ct_latest[i] = NULL;
614 	}
615 }
616 
617 /*
618  * Clear an LWP's contract template state.
619  */
620 void
621 lwp_ctmpl_clear(klwp_t *lwp)
622 {
623 	ct_template_t *tmpl;
624 	int i;
625 
626 	for (i = 0; i < ct_ntypes; i++) {
627 		if ((tmpl = lwp->lwp_ct_active[i]) != NULL) {
628 			ctmpl_free(tmpl);
629 			lwp->lwp_ct_active[i] = NULL;
630 		}
631 
632 		if (lwp->lwp_ct_latest[i] != NULL) {
633 			contract_rele(lwp->lwp_ct_latest[i]);
634 			lwp->lwp_ct_latest[i] = NULL;
635 		}
636 	}
637 }
638 
639 /*
640  * Individual lwp exit.
641  * If this is the last lwp, exit the whole process.
642  */
643 void
644 lwp_exit(void)
645 {
646 	kthread_t *t = curthread;
647 	klwp_t *lwp = ttolwp(t);
648 	proc_t *p = ttoproc(t);
649 
650 	ASSERT(MUTEX_HELD(&p->p_lock));
651 
652 	mutex_exit(&p->p_lock);
653 
654 #if defined(__sparc)
655 	/*
656 	 * Ensure that the user stack is fully abandoned..
657 	 */
658 	trash_user_windows();
659 #endif
660 
661 	tsd_exit();			/* free thread specific data */
662 
663 	kcpc_passivate();		/* Clean up performance counter state */
664 
665 	pollcleanup();
666 
667 	if (t->t_door)
668 		door_slam();
669 
670 	if (t->t_schedctl != NULL)
671 		schedctl_lwp_cleanup(t);
672 
673 	if (t->t_upimutex != NULL)
674 		upimutex_cleanup();
675 
676 	mutex_enter(&p->p_lock);
677 	lwp_cleanup();
678 
679 	/*
680 	 * When this process is dumping core, its lwps are held here
681 	 * until the core dump is finished. Then exitlwps() is called
682 	 * again to release these lwps so that they can finish exiting.
683 	 */
684 	if (p->p_flag & SCOREDUMP)
685 		stop(PR_SUSPENDED, SUSPEND_NORMAL);
686 
687 	/*
688 	 * Block the process against /proc now that we have really acquired
689 	 * p->p_lock (to decrement p_lwpcnt and manipulate p_tlist at least).
690 	 */
691 	prbarrier(p);
692 
693 	/*
694 	 * Call proc_exit() if this is the last non-daemon lwp in the process.
695 	 */
696 	if (!(t->t_proc_flag & TP_DAEMON) &&
697 	    p->p_lwpcnt == p->p_lwpdaemon + 1) {
698 		mutex_exit(&p->p_lock);
699 		if (proc_exit(CLD_EXITED, 0) == 0) {
700 			/* Restarting init. */
701 			return;
702 		}
703 
704 		/*
705 		 * proc_exit() returns a non-zero value when some other
706 		 * lwp got there first.  We just have to continue in
707 		 * lwp_exit().
708 		 */
709 		mutex_enter(&p->p_lock);
710 		ASSERT(curproc->p_flag & SEXITLWPS);
711 		prbarrier(p);
712 	}
713 
714 	DTRACE_PROC(lwp__exit);
715 
716 	/*
717 	 * If the lwp is a detached lwp or if the process is exiting,
718 	 * remove (lwp_hash_out()) the lwp from the lwp directory.
719 	 * Otherwise null out the lwp's le_thread pointer in the lwp
720 	 * directory so that other threads will see it as a zombie lwp.
721 	 */
722 	prlwpexit(t);		/* notify /proc */
723 	if (!(t->t_proc_flag & TP_TWAIT) || (p->p_flag & SEXITLWPS))
724 		lwp_hash_out(p, t->t_tid);
725 	else {
726 		ASSERT(!(t->t_proc_flag & TP_DAEMON));
727 		p->p_lwpdir[t->t_dslot].ld_entry->le_thread = NULL;
728 		p->p_zombcnt++;
729 		cv_broadcast(&p->p_lwpexit);
730 	}
731 	if (t->t_proc_flag & TP_DAEMON) {
732 		p->p_lwpdaemon--;
733 		t->t_proc_flag &= ~TP_DAEMON;
734 	}
735 	t->t_proc_flag &= ~TP_TWAIT;
736 
737 	/*
738 	 * Maintain accurate lwp count for task.max-lwps resource control.
739 	 */
740 	mutex_enter(&p->p_zone->zone_nlwps_lock);
741 	p->p_task->tk_nlwps--;
742 	p->p_task->tk_proj->kpj_nlwps--;
743 	p->p_zone->zone_nlwps--;
744 	mutex_exit(&p->p_zone->zone_nlwps_lock);
745 
746 	CL_EXIT(t);		/* tell the scheduler that t is exiting */
747 	ASSERT(p->p_lwpcnt != 0);
748 	p->p_lwpcnt--;
749 
750 	/*
751 	 * If all remaining non-daemon lwps are waiting in lwp_wait(),
752 	 * wake them up so someone can return EDEADLK.
753 	 * (See the block comment preceeding lwp_wait().)
754 	 */
755 	if (p->p_lwpcnt == p->p_lwpdaemon + (p->p_lwpwait - p->p_lwpdwait))
756 		cv_broadcast(&p->p_lwpexit);
757 
758 	t->t_proc_flag |= TP_LWPEXIT;
759 	term_mstate(t);
760 #ifndef NPROBE
761 	/* Kernel probe */
762 	if (t->t_tnf_tpdp)
763 		tnf_thread_exit();
764 #endif /* NPROBE */
765 
766 	t->t_forw->t_back = t->t_back;
767 	t->t_back->t_forw = t->t_forw;
768 	if (t == p->p_tlist)
769 		p->p_tlist = t->t_forw;
770 
771 	/*
772 	 * Clean up the signal state.
773 	 */
774 	if (t->t_sigqueue != NULL)
775 		sigdelq(p, t, 0);
776 	if (lwp->lwp_curinfo != NULL) {
777 		siginfofree(lwp->lwp_curinfo);
778 		lwp->lwp_curinfo = NULL;
779 	}
780 
781 	thread_rele(t);
782 
783 	/*
784 	 * Terminated lwps are associated with process zero and are put onto
785 	 * death-row by resume().  Avoid preemption after resetting t->t_procp.
786 	 */
787 	t->t_preempt++;
788 	t->t_procp = &p0;
789 
790 	/*
791 	 * Notify the HAT about the change of address space
792 	 */
793 	hat_thread_exit(t);
794 	/*
795 	 * When this is the last running lwp in this process and some lwp is
796 	 * waiting for this condition to become true, or this thread was being
797 	 * suspended, then the waiting lwp is awakened.
798 	 *
799 	 * Also, if the process is exiting, we may have a thread waiting in
800 	 * exitlwps() that needs to be notified.
801 	 */
802 	if (--p->p_lwprcnt == 0 || (t->t_proc_flag & TP_HOLDLWP) ||
803 	    (p->p_flag & SEXITLWPS))
804 		cv_broadcast(&p->p_holdlwps);
805 
806 	/*
807 	 * Need to drop p_lock so we can reacquire pidlock.
808 	 */
809 	mutex_exit(&p->p_lock);
810 	mutex_enter(&pidlock);
811 
812 	ASSERT(t != t->t_next);		/* t0 never exits */
813 	t->t_next->t_prev = t->t_prev;
814 	t->t_prev->t_next = t->t_next;
815 	cv_broadcast(&t->t_joincv);	/* wake up anyone in thread_join */
816 	mutex_exit(&pidlock);
817 
818 	lwp_pcb_exit();
819 
820 	if (t->t_ctx != NULL)
821 		exitctx(t);
822 
823 	t->t_state = TS_ZOMB;
824 	swtch_from_zombie();
825 	/* never returns */
826 }
827 
828 
829 /*
830  * Cleanup function for an exiting lwp.
831  * Called both from lwp_exit() and from proc_exit().
832  * p->p_lock is repeatedly released and grabbed in this function.
833  */
834 void
835 lwp_cleanup(void)
836 {
837 	kthread_t *t = curthread;
838 	proc_t *p = ttoproc(t);
839 
840 	ASSERT(MUTEX_HELD(&p->p_lock));
841 
842 	/* untimeout any lwp-bound realtime timers */
843 	if (p->p_itimer != NULL)
844 		timer_lwpexit();
845 
846 	/*
847 	 * If this is the /proc agent lwp that is exiting, readjust p_lwpid
848 	 * so it appears that the agent never existed, and clear p_agenttp.
849 	 */
850 	if (t == p->p_agenttp) {
851 		ASSERT(t->t_tid == p->p_lwpid);
852 		p->p_lwpid--;
853 		p->p_agenttp = NULL;
854 	}
855 
856 	/*
857 	 * Do lgroup bookkeeping to account for thread exiting.
858 	 */
859 	kpreempt_disable();
860 	lgrp_move_thread(t, NULL, 1);
861 	kpreempt_enable();
862 
863 	lwp_ctmpl_clear(ttolwp(t));
864 }
865 
866 int
867 lwp_suspend(kthread_t *t)
868 {
869 	int tid;
870 	proc_t *p = ttoproc(t);
871 
872 	ASSERT(MUTEX_HELD(&p->p_lock));
873 
874 	/*
875 	 * Set the thread's TP_HOLDLWP flag so it will stop in holdlwp().
876 	 * If an lwp is stopping itself, there is no need to wait.
877 	 */
878 top:
879 	t->t_proc_flag |= TP_HOLDLWP;
880 	if (t == curthread) {
881 		t->t_sig_check = 1;
882 	} else {
883 		/*
884 		 * Make sure the lwp stops promptly.
885 		 */
886 		thread_lock(t);
887 		t->t_sig_check = 1;
888 		/*
889 		 * XXX Should use virtual stop like /proc does instead of
890 		 * XXX waking the thread to get it to stop.
891 		 */
892 		if (t->t_state == TS_SLEEP && (t->t_flag & T_WAKEABLE))
893 			setrun_locked(t);
894 		else if (t->t_state == TS_ONPROC && t->t_cpu != CPU)
895 			poke_cpu(t->t_cpu->cpu_id);
896 		tid = t->t_tid;	 /* remember thread ID */
897 		/*
898 		 * Wait for lwp to stop
899 		 */
900 		while (!SUSPENDED(t)) {
901 			/*
902 			 * Drop the thread lock before waiting and reacquire it
903 			 * afterwards, so the thread can change its t_state
904 			 * field.
905 			 */
906 			thread_unlock(t);
907 
908 			/*
909 			 * Check if aborted by exitlwps().
910 			 */
911 			if (p->p_flag & SEXITLWPS)
912 				lwp_exit();
913 
914 			/*
915 			 * Cooperate with jobcontrol signals and /proc stopping
916 			 * by calling cv_wait_sig() to wait for the target
917 			 * lwp to stop.  Just using cv_wait() can lead to
918 			 * deadlock because, if some other lwp has stopped
919 			 * by either of these mechanisms, then p_lwprcnt will
920 			 * never become zero if we do a cv_wait().
921 			 */
922 			if (!cv_wait_sig(&p->p_holdlwps, &p->p_lock))
923 				return (EINTR);
924 
925 			/*
926 			 * Check to see if thread died while we were
927 			 * waiting for it to suspend.
928 			 */
929 			if (idtot(p, tid) == NULL)
930 				return (ESRCH);
931 
932 			thread_lock(t);
933 			/*
934 			 * If the TP_HOLDLWP flag went away, lwp_continue()
935 			 * or vfork() must have been called while we were
936 			 * waiting, so start over again.
937 			 */
938 			if ((t->t_proc_flag & TP_HOLDLWP) == 0) {
939 				thread_unlock(t);
940 				goto top;
941 			}
942 		}
943 		thread_unlock(t);
944 	}
945 	return (0);
946 }
947 
948 /*
949  * continue a lwp that's been stopped by lwp_suspend().
950  */
951 void
952 lwp_continue(kthread_t *t)
953 {
954 	proc_t *p = ttoproc(t);
955 	int was_suspended = t->t_proc_flag & TP_HOLDLWP;
956 
957 	ASSERT(MUTEX_HELD(&p->p_lock));
958 
959 	t->t_proc_flag &= ~TP_HOLDLWP;
960 	thread_lock(t);
961 	if (SUSPENDED(t) &&
962 	    !(p->p_flag & (SHOLDFORK | SHOLDFORK1 | SHOLDWATCH))) {
963 		p->p_lwprcnt++;
964 		t->t_schedflag |= TS_CSTART;
965 		setrun_locked(t);
966 	}
967 	thread_unlock(t);
968 	/*
969 	 * Wakeup anyone waiting for this thread to be suspended
970 	 */
971 	if (was_suspended)
972 		cv_broadcast(&p->p_holdlwps);
973 }
974 
975 /*
976  * ********************************
977  *  Miscellaneous lwp routines	  *
978  * ********************************
979  */
980 /*
981  * When a process is undergoing a forkall(), its p_flag is set to SHOLDFORK.
982  * This will cause the process's lwps to stop at a hold point.  A hold
983  * point is where a kernel thread has a flat stack.  This is at the
984  * return from a system call and at the return from a user level trap.
985  *
986  * When a process is undergoing a fork1() or vfork(), its p_flag is set to
987  * SHOLDFORK1.  This will cause the process's lwps to stop at a modified
988  * hold point.  The lwps in the process are not being cloned, so they
989  * are held at the usual hold points and also within issig_forreal().
990  * This has the side-effect that their system calls do not return
991  * showing EINTR.
992  *
993  * An lwp can also be held.  This is identified by the TP_HOLDLWP flag on
994  * the thread.  The TP_HOLDLWP flag is set in lwp_suspend(), where the active
995  * lwp is waiting for the target lwp to be stopped.
996  */
997 void
998 holdlwp(void)
999 {
1000 	proc_t *p = curproc;
1001 	kthread_t *t = curthread;
1002 
1003 	mutex_enter(&p->p_lock);
1004 	/*
1005 	 * Don't terminate immediately if the process is dumping core.
1006 	 * Once the process has dumped core, all lwps are terminated.
1007 	 */
1008 	if (!(p->p_flag & SCOREDUMP)) {
1009 		if ((p->p_flag & SEXITLWPS) || (t->t_proc_flag & TP_EXITLWP))
1010 			lwp_exit();
1011 	}
1012 	if (!(ISHOLD(p)) && !(p->p_flag & (SHOLDFORK1 | SHOLDWATCH))) {
1013 		mutex_exit(&p->p_lock);
1014 		return;
1015 	}
1016 	/*
1017 	 * stop() decrements p->p_lwprcnt and cv_signal()s &p->p_holdlwps
1018 	 * when p->p_lwprcnt becomes zero.
1019 	 */
1020 	stop(PR_SUSPENDED, SUSPEND_NORMAL);
1021 	if (p->p_flag & SEXITLWPS)
1022 		lwp_exit();
1023 	mutex_exit(&p->p_lock);
1024 }
1025 
1026 /*
1027  * Have all lwps within the process hold at a point where they are
1028  * cloneable (SHOLDFORK) or just safe w.r.t. fork1 (SHOLDFORK1).
1029  */
1030 int
1031 holdlwps(int holdflag)
1032 {
1033 	proc_t *p = curproc;
1034 
1035 	ASSERT(holdflag == SHOLDFORK || holdflag == SHOLDFORK1);
1036 	mutex_enter(&p->p_lock);
1037 	schedctl_finish_sigblock(curthread);
1038 again:
1039 	while (p->p_flag & (SEXITLWPS | SHOLDFORK | SHOLDFORK1 | SHOLDWATCH)) {
1040 		/*
1041 		 * If another lwp is doing a forkall() or proc_exit(), bail out.
1042 		 */
1043 		if (p->p_flag & (SEXITLWPS | SHOLDFORK)) {
1044 			mutex_exit(&p->p_lock);
1045 			return (0);
1046 		}
1047 		/*
1048 		 * Another lwp is doing a fork1() or is undergoing
1049 		 * watchpoint activity.  We hold here for it to complete.
1050 		 */
1051 		stop(PR_SUSPENDED, SUSPEND_NORMAL);
1052 	}
1053 	p->p_flag |= holdflag;
1054 	pokelwps(p);
1055 	--p->p_lwprcnt;
1056 	/*
1057 	 * Wait for the process to become quiescent (p->p_lwprcnt == 0).
1058 	 */
1059 	while (p->p_lwprcnt > 0) {
1060 		/*
1061 		 * Check if aborted by exitlwps().
1062 		 * Also check if SHOLDWATCH is set; it takes precedence.
1063 		 */
1064 		if (p->p_flag & (SEXITLWPS | SHOLDWATCH)) {
1065 			p->p_lwprcnt++;
1066 			p->p_flag &= ~holdflag;
1067 			cv_broadcast(&p->p_holdlwps);
1068 			goto again;
1069 		}
1070 		/*
1071 		 * Cooperate with jobcontrol signals and /proc stopping.
1072 		 * If some other lwp has stopped by either of these
1073 		 * mechanisms, then p_lwprcnt will never become zero
1074 		 * and the process will appear deadlocked unless we
1075 		 * stop here in sympathy with the other lwp before
1076 		 * doing the cv_wait() below.
1077 		 *
1078 		 * If the other lwp stops after we do the cv_wait(), it
1079 		 * will wake us up to loop around and do the sympathy stop.
1080 		 *
1081 		 * Since stop() drops p->p_lock, we must start from
1082 		 * the top again on returning from stop().
1083 		 */
1084 		if (p->p_stopsig | (curthread->t_proc_flag & TP_PRSTOP)) {
1085 			int whystop = p->p_stopsig? PR_JOBCONTROL :
1086 			    PR_REQUESTED;
1087 			p->p_lwprcnt++;
1088 			p->p_flag &= ~holdflag;
1089 			stop(whystop, p->p_stopsig);
1090 			goto again;
1091 		}
1092 		cv_wait(&p->p_holdlwps, &p->p_lock);
1093 	}
1094 	p->p_lwprcnt++;
1095 	p->p_flag &= ~holdflag;
1096 	mutex_exit(&p->p_lock);
1097 	return (1);
1098 }
1099 
1100 /*
1101  * See comments for holdwatch(), below.
1102  */
1103 static int
1104 holdcheck(int clearflags)
1105 {
1106 	proc_t *p = curproc;
1107 
1108 	/*
1109 	 * If we are trying to exit, that takes precedence over anything else.
1110 	 */
1111 	if (p->p_flag & SEXITLWPS) {
1112 		p->p_lwprcnt++;
1113 		p->p_flag &= ~clearflags;
1114 		lwp_exit();
1115 	}
1116 
1117 	/*
1118 	 * If another thread is calling fork1(), stop the current thread so the
1119 	 * other can complete.
1120 	 */
1121 	if (p->p_flag & SHOLDFORK1) {
1122 		p->p_lwprcnt++;
1123 		stop(PR_SUSPENDED, SUSPEND_NORMAL);
1124 		if (p->p_flag & SEXITLWPS) {
1125 			p->p_flag &= ~clearflags;
1126 			lwp_exit();
1127 		}
1128 		return (-1);
1129 	}
1130 
1131 	/*
1132 	 * If another thread is calling fork(), then indicate we are doing
1133 	 * watchpoint activity.  This will cause holdlwps() above to stop the
1134 	 * forking thread, at which point we can continue with watchpoint
1135 	 * activity.
1136 	 */
1137 	if (p->p_flag & SHOLDFORK) {
1138 		p->p_lwprcnt++;
1139 		while (p->p_flag & SHOLDFORK) {
1140 			p->p_flag |= SHOLDWATCH;
1141 			cv_broadcast(&p->p_holdlwps);
1142 			cv_wait(&p->p_holdlwps, &p->p_lock);
1143 			p->p_flag &= ~SHOLDWATCH;
1144 		}
1145 		return (-1);
1146 	}
1147 
1148 	return (0);
1149 }
1150 
1151 /*
1152  * Stop all lwps within the process, holding themselves in the kernel while the
1153  * active lwp undergoes watchpoint activity.  This is more complicated than
1154  * expected because stop() relies on calling holdwatch() in order to copyin data
1155  * from the user's address space.  A double barrier is used to prevent an
1156  * infinite loop.
1157  *
1158  * 	o The first thread into holdwatch() is the 'master' thread and does
1159  *        the following:
1160  *
1161  *              - Sets SHOLDWATCH on the current process
1162  *              - Sets TP_WATCHSTOP on the current thread
1163  *              - Waits for all threads to be either stopped or have
1164  *                TP_WATCHSTOP set.
1165  *              - Sets the SWATCHOK flag on the process
1166  *              - Unsets TP_WATCHSTOP
1167  *              - Waits for the other threads to completely stop
1168  *              - Unsets SWATCHOK
1169  *
1170  * 	o If SHOLDWATCH is already set when we enter this function, then another
1171  *        thread is already trying to stop this thread.  This 'slave' thread
1172  *        does the following:
1173  *
1174  *              - Sets TP_WATCHSTOP on the current thread
1175  *              - Waits for SWATCHOK flag to be set
1176  *              - Calls stop()
1177  *
1178  * 	o If SWATCHOK is set on the process, then this function immediately
1179  *        returns, as we must have been called via stop().
1180  *
1181  * In addition, there are other flags that take precedence over SHOLDWATCH:
1182  *
1183  * 	o If SEXITLWPS is set, exit immediately.
1184  *
1185  * 	o If SHOLDFORK1 is set, wait for fork1() to complete.
1186  *
1187  * 	o If SHOLDFORK is set, then watchpoint activity takes precedence In this
1188  *        case, set SHOLDWATCH, signalling the forking thread to stop first.
1189  *
1190  * 	o If the process is being stopped via /proc (TP_PRSTOP is set), then we
1191  *        stop the current thread.
1192  *
1193  * Returns 0 if all threads have been quiesced.  Returns non-zero if not all
1194  * threads were stopped, or the list of watched pages has changed.
1195  */
1196 int
1197 holdwatch(void)
1198 {
1199 	proc_t *p = curproc;
1200 	kthread_t *t = curthread;
1201 	int ret = 0;
1202 
1203 	mutex_enter(&p->p_lock);
1204 
1205 	p->p_lwprcnt--;
1206 
1207 	/*
1208 	 * Check for bail-out conditions as outlined above.
1209 	 */
1210 	if (holdcheck(0) != 0) {
1211 		mutex_exit(&p->p_lock);
1212 		return (-1);
1213 	}
1214 
1215 	if (!(p->p_flag & SHOLDWATCH)) {
1216 		/*
1217 		 * We are the master watchpoint thread.  Set SHOLDWATCH and poke
1218 		 * the other threads.
1219 		 */
1220 		p->p_flag |= SHOLDWATCH;
1221 		pokelwps(p);
1222 
1223 		/*
1224 		 * Wait for all threads to be stopped or have TP_WATCHSTOP set.
1225 		 */
1226 		while (pr_allstopped(p, 1) > 0) {
1227 			if (holdcheck(SHOLDWATCH) != 0) {
1228 				p->p_flag &= ~SHOLDWATCH;
1229 				mutex_exit(&p->p_lock);
1230 				return (-1);
1231 			}
1232 
1233 			cv_wait(&p->p_holdlwps, &p->p_lock);
1234 		}
1235 
1236 		/*
1237 		 * All threads are now stopped or in the process of stopping.
1238 		 * Set SWATCHOK and let them stop completely.
1239 		 */
1240 		p->p_flag |= SWATCHOK;
1241 		t->t_proc_flag &= ~TP_WATCHSTOP;
1242 		cv_broadcast(&p->p_holdlwps);
1243 
1244 		while (pr_allstopped(p, 0) > 0) {
1245 			/*
1246 			 * At first glance, it may appear that we don't need a
1247 			 * call to holdcheck() here.  But if the process gets a
1248 			 * SIGKILL signal, one of our stopped threads may have
1249 			 * been awakened and is waiting in exitlwps(), which
1250 			 * takes precedence over watchpoints.
1251 			 */
1252 			if (holdcheck(SHOLDWATCH | SWATCHOK) != 0) {
1253 				p->p_flag &= ~(SHOLDWATCH | SWATCHOK);
1254 				mutex_exit(&p->p_lock);
1255 				return (-1);
1256 			}
1257 
1258 			cv_wait(&p->p_holdlwps, &p->p_lock);
1259 		}
1260 
1261 		/*
1262 		 * All threads are now completely stopped.
1263 		 */
1264 		p->p_flag &= ~SWATCHOK;
1265 		p->p_flag &= ~SHOLDWATCH;
1266 		p->p_lwprcnt++;
1267 
1268 	} else if (!(p->p_flag & SWATCHOK)) {
1269 
1270 		/*
1271 		 * SHOLDWATCH is set, so another thread is trying to do
1272 		 * watchpoint activity.  Indicate this thread is stopping, and
1273 		 * wait for the OK from the master thread.
1274 		 */
1275 		t->t_proc_flag |= TP_WATCHSTOP;
1276 		cv_broadcast(&p->p_holdlwps);
1277 
1278 		while (!(p->p_flag & SWATCHOK)) {
1279 			if (holdcheck(0) != 0) {
1280 				t->t_proc_flag &= ~TP_WATCHSTOP;
1281 				mutex_exit(&p->p_lock);
1282 				return (-1);
1283 			}
1284 
1285 			cv_wait(&p->p_holdlwps, &p->p_lock);
1286 		}
1287 
1288 		/*
1289 		 * Once the master thread has given the OK, this thread can
1290 		 * actually call stop().
1291 		 */
1292 		t->t_proc_flag &= ~TP_WATCHSTOP;
1293 		p->p_lwprcnt++;
1294 
1295 		stop(PR_SUSPENDED, SUSPEND_NORMAL);
1296 
1297 		/*
1298 		 * It's not OK to do watchpoint activity, notify caller to
1299 		 * retry.
1300 		 */
1301 		ret = -1;
1302 
1303 	} else {
1304 
1305 		/*
1306 		 * The only way we can hit the case where SHOLDWATCH is set and
1307 		 * SWATCHOK is set is if we are triggering this from within a
1308 		 * stop() call.  Assert that this is the case.
1309 		 */
1310 
1311 		ASSERT(t->t_proc_flag & TP_STOPPING);
1312 		p->p_lwprcnt++;
1313 	}
1314 
1315 	mutex_exit(&p->p_lock);
1316 
1317 	return (ret);
1318 }
1319 
1320 /*
1321  * force all interruptible lwps to trap into the kernel.
1322  */
1323 void
1324 pokelwps(proc_t *p)
1325 {
1326 	kthread_t *t;
1327 
1328 	ASSERT(MUTEX_HELD(&p->p_lock));
1329 
1330 	t = p->p_tlist;
1331 	do {
1332 		if (t == curthread)
1333 			continue;
1334 		thread_lock(t);
1335 		aston(t);	/* make thread trap or do post_syscall */
1336 		if (t->t_state == TS_SLEEP) {
1337 			if (t->t_flag & T_WAKEABLE)
1338 				setrun_locked(t);
1339 		} else if (t->t_state == TS_STOPPED) {
1340 			/*
1341 			 * Ensure that proc_exit() is not blocked by lwps
1342 			 * that were stopped via jobcontrol or /proc.
1343 			 */
1344 			if (p->p_flag & SEXITLWPS) {
1345 				p->p_stopsig = 0;
1346 				t->t_schedflag |= (TS_XSTART | TS_PSTART);
1347 				setrun_locked(t);
1348 			}
1349 			/*
1350 			 * If we are holding lwps for a forkall(),
1351 			 * force lwps that have been suspended via
1352 			 * lwp_suspend() and are suspended inside
1353 			 * of a system call to proceed to their
1354 			 * holdlwp() points where they are clonable.
1355 			 */
1356 			if ((p->p_flag & SHOLDFORK) && SUSPENDED(t)) {
1357 				if ((t->t_schedflag & TS_CSTART) == 0) {
1358 					p->p_lwprcnt++;
1359 					t->t_schedflag |= TS_CSTART;
1360 					setrun_locked(t);
1361 				}
1362 			}
1363 		} else if (t->t_state == TS_ONPROC) {
1364 			if (t->t_cpu != CPU)
1365 				poke_cpu(t->t_cpu->cpu_id);
1366 		}
1367 		thread_unlock(t);
1368 	} while ((t = t->t_forw) != p->p_tlist);
1369 }
1370 
1371 /*
1372  * undo the effects of holdlwps() or holdwatch().
1373  */
1374 void
1375 continuelwps(proc_t *p)
1376 {
1377 	kthread_t *t;
1378 
1379 	/*
1380 	 * If this flag is set, then the original holdwatch() didn't actually
1381 	 * stop the process.  See comments for holdwatch().
1382 	 */
1383 	if (p->p_flag & SWATCHOK) {
1384 		ASSERT(curthread->t_proc_flag & TP_STOPPING);
1385 		return;
1386 	}
1387 
1388 	ASSERT(MUTEX_HELD(&p->p_lock));
1389 	ASSERT((p->p_flag & (SHOLDFORK | SHOLDFORK1 | SHOLDWATCH)) == 0);
1390 
1391 	t = p->p_tlist;
1392 	do {
1393 		thread_lock(t);		/* SUSPENDED looks at t_schedflag */
1394 		if (SUSPENDED(t) && !(t->t_proc_flag & TP_HOLDLWP)) {
1395 			p->p_lwprcnt++;
1396 			t->t_schedflag |= TS_CSTART;
1397 			setrun_locked(t);
1398 		}
1399 		thread_unlock(t);
1400 	} while ((t = t->t_forw) != p->p_tlist);
1401 }
1402 
1403 /*
1404  * Force all other LWPs in the current process other than the caller to exit,
1405  * and then cv_wait() on p_holdlwps for them to exit.  The exitlwps() function
1406  * is typically used in these situations:
1407  *
1408  *   (a) prior to an exec() system call
1409  *   (b) prior to dumping a core file
1410  *   (c) prior to a uadmin() shutdown
1411  *
1412  * If the 'coredump' flag is set, other LWPs are quiesced but not destroyed.
1413  * Multiple threads in the process can call this function at one time by
1414  * triggering execs or core dumps simultaneously, so the SEXITLWPS bit is used
1415  * to declare one particular thread the winner who gets to kill the others.
1416  * If a thread wins the exitlwps() dance, zero is returned; otherwise an
1417  * appropriate errno value is returned to caller for its system call to return.
1418  */
1419 int
1420 exitlwps(int coredump)
1421 {
1422 	proc_t *p = curproc;
1423 	int heldcnt;
1424 
1425 	if (curthread->t_door)
1426 		door_slam();
1427 	if (p->p_door_list)
1428 		door_revoke_all();
1429 	if (curthread->t_schedctl != NULL)
1430 		schedctl_lwp_cleanup(curthread);
1431 
1432 	/*
1433 	 * Ensure that before starting to wait for other lwps to exit,
1434 	 * cleanup all upimutexes held by curthread. Otherwise, some other
1435 	 * lwp could be waiting (uninterruptibly) for a upimutex held by
1436 	 * curthread, and the call to pokelwps() below would deadlock.
1437 	 * Even if a blocked upimutex_lock is made interruptible,
1438 	 * curthread's upimutexes need to be unlocked: do it here.
1439 	 */
1440 	if (curthread->t_upimutex != NULL)
1441 		upimutex_cleanup();
1442 
1443 	/*
1444 	 * Grab p_lock in order to check and set SEXITLWPS to declare a winner.
1445 	 * We must also block any further /proc access from this point forward.
1446 	 */
1447 	mutex_enter(&p->p_lock);
1448 	prbarrier(p);
1449 
1450 	if (p->p_flag & SEXITLWPS) {
1451 		mutex_exit(&p->p_lock);
1452 		aston(curthread);	/* force a trip through post_syscall */
1453 		return (set_errno(EINTR));
1454 	}
1455 
1456 	p->p_flag |= SEXITLWPS;
1457 	if (coredump)		/* tell other lwps to stop, not exit */
1458 		p->p_flag |= SCOREDUMP;
1459 
1460 	/*
1461 	 * Give precedence to exitlwps() if a holdlwps() is
1462 	 * in progress. The lwp doing the holdlwps() operation
1463 	 * is aborted when it is awakened.
1464 	 */
1465 	while (p->p_flag & (SHOLDFORK | SHOLDFORK1 | SHOLDWATCH)) {
1466 		cv_broadcast(&p->p_holdlwps);
1467 		cv_wait(&p->p_holdlwps, &p->p_lock);
1468 		prbarrier(p);
1469 	}
1470 	p->p_flag |= SHOLDFORK;
1471 	pokelwps(p);
1472 
1473 	/*
1474 	 * Wait for process to become quiescent.
1475 	 */
1476 	--p->p_lwprcnt;
1477 	while (p->p_lwprcnt > 0) {
1478 		cv_wait(&p->p_holdlwps, &p->p_lock);
1479 		prbarrier(p);
1480 	}
1481 	p->p_lwprcnt++;
1482 	ASSERT(p->p_lwprcnt == 1);
1483 
1484 	/*
1485 	 * The SCOREDUMP flag puts the process into a quiescent
1486 	 * state.  The process's lwps remain attached to this
1487 	 * process until exitlwps() is called again without the
1488 	 * 'coredump' flag set, then the lwps are terminated
1489 	 * and the process can exit.
1490 	 */
1491 	if (coredump) {
1492 		p->p_flag &= ~(SCOREDUMP | SHOLDFORK | SEXITLWPS);
1493 		goto out;
1494 	}
1495 
1496 	/*
1497 	 * Determine if there are any lwps left dangling in
1498 	 * the stopped state.  This happens when exitlwps()
1499 	 * aborts a holdlwps() operation.
1500 	 */
1501 	p->p_flag &= ~SHOLDFORK;
1502 	if ((heldcnt = p->p_lwpcnt) > 1) {
1503 		kthread_t *t;
1504 		for (t = curthread->t_forw; --heldcnt > 0; t = t->t_forw) {
1505 			t->t_proc_flag &= ~TP_TWAIT;
1506 			lwp_continue(t);
1507 		}
1508 	}
1509 
1510 	/*
1511 	 * Wait for all other lwps to exit.
1512 	 */
1513 	--p->p_lwprcnt;
1514 	while (p->p_lwpcnt > 1) {
1515 		cv_wait(&p->p_holdlwps, &p->p_lock);
1516 		prbarrier(p);
1517 	}
1518 	++p->p_lwprcnt;
1519 	ASSERT(p->p_lwpcnt == 1 && p->p_lwprcnt == 1);
1520 
1521 	p->p_flag &= ~SEXITLWPS;
1522 	curthread->t_proc_flag &= ~TP_TWAIT;
1523 
1524 out:
1525 	if (!coredump && p->p_zombcnt) {	/* cleanup the zombie lwps */
1526 		lwpdir_t *ldp;
1527 		lwpent_t *lep;
1528 		int i;
1529 
1530 		for (ldp = p->p_lwpdir, i = 0; i < p->p_lwpdir_sz; i++, ldp++) {
1531 			lep = ldp->ld_entry;
1532 			if (lep != NULL && lep->le_thread != curthread) {
1533 				ASSERT(lep->le_thread == NULL);
1534 				p->p_zombcnt--;
1535 				lwp_hash_out(p, lep->le_lwpid);
1536 			}
1537 		}
1538 		ASSERT(p->p_zombcnt == 0);
1539 	}
1540 
1541 	/*
1542 	 * If some other LWP in the process wanted us to suspend ourself,
1543 	 * then we will not do it.  The other LWP is now terminated and
1544 	 * no one will ever continue us again if we suspend ourself.
1545 	 */
1546 	curthread->t_proc_flag &= ~TP_HOLDLWP;
1547 	p->p_flag &= ~(SHOLDFORK | SHOLDFORK1 | SHOLDWATCH | SLWPWRAP);
1548 	mutex_exit(&p->p_lock);
1549 	return (0);
1550 }
1551 
1552 /*
1553  * duplicate a lwp.
1554  */
1555 klwp_t *
1556 forklwp(klwp_t *lwp, proc_t *cp, id_t lwpid)
1557 {
1558 	klwp_t *clwp;
1559 	void *tregs, *tfpu;
1560 	kthread_t *t = lwptot(lwp);
1561 	kthread_t *ct;
1562 	proc_t *p = lwptoproc(lwp);
1563 	int cid;
1564 	void *bufp;
1565 	int val;
1566 
1567 	ASSERT(p == curproc);
1568 	ASSERT(t == curthread || (SUSPENDED(t) && lwp->lwp_asleep == 0));
1569 
1570 #if defined(__sparc)
1571 	if (t == curthread)
1572 		(void) flush_user_windows_to_stack(NULL);
1573 #endif
1574 
1575 	if (t == curthread)
1576 		/* copy args out of registers first */
1577 		(void) save_syscall_args();
1578 	clwp = lwp_create(cp->p_lwpcnt == 0 ? lwp_rtt_initial : lwp_rtt,
1579 	    NULL, 0, cp, TS_STOPPED, t->t_pri, &t->t_hold, NOCLASS, lwpid);
1580 	if (clwp == NULL)
1581 		return (NULL);
1582 
1583 	/*
1584 	 * most of the parent's lwp can be copied to its duplicate,
1585 	 * except for the fields that are unique to each lwp, like
1586 	 * lwp_thread, lwp_procp, lwp_regs, and lwp_ap.
1587 	 */
1588 	ct = clwp->lwp_thread;
1589 	tregs = clwp->lwp_regs;
1590 	tfpu = clwp->lwp_fpu;
1591 
1592 	/* copy parent lwp to child lwp */
1593 	*clwp = *lwp;
1594 
1595 	/* fix up child's lwp */
1596 
1597 	clwp->lwp_pcb.pcb_flags = 0;
1598 #if defined(__sparc)
1599 	clwp->lwp_pcb.pcb_step = STEP_NONE;
1600 #endif
1601 	clwp->lwp_cursig = 0;
1602 	clwp->lwp_extsig = 0;
1603 	clwp->lwp_curinfo = (struct sigqueue *)0;
1604 	clwp->lwp_thread = ct;
1605 	ct->t_sysnum = t->t_sysnum;
1606 	clwp->lwp_regs = tregs;
1607 	clwp->lwp_fpu = tfpu;
1608 	clwp->lwp_ap = clwp->lwp_arg;
1609 	clwp->lwp_procp = cp;
1610 	bzero(clwp->lwp_timer, sizeof (clwp->lwp_timer));
1611 	init_mstate(ct, LMS_STOPPED);
1612 	bzero(&clwp->lwp_ru, sizeof (clwp->lwp_ru));
1613 	clwp->lwp_lastfault = 0;
1614 	clwp->lwp_lastfaddr = 0;
1615 
1616 	/* copy parent's struct regs to child. */
1617 	lwp_forkregs(lwp, clwp);
1618 
1619 	/*
1620 	 * fork device context, if any.
1621 	 *
1622 	 * Someday we could do the work to support the possibility of
1623 	 * forkctx() or lwp_createctx() failing.  Currently, this would
1624 	 * only be needed on x86 for the occasional process using a
1625 	 * private LDT.
1626 	 */
1627 	if (t->t_ctx)
1628 		forkctx(t, ct);
1629 
1630 	/* fix door state in the child */
1631 	if (t->t_door)
1632 		door_fork(t, ct);
1633 
1634 	/* copy current contract templates, clear latest contracts */
1635 	lwp_ctmpl_copy(clwp, lwp);
1636 
1637 	mutex_enter(&cp->p_lock);
1638 	/* lwp_create() set the TP_HOLDLWP flag */
1639 	if (!(t->t_proc_flag & TP_HOLDLWP))
1640 		ct->t_proc_flag &= ~TP_HOLDLWP;
1641 	if (cp->p_flag & SMSACCT)
1642 		ct->t_proc_flag |= TP_MSACCT;
1643 	mutex_exit(&cp->p_lock);
1644 
1645 retry:
1646 	cid = t->t_cid;
1647 
1648 	val = CL_ALLOC(&bufp, cid, KM_SLEEP);
1649 	ASSERT(val == 0);
1650 
1651 	mutex_enter(&p->p_lock);
1652 	if (cid != t->t_cid) {
1653 		/*
1654 		 * Someone just changed this thread's scheduling class,
1655 		 * so try pre-allocating the buffer again.  Hopefully we
1656 		 * don't hit this often.
1657 		 */
1658 		mutex_exit(&p->p_lock);
1659 		CL_FREE(cid, bufp);
1660 		goto retry;
1661 	}
1662 
1663 	ct->t_unpark = t->t_unpark;
1664 	ct->t_clfuncs = t->t_clfuncs;
1665 	CL_FORK(t, ct, bufp);
1666 	ct->t_cid = t->t_cid;	/* after data allocated so prgetpsinfo works */
1667 	mutex_exit(&p->p_lock);
1668 
1669 	return (clwp);
1670 }
1671 
1672 /*
1673  * Add a new lwp entry to the lwp directory and to the lwpid hash table.
1674  */
1675 void
1676 lwp_hash_in(proc_t *p, lwpent_t *lep)
1677 {
1678 	lwpdir_t **ldpp;
1679 	lwpdir_t *ldp;
1680 	kthread_t *t;
1681 
1682 	/*
1683 	 * Allocate a directory element from the free list.
1684 	 * Code elsewhere guarantees a free slot.
1685 	 */
1686 	ldp = p->p_lwpfree;
1687 	p->p_lwpfree = ldp->ld_next;
1688 	ASSERT(ldp->ld_entry == NULL);
1689 	ldp->ld_entry = lep;
1690 
1691 	/*
1692 	 * Insert it into the lwpid hash table.
1693 	 */
1694 	ldpp = &p->p_tidhash[TIDHASH(p, lep->le_lwpid)];
1695 	ldp->ld_next = *ldpp;
1696 	*ldpp = ldp;
1697 
1698 	/*
1699 	 * Set the active thread's directory slot entry.
1700 	 */
1701 	if ((t = lep->le_thread) != NULL) {
1702 		ASSERT(lep->le_lwpid == t->t_tid);
1703 		t->t_dslot = (int)(ldp - p->p_lwpdir);
1704 	}
1705 }
1706 
1707 /*
1708  * Remove an lwp from the lwpid hash table and free its directory entry.
1709  * This is done when a detached lwp exits in lwp_exit() or
1710  * when a non-detached lwp is waited for in lwp_wait() or
1711  * when a zombie lwp is detached in lwp_detach().
1712  */
1713 void
1714 lwp_hash_out(proc_t *p, id_t lwpid)
1715 {
1716 	lwpdir_t **ldpp;
1717 	lwpdir_t *ldp;
1718 	lwpent_t *lep;
1719 
1720 	for (ldpp = &p->p_tidhash[TIDHASH(p, lwpid)];
1721 	    (ldp = *ldpp) != NULL; ldpp = &ldp->ld_next) {
1722 		lep = ldp->ld_entry;
1723 		if (lep->le_lwpid == lwpid) {
1724 			prlwpfree(p, lep);	/* /proc deals with le_trace */
1725 			*ldpp = ldp->ld_next;
1726 			ldp->ld_entry = NULL;
1727 			ldp->ld_next = p->p_lwpfree;
1728 			p->p_lwpfree = ldp;
1729 			kmem_free(lep, sizeof (*lep));
1730 			break;
1731 		}
1732 	}
1733 }
1734 
1735 /*
1736  * Lookup an lwp in the lwpid hash table by lwpid.
1737  */
1738 lwpdir_t *
1739 lwp_hash_lookup(proc_t *p, id_t lwpid)
1740 {
1741 	lwpdir_t *ldp;
1742 
1743 	/*
1744 	 * The process may be exiting, after p_tidhash has been set to NULL in
1745 	 * proc_exit() but before prfee() has been called.  Return failure in
1746 	 * this case.
1747 	 */
1748 	if (p->p_tidhash == NULL)
1749 		return (NULL);
1750 
1751 	for (ldp = p->p_tidhash[TIDHASH(p, lwpid)];
1752 	    ldp != NULL; ldp = ldp->ld_next) {
1753 		if (ldp->ld_entry->le_lwpid == lwpid)
1754 			return (ldp);
1755 	}
1756 
1757 	return (NULL);
1758 }
1759 
1760 /*
1761  * Update the indicated LWP usage statistic for the current LWP.
1762  */
1763 void
1764 lwp_stat_update(lwp_stat_id_t lwp_stat_id, long inc)
1765 {
1766 	klwp_t *lwp = ttolwp(curthread);
1767 
1768 	if (lwp == NULL)
1769 		return;
1770 
1771 	switch (lwp_stat_id) {
1772 	case LWP_STAT_INBLK:
1773 		lwp->lwp_ru.inblock += inc;
1774 		break;
1775 	case LWP_STAT_OUBLK:
1776 		lwp->lwp_ru.oublock += inc;
1777 		break;
1778 	case LWP_STAT_MSGRCV:
1779 		lwp->lwp_ru.msgrcv += inc;
1780 		break;
1781 	case LWP_STAT_MSGSND:
1782 		lwp->lwp_ru.msgsnd += inc;
1783 		break;
1784 	default:
1785 		panic("lwp_stat_update: invalid lwp_stat_id 0x%x", lwp_stat_id);
1786 	}
1787 }
1788