xref: /illumos-gate/usr/src/uts/common/os/lwp.c (revision 088d69f8)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 /*
28  * Copyright (c) 2013, Joyent, Inc. All rights reserved.
29  */
30 
31 #include <sys/param.h>
32 #include <sys/types.h>
33 #include <sys/sysmacros.h>
34 #include <sys/systm.h>
35 #include <sys/thread.h>
36 #include <sys/proc.h>
37 #include <sys/task.h>
38 #include <sys/project.h>
39 #include <sys/signal.h>
40 #include <sys/errno.h>
41 #include <sys/vmparam.h>
42 #include <sys/stack.h>
43 #include <sys/procfs.h>
44 #include <sys/prsystm.h>
45 #include <sys/cpuvar.h>
46 #include <sys/kmem.h>
47 #include <sys/vtrace.h>
48 #include <sys/door.h>
49 #include <vm/seg_kp.h>
50 #include <sys/debug.h>
51 #include <sys/tnf.h>
52 #include <sys/schedctl.h>
53 #include <sys/poll.h>
54 #include <sys/copyops.h>
55 #include <sys/lwp_upimutex_impl.h>
56 #include <sys/cpupart.h>
57 #include <sys/lgrp.h>
58 #include <sys/rctl.h>
59 #include <sys/contract_impl.h>
60 #include <sys/cpc_impl.h>
61 #include <sys/sdt.h>
62 #include <sys/cmn_err.h>
63 #include <sys/brand.h>
64 #include <sys/cyclic.h>
65 #include <sys/pool.h>
66 
67 /* hash function for the lwpid hash table, p->p_tidhash[] */
68 #define	TIDHASH(tid, hash_sz)	((tid) & ((hash_sz) - 1))
69 
70 void *segkp_lwp;		/* cookie for pool of segkp resources */
71 extern void reapq_move_lq_to_tq(kthread_t *);
72 extern void freectx_ctx(struct ctxop *);
73 
74 /*
75  * Create a kernel thread associated with a particular system process.  Give
76  * it an LWP so that microstate accounting will be available for it.
77  */
78 kthread_t *
lwp_kernel_create(proc_t * p,void (* proc)(),void * arg,int state,pri_t pri)79 lwp_kernel_create(proc_t *p, void (*proc)(), void *arg, int state, pri_t pri)
80 {
81 	klwp_t *lwp;
82 
83 	VERIFY((p->p_flag & SSYS) != 0);
84 
85 	lwp = lwp_create(proc, arg, 0, p, state, pri, &t0.t_hold, syscid, 0);
86 
87 	VERIFY(lwp != NULL);
88 
89 	return (lwptot(lwp));
90 }
91 
92 /*
93  * Create a thread that appears to be stopped at sys_rtt.
94  */
95 klwp_t *
lwp_create(void (* proc)(),caddr_t arg,size_t len,proc_t * p,int state,int pri,const k_sigset_t * smask,int cid,id_t lwpid)96 lwp_create(void (*proc)(), caddr_t arg, size_t len, proc_t *p,
97     int state, int pri, const k_sigset_t *smask, int cid, id_t lwpid)
98 {
99 	klwp_t *lwp = NULL;
100 	kthread_t *t;
101 	kthread_t *tx;
102 	cpupart_t *oldpart = NULL;
103 	size_t	stksize;
104 	caddr_t lwpdata = NULL;
105 	processorid_t	binding;
106 	int err = 0;
107 	kproject_t *oldkpj, *newkpj;
108 	void *bufp = NULL;
109 	klwp_t *curlwp;
110 	lwpent_t *lep;
111 	lwpdir_t *old_dir = NULL;
112 	uint_t old_dirsz = 0;
113 	tidhash_t *old_hash = NULL;
114 	uint_t old_hashsz = 0;
115 	ret_tidhash_t *ret_tidhash = NULL;
116 	int i;
117 	int rctlfail = 0;
118 	boolean_t branded = 0;
119 	struct ctxop *ctx = NULL;
120 
121 	ASSERT(cid != sysdccid);	/* system threads must start in SYS */
122 
123 	ASSERT(p != &p0);		/* No new LWPs in p0. */
124 
125 	mutex_enter(&p->p_lock);
126 	mutex_enter(&p->p_zone->zone_nlwps_lock);
127 	/*
128 	 * don't enforce rctl limits on system processes
129 	 */
130 	if (!CLASS_KERNEL(cid)) {
131 		if (p->p_task->tk_nlwps >= p->p_task->tk_nlwps_ctl)
132 			if (rctl_test(rc_task_lwps, p->p_task->tk_rctls, p,
133 			    1, 0) & RCT_DENY)
134 				rctlfail = 1;
135 		if (p->p_task->tk_proj->kpj_nlwps >=
136 		    p->p_task->tk_proj->kpj_nlwps_ctl)
137 			if (rctl_test(rc_project_nlwps,
138 			    p->p_task->tk_proj->kpj_rctls, p, 1, 0)
139 			    & RCT_DENY)
140 				rctlfail = 1;
141 		if (p->p_zone->zone_nlwps >= p->p_zone->zone_nlwps_ctl)
142 			if (rctl_test(rc_zone_nlwps, p->p_zone->zone_rctls, p,
143 			    1, 0) & RCT_DENY)
144 				rctlfail = 1;
145 	}
146 	if (rctlfail) {
147 		mutex_exit(&p->p_zone->zone_nlwps_lock);
148 		mutex_exit(&p->p_lock);
149 		atomic_inc_32(&p->p_zone->zone_ffcap);
150 		return (NULL);
151 	}
152 	p->p_task->tk_nlwps++;
153 	p->p_task->tk_proj->kpj_nlwps++;
154 	p->p_zone->zone_nlwps++;
155 	mutex_exit(&p->p_zone->zone_nlwps_lock);
156 	mutex_exit(&p->p_lock);
157 
158 	curlwp = ttolwp(curthread);
159 	if (curlwp == NULL || (stksize = curlwp->lwp_childstksz) == 0)
160 		stksize = lwp_default_stksize;
161 
162 	if (CLASS_KERNEL(cid)) {
163 		/*
164 		 * Since we are creating an LWP in an SSYS process, we do not
165 		 * inherit anything from the current thread's LWP.  We set
166 		 * stksize and lwpdata to 0 in order to let thread_create()
167 		 * allocate a regular kernel thread stack for this thread.
168 		 */
169 		curlwp = NULL;
170 		stksize = 0;
171 		lwpdata = NULL;
172 
173 	} else if (stksize == lwp_default_stksize) {
174 		/*
175 		 * Try to reuse an <lwp,stack> from the LWP deathrow.
176 		 */
177 		if (lwp_reapcnt > 0) {
178 			mutex_enter(&reaplock);
179 			if ((t = lwp_deathrow) != NULL) {
180 				ASSERT(t->t_swap);
181 				lwp_deathrow = t->t_forw;
182 				lwp_reapcnt--;
183 				lwpdata = t->t_swap;
184 				lwp = t->t_lwp;
185 				ctx = t->t_ctx;
186 				t->t_swap = NULL;
187 				t->t_lwp = NULL;
188 				t->t_ctx = NULL;
189 				reapq_move_lq_to_tq(t);
190 			}
191 			mutex_exit(&reaplock);
192 			if (lwp != NULL) {
193 				lwp_stk_fini(lwp);
194 			}
195 			if (ctx != NULL) {
196 				freectx_ctx(ctx);
197 			}
198 		}
199 		if (lwpdata == NULL &&
200 		    (lwpdata = (caddr_t)segkp_cache_get(segkp_lwp)) == NULL) {
201 			mutex_enter(&p->p_lock);
202 			mutex_enter(&p->p_zone->zone_nlwps_lock);
203 			p->p_task->tk_nlwps--;
204 			p->p_task->tk_proj->kpj_nlwps--;
205 			p->p_zone->zone_nlwps--;
206 			mutex_exit(&p->p_zone->zone_nlwps_lock);
207 			mutex_exit(&p->p_lock);
208 			atomic_inc_32(&p->p_zone->zone_ffnomem);
209 			return (NULL);
210 		}
211 	} else {
212 		stksize = roundup(stksize, PAGESIZE);
213 		if ((lwpdata = (caddr_t)segkp_get(segkp, stksize,
214 		    (KPD_NOWAIT | KPD_HASREDZONE | KPD_LOCKED))) == NULL) {
215 			mutex_enter(&p->p_lock);
216 			mutex_enter(&p->p_zone->zone_nlwps_lock);
217 			p->p_task->tk_nlwps--;
218 			p->p_task->tk_proj->kpj_nlwps--;
219 			p->p_zone->zone_nlwps--;
220 			mutex_exit(&p->p_zone->zone_nlwps_lock);
221 			mutex_exit(&p->p_lock);
222 			atomic_inc_32(&p->p_zone->zone_ffnomem);
223 			return (NULL);
224 		}
225 	}
226 
227 	/*
228 	 * Create a thread, initializing the stack pointer
229 	 */
230 	t = thread_create(lwpdata, stksize, NULL, NULL, 0, p, TS_STOPPED, pri);
231 
232 	/*
233 	 * If a non-NULL stack base is passed in, thread_create() assumes
234 	 * that the stack might be statically allocated (as opposed to being
235 	 * allocated from segkp), and so it does not set t_swap.  Since
236 	 * the lwpdata was allocated from segkp, we must set t_swap to point
237 	 * to it ourselves.
238 	 *
239 	 * This would be less confusing if t_swap had a better name; it really
240 	 * indicates that the stack is allocated from segkp, regardless of
241 	 * whether or not it is swappable.
242 	 */
243 	if (lwpdata != NULL) {
244 		ASSERT(!CLASS_KERNEL(cid));
245 		ASSERT(t->t_swap == NULL);
246 		t->t_swap = lwpdata;	/* Start of page-able data */
247 	}
248 
249 	/*
250 	 * If the stack and lwp can be reused, mark the thread as such.
251 	 * When we get to reapq_add() from resume_from_zombie(), these
252 	 * threads will go onto lwp_deathrow instead of thread_deathrow.
253 	 */
254 	if (!CLASS_KERNEL(cid) && stksize == lwp_default_stksize)
255 		t->t_flag |= T_LWPREUSE;
256 
257 	if (lwp == NULL)
258 		lwp = kmem_cache_alloc(lwp_cache, KM_SLEEP);
259 	bzero(lwp, sizeof (*lwp));
260 	t->t_lwp = lwp;
261 
262 	t->t_hold = *smask;
263 	lwp->lwp_thread = t;
264 	lwp->lwp_procp = p;
265 	lwp->lwp_sigaltstack.ss_flags = SS_DISABLE;
266 	if (curlwp != NULL && curlwp->lwp_childstksz != 0)
267 		lwp->lwp_childstksz = curlwp->lwp_childstksz;
268 
269 	t->t_stk = lwp_stk_init(lwp, t->t_stk);
270 	thread_load(t, proc, arg, len);
271 
272 	/*
273 	 * Allocate the SIGPROF buffer if ITIMER_REALPROF is in effect.
274 	 */
275 	if (p->p_rprof_cyclic != CYCLIC_NONE)
276 		t->t_rprof = kmem_zalloc(sizeof (struct rprof), KM_SLEEP);
277 
278 	if (cid != NOCLASS)
279 		(void) CL_ALLOC(&bufp, cid, KM_SLEEP);
280 
281 	/*
282 	 * Allocate an lwp directory entry for the new lwp.
283 	 */
284 	lep = kmem_zalloc(sizeof (*lep), KM_SLEEP);
285 
286 	mutex_enter(&p->p_lock);
287 grow:
288 	/*
289 	 * Grow the lwp (thread) directory and lwpid hash table if necessary.
290 	 * A note on the growth algorithm:
291 	 *	The new lwp directory size is computed as:
292 	 *		new = 2 * old + 2
293 	 *	Starting with an initial size of 2 (see exec_common()),
294 	 *	this yields numbers that are a power of two minus 2:
295 	 *		2, 6, 14, 30, 62, 126, 254, 510, 1022, ...
296 	 *	The size of the lwpid hash table must be a power of two
297 	 *	and must be commensurate in size with the lwp directory
298 	 *	so that hash bucket chains remain short.  Therefore,
299 	 *	the lwpid hash table size is computed as:
300 	 *		hashsz = (dirsz + 2) / 2
301 	 *	which leads to these hash table sizes corresponding to
302 	 *	the above directory sizes:
303 	 *		2, 4, 8, 16, 32, 64, 128, 256, 512, ...
304 	 * A note on growing the hash table:
305 	 *	For performance reasons, code in lwp_unpark() does not
306 	 *	acquire curproc->p_lock when searching the hash table.
307 	 *	Rather, it calls lwp_hash_lookup_and_lock() which
308 	 *	acquires only the individual hash bucket lock, taking
309 	 *	care to deal with reallocation of the hash table
310 	 *	during the time it takes to acquire the lock.
311 	 *
312 	 *	This is sufficient to protect the integrity of the
313 	 *	hash table, but it requires us to acquire all of the
314 	 *	old hash bucket locks before growing the hash table
315 	 *	and to release them afterwards.  It also requires us
316 	 *	not to free the old hash table because some thread
317 	 *	in lwp_hash_lookup_and_lock() might still be trying
318 	 *	to acquire the old bucket lock.
319 	 *
320 	 *	So we adopt the tactic of keeping all of the retired
321 	 *	hash tables on a linked list, so they can be safely
322 	 *	freed when the process exits or execs.
323 	 *
324 	 *	Because the hash table grows in powers of two, the
325 	 *	total size of all of the hash tables will be slightly
326 	 *	less than twice the size of the largest hash table.
327 	 */
328 	while (p->p_lwpfree == NULL) {
329 		uint_t dirsz = p->p_lwpdir_sz;
330 		lwpdir_t *new_dir;
331 		uint_t new_dirsz;
332 		lwpdir_t *ldp;
333 		tidhash_t *new_hash;
334 		uint_t new_hashsz;
335 
336 		mutex_exit(&p->p_lock);
337 
338 		/*
339 		 * Prepare to remember the old p_tidhash for later
340 		 * kmem_free()ing when the process exits or execs.
341 		 */
342 		if (ret_tidhash == NULL)
343 			ret_tidhash = kmem_zalloc(sizeof (ret_tidhash_t),
344 			    KM_SLEEP);
345 		if (old_dir != NULL)
346 			kmem_free(old_dir, old_dirsz * sizeof (*old_dir));
347 		if (old_hash != NULL)
348 			kmem_free(old_hash, old_hashsz * sizeof (*old_hash));
349 
350 		new_dirsz = 2 * dirsz + 2;
351 		new_dir = kmem_zalloc(new_dirsz * sizeof (lwpdir_t), KM_SLEEP);
352 		for (ldp = new_dir, i = 1; i < new_dirsz; i++, ldp++)
353 			ldp->ld_next = ldp + 1;
354 		new_hashsz = (new_dirsz + 2) / 2;
355 		new_hash = kmem_zalloc(new_hashsz * sizeof (tidhash_t),
356 		    KM_SLEEP);
357 
358 		mutex_enter(&p->p_lock);
359 		if (p == curproc)
360 			prbarrier(p);
361 
362 		if (dirsz != p->p_lwpdir_sz || p->p_lwpfree != NULL) {
363 			/*
364 			 * Someone else beat us to it or some lwp exited.
365 			 * Set up to free our memory and take a lap.
366 			 */
367 			old_dir = new_dir;
368 			old_dirsz = new_dirsz;
369 			old_hash = new_hash;
370 			old_hashsz = new_hashsz;
371 		} else {
372 			/*
373 			 * For the benefit of lwp_hash_lookup_and_lock(),
374 			 * called from lwp_unpark(), which searches the
375 			 * tid hash table without acquiring p->p_lock,
376 			 * we must acquire all of the tid hash table
377 			 * locks before replacing p->p_tidhash.
378 			 */
379 			old_hash = p->p_tidhash;
380 			old_hashsz = p->p_tidhash_sz;
381 			for (i = 0; i < old_hashsz; i++) {
382 				mutex_enter(&old_hash[i].th_lock);
383 				mutex_enter(&new_hash[i].th_lock);
384 			}
385 
386 			/*
387 			 * We simply hash in all of the old directory entries.
388 			 * This works because the old directory has no empty
389 			 * slots and the new hash table starts out empty.
390 			 * This reproduces the original directory ordering
391 			 * (required for /proc directory semantics).
392 			 */
393 			old_dir = p->p_lwpdir;
394 			old_dirsz = p->p_lwpdir_sz;
395 			p->p_lwpdir = new_dir;
396 			p->p_lwpfree = new_dir;
397 			p->p_lwpdir_sz = new_dirsz;
398 			for (ldp = old_dir, i = 0; i < old_dirsz; i++, ldp++)
399 				lwp_hash_in(p, ldp->ld_entry,
400 				    new_hash, new_hashsz, 0);
401 
402 			/*
403 			 * Remember the old hash table along with all
404 			 * of the previously-remembered hash tables.
405 			 * We will free them at process exit or exec.
406 			 */
407 			ret_tidhash->rth_tidhash = old_hash;
408 			ret_tidhash->rth_tidhash_sz = old_hashsz;
409 			ret_tidhash->rth_next = p->p_ret_tidhash;
410 			p->p_ret_tidhash = ret_tidhash;
411 
412 			/*
413 			 * Now establish the new tid hash table.
414 			 * As soon as we assign p->p_tidhash,
415 			 * code in lwp_unpark() can start using it.
416 			 */
417 			membar_producer();
418 			p->p_tidhash = new_hash;
419 
420 			/*
421 			 * It is necessary that p_tidhash reach global
422 			 * visibility before p_tidhash_sz.  Otherwise,
423 			 * code in lwp_hash_lookup_and_lock() could
424 			 * index into the old p_tidhash using the new
425 			 * p_tidhash_sz and thereby access invalid data.
426 			 */
427 			membar_producer();
428 			p->p_tidhash_sz = new_hashsz;
429 
430 			/*
431 			 * Release the locks; allow lwp_unpark() to carry on.
432 			 */
433 			for (i = 0; i < old_hashsz; i++) {
434 				mutex_exit(&old_hash[i].th_lock);
435 				mutex_exit(&new_hash[i].th_lock);
436 			}
437 
438 			/*
439 			 * Avoid freeing these objects below.
440 			 */
441 			ret_tidhash = NULL;
442 			old_hash = NULL;
443 			old_hashsz = 0;
444 		}
445 	}
446 
447 	/*
448 	 * Block the process against /proc while we manipulate p->p_tlist,
449 	 * unless lwp_create() was called by /proc for the PCAGENT operation.
450 	 * We want to do this early enough so that we don't drop p->p_lock
451 	 * until the thread is put on the p->p_tlist.
452 	 */
453 	if (p == curproc) {
454 		prbarrier(p);
455 		/*
456 		 * If the current lwp has been requested to stop, do so now.
457 		 * Otherwise we have a race condition between /proc attempting
458 		 * to stop the process and this thread creating a new lwp
459 		 * that was not seen when the /proc PCSTOP request was issued.
460 		 * We rely on stop() to call prbarrier(p) before returning.
461 		 */
462 		while ((curthread->t_proc_flag & TP_PRSTOP) &&
463 		    !ttolwp(curthread)->lwp_nostop) {
464 			/*
465 			 * We called pool_barrier_enter() before calling
466 			 * here to lwp_create(). We have to call
467 			 * pool_barrier_exit() before stopping.
468 			 */
469 			pool_barrier_exit();
470 			prbarrier(p);
471 			stop(PR_REQUESTED, 0);
472 			/*
473 			 * And we have to repeat the call to
474 			 * pool_barrier_enter after stopping.
475 			 */
476 			pool_barrier_enter();
477 			prbarrier(p);
478 		}
479 
480 		/*
481 		 * If process is exiting, there could be a race between
482 		 * the agent lwp creation and the new lwp currently being
483 		 * created. So to prevent this race lwp creation is failed
484 		 * if the process is exiting.
485 		 */
486 		if (p->p_flag & (SEXITLWPS|SKILLED)) {
487 			err = 1;
488 			goto error;
489 		}
490 
491 		/*
492 		 * Since we might have dropped p->p_lock, the
493 		 * lwp directory free list might have changed.
494 		 */
495 		if (p->p_lwpfree == NULL)
496 			goto grow;
497 	}
498 
499 	kpreempt_disable();	/* can't grab cpu_lock here */
500 
501 	/*
502 	 * Inherit processor and processor set bindings from curthread.
503 	 *
504 	 * For kernel LWPs, we do not inherit processor set bindings at
505 	 * process creation time (i.e. when p != curproc).  After the
506 	 * kernel process is created, any subsequent LWPs must be created
507 	 * by threads in the kernel process, at which point we *will*
508 	 * inherit processor set bindings.
509 	 */
510 	if (CLASS_KERNEL(cid) && p != curproc) {
511 		t->t_bind_cpu = binding = PBIND_NONE;
512 		t->t_cpupart = oldpart = &cp_default;
513 		t->t_bind_pset = PS_NONE;
514 		t->t_bindflag = (uchar_t)default_binding_mode;
515 	} else {
516 		binding = curthread->t_bind_cpu;
517 		t->t_bind_cpu = binding;
518 		oldpart = t->t_cpupart;
519 		t->t_cpupart = curthread->t_cpupart;
520 		t->t_bind_pset = curthread->t_bind_pset;
521 		t->t_bindflag = curthread->t_bindflag |
522 		    (uchar_t)default_binding_mode;
523 	}
524 
525 	/*
526 	 * thread_create() initializes this thread's home lgroup to the root.
527 	 * Choose a more suitable lgroup, since this thread is associated
528 	 * with an lwp.
529 	 */
530 	ASSERT(oldpart != NULL);
531 	if (binding != PBIND_NONE && t->t_affinitycnt == 0) {
532 		t->t_bound_cpu = cpu[binding];
533 		if (t->t_lpl != t->t_bound_cpu->cpu_lpl)
534 			lgrp_move_thread(t, t->t_bound_cpu->cpu_lpl, 1);
535 	} else if (CLASS_KERNEL(cid)) {
536 		/*
537 		 * Kernel threads are always in the root lgrp.
538 		 */
539 		lgrp_move_thread(t,
540 		    &t->t_cpupart->cp_lgrploads[LGRP_ROOTID], 1);
541 	} else {
542 		lgrp_move_thread(t, lgrp_choose(t, t->t_cpupart), 1);
543 	}
544 
545 	kpreempt_enable();
546 
547 	/*
548 	 * make sure lpl points to our own partition
549 	 */
550 	ASSERT(t->t_lpl >= t->t_cpupart->cp_lgrploads);
551 	ASSERT(t->t_lpl < t->t_cpupart->cp_lgrploads +
552 	    t->t_cpupart->cp_nlgrploads);
553 
554 	/*
555 	 * It is safe to point the thread to the new project without holding it
556 	 * since we're holding the target process' p_lock here and therefore
557 	 * we're guaranteed that it will not move to another project.
558 	 */
559 	newkpj = p->p_task->tk_proj;
560 	oldkpj = ttoproj(t);
561 	if (newkpj != oldkpj) {
562 		t->t_proj = newkpj;
563 		(void) project_hold(newkpj);
564 		project_rele(oldkpj);
565 	}
566 
567 	if (cid != NOCLASS) {
568 		/*
569 		 * If the lwp is being created in the current process
570 		 * and matches the current thread's scheduling class,
571 		 * we should propagate the current thread's scheduling
572 		 * parameters by calling CL_FORK.  Otherwise just use
573 		 * the defaults by calling CL_ENTERCLASS.
574 		 */
575 		if (p != curproc || curthread->t_cid != cid) {
576 			err = CL_ENTERCLASS(t, cid, NULL, NULL, bufp);
577 			t->t_pri = pri;	/* CL_ENTERCLASS may have changed it */
578 			/*
579 			 * We don't call schedctl_set_cidpri(t) here
580 			 * because the schedctl data is not yet set
581 			 * up for the newly-created lwp.
582 			 */
583 		} else {
584 			t->t_clfuncs = &(sclass[cid].cl_funcs->thread);
585 			err = CL_FORK(curthread, t, bufp);
586 			t->t_cid = cid;
587 		}
588 		if (err) {
589 			atomic_inc_32(&p->p_zone->zone_ffmisc);
590 			goto error;
591 		} else {
592 			bufp = NULL;
593 		}
594 	}
595 
596 	/*
597 	 * If we were given an lwpid then use it, else allocate one.
598 	 */
599 	if (lwpid != 0)
600 		t->t_tid = lwpid;
601 	else {
602 		/*
603 		 * lwp/thread id 0 is never valid; reserved for special checks.
604 		 * lwp/thread id 1 is reserved for the main thread.
605 		 * Start again at 2 when INT_MAX has been reached
606 		 * (id_t is a signed 32-bit integer).
607 		 */
608 		id_t prev_id = p->p_lwpid;	/* last allocated tid */
609 
610 		do {			/* avoid lwpid duplication */
611 			if (p->p_lwpid == INT_MAX) {
612 				p->p_flag |= SLWPWRAP;
613 				p->p_lwpid = 1;
614 			}
615 			if ((t->t_tid = ++p->p_lwpid) == prev_id) {
616 				/*
617 				 * All lwpids are allocated; fail the request.
618 				 */
619 				err = 1;
620 				atomic_inc_32(&p->p_zone->zone_ffnoproc);
621 				goto error;
622 			}
623 			/*
624 			 * We only need to worry about colliding with an id
625 			 * that's already in use if this process has
626 			 * cycled through all available lwp ids.
627 			 */
628 			if ((p->p_flag & SLWPWRAP) == 0)
629 				break;
630 		} while (lwp_hash_lookup(p, t->t_tid) != NULL);
631 	}
632 
633 	/*
634 	 * If this is a branded process, let the brand do any necessary lwp
635 	 * initialization.
636 	 */
637 	if (PROC_IS_BRANDED(p)) {
638 		if (BROP(p)->b_initlwp(lwp)) {
639 			err = 1;
640 			atomic_inc_32(&p->p_zone->zone_ffmisc);
641 			goto error;
642 		}
643 		branded = 1;
644 	}
645 
646 	if (t->t_tid == 1) {
647 		kpreempt_disable();
648 		ASSERT(t->t_lpl != NULL);
649 		p->p_t1_lgrpid = t->t_lpl->lpl_lgrpid;
650 		kpreempt_enable();
651 		if (p->p_tr_lgrpid != LGRP_NONE &&
652 		    p->p_tr_lgrpid != p->p_t1_lgrpid) {
653 			lgrp_update_trthr_migrations(1);
654 		}
655 	}
656 
657 	p->p_lwpcnt++;
658 	t->t_waitfor = -1;
659 
660 	/*
661 	 * Turn microstate accounting on for thread if on for process.
662 	 */
663 	if (p->p_flag & SMSACCT)
664 		t->t_proc_flag |= TP_MSACCT;
665 
666 	/*
667 	 * If the process has watchpoints, mark the new thread as such.
668 	 */
669 	if (pr_watch_active(p))
670 		watch_enable(t);
671 
672 	/*
673 	 * The lwp is being created in the stopped state.
674 	 * We set all the necessary flags to indicate that fact here.
675 	 * We omit the TS_CREATE flag from t_schedflag so that the lwp
676 	 * cannot be set running until the caller is finished with it,
677 	 * even if lwp_continue() is called on it after we drop p->p_lock.
678 	 * When the caller is finished with the newly-created lwp,
679 	 * the caller must call lwp_create_done() to allow the lwp
680 	 * to be set running.  If the TP_HOLDLWP is left set, the
681 	 * lwp will suspend itself after reaching system call exit.
682 	 */
683 	init_mstate(t, LMS_STOPPED);
684 	t->t_proc_flag |= TP_HOLDLWP;
685 	t->t_schedflag |= (TS_ALLSTART & ~(TS_CSTART | TS_CREATE));
686 	t->t_whystop = PR_SUSPENDED;
687 	t->t_whatstop = SUSPEND_NORMAL;
688 	t->t_sig_check = 1;	/* ensure that TP_HOLDLWP is honored */
689 
690 	/*
691 	 * Set system call processing flags in case tracing or profiling
692 	 * is set.  The first system call will evaluate these and turn
693 	 * them off if they aren't needed.
694 	 */
695 	t->t_pre_sys = 1;
696 	t->t_post_sys = 1;
697 
698 	/*
699 	 * Insert the new thread into the list of all threads.
700 	 */
701 	if ((tx = p->p_tlist) == NULL) {
702 		t->t_back = t;
703 		t->t_forw = t;
704 		p->p_tlist = t;
705 	} else {
706 		t->t_forw = tx;
707 		t->t_back = tx->t_back;
708 		tx->t_back->t_forw = t;
709 		tx->t_back = t;
710 	}
711 
712 	/*
713 	 * Insert the new lwp into an lwp directory slot position
714 	 * and into the lwpid hash table.
715 	 */
716 	lep->le_thread = t;
717 	lep->le_lwpid = t->t_tid;
718 	lep->le_start = t->t_start;
719 	lwp_hash_in(p, lep, p->p_tidhash, p->p_tidhash_sz, 1);
720 
721 	lwp_fp_init(lwp);
722 
723 	if (state == TS_RUN) {
724 		/*
725 		 * We set the new lwp running immediately.
726 		 */
727 		t->t_proc_flag &= ~TP_HOLDLWP;
728 		lwp_create_done(t);
729 	}
730 
731 error:
732 	if (err) {
733 		if (CLASS_KERNEL(cid)) {
734 			/*
735 			 * This should only happen if a system process runs
736 			 * out of lwpids, which shouldn't occur.
737 			 */
738 			panic("Failed to create a system LWP");
739 		}
740 		/*
741 		 * We have failed to create an lwp, so decrement the number
742 		 * of lwps in the task and let the lgroup load averages know
743 		 * that this thread isn't going to show up.
744 		 */
745 		kpreempt_disable();
746 		lgrp_move_thread(t, NULL, 1);
747 		kpreempt_enable();
748 
749 		ASSERT(MUTEX_HELD(&p->p_lock));
750 		mutex_enter(&p->p_zone->zone_nlwps_lock);
751 		p->p_task->tk_nlwps--;
752 		p->p_task->tk_proj->kpj_nlwps--;
753 		p->p_zone->zone_nlwps--;
754 		mutex_exit(&p->p_zone->zone_nlwps_lock);
755 		if (cid != NOCLASS && bufp != NULL)
756 			CL_FREE(cid, bufp);
757 
758 		if (branded)
759 			BROP(p)->b_freelwp(lwp);
760 
761 		mutex_exit(&p->p_lock);
762 		t->t_state = TS_FREE;
763 		thread_rele(t);
764 
765 		/*
766 		 * We need to remove t from the list of all threads
767 		 * because thread_exit()/lwp_exit() isn't called on t.
768 		 */
769 		mutex_enter(&pidlock);
770 		ASSERT(t != t->t_next);		/* t0 never exits */
771 		t->t_next->t_prev = t->t_prev;
772 		t->t_prev->t_next = t->t_next;
773 		mutex_exit(&pidlock);
774 
775 		thread_free(t);
776 		kmem_free(lep, sizeof (*lep));
777 		lwp = NULL;
778 	} else {
779 		mutex_exit(&p->p_lock);
780 	}
781 
782 	if (old_dir != NULL)
783 		kmem_free(old_dir, old_dirsz * sizeof (*old_dir));
784 	if (old_hash != NULL)
785 		kmem_free(old_hash, old_hashsz * sizeof (*old_hash));
786 	if (ret_tidhash != NULL)
787 		kmem_free(ret_tidhash, sizeof (ret_tidhash_t));
788 
789 	DTRACE_PROC1(lwp__create, kthread_t *, t);
790 	return (lwp);
791 }
792 
793 /*
794  * lwp_create_done() is called by the caller of lwp_create() to set the
795  * newly-created lwp running after the caller has finished manipulating it.
796  */
797 void
lwp_create_done(kthread_t * t)798 lwp_create_done(kthread_t *t)
799 {
800 	proc_t *p = ttoproc(t);
801 
802 	ASSERT(MUTEX_HELD(&p->p_lock));
803 
804 	/*
805 	 * We set the TS_CREATE and TS_CSTART flags and call setrun_locked().
806 	 * (The absence of the TS_CREATE flag prevents the lwp from running
807 	 * until we are finished with it, even if lwp_continue() is called on
808 	 * it by some other lwp in the process or elsewhere in the kernel.)
809 	 */
810 	thread_lock(t);
811 	ASSERT(t->t_state == TS_STOPPED && !(t->t_schedflag & TS_CREATE));
812 	/*
813 	 * If TS_CSTART is set, lwp_continue(t) has been called and
814 	 * has already incremented p_lwprcnt; avoid doing this twice.
815 	 */
816 	if (!(t->t_schedflag & TS_CSTART))
817 		p->p_lwprcnt++;
818 	t->t_schedflag |= (TS_CSTART | TS_CREATE);
819 	setrun_locked(t);
820 	thread_unlock(t);
821 }
822 
823 /*
824  * Copy an LWP's active templates, and clear the latest contracts.
825  */
826 void
lwp_ctmpl_copy(klwp_t * dst,klwp_t * src)827 lwp_ctmpl_copy(klwp_t *dst, klwp_t *src)
828 {
829 	int i;
830 
831 	for (i = 0; i < ct_ntypes; i++) {
832 		dst->lwp_ct_active[i] = ctmpl_dup(src->lwp_ct_active[i]);
833 		dst->lwp_ct_latest[i] = NULL;
834 	}
835 }
836 
837 /*
838  * Clear an LWP's contract template state.
839  */
840 void
lwp_ctmpl_clear(klwp_t * lwp)841 lwp_ctmpl_clear(klwp_t *lwp)
842 {
843 	ct_template_t *tmpl;
844 	int i;
845 
846 	for (i = 0; i < ct_ntypes; i++) {
847 		if ((tmpl = lwp->lwp_ct_active[i]) != NULL) {
848 			ctmpl_free(tmpl);
849 			lwp->lwp_ct_active[i] = NULL;
850 		}
851 
852 		if (lwp->lwp_ct_latest[i] != NULL) {
853 			contract_rele(lwp->lwp_ct_latest[i]);
854 			lwp->lwp_ct_latest[i] = NULL;
855 		}
856 	}
857 }
858 
859 /*
860  * Individual lwp exit.
861  * If this is the last lwp, exit the whole process.
862  */
863 void
lwp_exit(void)864 lwp_exit(void)
865 {
866 	kthread_t *t = curthread;
867 	klwp_t *lwp = ttolwp(t);
868 	proc_t *p = ttoproc(t);
869 
870 	ASSERT(MUTEX_HELD(&p->p_lock));
871 
872 	mutex_exit(&p->p_lock);
873 
874 #if defined(__sparc)
875 	/*
876 	 * Ensure that the user stack is fully abandoned..
877 	 */
878 	trash_user_windows();
879 #endif
880 
881 	tsd_exit();			/* free thread specific data */
882 
883 	kcpc_passivate();		/* Clean up performance counter state */
884 
885 	pollcleanup();
886 
887 	if (t->t_door)
888 		door_slam();
889 
890 	if (t->t_schedctl != NULL)
891 		schedctl_lwp_cleanup(t);
892 
893 	if (t->t_upimutex != NULL)
894 		upimutex_cleanup();
895 
896 	/*
897 	 * Perform any brand specific exit processing, then release any
898 	 * brand data associated with the lwp
899 	 */
900 	if (PROC_IS_BRANDED(p))
901 		BROP(p)->b_lwpexit(lwp);
902 
903 	lwp_pcb_exit();
904 
905 	mutex_enter(&p->p_lock);
906 	lwp_cleanup();
907 
908 	/*
909 	 * When this process is dumping core, its lwps are held here
910 	 * until the core dump is finished. Then exitlwps() is called
911 	 * again to release these lwps so that they can finish exiting.
912 	 */
913 	if (p->p_flag & SCOREDUMP)
914 		stop(PR_SUSPENDED, SUSPEND_NORMAL);
915 
916 	/*
917 	 * Block the process against /proc now that we have really acquired
918 	 * p->p_lock (to decrement p_lwpcnt and manipulate p_tlist at least).
919 	 */
920 	prbarrier(p);
921 
922 	/*
923 	 * Call proc_exit() if this is the last non-daemon lwp in the process.
924 	 */
925 	if (!(t->t_proc_flag & TP_DAEMON) &&
926 	    p->p_lwpcnt == p->p_lwpdaemon + 1) {
927 		mutex_exit(&p->p_lock);
928 		if (proc_exit(CLD_EXITED, 0) == 0) {
929 			/* Restarting init. */
930 			return;
931 		}
932 
933 		/*
934 		 * proc_exit() returns a non-zero value when some other
935 		 * lwp got there first.  We just have to continue in
936 		 * lwp_exit().
937 		 */
938 		mutex_enter(&p->p_lock);
939 		ASSERT(curproc->p_flag & SEXITLWPS);
940 		prbarrier(p);
941 	}
942 
943 	DTRACE_PROC(lwp__exit);
944 
945 	/*
946 	 * If the lwp is a detached lwp or if the process is exiting,
947 	 * remove (lwp_hash_out()) the lwp from the lwp directory.
948 	 * Otherwise null out the lwp's le_thread pointer in the lwp
949 	 * directory so that other threads will see it as a zombie lwp.
950 	 */
951 	prlwpexit(t);		/* notify /proc */
952 	if (!(t->t_proc_flag & TP_TWAIT) || (p->p_flag & SEXITLWPS))
953 		lwp_hash_out(p, t->t_tid);
954 	else {
955 		ASSERT(!(t->t_proc_flag & TP_DAEMON));
956 		p->p_lwpdir[t->t_dslot].ld_entry->le_thread = NULL;
957 		p->p_zombcnt++;
958 		cv_broadcast(&p->p_lwpexit);
959 	}
960 	if (t->t_proc_flag & TP_DAEMON) {
961 		p->p_lwpdaemon--;
962 		t->t_proc_flag &= ~TP_DAEMON;
963 	}
964 	t->t_proc_flag &= ~TP_TWAIT;
965 
966 	/*
967 	 * Maintain accurate lwp count for task.max-lwps resource control.
968 	 */
969 	mutex_enter(&p->p_zone->zone_nlwps_lock);
970 	p->p_task->tk_nlwps--;
971 	p->p_task->tk_proj->kpj_nlwps--;
972 	p->p_zone->zone_nlwps--;
973 	mutex_exit(&p->p_zone->zone_nlwps_lock);
974 
975 	CL_EXIT(t);		/* tell the scheduler that t is exiting */
976 	ASSERT(p->p_lwpcnt != 0);
977 	p->p_lwpcnt--;
978 
979 	/*
980 	 * If all remaining non-daemon lwps are waiting in lwp_wait(),
981 	 * wake them up so someone can return EDEADLK.
982 	 * (See the block comment preceeding lwp_wait().)
983 	 */
984 	if (p->p_lwpcnt == p->p_lwpdaemon + (p->p_lwpwait - p->p_lwpdwait))
985 		cv_broadcast(&p->p_lwpexit);
986 
987 	t->t_proc_flag |= TP_LWPEXIT;
988 	term_mstate(t);
989 
990 #ifndef NPROBE
991 	/* Kernel probe */
992 	if (t->t_tnf_tpdp)
993 		tnf_thread_exit();
994 #endif /* NPROBE */
995 
996 	t->t_forw->t_back = t->t_back;
997 	t->t_back->t_forw = t->t_forw;
998 	if (t == p->p_tlist)
999 		p->p_tlist = t->t_forw;
1000 
1001 	/*
1002 	 * Clean up the signal state.
1003 	 */
1004 	if (t->t_sigqueue != NULL)
1005 		sigdelq(p, t, 0);
1006 	if (lwp->lwp_curinfo != NULL) {
1007 		siginfofree(lwp->lwp_curinfo);
1008 		lwp->lwp_curinfo = NULL;
1009 	}
1010 
1011 	/*
1012 	 * If we have spymaster information (that is, if we're an agent LWP),
1013 	 * free that now.
1014 	 */
1015 	if (lwp->lwp_spymaster != NULL) {
1016 		kmem_free(lwp->lwp_spymaster, sizeof (psinfo_t));
1017 		lwp->lwp_spymaster = NULL;
1018 	}
1019 
1020 	thread_rele(t);
1021 
1022 	/*
1023 	 * Terminated lwps are associated with process zero and are put onto
1024 	 * death-row by resume().  Avoid preemption after resetting t->t_procp.
1025 	 */
1026 	t->t_preempt++;
1027 
1028 	if (t->t_ctx != NULL)
1029 		exitctx(t);
1030 	if (p->p_pctx != NULL)
1031 		exitpctx(p);
1032 
1033 	t->t_procp = &p0;
1034 
1035 	/*
1036 	 * Notify the HAT about the change of address space
1037 	 */
1038 	hat_thread_exit(t);
1039 	/*
1040 	 * When this is the last running lwp in this process and some lwp is
1041 	 * waiting for this condition to become true, or this thread was being
1042 	 * suspended, then the waiting lwp is awakened.
1043 	 *
1044 	 * Also, if the process is exiting, we may have a thread waiting in
1045 	 * exitlwps() that needs to be notified.
1046 	 */
1047 	if (--p->p_lwprcnt == 0 || (t->t_proc_flag & TP_HOLDLWP) ||
1048 	    (p->p_flag & SEXITLWPS))
1049 		cv_broadcast(&p->p_holdlwps);
1050 
1051 	/*
1052 	 * Need to drop p_lock so we can reacquire pidlock.
1053 	 */
1054 	mutex_exit(&p->p_lock);
1055 	mutex_enter(&pidlock);
1056 
1057 	ASSERT(t != t->t_next);		/* t0 never exits */
1058 	t->t_next->t_prev = t->t_prev;
1059 	t->t_prev->t_next = t->t_next;
1060 	cv_broadcast(&t->t_joincv);	/* wake up anyone in thread_join */
1061 	mutex_exit(&pidlock);
1062 
1063 	t->t_state = TS_ZOMB;
1064 	swtch_from_zombie();
1065 	/* never returns */
1066 }
1067 
1068 
1069 /*
1070  * Cleanup function for an exiting lwp.
1071  * Called both from lwp_exit() and from proc_exit().
1072  * p->p_lock is repeatedly released and grabbed in this function.
1073  */
1074 void
lwp_cleanup(void)1075 lwp_cleanup(void)
1076 {
1077 	kthread_t *t = curthread;
1078 	proc_t *p = ttoproc(t);
1079 
1080 	ASSERT(MUTEX_HELD(&p->p_lock));
1081 
1082 	/* untimeout any lwp-bound realtime timers */
1083 	if (p->p_itimer != NULL)
1084 		timer_lwpexit();
1085 
1086 	/*
1087 	 * If this is the /proc agent lwp that is exiting, readjust p_lwpid
1088 	 * so it appears that the agent never existed, and clear p_agenttp.
1089 	 */
1090 	if (t == p->p_agenttp) {
1091 		ASSERT(t->t_tid == p->p_lwpid);
1092 		p->p_lwpid--;
1093 		p->p_agenttp = NULL;
1094 	}
1095 
1096 	/*
1097 	 * Do lgroup bookkeeping to account for thread exiting.
1098 	 */
1099 	kpreempt_disable();
1100 	lgrp_move_thread(t, NULL, 1);
1101 	if (t->t_tid == 1) {
1102 		p->p_t1_lgrpid = LGRP_NONE;
1103 	}
1104 	kpreempt_enable();
1105 
1106 	lwp_ctmpl_clear(ttolwp(t));
1107 }
1108 
1109 int
lwp_suspend(kthread_t * t)1110 lwp_suspend(kthread_t *t)
1111 {
1112 	int tid;
1113 	proc_t *p = ttoproc(t);
1114 
1115 	ASSERT(MUTEX_HELD(&p->p_lock));
1116 
1117 	/*
1118 	 * Set the thread's TP_HOLDLWP flag so it will stop in holdlwp().
1119 	 * If an lwp is stopping itself, there is no need to wait.
1120 	 */
1121 top:
1122 	t->t_proc_flag |= TP_HOLDLWP;
1123 	if (t == curthread) {
1124 		t->t_sig_check = 1;
1125 	} else {
1126 		/*
1127 		 * Make sure the lwp stops promptly.
1128 		 */
1129 		thread_lock(t);
1130 		t->t_sig_check = 1;
1131 		/*
1132 		 * XXX Should use virtual stop like /proc does instead of
1133 		 * XXX waking the thread to get it to stop.
1134 		 */
1135 		if (ISWAKEABLE(t) || ISWAITING(t)) {
1136 			setrun_locked(t);
1137 		} else if (t->t_state == TS_ONPROC && t->t_cpu != CPU) {
1138 			poke_cpu(t->t_cpu->cpu_id);
1139 		}
1140 
1141 		tid = t->t_tid;	 /* remember thread ID */
1142 		/*
1143 		 * Wait for lwp to stop
1144 		 */
1145 		while (!SUSPENDED(t)) {
1146 			/*
1147 			 * Drop the thread lock before waiting and reacquire it
1148 			 * afterwards, so the thread can change its t_state
1149 			 * field.
1150 			 */
1151 			thread_unlock(t);
1152 
1153 			/*
1154 			 * Check if aborted by exitlwps().
1155 			 */
1156 			if (p->p_flag & SEXITLWPS)
1157 				lwp_exit();
1158 
1159 			/*
1160 			 * Cooperate with jobcontrol signals and /proc stopping
1161 			 * by calling cv_wait_sig() to wait for the target
1162 			 * lwp to stop.  Just using cv_wait() can lead to
1163 			 * deadlock because, if some other lwp has stopped
1164 			 * by either of these mechanisms, then p_lwprcnt will
1165 			 * never become zero if we do a cv_wait().
1166 			 */
1167 			if (!cv_wait_sig(&p->p_holdlwps, &p->p_lock))
1168 				return (EINTR);
1169 
1170 			/*
1171 			 * Check to see if thread died while we were
1172 			 * waiting for it to suspend.
1173 			 */
1174 			if (idtot(p, tid) == NULL)
1175 				return (ESRCH);
1176 
1177 			thread_lock(t);
1178 			/*
1179 			 * If the TP_HOLDLWP flag went away, lwp_continue()
1180 			 * or vfork() must have been called while we were
1181 			 * waiting, so start over again.
1182 			 */
1183 			if ((t->t_proc_flag & TP_HOLDLWP) == 0) {
1184 				thread_unlock(t);
1185 				goto top;
1186 			}
1187 		}
1188 		thread_unlock(t);
1189 	}
1190 	return (0);
1191 }
1192 
1193 /*
1194  * continue a lwp that's been stopped by lwp_suspend().
1195  */
1196 void
lwp_continue(kthread_t * t)1197 lwp_continue(kthread_t *t)
1198 {
1199 	proc_t *p = ttoproc(t);
1200 	int was_suspended = t->t_proc_flag & TP_HOLDLWP;
1201 
1202 	ASSERT(MUTEX_HELD(&p->p_lock));
1203 
1204 	t->t_proc_flag &= ~TP_HOLDLWP;
1205 	thread_lock(t);
1206 	if (SUSPENDED(t) &&
1207 	    !(p->p_flag & (SHOLDFORK | SHOLDFORK1 | SHOLDWATCH))) {
1208 		p->p_lwprcnt++;
1209 		t->t_schedflag |= TS_CSTART;
1210 		setrun_locked(t);
1211 	}
1212 	thread_unlock(t);
1213 	/*
1214 	 * Wakeup anyone waiting for this thread to be suspended
1215 	 */
1216 	if (was_suspended)
1217 		cv_broadcast(&p->p_holdlwps);
1218 }
1219 
1220 /*
1221  * ********************************
1222  *  Miscellaneous lwp routines	  *
1223  * ********************************
1224  */
1225 /*
1226  * When a process is undergoing a forkall(), its p_flag is set to SHOLDFORK.
1227  * This will cause the process's lwps to stop at a hold point.  A hold
1228  * point is where a kernel thread has a flat stack.  This is at the
1229  * return from a system call and at the return from a user level trap.
1230  *
1231  * When a process is undergoing a fork1() or vfork(), its p_flag is set to
1232  * SHOLDFORK1.  This will cause the process's lwps to stop at a modified
1233  * hold point.  The lwps in the process are not being cloned, so they
1234  * are held at the usual hold points and also within issig_forreal().
1235  * This has the side-effect that their system calls do not return
1236  * showing EINTR.
1237  *
1238  * An lwp can also be held.  This is identified by the TP_HOLDLWP flag on
1239  * the thread.  The TP_HOLDLWP flag is set in lwp_suspend(), where the active
1240  * lwp is waiting for the target lwp to be stopped.
1241  */
1242 void
holdlwp(void)1243 holdlwp(void)
1244 {
1245 	proc_t *p = curproc;
1246 	kthread_t *t = curthread;
1247 
1248 	mutex_enter(&p->p_lock);
1249 	/*
1250 	 * Don't terminate immediately if the process is dumping core.
1251 	 * Once the process has dumped core, all lwps are terminated.
1252 	 */
1253 	if (!(p->p_flag & SCOREDUMP)) {
1254 		if ((p->p_flag & SEXITLWPS) || (t->t_proc_flag & TP_EXITLWP))
1255 			lwp_exit();
1256 	}
1257 	if (!(ISHOLD(p)) && !(p->p_flag & (SHOLDFORK1 | SHOLDWATCH))) {
1258 		mutex_exit(&p->p_lock);
1259 		return;
1260 	}
1261 	/*
1262 	 * stop() decrements p->p_lwprcnt and cv_signal()s &p->p_holdlwps
1263 	 * when p->p_lwprcnt becomes zero.
1264 	 */
1265 	stop(PR_SUSPENDED, SUSPEND_NORMAL);
1266 	if (p->p_flag & SEXITLWPS)
1267 		lwp_exit();
1268 	mutex_exit(&p->p_lock);
1269 }
1270 
1271 /*
1272  * Have all lwps within the process hold at a point where they are
1273  * cloneable (SHOLDFORK) or just safe w.r.t. fork1 (SHOLDFORK1).
1274  */
1275 int
holdlwps(int holdflag)1276 holdlwps(int holdflag)
1277 {
1278 	proc_t *p = curproc;
1279 
1280 	ASSERT(holdflag == SHOLDFORK || holdflag == SHOLDFORK1);
1281 	mutex_enter(&p->p_lock);
1282 	schedctl_finish_sigblock(curthread);
1283 again:
1284 	while (p->p_flag & (SEXITLWPS | SHOLDFORK | SHOLDFORK1 | SHOLDWATCH)) {
1285 		/*
1286 		 * If another lwp is doing a forkall() or proc_exit(), bail out.
1287 		 */
1288 		if (p->p_flag & (SEXITLWPS | SHOLDFORK)) {
1289 			mutex_exit(&p->p_lock);
1290 			return (0);
1291 		}
1292 		/*
1293 		 * Another lwp is doing a fork1() or is undergoing
1294 		 * watchpoint activity.  We hold here for it to complete.
1295 		 */
1296 		stop(PR_SUSPENDED, SUSPEND_NORMAL);
1297 	}
1298 	p->p_flag |= holdflag;
1299 	pokelwps(p);
1300 	--p->p_lwprcnt;
1301 	/*
1302 	 * Wait for the process to become quiescent (p->p_lwprcnt == 0).
1303 	 */
1304 	while (p->p_lwprcnt > 0) {
1305 		/*
1306 		 * Check if aborted by exitlwps().
1307 		 * Also check if SHOLDWATCH is set; it takes precedence.
1308 		 */
1309 		if (p->p_flag & (SEXITLWPS | SHOLDWATCH)) {
1310 			p->p_lwprcnt++;
1311 			p->p_flag &= ~holdflag;
1312 			cv_broadcast(&p->p_holdlwps);
1313 			goto again;
1314 		}
1315 		/*
1316 		 * Cooperate with jobcontrol signals and /proc stopping.
1317 		 * If some other lwp has stopped by either of these
1318 		 * mechanisms, then p_lwprcnt will never become zero
1319 		 * and the process will appear deadlocked unless we
1320 		 * stop here in sympathy with the other lwp before
1321 		 * doing the cv_wait() below.
1322 		 *
1323 		 * If the other lwp stops after we do the cv_wait(), it
1324 		 * will wake us up to loop around and do the sympathy stop.
1325 		 *
1326 		 * Since stop() drops p->p_lock, we must start from
1327 		 * the top again on returning from stop().
1328 		 */
1329 		if (p->p_stopsig | (curthread->t_proc_flag & TP_PRSTOP)) {
1330 			int whystop = p->p_stopsig? PR_JOBCONTROL :
1331 			    PR_REQUESTED;
1332 			p->p_lwprcnt++;
1333 			p->p_flag &= ~holdflag;
1334 			stop(whystop, p->p_stopsig);
1335 			goto again;
1336 		}
1337 		cv_wait(&p->p_holdlwps, &p->p_lock);
1338 	}
1339 	p->p_lwprcnt++;
1340 	p->p_flag &= ~holdflag;
1341 	mutex_exit(&p->p_lock);
1342 	return (1);
1343 }
1344 
1345 /*
1346  * See comments for holdwatch(), below.
1347  */
1348 static int
holdcheck(int clearflags)1349 holdcheck(int clearflags)
1350 {
1351 	proc_t *p = curproc;
1352 
1353 	/*
1354 	 * If we are trying to exit, that takes precedence over anything else.
1355 	 */
1356 	if (p->p_flag & SEXITLWPS) {
1357 		p->p_lwprcnt++;
1358 		p->p_flag &= ~clearflags;
1359 		lwp_exit();
1360 	}
1361 
1362 	/*
1363 	 * If another thread is calling fork1(), stop the current thread so the
1364 	 * other can complete.
1365 	 */
1366 	if (p->p_flag & SHOLDFORK1) {
1367 		p->p_lwprcnt++;
1368 		stop(PR_SUSPENDED, SUSPEND_NORMAL);
1369 		if (p->p_flag & SEXITLWPS) {
1370 			p->p_flag &= ~clearflags;
1371 			lwp_exit();
1372 		}
1373 		return (-1);
1374 	}
1375 
1376 	/*
1377 	 * If another thread is calling fork(), then indicate we are doing
1378 	 * watchpoint activity.  This will cause holdlwps() above to stop the
1379 	 * forking thread, at which point we can continue with watchpoint
1380 	 * activity.
1381 	 */
1382 	if (p->p_flag & SHOLDFORK) {
1383 		p->p_lwprcnt++;
1384 		while (p->p_flag & SHOLDFORK) {
1385 			p->p_flag |= SHOLDWATCH;
1386 			cv_broadcast(&p->p_holdlwps);
1387 			cv_wait(&p->p_holdlwps, &p->p_lock);
1388 			p->p_flag &= ~SHOLDWATCH;
1389 		}
1390 		return (-1);
1391 	}
1392 
1393 	return (0);
1394 }
1395 
1396 /*
1397  * Stop all lwps within the process, holding themselves in the kernel while the
1398  * active lwp undergoes watchpoint activity.  This is more complicated than
1399  * expected because stop() relies on calling holdwatch() in order to copyin data
1400  * from the user's address space.  A double barrier is used to prevent an
1401  * infinite loop.
1402  *
1403  * 	o The first thread into holdwatch() is the 'master' thread and does
1404  *        the following:
1405  *
1406  *              - Sets SHOLDWATCH on the current process
1407  *              - Sets TP_WATCHSTOP on the current thread
1408  *              - Waits for all threads to be either stopped or have
1409  *                TP_WATCHSTOP set.
1410  *              - Sets the SWATCHOK flag on the process
1411  *              - Unsets TP_WATCHSTOP
1412  *              - Waits for the other threads to completely stop
1413  *              - Unsets SWATCHOK
1414  *
1415  * 	o If SHOLDWATCH is already set when we enter this function, then another
1416  *        thread is already trying to stop this thread.  This 'slave' thread
1417  *        does the following:
1418  *
1419  *              - Sets TP_WATCHSTOP on the current thread
1420  *              - Waits for SWATCHOK flag to be set
1421  *              - Calls stop()
1422  *
1423  * 	o If SWATCHOK is set on the process, then this function immediately
1424  *        returns, as we must have been called via stop().
1425  *
1426  * In addition, there are other flags that take precedence over SHOLDWATCH:
1427  *
1428  * 	o If SEXITLWPS is set, exit immediately.
1429  *
1430  * 	o If SHOLDFORK1 is set, wait for fork1() to complete.
1431  *
1432  * 	o If SHOLDFORK is set, then watchpoint activity takes precedence In this
1433  *        case, set SHOLDWATCH, signalling the forking thread to stop first.
1434  *
1435  * 	o If the process is being stopped via /proc (TP_PRSTOP is set), then we
1436  *        stop the current thread.
1437  *
1438  * Returns 0 if all threads have been quiesced.  Returns non-zero if not all
1439  * threads were stopped, or the list of watched pages has changed.
1440  */
1441 int
holdwatch(void)1442 holdwatch(void)
1443 {
1444 	proc_t *p = curproc;
1445 	kthread_t *t = curthread;
1446 	int ret = 0;
1447 
1448 	mutex_enter(&p->p_lock);
1449 
1450 	p->p_lwprcnt--;
1451 
1452 	/*
1453 	 * Check for bail-out conditions as outlined above.
1454 	 */
1455 	if (holdcheck(0) != 0) {
1456 		mutex_exit(&p->p_lock);
1457 		return (-1);
1458 	}
1459 
1460 	if (!(p->p_flag & SHOLDWATCH)) {
1461 		/*
1462 		 * We are the master watchpoint thread.  Set SHOLDWATCH and poke
1463 		 * the other threads.
1464 		 */
1465 		p->p_flag |= SHOLDWATCH;
1466 		pokelwps(p);
1467 
1468 		/*
1469 		 * Wait for all threads to be stopped or have TP_WATCHSTOP set.
1470 		 */
1471 		while (pr_allstopped(p, 1) > 0) {
1472 			if (holdcheck(SHOLDWATCH) != 0) {
1473 				p->p_flag &= ~SHOLDWATCH;
1474 				mutex_exit(&p->p_lock);
1475 				return (-1);
1476 			}
1477 
1478 			cv_wait(&p->p_holdlwps, &p->p_lock);
1479 		}
1480 
1481 		/*
1482 		 * All threads are now stopped or in the process of stopping.
1483 		 * Set SWATCHOK and let them stop completely.
1484 		 */
1485 		p->p_flag |= SWATCHOK;
1486 		t->t_proc_flag &= ~TP_WATCHSTOP;
1487 		cv_broadcast(&p->p_holdlwps);
1488 
1489 		while (pr_allstopped(p, 0) > 0) {
1490 			/*
1491 			 * At first glance, it may appear that we don't need a
1492 			 * call to holdcheck() here.  But if the process gets a
1493 			 * SIGKILL signal, one of our stopped threads may have
1494 			 * been awakened and is waiting in exitlwps(), which
1495 			 * takes precedence over watchpoints.
1496 			 */
1497 			if (holdcheck(SHOLDWATCH | SWATCHOK) != 0) {
1498 				p->p_flag &= ~(SHOLDWATCH | SWATCHOK);
1499 				mutex_exit(&p->p_lock);
1500 				return (-1);
1501 			}
1502 
1503 			cv_wait(&p->p_holdlwps, &p->p_lock);
1504 		}
1505 
1506 		/*
1507 		 * All threads are now completely stopped.
1508 		 */
1509 		p->p_flag &= ~SWATCHOK;
1510 		p->p_flag &= ~SHOLDWATCH;
1511 		p->p_lwprcnt++;
1512 
1513 	} else if (!(p->p_flag & SWATCHOK)) {
1514 
1515 		/*
1516 		 * SHOLDWATCH is set, so another thread is trying to do
1517 		 * watchpoint activity.  Indicate this thread is stopping, and
1518 		 * wait for the OK from the master thread.
1519 		 */
1520 		t->t_proc_flag |= TP_WATCHSTOP;
1521 		cv_broadcast(&p->p_holdlwps);
1522 
1523 		while (!(p->p_flag & SWATCHOK)) {
1524 			if (holdcheck(0) != 0) {
1525 				t->t_proc_flag &= ~TP_WATCHSTOP;
1526 				mutex_exit(&p->p_lock);
1527 				return (-1);
1528 			}
1529 
1530 			cv_wait(&p->p_holdlwps, &p->p_lock);
1531 		}
1532 
1533 		/*
1534 		 * Once the master thread has given the OK, this thread can
1535 		 * actually call stop().
1536 		 */
1537 		t->t_proc_flag &= ~TP_WATCHSTOP;
1538 		p->p_lwprcnt++;
1539 
1540 		stop(PR_SUSPENDED, SUSPEND_NORMAL);
1541 
1542 		/*
1543 		 * It's not OK to do watchpoint activity, notify caller to
1544 		 * retry.
1545 		 */
1546 		ret = -1;
1547 
1548 	} else {
1549 
1550 		/*
1551 		 * The only way we can hit the case where SHOLDWATCH is set and
1552 		 * SWATCHOK is set is if we are triggering this from within a
1553 		 * stop() call.  Assert that this is the case.
1554 		 */
1555 
1556 		ASSERT(t->t_proc_flag & TP_STOPPING);
1557 		p->p_lwprcnt++;
1558 	}
1559 
1560 	mutex_exit(&p->p_lock);
1561 
1562 	return (ret);
1563 }
1564 
1565 /*
1566  * force all interruptible lwps to trap into the kernel.
1567  */
1568 void
pokelwps(proc_t * p)1569 pokelwps(proc_t *p)
1570 {
1571 	kthread_t *t;
1572 
1573 	ASSERT(MUTEX_HELD(&p->p_lock));
1574 
1575 	t = p->p_tlist;
1576 	do {
1577 		if (t == curthread)
1578 			continue;
1579 		thread_lock(t);
1580 		aston(t);	/* make thread trap or do post_syscall */
1581 		if (ISWAKEABLE(t) || ISWAITING(t)) {
1582 			setrun_locked(t);
1583 		} else if (t->t_state == TS_STOPPED) {
1584 			/*
1585 			 * Ensure that proc_exit() is not blocked by lwps
1586 			 * that were stopped via jobcontrol or /proc.
1587 			 */
1588 			if (p->p_flag & SEXITLWPS) {
1589 				p->p_stopsig = 0;
1590 				t->t_schedflag |= (TS_XSTART | TS_PSTART);
1591 				setrun_locked(t);
1592 			}
1593 			/*
1594 			 * If we are holding lwps for a forkall(),
1595 			 * force lwps that have been suspended via
1596 			 * lwp_suspend() and are suspended inside
1597 			 * of a system call to proceed to their
1598 			 * holdlwp() points where they are clonable.
1599 			 */
1600 			if ((p->p_flag & SHOLDFORK) && SUSPENDED(t)) {
1601 				if ((t->t_schedflag & TS_CSTART) == 0) {
1602 					p->p_lwprcnt++;
1603 					t->t_schedflag |= TS_CSTART;
1604 					setrun_locked(t);
1605 				}
1606 			}
1607 		} else if (t->t_state == TS_ONPROC) {
1608 			if (t->t_cpu != CPU)
1609 				poke_cpu(t->t_cpu->cpu_id);
1610 		}
1611 		thread_unlock(t);
1612 	} while ((t = t->t_forw) != p->p_tlist);
1613 }
1614 
1615 /*
1616  * undo the effects of holdlwps() or holdwatch().
1617  */
1618 void
continuelwps(proc_t * p)1619 continuelwps(proc_t *p)
1620 {
1621 	kthread_t *t;
1622 
1623 	/*
1624 	 * If this flag is set, then the original holdwatch() didn't actually
1625 	 * stop the process.  See comments for holdwatch().
1626 	 */
1627 	if (p->p_flag & SWATCHOK) {
1628 		ASSERT(curthread->t_proc_flag & TP_STOPPING);
1629 		return;
1630 	}
1631 
1632 	ASSERT(MUTEX_HELD(&p->p_lock));
1633 	ASSERT((p->p_flag & (SHOLDFORK | SHOLDFORK1 | SHOLDWATCH)) == 0);
1634 
1635 	t = p->p_tlist;
1636 	do {
1637 		thread_lock(t);		/* SUSPENDED looks at t_schedflag */
1638 		if (SUSPENDED(t) && !(t->t_proc_flag & TP_HOLDLWP)) {
1639 			p->p_lwprcnt++;
1640 			t->t_schedflag |= TS_CSTART;
1641 			setrun_locked(t);
1642 		}
1643 		thread_unlock(t);
1644 	} while ((t = t->t_forw) != p->p_tlist);
1645 }
1646 
1647 /*
1648  * Force all other LWPs in the current process other than the caller to exit,
1649  * and then cv_wait() on p_holdlwps for them to exit.  The exitlwps() function
1650  * is typically used in these situations:
1651  *
1652  *   (a) prior to an exec() system call
1653  *   (b) prior to dumping a core file
1654  *   (c) prior to a uadmin() shutdown
1655  *
1656  * If the 'coredump' flag is set, other LWPs are quiesced but not destroyed.
1657  * Multiple threads in the process can call this function at one time by
1658  * triggering execs or core dumps simultaneously, so the SEXITLWPS bit is used
1659  * to declare one particular thread the winner who gets to kill the others.
1660  * If a thread wins the exitlwps() dance, zero is returned; otherwise an
1661  * appropriate errno value is returned to caller for its system call to return.
1662  */
1663 int
exitlwps(int coredump)1664 exitlwps(int coredump)
1665 {
1666 	proc_t *p = curproc;
1667 	int heldcnt;
1668 
1669 	if (curthread->t_door)
1670 		door_slam();
1671 	if (p->p_door_list)
1672 		door_revoke_all();
1673 	if (curthread->t_schedctl != NULL)
1674 		schedctl_lwp_cleanup(curthread);
1675 
1676 	/*
1677 	 * Ensure that before starting to wait for other lwps to exit,
1678 	 * cleanup all upimutexes held by curthread. Otherwise, some other
1679 	 * lwp could be waiting (uninterruptibly) for a upimutex held by
1680 	 * curthread, and the call to pokelwps() below would deadlock.
1681 	 * Even if a blocked upimutex_lock is made interruptible,
1682 	 * curthread's upimutexes need to be unlocked: do it here.
1683 	 */
1684 	if (curthread->t_upimutex != NULL)
1685 		upimutex_cleanup();
1686 
1687 	/*
1688 	 * Grab p_lock in order to check and set SEXITLWPS to declare a winner.
1689 	 * We must also block any further /proc access from this point forward.
1690 	 */
1691 	mutex_enter(&p->p_lock);
1692 	prbarrier(p);
1693 
1694 	if (p->p_flag & SEXITLWPS) {
1695 		mutex_exit(&p->p_lock);
1696 		aston(curthread);	/* force a trip through post_syscall */
1697 		return (set_errno(EINTR));
1698 	}
1699 
1700 	p->p_flag |= SEXITLWPS;
1701 	if (coredump)		/* tell other lwps to stop, not exit */
1702 		p->p_flag |= SCOREDUMP;
1703 
1704 	/*
1705 	 * Give precedence to exitlwps() if a holdlwps() is
1706 	 * in progress. The lwp doing the holdlwps() operation
1707 	 * is aborted when it is awakened.
1708 	 */
1709 	while (p->p_flag & (SHOLDFORK | SHOLDFORK1 | SHOLDWATCH)) {
1710 		cv_broadcast(&p->p_holdlwps);
1711 		cv_wait(&p->p_holdlwps, &p->p_lock);
1712 		prbarrier(p);
1713 	}
1714 	p->p_flag |= SHOLDFORK;
1715 	pokelwps(p);
1716 
1717 	/*
1718 	 * Wait for process to become quiescent.
1719 	 */
1720 	--p->p_lwprcnt;
1721 	while (p->p_lwprcnt > 0) {
1722 		cv_wait(&p->p_holdlwps, &p->p_lock);
1723 		prbarrier(p);
1724 	}
1725 	p->p_lwprcnt++;
1726 	ASSERT(p->p_lwprcnt == 1);
1727 
1728 	/*
1729 	 * The SCOREDUMP flag puts the process into a quiescent
1730 	 * state.  The process's lwps remain attached to this
1731 	 * process until exitlwps() is called again without the
1732 	 * 'coredump' flag set, then the lwps are terminated
1733 	 * and the process can exit.
1734 	 */
1735 	if (coredump) {
1736 		p->p_flag &= ~(SCOREDUMP | SHOLDFORK | SEXITLWPS);
1737 		goto out;
1738 	}
1739 
1740 	/*
1741 	 * Determine if there are any lwps left dangling in
1742 	 * the stopped state.  This happens when exitlwps()
1743 	 * aborts a holdlwps() operation.
1744 	 */
1745 	p->p_flag &= ~SHOLDFORK;
1746 	if ((heldcnt = p->p_lwpcnt) > 1) {
1747 		kthread_t *t;
1748 		for (t = curthread->t_forw; --heldcnt > 0; t = t->t_forw) {
1749 			t->t_proc_flag &= ~TP_TWAIT;
1750 			lwp_continue(t);
1751 		}
1752 	}
1753 
1754 	/*
1755 	 * Wait for all other lwps to exit.
1756 	 */
1757 	--p->p_lwprcnt;
1758 	while (p->p_lwpcnt > 1) {
1759 		cv_wait(&p->p_holdlwps, &p->p_lock);
1760 		prbarrier(p);
1761 	}
1762 	++p->p_lwprcnt;
1763 	ASSERT(p->p_lwpcnt == 1 && p->p_lwprcnt == 1);
1764 
1765 	p->p_flag &= ~SEXITLWPS;
1766 	curthread->t_proc_flag &= ~TP_TWAIT;
1767 
1768 out:
1769 	if (!coredump && p->p_zombcnt) {	/* cleanup the zombie lwps */
1770 		lwpdir_t *ldp;
1771 		lwpent_t *lep;
1772 		int i;
1773 
1774 		for (ldp = p->p_lwpdir, i = 0; i < p->p_lwpdir_sz; i++, ldp++) {
1775 			lep = ldp->ld_entry;
1776 			if (lep != NULL && lep->le_thread != curthread) {
1777 				ASSERT(lep->le_thread == NULL);
1778 				p->p_zombcnt--;
1779 				lwp_hash_out(p, lep->le_lwpid);
1780 			}
1781 		}
1782 		ASSERT(p->p_zombcnt == 0);
1783 	}
1784 
1785 	/*
1786 	 * If some other LWP in the process wanted us to suspend ourself,
1787 	 * then we will not do it.  The other LWP is now terminated and
1788 	 * no one will ever continue us again if we suspend ourself.
1789 	 */
1790 	curthread->t_proc_flag &= ~TP_HOLDLWP;
1791 	p->p_flag &= ~(SHOLDFORK | SHOLDFORK1 | SHOLDWATCH | SLWPWRAP);
1792 	mutex_exit(&p->p_lock);
1793 	return (0);
1794 }
1795 
1796 /*
1797  * duplicate a lwp.
1798  */
1799 klwp_t *
forklwp(klwp_t * lwp,proc_t * cp,id_t lwpid)1800 forklwp(klwp_t *lwp, proc_t *cp, id_t lwpid)
1801 {
1802 	klwp_t *clwp;
1803 	void *tregs, *tfpu;
1804 	kthread_t *t = lwptot(lwp);
1805 	kthread_t *ct;
1806 	proc_t *p = lwptoproc(lwp);
1807 	int cid;
1808 	void *bufp;
1809 	void *brand_data;
1810 	int val;
1811 
1812 	ASSERT(p == curproc);
1813 	ASSERT(t == curthread || (SUSPENDED(t) && lwp->lwp_asleep == 0));
1814 
1815 #if defined(__sparc)
1816 	if (t == curthread)
1817 		(void) flush_user_windows_to_stack(NULL);
1818 #endif
1819 
1820 	if (t == curthread)
1821 		/* copy args out of registers first */
1822 		(void) save_syscall_args();
1823 
1824 	clwp = lwp_create(cp->p_lwpcnt == 0 ? lwp_rtt_initial : lwp_rtt,
1825 	    NULL, 0, cp, TS_STOPPED, t->t_pri, &t->t_hold, NOCLASS, lwpid);
1826 	if (clwp == NULL)
1827 		return (NULL);
1828 
1829 	/*
1830 	 * most of the parent's lwp can be copied to its duplicate,
1831 	 * except for the fields that are unique to each lwp, like
1832 	 * lwp_thread, lwp_procp, lwp_regs, and lwp_ap.
1833 	 */
1834 	ct = clwp->lwp_thread;
1835 	tregs = clwp->lwp_regs;
1836 	tfpu = clwp->lwp_fpu;
1837 	brand_data = clwp->lwp_brand;
1838 
1839 	/*
1840 	 * Copy parent lwp to child lwp.  Hold child's p_lock to prevent
1841 	 * mstate_aggr_state() from reading stale mstate entries copied
1842 	 * from lwp to clwp.
1843 	 */
1844 	mutex_enter(&cp->p_lock);
1845 	*clwp = *lwp;
1846 
1847 	/* clear microstate and resource usage data in new lwp */
1848 	init_mstate(ct, LMS_STOPPED);
1849 	bzero(&clwp->lwp_ru, sizeof (clwp->lwp_ru));
1850 	mutex_exit(&cp->p_lock);
1851 
1852 	/* fix up child's lwp */
1853 
1854 	clwp->lwp_pcb.pcb_flags = 0;
1855 #if defined(__sparc)
1856 	clwp->lwp_pcb.pcb_step = STEP_NONE;
1857 #endif
1858 	clwp->lwp_cursig = 0;
1859 	clwp->lwp_extsig = 0;
1860 	clwp->lwp_curinfo = (struct sigqueue *)0;
1861 	clwp->lwp_thread = ct;
1862 	ct->t_sysnum = t->t_sysnum;
1863 	clwp->lwp_regs = tregs;
1864 	clwp->lwp_fpu = tfpu;
1865 	clwp->lwp_brand = brand_data;
1866 	clwp->lwp_ap = clwp->lwp_arg;
1867 	clwp->lwp_procp = cp;
1868 	bzero(clwp->lwp_timer, sizeof (clwp->lwp_timer));
1869 	clwp->lwp_lastfault = 0;
1870 	clwp->lwp_lastfaddr = 0;
1871 
1872 	/* copy parent's struct regs to child. */
1873 	lwp_forkregs(lwp, clwp);
1874 
1875 	/*
1876 	 * Fork thread context ops, if any.
1877 	 */
1878 	if (t->t_ctx)
1879 		forkctx(t, ct);
1880 
1881 	/* fix door state in the child */
1882 	if (t->t_door)
1883 		door_fork(t, ct);
1884 
1885 	/* copy current contract templates, clear latest contracts */
1886 	lwp_ctmpl_copy(clwp, lwp);
1887 
1888 	mutex_enter(&cp->p_lock);
1889 	/* lwp_create() set the TP_HOLDLWP flag */
1890 	if (!(t->t_proc_flag & TP_HOLDLWP))
1891 		ct->t_proc_flag &= ~TP_HOLDLWP;
1892 	if (cp->p_flag & SMSACCT)
1893 		ct->t_proc_flag |= TP_MSACCT;
1894 	mutex_exit(&cp->p_lock);
1895 
1896 	/* Allow brand to propagate brand-specific state */
1897 	if (PROC_IS_BRANDED(p))
1898 		BROP(p)->b_forklwp(lwp, clwp);
1899 
1900 retry:
1901 	cid = t->t_cid;
1902 
1903 	val = CL_ALLOC(&bufp, cid, KM_SLEEP);
1904 	ASSERT(val == 0);
1905 
1906 	mutex_enter(&p->p_lock);
1907 	if (cid != t->t_cid) {
1908 		/*
1909 		 * Someone just changed this thread's scheduling class,
1910 		 * so try pre-allocating the buffer again.  Hopefully we
1911 		 * don't hit this often.
1912 		 */
1913 		mutex_exit(&p->p_lock);
1914 		CL_FREE(cid, bufp);
1915 		goto retry;
1916 	}
1917 
1918 	ct->t_unpark = t->t_unpark;
1919 	ct->t_clfuncs = t->t_clfuncs;
1920 	CL_FORK(t, ct, bufp);
1921 	ct->t_cid = t->t_cid;	/* after data allocated so prgetpsinfo works */
1922 	mutex_exit(&p->p_lock);
1923 
1924 	return (clwp);
1925 }
1926 
1927 /*
1928  * Add a new lwp entry to the lwp directory and to the lwpid hash table.
1929  */
1930 void
lwp_hash_in(proc_t * p,lwpent_t * lep,tidhash_t * tidhash,uint_t tidhash_sz,int do_lock)1931 lwp_hash_in(proc_t *p, lwpent_t *lep, tidhash_t *tidhash, uint_t tidhash_sz,
1932     int do_lock)
1933 {
1934 	tidhash_t *thp = &tidhash[TIDHASH(lep->le_lwpid, tidhash_sz)];
1935 	lwpdir_t **ldpp;
1936 	lwpdir_t *ldp;
1937 	kthread_t *t;
1938 
1939 	/*
1940 	 * Allocate a directory element from the free list.
1941 	 * Code elsewhere guarantees a free slot.
1942 	 */
1943 	ldp = p->p_lwpfree;
1944 	p->p_lwpfree = ldp->ld_next;
1945 	ASSERT(ldp->ld_entry == NULL);
1946 	ldp->ld_entry = lep;
1947 
1948 	if (do_lock)
1949 		mutex_enter(&thp->th_lock);
1950 
1951 	/*
1952 	 * Insert it into the lwpid hash table.
1953 	 */
1954 	ldpp = &thp->th_list;
1955 	ldp->ld_next = *ldpp;
1956 	*ldpp = ldp;
1957 
1958 	/*
1959 	 * Set the active thread's directory slot entry.
1960 	 */
1961 	if ((t = lep->le_thread) != NULL) {
1962 		ASSERT(lep->le_lwpid == t->t_tid);
1963 		t->t_dslot = (int)(ldp - p->p_lwpdir);
1964 	}
1965 
1966 	if (do_lock)
1967 		mutex_exit(&thp->th_lock);
1968 }
1969 
1970 /*
1971  * Remove an lwp from the lwpid hash table and free its directory entry.
1972  * This is done when a detached lwp exits in lwp_exit() or
1973  * when a non-detached lwp is waited for in lwp_wait() or
1974  * when a zombie lwp is detached in lwp_detach().
1975  */
1976 void
lwp_hash_out(proc_t * p,id_t lwpid)1977 lwp_hash_out(proc_t *p, id_t lwpid)
1978 {
1979 	tidhash_t *thp = &p->p_tidhash[TIDHASH(lwpid, p->p_tidhash_sz)];
1980 	lwpdir_t **ldpp;
1981 	lwpdir_t *ldp;
1982 	lwpent_t *lep;
1983 
1984 	mutex_enter(&thp->th_lock);
1985 	for (ldpp = &thp->th_list;
1986 	    (ldp = *ldpp) != NULL; ldpp = &ldp->ld_next) {
1987 		lep = ldp->ld_entry;
1988 		if (lep->le_lwpid == lwpid) {
1989 			prlwpfree(p, lep);	/* /proc deals with le_trace */
1990 			*ldpp = ldp->ld_next;
1991 			ldp->ld_entry = NULL;
1992 			ldp->ld_next = p->p_lwpfree;
1993 			p->p_lwpfree = ldp;
1994 			kmem_free(lep, sizeof (*lep));
1995 			break;
1996 		}
1997 	}
1998 	mutex_exit(&thp->th_lock);
1999 }
2000 
2001 /*
2002  * Lookup an lwp in the lwpid hash table by lwpid.
2003  */
2004 lwpdir_t *
lwp_hash_lookup(proc_t * p,id_t lwpid)2005 lwp_hash_lookup(proc_t *p, id_t lwpid)
2006 {
2007 	tidhash_t *thp;
2008 	lwpdir_t *ldp;
2009 
2010 	/*
2011 	 * The process may be exiting, after p_tidhash has been set to NULL in
2012 	 * proc_exit() but before prfee() has been called.  Return failure in
2013 	 * this case.
2014 	 */
2015 	if (p->p_tidhash == NULL)
2016 		return (NULL);
2017 
2018 	thp = &p->p_tidhash[TIDHASH(lwpid, p->p_tidhash_sz)];
2019 	for (ldp = thp->th_list; ldp != NULL; ldp = ldp->ld_next) {
2020 		if (ldp->ld_entry->le_lwpid == lwpid)
2021 			return (ldp);
2022 	}
2023 
2024 	return (NULL);
2025 }
2026 
2027 /*
2028  * Same as lwp_hash_lookup(), but acquire and return
2029  * the tid hash table entry lock on success.
2030  */
2031 lwpdir_t *
lwp_hash_lookup_and_lock(proc_t * p,id_t lwpid,kmutex_t ** mpp)2032 lwp_hash_lookup_and_lock(proc_t *p, id_t lwpid, kmutex_t **mpp)
2033 {
2034 	tidhash_t *tidhash;
2035 	uint_t tidhash_sz;
2036 	tidhash_t *thp;
2037 	lwpdir_t *ldp;
2038 
2039 top:
2040 	tidhash_sz = p->p_tidhash_sz;
2041 	membar_consumer();
2042 	if ((tidhash = p->p_tidhash) == NULL)
2043 		return (NULL);
2044 
2045 	thp = &tidhash[TIDHASH(lwpid, tidhash_sz)];
2046 	mutex_enter(&thp->th_lock);
2047 
2048 	/*
2049 	 * Since we are not holding p->p_lock, the tid hash table
2050 	 * may have changed.  If so, start over.  If not, then
2051 	 * it cannot change until after we drop &thp->th_lock;
2052 	 */
2053 	if (tidhash != p->p_tidhash || tidhash_sz != p->p_tidhash_sz) {
2054 		mutex_exit(&thp->th_lock);
2055 		goto top;
2056 	}
2057 
2058 	for (ldp = thp->th_list; ldp != NULL; ldp = ldp->ld_next) {
2059 		if (ldp->ld_entry->le_lwpid == lwpid) {
2060 			*mpp = &thp->th_lock;
2061 			return (ldp);
2062 		}
2063 	}
2064 
2065 	mutex_exit(&thp->th_lock);
2066 	return (NULL);
2067 }
2068 
2069 /*
2070  * Update the indicated LWP usage statistic for the current LWP.
2071  */
2072 void
lwp_stat_update(lwp_stat_id_t lwp_stat_id,long inc)2073 lwp_stat_update(lwp_stat_id_t lwp_stat_id, long inc)
2074 {
2075 	klwp_t *lwp = ttolwp(curthread);
2076 
2077 	if (lwp == NULL)
2078 		return;
2079 
2080 	switch (lwp_stat_id) {
2081 	case LWP_STAT_INBLK:
2082 		lwp->lwp_ru.inblock += inc;
2083 		break;
2084 	case LWP_STAT_OUBLK:
2085 		lwp->lwp_ru.oublock += inc;
2086 		break;
2087 	case LWP_STAT_MSGRCV:
2088 		lwp->lwp_ru.msgrcv += inc;
2089 		break;
2090 	case LWP_STAT_MSGSND:
2091 		lwp->lwp_ru.msgsnd += inc;
2092 		break;
2093 	default:
2094 		panic("lwp_stat_update: invalid lwp_stat_id 0x%x", lwp_stat_id);
2095 	}
2096 }
2097