xref: /illumos-gate/usr/src/uts/common/os/lwp.c (revision 088d69f8)

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * Copyright (c) 2013, Joyent, Inc. All rights reserved.
 */

#include <sys/param.h>
#include <sys/types.h>
#include <sys/sysmacros.h>
#include <sys/systm.h>
#include <sys/thread.h>
#include <sys/proc.h>
#include <sys/task.h>
#include <sys/project.h>
#include <sys/signal.h>
#include <sys/errno.h>
#include <sys/vmparam.h>
#include <sys/stack.h>
#include <sys/procfs.h>
#include <sys/prsystm.h>
#include <sys/cpuvar.h>
#include <sys/kmem.h>
#include <sys/vtrace.h>
#include <sys/door.h>
#include <vm/seg_kp.h>
#include <sys/debug.h>
#include <sys/tnf.h>
#include <sys/schedctl.h>
#include <sys/poll.h>
#include <sys/copyops.h>
#include <sys/lwp_upimutex_impl.h>
#include <sys/cpupart.h>
#include <sys/lgrp.h>
#include <sys/rctl.h>
#include <sys/contract_impl.h>
#include <sys/cpc_impl.h>
#include <sys/sdt.h>
#include <sys/cmn_err.h>
#include <sys/brand.h>
#include <sys/cyclic.h>
#include <sys/pool.h>

/* hash function for the lwpid hash table, p->p_tidhash[] */
#define	TIDHASH(tid, hash_sz)	((tid) & ((hash_sz) - 1))

void *segkp_lwp;		/* cookie for pool of segkp resources */
extern void reapq_move_lq_to_tq(kthread_t *);
extern void freectx_ctx(struct ctxop *);

/*
 * Create a kernel thread associated with a particular system process.  Give
 * it an LWP so that microstate accounting will be available for it.
 */
kthread_t *
lwp_kernel_create(proc_t *p, void (*proc)(), void *arg, int state, pri_t pri)
{
	klwp_t *lwp;

	VERIFY((p->p_flag & SSYS) != 0);

	lwp = lwp_create(proc, arg, 0, p, state, pri, &t0.t_hold, syscid, 0);

	VERIFY(lwp != NULL);

	return (lwptot(lwp));
}

/*
 * Create a thread that appears to be stopped at sys_rtt.
 */
klwp_t *
lwp_create(void (*proc)(), caddr_t arg, size_t len, proc_t *p,
    int state, int pri, const k_sigset_t *smask, int cid, id_t lwpid)
{
	klwp_t *lwp = NULL;
	kthread_t *t;
	kthread_t *tx;
	cpupart_t *oldpart = NULL;
	size_t	stksize;
	caddr_t lwpdata = NULL;
	processorid_t	binding;
	int err = 0;
	kproject_t *oldkpj, *newkpj;
	void *bufp = NULL;
	klwp_t *curlwp;
	lwpent_t *lep;
	lwpdir_t *old_dir = NULL;
	uint_t old_dirsz = 0;
	tidhash_t *old_hash = NULL;
	uint_t old_hashsz = 0;
	ret_tidhash_t *ret_tidhash = NULL;
	int i;
	int rctlfail = 0;
	boolean_t branded = 0;
	struct ctxop *ctx = NULL;

	ASSERT(cid != sysdccid);	/* system threads must start in SYS */

	ASSERT(p != &p0);		/* No new LWPs in p0. */

	mutex_enter(&p->p_lock);
	mutex_enter(&p->p_zone->zone_nlwps_lock);
	/*
	 * don't enforce rctl limits on system processes
	 */
	if (!CLASS_KERNEL(cid)) {
		if (p->p_task->tk_nlwps >= p->p_task->tk_nlwps_ctl)
			if (rctl_test(rc_task_lwps, p->p_task->tk_rctls, p,
			    1, 0) & RCT_DENY)
				rctlfail = 1;
		if (p->p_task->tk_proj->kpj_nlwps >=
		    p->p_task->tk_proj->kpj_nlwps_ctl)
			if (rctl_test(rc_project_nlwps,
			    p->p_task->tk_proj->kpj_rctls, p, 1, 0)
			    & RCT_DENY)
				rctlfail = 1;
		if (p->p_zone->zone_nlwps >= p->p_zone->zone_nlwps_ctl)
			if (rctl_test(rc_zone_nlwps, p->p_zone->zone_rctls, p,
			    1, 0) & RCT_DENY)
				rctlfail = 1;
	}
	if (rctlfail) {
		mutex_exit(&p->p_zone->zone_nlwps_lock);
		mutex_exit(&p->p_lock);
		atomic_inc_32(&p->p_zone->zone_ffcap);
		return (NULL);
	}
	p->p_task->tk_nlwps++;
	p->p_task->tk_proj->kpj_nlwps++;
	p->p_zone->zone_nlwps++;
	mutex_exit(&p->p_zone->zone_nlwps_lock);
	mutex_exit(&p->p_lock);

	curlwp = ttolwp(curthread);
	if (curlwp == NULL || (stksize = curlwp->lwp_childstksz) == 0)
		stksize = lwp_default_stksize;

	if (CLASS_KERNEL(cid)) {
		/*
		 * Since we are creating an LWP in an SSYS process, we do not
		 * inherit anything from the current thread's LWP.  We set
		 * stksize and lwpdata to 0 in order to let thread_create()
		 * allocate a regular kernel thread stack for this thread.
		 */
		curlwp = NULL;
		stksize = 0;
		lwpdata = NULL;

	} else if (stksize == lwp_default_stksize) {
		/*
		 * Try to reuse an <lwp,stack> from the LWP deathrow.
		 */
		if (lwp_reapcnt > 0) {
			mutex_enter(&reaplock);
			if ((t = lwp_deathrow) != NULL) {
				ASSERT(t->t_swap);
				lwp_deathrow = t->t_forw;
				lwp_reapcnt--;
				lwpdata = t->t_swap;
				lwp = t->t_lwp;
				ctx = t->t_ctx;
				t->t_swap = NULL;
				t->t_lwp = NULL;
				t->t_ctx = NULL;
				reapq_move_lq_to_tq(t);
			}
			mutex_exit(&reaplock);
			if (lwp != NULL) {
				lwp_stk_fini(lwp);
			}
			if (ctx != NULL) {
				freectx_ctx(ctx);
			}
		}
		if (lwpdata == NULL &&
		    (lwpdata = (caddr_t)segkp_cache_get(segkp_lwp)) == NULL) {
			mutex_enter(&p->p_lock);
			mutex_enter(&p->p_zone->zone_nlwps_lock);
			p->p_task->tk_nlwps--;
			p->p_task->tk_proj->kpj_nlwps--;
			p->p_zone->zone_nlwps--;
			mutex_exit(&p->p_zone->zone_nlwps_lock);
			mutex_exit(&p->p_lock);
			atomic_inc_32(&p->p_zone->zone_ffnomem);
			return (NULL);
		}
	} else {
		stksize = roundup(stksize, PAGESIZE);
		if ((lwpdata = (caddr_t)segkp_get(segkp, stksize,
		    (KPD_NOWAIT | KPD_HASREDZONE | KPD_LOCKED))) == NULL) {
			mutex_enter(&p->p_lock);
			mutex_enter(&p->p_zone->zone_nlwps_lock);
			p->p_task->tk_nlwps--;
			p->p_task->tk_proj->kpj_nlwps--;
			p->p_zone->zone_nlwps--;
			mutex_exit(&p->p_zone->zone_nlwps_lock);
			mutex_exit(&p->p_lock);
			atomic_inc_32(&p->p_zone->zone_ffnomem);
			return (NULL);
		}
	}

	/*
	 * Create a thread, initializing the stack pointer
	 */
	t = thread_create(lwpdata, stksize, NULL, NULL, 0, p, TS_STOPPED, pri);

	/*
	 * If a non-NULL stack base is passed in, thread_create() assumes
	 * that the stack might be statically allocated (as opposed to being
	 * allocated from segkp), and so it does not set t_swap.  Since
	 * the lwpdata was allocated from segkp, we must set t_swap to point
	 * to it ourselves.
	 *
	 * This would be less confusing if t_swap had a better name; it really
	 * indicates that the stack is allocated from segkp, regardless of
	 * whether or not it is swappable.
	 */
	if (lwpdata != NULL) {
		ASSERT(!CLASS_KERNEL(cid));
		ASSERT(t->t_swap == NULL);
		t->t_swap = lwpdata;	/* Start of page-able data */
	}

	/*
	 * If the stack and lwp can be reused, mark the thread as such.
	 * When we get to reapq_add() from resume_from_zombie(), these
	 * threads will go onto lwp_deathrow instead of thread_deathrow.
	 */
	if (!CLASS_KERNEL(cid) && stksize == lwp_default_stksize)
		t->t_flag |= T_LWPREUSE;

	if (lwp == NULL)
		lwp = kmem_cache_alloc(lwp_cache, KM_SLEEP);
	bzero(lwp, sizeof (*lwp));
	t->t_lwp = lwp;

	t->t_hold = *smask;
	lwp->lwp_thread = t;
	lwp->lwp_procp = p;
	lwp->lwp_sigaltstack.ss_flags = SS_DISABLE;
	if (curlwp != NULL && curlwp->lwp_childstksz != 0)
		lwp->lwp_childstksz = curlwp->lwp_childstksz;

	t->t_stk = lwp_stk_init(lwp, t->t_stk);
	thread_load(t, proc, arg, len);

	/*
	 * Allocate the SIGPROF buffer if ITIMER_REALPROF is in effect.
	 */
	if (p->p_rprof_cyclic != CYCLIC_NONE)
		t->t_rprof = kmem_zalloc(sizeof (struct rprof), KM_SLEEP);

	if (cid != NOCLASS)
		(void) CL_ALLOC(&bufp, cid, KM_SLEEP);

	/*
	 * Allocate an lwp directory entry for the new lwp.
	 */
	lep = kmem_zalloc(sizeof (*lep), KM_SLEEP);

	mutex_enter(&p->p_lock);
grow:
	/*
	 * Grow the lwp (thread) directory and lwpid hash table if necessary.
	 * A note on the growth algorithm:
	 *	The new lwp directory size is computed as:
	 *		new = 2 * old + 2
	 *	Starting with an initial size of 2 (see exec_common()),
	 *	this yields numbers that are a power of two minus 2:
	 *		2, 6, 14, 30, 62, 126, 254, 510, 1022, ...
	 *	The size of the lwpid hash table must be a power of two
	 *	and must be commensurate in size with the lwp directory
	 *	so that hash bucket chains remain short.  Therefore,
	 *	the lwpid hash table size is computed as:
	 *		hashsz = (dirsz + 2) / 2
	 *	which leads to these hash table sizes corresponding to
	 *	the above directory sizes:
	 *		2, 4, 8, 16, 32, 64, 128, 256, 512, ...
	 * A note on growing the hash table:
	 *	For performance reasons, code in lwp_unpark() does not
	 *	acquire curproc->p_lock when searching the hash table.
	 *	Rather, it calls lwp_hash_lookup_and_lock() which
	 *	acquires only the individual hash bucket lock, taking
	 *	care to deal with reallocation of the hash table
	 *	during the time it takes to acquire the lock.
	 *
	 *	This is sufficient to protect the integrity of the
	 *	hash table, but it requires us to acquire all of the
	 *	old hash bucket locks before growing the hash table
	 *	and to release them afterwards.  It also requires us
	 *	not to free the old hash table because some thread
	 *	in lwp_hash_lookup_and_lock() might still be trying
	 *	to acquire the old bucket lock.
	 *
	 *	So we adopt the tactic of keeping all of the retired
	 *	hash tables on a linked list, so they can be safely
	 *	freed when the process exits or execs.
	 *
	 *	Because the hash table grows in powers of two, the
	 *	total size of all of the hash tables will be slightly
	 *	less than twice the size of the largest hash table.
	 */
	while (p->p_lwpfree == NULL) {
		uint_t dirsz = p->p_lwpdir_sz;
		lwpdir_t *new_dir;
		uint_t new_dirsz;
		lwpdir_t *ldp;
		tidhash_t *new_hash;
		uint_t new_hashsz;

		mutex_exit(&p->p_lock);

		/*
		 * Prepare to remember the old p_tidhash for later
		 * kmem_free()ing when the process exits or execs.
		 */
		if (ret_tidhash == NULL)
			ret_tidhash = kmem_zalloc(sizeof (ret_tidhash_t),
			    KM_SLEEP);
		if (old_dir != NULL)
			kmem_free(old_dir, old_dirsz * sizeof (*old_dir));
		if (old_hash != NULL)
			kmem_free(old_hash, old_hashsz * sizeof (*old_hash));

		new_dirsz = 2 * dirsz + 2;
		new_dir = kmem_zalloc(new_dirsz * sizeof (lwpdir_t), KM_SLEEP);
		for (ldp = new_dir, i = 1; i < new_dirsz; i++, ldp++)
			ldp->ld_next = ldp + 1;
		new_hashsz = (new_dirsz + 2) / 2;
		new_hash = kmem_zalloc(new_hashsz * sizeof (tidhash_t),
		    KM_SLEEP);

		mutex_enter(&p->p_lock);
		if (p == curproc)
			prbarrier(p);

		if (dirsz != p->p_lwpdir_sz || p->p_lwpfree != NULL) {
			/*
			 * Someone else beat us to it or some lwp exited.
			 * Set up to free our memory and take a lap.
			 */
			old_dir = new_dir;
			old_dirsz = new_dirsz;
			old_hash = new_hash;
			old_hashsz = new_hashsz;
		} else {
			/*
			 * For the benefit of lwp_hash_lookup_and_lock(),
			 * called from lwp_unpark(), which searches the
			 * tid hash table without acquiring p->p_lock,
			 * we must acquire all of the tid hash table
			 * locks before replacing p->p_tidhash.
			 */
			old_hash = p->p_tidhash;
			old_hashsz = p->p_tidhash_sz;
			for (i = 0; i < old_hashsz; i++) {
				mutex_enter(&old_hash[i].th_lock);
				mutex_enter(&new_hash[i].th_lock);
			}

			/*
			 * We simply hash in all of the old directory entries.
			 * This works because the old directory has no empty
			 * slots and the new hash table starts out empty.
			 * This reproduces the original directory ordering
			 * (required for /proc directory semantics).
			 */
			old_dir = p->p_lwpdir;
			old_dirsz = p->p_lwpdir_sz;
			p->p_lwpdir = new_dir;
			p->p_lwpfree = new_dir;
			p->p_lwpdir_sz = new_dirsz;
			for (ldp = old_dir, i = 0; i < old_dirsz; i++, ldp++)
				lwp_hash_in(p, ldp->ld_entry,
				    new_hash, new_hashsz, 0);

			/*
			 * Remember the old hash table along with all
			 * of the previously-remembered hash tables.
			 * We will free them at process exit or exec.
			 */
			ret_tidhash->rth_tidhash = old_hash;
			ret_tidhash->rth_tidhash_sz = old_hashsz;
			ret_tidhash->rth_next = p->p_ret_tidhash;
			p->p_ret_tidhash = ret_tidhash;

			/*
			 * Now establish the new tid hash table.
			 * As soon as we assign p->p_tidhash,
			 * code in lwp_unpark() can start using it.
			 */
			membar_producer();
			p->p_tidhash = new_hash;

			/*
			 * It is necessary that p_tidhash reach global
			 * visibility before p_tidhash_sz.  Otherwise,
			 * code in lwp_hash_lookup_and_lock() could
			 * index into the old p_tidhash using the new
			 * p_tidhash_sz and thereby access invalid data.
			 */
			membar_producer();
			p->p_tidhash_sz = new_hashsz;

			/*
			 * Release the locks; allow lwp_unpark() to carry on.
			 */
			for (i = 0; i < old_hashsz; i++) {
				mutex_exit(&old_hash[i].th_lock);
				mutex_exit(&new_hash[i].th_lock);
			}

			/*
			 * Avoid freeing these objects below.
			 */
			ret_tidhash = NULL;
			old_hash = NULL;
			old_hashsz = 0;
		}
	}

	/*
	 * Block the process against /proc while we manipulate p->p_tlist,
	 * unless lwp_create() was called by /proc for the PCAGENT operation.
	 * We want to do this early enough so that we don't drop p->p_lock
	 * until the thread is put on the p->p_tlist.
	 */
	if (p == curproc) {
		prbarrier(p);
		/*
		 * If the current lwp has been requested to stop, do so now.
		 * Otherwise we have a race condition between /proc attempting
		 * to stop the process and this thread creating a new lwp
		 * that was not seen when the /proc PCSTOP request was issued.
		 * We rely on stop() to call prbarrier(p) before returning.
		 */
		while ((curthread->t_proc_flag & TP_PRSTOP) &&
		    !ttolwp(curthread)->lwp_nostop) {
			/*
			 * We called pool_barrier_enter() before calling
			 * here to lwp_create(). We have to call
			 * pool_barrier_exit() before stopping.
			 */
			pool_barrier_exit();
			prbarrier(p);
			stop(PR_REQUESTED, 0);
			/*
			 * And we have to repeat the call to
			 * pool_barrier_enter after stopping.
			 */
			pool_barrier_enter();
			prbarrier(p);
		}

		/*
		 * If process is exiting, there could be a race between
		 * the agent lwp creation and the new lwp currently being
		 * created. So to prevent this race lwp creation is failed
		 * if the process is exiting.
		 */
		if (p->p_flag & (SEXITLWPS|SKILLED)) {
			err = 1;
			goto error;
		}

		/*
		 * Since we might have dropped p->p_lock, the
		 * lwp directory free list might have changed.
		 */
		if (p->p_lwpfree == NULL)
			goto grow;
	}

	kpreempt_disable();	/* can't grab cpu_lock here */

	/*
	 * Inherit processor and processor set bindings from curthread.
	 *
	 * For kernel LWPs, we do not inherit processor set bindings at
	 * process creation time (i.e. when p != curproc).  After the
	 * kernel process is created, any subsequent LWPs must be created
	 * by threads in the kernel process, at which point we *will*
	 * inherit processor set bindings.
	 */
	if (CLASS_KERNEL(cid) && p != curproc) {
		t->t_bind_cpu = binding = PBIND_NONE;
		t->t_cpupart = oldpart = &cp_default;
		t->t_bind_pset = PS_NONE;
		t->t_bindflag = (uchar_t)default_binding_mode;
	} else {
		binding = curthread->t_bind_cpu;
		t->t_bind_cpu = binding;
		oldpart = t->t_cpupart;
		t->t_cpupart = curthread->t_cpupart;
		t->t_bind_pset = curthread->t_bind_pset;
		t->t_bindflag = curthread->t_bindflag |
		    (uchar_t)default_binding_mode;
	}

	/*
	 * thread_create() initializes this thread's home lgroup to the root.
	 * Choose a more suitable lgroup, since this thread is associated
	 * with an lwp.
	 */
	ASSERT(oldpart != NULL);
	if (binding != PBIND_NONE && t->t_affinitycnt == 0) {
		t->t_bound_cpu = cpu[binding];
		if (t->t_lpl != t->t_bound_cpu->cpu_lpl)
			lgrp_move_thread(t, t->t_bound_cpu->cpu_lpl, 1);
	} else if (CLASS_KERNEL(cid)) {
		/*
		 * Kernel threads are always in the root lgrp.
		 */
		lgrp_move_thread(t,
		    &t->t_cpupart->cp_lgrploads[LGRP_ROOTID], 1);
	} else {
		lgrp_move_thread(t, lgrp_choose(t, t->t_cpupart), 1);
	}

	kpreempt_enable();

	/*
	 * make sure lpl points to our own partition
	 */
	ASSERT(t->t_lpl >= t->t_cpupart->cp_lgrploads);
	ASSERT(t->t_lpl < t->t_cpupart->cp_lgrploads +
	    t->t_cpupart->cp_nlgrploads);

	/*
	 * It is safe to point the thread to the new project without holding it
	 * since we're holding the target process' p_lock here and therefore
	 * we're guaranteed that it will not move to another project.
	 */
	newkpj = p->p_task->tk_proj;
	oldkpj = ttoproj(t);
	if (newkpj != oldkpj) {
		t->t_proj = newkpj;
		(void) project_hold(newkpj);
		project_rele(oldkpj);
	}

	if (cid != NOCLASS) {
		/*
		 * If the lwp is being created in the current process
		 * and matches the current thread's scheduling class,
		 * we should propagate the current thread's scheduling
		 * parameters by calling CL_FORK.  Otherwise just use
		 * the defaults by calling CL_ENTERCLASS.
		 */
		if (p != curproc || curthread->t_cid != cid) {
			err = CL_ENTERCLASS(t, cid, NULL, NULL, bufp);
			t->t_pri = pri;	/* CL_ENTERCLASS may have changed it */
			/*
			 * We don't call schedctl_set_cidpri(t) here
			 * because the schedctl data is not yet set
			 * up for the newly-created lwp.
			 */
		} else {
			t->t_clfuncs = &(sclass[cid].cl_funcs->thread);
			err = CL_FORK(curthread, t, bufp);
			t->t_cid = cid;
		}
		if (err) {
			atomic_inc_32(&p->p_zone->zone_ffmisc);
			goto error;
		} else {
			bufp = NULL;
		}
	}

	/*
	 * If we were given an lwpid then use it, else allocate one.
	 */
	if (lwpid != 0)
		t->t_tid = lwpid;
	else {
		/*
		 * lwp/thread id 0 is never valid; reserved for special checks.
		 * lwp/thread id 1 is reserved for the main thread.
		 * Start again at 2 when INT_MAX has been reached
		 * (id_t is a signed 32-bit integer).
		 */
		id_t prev_id = p->p_lwpid;	/* last allocated tid */

		do {			/* avoid lwpid duplication */
			if (p->p_lwpid == INT_MAX) {
				p->p_flag |= SLWPWRAP;
				p->p_lwpid = 1;
			}
			if ((t->t_tid = ++p->p_lwpid) == prev_id) {
				/*
				 * All lwpids are allocated; fail the request.
				 */
				err = 1;
				atomic_inc_32(&p->p_zone->zone_ffnoproc);
				goto error;
			}
			/*
			 * We only need to worry about colliding with an id
			 * that's already in use if this process has
			 * cycled through all available lwp ids.
			 */
			if ((p->p_flag & SLWPWRAP) == 0)
				break;
		} while (lwp_hash_lookup(p, t->t_tid) != NULL);
	}

	/*
	 * If this is a branded process, let the brand do any necessary lwp
	 * initialization.
	 */
	if (PROC_IS_BRANDED(p)) {
		if (BROP(p)->b_initlwp(lwp)) {
			err = 1;
			atomic_inc_32(&p->p_zone->zone_ffmisc);
			goto error;
		}
		branded = 1;
	}

	if (t->t_tid == 1) {
		kpreempt_disable();
		ASSERT(t->t_lpl != NULL);
		p->p_t1_lgrpid = t->t_lpl->lpl_lgrpid;
		kpreempt_enable();
		if (p->p_tr_lgrpid != LGRP_NONE &&
		    p->p_tr_lgrpid != p->p_t1_lgrpid) {
			lgrp_update_trthr_migrations(1);
		}
	}

	p->p_lwpcnt++;
	t->t_waitfor = -1;

	/*
	 * Turn microstate accounting on for thread if on for process.
	 */
	if (p->p_flag & SMSACCT)
		t->t_proc_flag |= TP_MSACCT;

	/*
	 * If the process has watchpoints, mark the new thread as such.
	 */
	if (pr_watch_active(p))
		watch_enable(t);

	/*
	 * The lwp is being created in the stopped state.
	 * We set all the necessary flags to indicate that fact here.
	 * We omit the TS_CREATE flag from t_schedflag so that the lwp
	 * cannot be set running until the caller is finished with it,
	 * even if lwp_continue() is called on it after we drop p->p_lock.
	 * When the caller is finished with the newly-created lwp,
	 * the caller must call lwp_create_done() to allow the lwp
	 * to be set running.  If the TP_HOLDLWP is left set, the
	 * lwp will suspend itself after reaching system call exit.
	 */
	init_mstate(t, LMS_STOPPED);
	t->t_proc_flag |= TP_HOLDLWP;
	t->t_schedflag |= (TS_ALLSTART & ~(TS_CSTART | TS_CREATE));
	t->t_whystop = PR_SUSPENDED;
	t->t_whatstop = SUSPEND_NORMAL;
	t->t_sig_check = 1;	/* ensure that TP_HOLDLWP is honored */

	/*
	 * Set system call processing flags in case tracing or profiling
	 * is set.  The first system call will evaluate these and turn
	 * them off if they aren't needed.
	 */
	t->t_pre_sys = 1;
	t->t_post_sys = 1;

	/*
	 * Insert the new thread into the list of all threads.
	 */
	if ((tx = p->p_tlist) == NULL) {
		t->t_back = t;
		t->t_forw = t;
		p->p_tlist = t;
	} else {
		t->t_forw = tx;
		t->t_back = tx->t_back;
		tx->t_back->t_forw = t;
		tx->t_back = t;
	}

	/*
	 * Insert the new lwp into an lwp directory slot position
	 * and into the lwpid hash table.
	 */
	lep->le_thread = t;
	lep->le_lwpid = t->t_tid;
	lep->le_start = t->t_start;
	lwp_hash_in(p, lep, p->p_tidhash, p->p_tidhash_sz, 1);

	lwp_fp_init(lwp);

	if (state == TS_RUN) {
		/*
		 * We set the new lwp running immediately.
		 */
		t->t_proc_flag &= ~TP_HOLDLWP;
		lwp_create_done(t);
	}

error:
	if (err) {
		if (CLASS_KERNEL(cid)) {
			/*
			 * This should only happen if a system process runs
			 * out of lwpids, which shouldn't occur.
			 */
			panic("Failed to create a system LWP");
		}
		/*
		 * We have failed to create an lwp, so decrement the number
		 * of lwps in the task and let the lgroup load averages know
		 * that this thread isn't going to show up.
		 */
		kpreempt_disable();
		lgrp_move_thread(t, NULL, 1);
		kpreempt_enable();

		ASSERT(MUTEX_HELD(&p->p_lock));
		mutex_enter(&p->p_zone->zone_nlwps_lock);
		p->p_task->tk_nlwps--;
		p->p_task->tk_proj->kpj_nlwps--;
		p->p_zone->zone_nlwps--;
		mutex_exit(&p->p_zone->zone_nlwps_lock);
		if (cid != NOCLASS && bufp != NULL)
			CL_FREE(cid, bufp);

		if (branded)
			BROP(p)->b_freelwp(lwp);

		mutex_exit(&p->p_lock);
		t->t_state = TS_FREE;
		thread_rele(t);

		/*
		 * We need to remove t from the list of all threads
		 * because thread_exit()/lwp_exit() isn't called on t.
		 */
		mutex_enter(&pidlock);
		ASSERT(t != t->t_next);		/* t0 never exits */
		t->t_next->t_prev = t->t_prev;
		t->t_prev->t_next = t->t_next;
		mutex_exit(&pidlock);

		thread_free(t);
		kmem_free(lep, sizeof (*lep));
		lwp = NULL;
	} else {
		mutex_exit(&p->p_lock);
	}

	if (old_dir != NULL)
		kmem_free(old_dir, old_dirsz * sizeof (*old_dir));
	if (old_hash != NULL)
		kmem_free(old_hash, old_hashsz * sizeof (*old_hash));
	if (ret_tidhash != NULL)
		kmem_free(ret_tidhash, sizeof (ret_tidhash_t));

	DTRACE_PROC1(lwp__create, kthread_t *, t);
	return (lwp);
}

/*
 * lwp_create_done() is called by the caller of lwp_create() to set the
 * newly-created lwp running after the caller has finished manipulating it.
 */
void
lwp_create_done(kthread_t *t)
{
	proc_t *p = ttoproc(t);

	ASSERT(MUTEX_HELD(&p->p_lock));

	/*
	 * We set the TS_CREATE and TS_CSTART flags and call setrun_locked().
	 * (The absence of the TS_CREATE flag prevents the lwp from running
	 * until we are finished with it, even if lwp_continue() is called on
	 * it by some other lwp in the process or elsewhere in the kernel.)
	 */
	thread_lock(t);
	ASSERT(t->t_state == TS_STOPPED && !(t->t_schedflag & TS_CREATE));
	/*
	 * If TS_CSTART is set, lwp_continue(t) has been called and
	 * has already incremented p_lwprcnt; avoid doing this twice.
	 */
	if (!(t->t_schedflag & TS_CSTART))
		p->p_lwprcnt++;
	t->t_schedflag |= (TS_CSTART | TS_CREATE);
	setrun_locked(t);
	thread_unlock(t);
}

/*
 * Copy an LWP's active templates, and clear the latest contracts.
 */
void
lwp_ctmpl_copy(klwp_t *dst, klwp_t *src)
{
	int i;

	for (i = 0; i < ct_ntypes; i++) {
		dst->lwp_ct_active[i] = ctmpl_dup(src->lwp_ct_active[i]);
		dst->lwp_ct_latest[i] = NULL;
	}
}

/*
 * Clear an LWP's contract template state.
 */
void
lwp_ctmpl_clear(klwp_t *lwp)
{
	ct_template_t *tmpl;
	int i;

	for (i = 0; i < ct_ntypes; i++) {
		if ((tmpl = lwp->lwp_ct_active[i]) != NULL) {
			ctmpl_free(tmpl);
			lwp->lwp_ct_active[i] = NULL;
		}

		if (lwp->lwp_ct_latest[i] != NULL) {
			contract_rele(lwp->lwp_ct_latest[i]);
			lwp->lwp_ct_latest[i] = NULL;
		}
	}
}

/*
 * Individual lwp exit.
 * If this is the last lwp, exit the whole process.
 */
void
lwp_exit(void)
{
	kthread_t *t = curthread;
	klwp_t *lwp = ttolwp(t);
	proc_t *p = ttoproc(t);

	ASSERT(MUTEX_HELD(&p->p_lock));

	mutex_exit(&p->p_lock);

#if defined(__sparc)
	/*
	 * Ensure that the user stack is fully abandoned..
	 */
	trash_user_windows();
#endif

	tsd_exit();			/* free thread specific data */

	kcpc_passivate();		/* Clean up performance counter state */

	pollcleanup();

	if (t->t_door)
		door_slam();

	if (t->t_schedctl != NULL)
		schedctl_lwp_cleanup(t);

	if (t->t_upimutex != NULL)
		upimutex_cleanup();

	/*
	 * Perform any brand specific exit processing, then release any
	 * brand data associated with the lwp
	 */
	if (PROC_IS_BRANDED(p))
		BROP(p)->b_lwpexit(lwp);

	lwp_pcb_exit();

	mutex_enter(&p->p_lock);
	lwp_cleanup();

	/*
	 * When this process is dumping core, its lwps are held here
	 * until the core dump is finished. Then exitlwps() is called
	 * again to release these lwps so that they can finish exiting.
	 */
	if (p->p_flag & SCOREDUMP)
		stop(PR_SUSPENDED, SUSPEND_NORMAL);

	/*
	 * Block the process against /proc now that we have really acquired
	 * p->p_lock (to decrement p_lwpcnt and manipulate p_tlist at least).
	 */
	prbarrier(p);

	/*
	 * Call proc_exit() if this is the last non-daemon lwp in the process.
	 */
	if (!(t->t_proc_flag & TP_DAEMON) &&
	    p->p_lwpcnt == p->p_lwpdaemon + 1) {
		mutex_exit(&p->p_lock);
		if (proc_exit(CLD_EXITED, 0) == 0) {
			/* Restarting init. */
			return;
		}

		/*
		 * proc_exit() returns a non-zero value when some other
		 * lwp got there first.  We just have to continue in
		 * lwp_exit().
		 */
		mutex_enter(&p->p_lock);
		ASSERT(curproc->p_flag & SEXITLWPS);
		prbarrier(p);
	}

	DTRACE_PROC(lwp__exit);

	/*
	 * If the lwp is a detached lwp or if the process is exiting,
	 * remove (lwp_hash_out()) the lwp from the lwp directory.
	 * Otherwise null out the lwp's le_thread pointer in the lwp
	 * directory so that other threads will see it as a zombie lwp.
	 */
	prlwpexit(t);		/* notify /proc */
	if (!(t->t_proc_flag & TP_TWAIT) || (p->p_flag & SEXITLWPS))
		lwp_hash_out(p, t->t_tid);
	else {
		ASSERT(!(t->t_proc_flag & TP_DAEMON));
		p->p_lwpdir[t->t_dslot].ld_entry->le_thread = NULL;
		p->p_zombcnt++;
		cv_broadcast(&p->p_lwpexit);
	}
	if (t->t_proc_flag & TP_DAEMON) {
		p->p_lwpdaemon--;
		t->t_proc_flag &= ~TP_DAEMON;
	}
	t->t_proc_flag &= ~TP_TWAIT;

	/*
	 * Maintain accurate lwp count for task.max-lwps resource control.
	 */
	mutex_enter(&p->p_zone->zone_nlwps_lock);
	p->p_task->tk_nlwps--;
	p->p_task->tk_proj->kpj_nlwps--;
	p->p_zone->zone_nlwps--;
	mutex_exit(&p->p_zone->zone_nlwps_lock);

	CL_EXIT(t);		/* tell the scheduler that t is exiting */
	ASSERT(p->p_lwpcnt != 0);
	p->p_lwpcnt--;

	/*
	 * If all remaining non-daemon lwps are waiting in lwp_wait(),
	 * wake them up so someone can return EDEADLK.
	 * (See the block comment preceeding lwp_wait().)
	 */
	if (p->p_lwpcnt == p->p_lwpdaemon + (p->p_lwpwait - p->p_lwpdwait))
		cv_broadcast(&p->p_lwpexit);

	t->t_proc_flag |= TP_LWPEXIT;
	term_mstate(t);

#ifndef NPROBE
	/* Kernel probe */
	if (t->t_tnf_tpdp)
		tnf_thread_exit();
#endif /* NPROBE */

	t->t_forw->t_back = t->t_back;
	t->t_back->t_forw = t->t_forw;
	if (t == p->p_tlist)
		p->p_tlist = t->t_forw;

	/*
	 * Clean up the signal state.
	 */
	if (t->t_sigqueue != NULL)
		sigdelq(p, t, 0);
	if (lwp->lwp_curinfo != NULL) {
		siginfofree(lwp->lwp_curinfo);
		lwp->lwp_curinfo = NULL;
	}

	/*
	 * If we have spymaster information (that is, if we're an agent LWP),
	 * free that now.
	 */
	if (lwp->lwp_spymaster != NULL) {
		kmem_free(lwp->lwp_spymaster, sizeof (psinfo_t));
		lwp->lwp_spymaster = NULL;
	}

	thread_rele(t);

	/*
	 * Terminated lwps are associated with process zero and are put onto
	 * death-row by resume().  Avoid preemption after resetting t->t_procp.
	 */
	t->t_preempt++;

	if (t->t_ctx != NULL)
		exitctx(t);
	if (p->p_pctx != NULL)
		exitpctx(p);

	t->t_procp = &p0;

	/*
	 * Notify the HAT about the change of address space
	 */
	hat_thread_exit(t);
	/*
	 * When this is the last running lwp in this process and some lwp is
	 * waiting for this condition to become true, or this thread was being
	 * suspended, then the waiting lwp is awakened.
	 *
	 * Also, if the process is exiting, we may have a thread waiting in
	 * exitlwps() that needs to be notified.
	 */
	if (--p->p_lwprcnt == 0 || (t->t_proc_flag & TP_HOLDLWP) ||
	    (p->p_flag & SEXITLWPS))
		cv_broadcast(&p->p_holdlwps);

	/*
	 * Need to drop p_lock so we can reacquire pidlock.
	 */
	mutex_exit(&p->p_lock);
	mutex_enter(&pidlock);

	ASSERT(t != t->t_next);		/* t0 never exits */
	t->t_next->t_prev =