17c478bdstevel@tonic-gate/*
27c478bdstevel@tonic-gate * CDDL HEADER START
37c478bdstevel@tonic-gate *
47c478bdstevel@tonic-gate * The contents of this file are subject to the terms of the
53470957raf * Common Development and Distribution License (the "License").
63470957raf * You may not use this file except in compliance with the License.
77c478bdstevel@tonic-gate *
87c478bdstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bdstevel@tonic-gate * or http://www.opensolaris.org/os/licensing.
107c478bdstevel@tonic-gate * See the License for the specific language governing permissions
117c478bdstevel@tonic-gate * and limitations under the License.
127c478bdstevel@tonic-gate *
137c478bdstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each
147c478bdstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bdstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the
167c478bdstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying
177c478bdstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bdstevel@tonic-gate *
197c478bdstevel@tonic-gate * CDDL HEADER END
207c478bdstevel@tonic-gate */
210ec5755raf
227c478bdstevel@tonic-gate/*
2323a1cceRoger A. Faulkner * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved.
247c478bdstevel@tonic-gate */
258a7aa2aYuri Pankov
264f364e7Robert Mustacchi/*
27fc2512cRobert Mustacchi * Copyright 2016 Joyent, Inc.
288a7aa2aYuri Pankov * Copyright 2018 Nexenta Systems, Inc.
294f364e7Robert Mustacchi */
307c478bdstevel@tonic-gate
317c478bdstevel@tonic-gate#ifndef _THR_UBERDATA_H
327c478bdstevel@tonic-gate#define	_THR_UBERDATA_H
337c478bdstevel@tonic-gate
347c478bdstevel@tonic-gate#include <stdlib.h>
357c478bdstevel@tonic-gate#include <unistd.h>
367c478bdstevel@tonic-gate#include <sys/types.h>
377c478bdstevel@tonic-gate#include <fcntl.h>
387c478bdstevel@tonic-gate#include <string.h>
397c478bdstevel@tonic-gate#include <signal.h>
407c478bdstevel@tonic-gate#include <ucontext.h>
417c478bdstevel@tonic-gate#include <thread.h>
427c478bdstevel@tonic-gate#include <pthread.h>
437257d1braf#include <atomic.h>
447c478bdstevel@tonic-gate#include <link.h>
457c478bdstevel@tonic-gate#include <sys/resource.h>
467c478bdstevel@tonic-gate#include <sys/lwp.h>
477c478bdstevel@tonic-gate#include <errno.h>
487c478bdstevel@tonic-gate#include <sys/asm_linkage.h>
497c478bdstevel@tonic-gate#include <sys/regset.h>
507c478bdstevel@tonic-gate#include <sys/fcntl.h>
517c478bdstevel@tonic-gate#include <sys/mman.h>
527c478bdstevel@tonic-gate#include <synch.h>
537c478bdstevel@tonic-gate#include <door.h>
547c478bdstevel@tonic-gate#include <limits.h>
557c478bdstevel@tonic-gate#include <sys/synch32.h>
567c478bdstevel@tonic-gate#include <schedctl.h>
577c478bdstevel@tonic-gate#include <sys/priocntl.h>
587c478bdstevel@tonic-gate#include <thread_db.h>
59f841f6araf#include <setjmp.h>
60ab61854John Levon#include <sys/thread.h>
617c478bdstevel@tonic-gate#include "libc_int.h"
627c478bdstevel@tonic-gate#include "tdb_agent.h"
63f841f6araf#include "thr_debug.h"
643470957raf
657c478bdstevel@tonic-gate/*
667c478bdstevel@tonic-gate * This is an implementation-specific include file for threading support.
677c478bdstevel@tonic-gate * It is not to be seen by the clients of the library.
687c478bdstevel@tonic-gate *
697c478bdstevel@tonic-gate * This file also describes uberdata in libc.
707c478bdstevel@tonic-gate *
717c478bdstevel@tonic-gate * The term "uberdata" refers to data that is unique and visible across
727c478bdstevel@tonic-gate * all link maps.  The name is meant to imply that such data is truly
737c478bdstevel@tonic-gate * global, not just locally global to a particular link map.
747c478bdstevel@tonic-gate *
757c478bdstevel@tonic-gate * See the Linker and Libraries Guide for a full description of alternate
767c478bdstevel@tonic-gate * link maps and how they are set up and used.
777c478bdstevel@tonic-gate *
787c478bdstevel@tonic-gate * Alternate link maps implement multiple global namespaces within a single
797c478bdstevel@tonic-gate * process.  There may be multiple instances of identical dynamic libraries
807c478bdstevel@tonic-gate * loaded in a process's address space at the same time, each on a different
817c478bdstevel@tonic-gate * link map (as determined by the dynamic linker), each with its own set of
827c478bdstevel@tonic-gate * global variables.  Which particular instance of a global variable is seen
837c478bdstevel@tonic-gate * by a thread running in the process is determined by the link map on which
847c478bdstevel@tonic-gate * the thread happens to be executing at the time.
857c478bdstevel@tonic-gate *
867c478bdstevel@tonic-gate * However, there are aspects of a process that are unique across all
877c478bdstevel@tonic-gate * link maps, in particular the structures used to implement threads
887c478bdstevel@tonic-gate * of control (in Sparc terminology, there is only one %g7 regardless
897c478bdstevel@tonic-gate * of the link map on which the thread is executing).
907c478bdstevel@tonic-gate *
917c478bdstevel@tonic-gate * All uberdata is referenced from a base pointer in the thread's ulwp_t
927c478bdstevel@tonic-gate * structure (which is also uberdata).  All allocations and deallocations
937c478bdstevel@tonic-gate * of uberdata are made via the uberdata-aware lmalloc() and lfree()
947c478bdstevel@tonic-gate * interfaces (malloc() and free() are simply locally-global).
957c478bdstevel@tonic-gate */
967c478bdstevel@tonic-gate
977c478bdstevel@tonic-gate/*
987c478bdstevel@tonic-gate * Special libc-private access to errno.
997c478bdstevel@tonic-gate * We do this so that references to errno do not invoke the dynamic linker.
1007c478bdstevel@tonic-gate */
1017c478bdstevel@tonic-gate#undef errno
1027c478bdstevel@tonic-gate#define	errno (*curthread->ul_errnop)
1037c478bdstevel@tonic-gate
1047c478bdstevel@tonic-gate/*
1057c478bdstevel@tonic-gate * See <sys/synch32.h> for the reasons for these values
1067c478bdstevel@tonic-gate * and why they are different for sparc and intel.
1077c478bdstevel@tonic-gate */
1087c478bdstevel@tonic-gate#if defined(__sparc)
10931db3c2raf
1107c478bdstevel@tonic-gate/* lock.lock64.pad[x]	   4 5 6 7 */
1117c478bdstevel@tonic-gate#define	LOCKMASK	0xff000000
1127c478bdstevel@tonic-gate#define	WAITERMASK	0x000000ff
1135d1dd9araf#define	SPINNERMASK	0x00ff0000
1145d1dd9araf#define	SPINNERSHIFT	16
1157c478bdstevel@tonic-gate#define	WAITER		0x00000001
1167c478bdstevel@tonic-gate#define	LOCKSET		0xff
1177c478bdstevel@tonic-gate#define	LOCKCLEAR	0
11831db3c2raf
11931db3c2raf#define	PIDSHIFT	32
12031db3c2raf#define	LOCKMASK64	0xffffffffff000000ULL
12131db3c2raf#define	LOCKBYTE64	0x00000000ff000000ULL
12231db3c2raf#define	WAITERMASK64	0x00000000000000ffULL
12331db3c2raf#define	SPINNERMASK64	0x0000000000ff0000ULL
12431db3c2raf
1250ec5755raf#elif defined(__x86)
12631db3c2raf
1277c478bdstevel@tonic-gate/* lock.lock64.pad[x]	   7 6 5 4 */
1287c478bdstevel@tonic-gate#define	LOCKMASK	0xff000000
1297c478bdstevel@tonic-gate#define	WAITERMASK	0x00ff0000
1305d1dd9araf#define	SPINNERMASK	0x0000ff00
1315d1dd9araf#define	SPINNERSHIFT	8
1327c478bdstevel@tonic-gate#define	WAITER		0x00010000
1337c478bdstevel@tonic-gate#define	LOCKSET		0x01
1347c478bdstevel@tonic-gate#define	LOCKCLEAR	0
13531db3c2raf
13631db3c2raf#define	PIDSHIFT	0
13731db3c2raf#define	LOCKMASK64	0xff000000ffffffffULL
13831db3c2raf#define	LOCKBYTE64	0x0100000000000000ULL
13931db3c2raf#define	WAITERMASK64	0x00ff000000000000ULL
14031db3c2raf#define	SPINNERMASK64	0x0000ff0000000000ULL
14131db3c2raf
1427c478bdstevel@tonic-gate#else
1430ec5755raf#error "neither __sparc nor __x86 is defined"
1447c478bdstevel@tonic-gate#endif
1457c478bdstevel@tonic-gate
1467c478bdstevel@tonic-gate/*
1477c478bdstevel@tonic-gate * Fetch the owner of a USYNC_THREAD mutex.
1487c478bdstevel@tonic-gate * Don't use this with process-shared mutexes;
1497c478bdstevel@tonic-gate * the owing thread may be in a different process.
1507c478bdstevel@tonic-gate */
1517c478bdstevel@tonic-gate#define	MUTEX_OWNER(mp)	((ulwp_t *)(uintptr_t)(mp)->mutex_owner)
1527c478bdstevel@tonic-gate
1537c478bdstevel@tonic-gate/*
154883492draf * Test if a thread owns a process-private (USYNC_THREAD) mutex.
155883492draf * This is inappropriate for a process-shared (USYNC_PROCESS) mutex.
1567c478bdstevel@tonic-gate * The 'mp' argument must not have side-effects since it is evaluated twice.
1577c478bdstevel@tonic-gate */
1587c478bdstevel@tonic-gate#define	MUTEX_OWNED(mp, thrp)	\
1597c478bdstevel@tonic-gate	((mp)->mutex_lockw != 0 && MUTEX_OWNER(mp) == thrp)
1607c478bdstevel@tonic-gate
1617c478bdstevel@tonic-gate
1627c478bdstevel@tonic-gate/*
1637c478bdstevel@tonic-gate * uberflags.uf_tdb_register_sync is an interface with libc_db to enable the
1647c478bdstevel@tonic-gate * collection of lock statistics by a debugger or other collecting tool.
1657c478bdstevel@tonic-gate *
1667c478bdstevel@tonic-gate * uberflags.uf_thread_error_detection is set by an environment variable:
1677c478bdstevel@tonic-gate *	_THREAD_ERROR_DETECTION
1687c478bdstevel@tonic-gate *		0 == no detection of locking primitive errors.
1697c478bdstevel@tonic-gate *		1 == detect errors and issue a warning message.
1707c478bdstevel@tonic-gate *		2 == detect errors, issue a warning message, and dump core.
1717c478bdstevel@tonic-gate *
1727c478bdstevel@tonic-gate * We bundle these together in uberflags.uf_trs_ted to make a test of either
1737c478bdstevel@tonic-gate * being non-zero a single memory reference (for speed of mutex_lock(), etc).
1747c478bdstevel@tonic-gate *
1757c478bdstevel@tonic-gate * uberflags.uf_mt is set non-zero when the first thread (in addition
1767c478bdstevel@tonic-gate * to the main thread) is created.
1777c478bdstevel@tonic-gate *
1787c478bdstevel@tonic-gate * We bundle all these flags together in uberflags.uf_all to make a test
1797c478bdstevel@tonic-gate * of any being non-zero a single memory reference (again, for speed).
1807c478bdstevel@tonic-gate */
1817c478bdstevel@tonic-gatetypedef union {
1827c478bdstevel@tonic-gate	int	uf_all;			/* combined all flags */
1837c478bdstevel@tonic-gate	struct {
1847c478bdstevel@tonic-gate		short	h_pad;
1857c478bdstevel@tonic-gate		short	h_trs_ted;	/* combined reg sync & error detect */
1867c478bdstevel@tonic-gate	} uf_h;
1877c478bdstevel@tonic-gate	struct {
1887c478bdstevel@tonic-gate		char	x_mt;
1897c478bdstevel@tonic-gate		char	x_pad;
1907c478bdstevel@tonic-gate		char	x_tdb_register_sync;
1917c478bdstevel@tonic-gate		char	x_thread_error_detection;
1927c478bdstevel@tonic-gate	} uf_x;
1937c478bdstevel@tonic-gate} uberflags_t;
1947c478bdstevel@tonic-gate
1957c478bdstevel@tonic-gate#define	uf_mt				uf_x.x_mt
1967c478bdstevel@tonic-gate#define	uf_tdb_register_sync		uf_x.x_tdb_register_sync
1977c478bdstevel@tonic-gate#define	uf_thread_error_detection	uf_x.x_thread_error_detection
1987c478bdstevel@tonic-gate#define	uf_trs_ted			uf_h.h_trs_ted	/* both of the above */
1997c478bdstevel@tonic-gate
2007c478bdstevel@tonic-gate/*
2017c478bdstevel@tonic-gate * NOTE WELL:
2027c478bdstevel@tonic-gate * To enable further optimization, the "ul_schedctl_called" member
2037c478bdstevel@tonic-gate * of the ulwp_t structure (below) serves double-duty:
2047c478bdstevel@tonic-gate *	1. If NULL, it means that the thread must call __schedctl()
2057c478bdstevel@tonic-gate *	   to set up its schedctl mappings before acquiring a mutex.
2067c478bdstevel@tonic-gate *	   This is required by the implementation of adaptive mutex locking.
2077c478bdstevel@tonic-gate *	2. If non-NULL, it points to uberdata.uberflags, so that tests of
2087c478bdstevel@tonic-gate *	   uberflags can be made without additional memory references.
2097c478bdstevel@tonic-gate * This allows the common case of _mutex_lock() and _mutex_unlock() for
2107c478bdstevel@tonic-gate * USYNC_THREAD mutexes with no error detection and no lock statistics
2117c478bdstevel@tonic-gate * to be optimized for speed.
2127c478bdstevel@tonic-gate */
2137c478bdstevel@tonic-gate
2147c478bdstevel@tonic-gate/* double the default stack size for 64-bit processes */
2157c478bdstevel@tonic-gate#ifdef _LP64
2167c478bdstevel@tonic-gate#define	MINSTACK	(8 * 1024)
2177c478bdstevel@tonic-gate#define	DEFAULTSTACK	(2 * 1024 * 1024)
2187c478bdstevel@tonic-gate#else
2197c478bdstevel@tonic-gate#define	MINSTACK	(4 * 1024)
2207c478bdstevel@tonic-gate#define	DEFAULTSTACK	(1024 * 1024)
2217c478bdstevel@tonic-gate#endif
2227c478bdstevel@tonic-gate
2237c478bdstevel@tonic-gate#define	MUTEX_TRY	0
2247c478bdstevel@tonic-gate#define	MUTEX_LOCK	1
225d4204c8raf#define	MUTEX_NOCEIL	0x40
2267c478bdstevel@tonic-gate
2270ec5755raf#if defined(__x86)
2287c478bdstevel@tonic-gate
2297c478bdstevel@tonic-gatetypedef struct {	/* structure returned by fnstenv */
2307c478bdstevel@tonic-gate	int	fctrl;		/* control word */
2317c478bdstevel@tonic-gate	int	fstat;		/* status word (flags, etc) */
2327c478bdstevel@tonic-gate	int	ftag;		/* tag of which regs busy */
2337c478bdstevel@tonic-gate	int	misc[4];	/* other stuff, 28 bytes total */
2347c478bdstevel@tonic-gate} fpuenv_t;
2357c478bdstevel@tonic-gate
2367c478bdstevel@tonic-gate#ifdef _SYSCALL32
2377c478bdstevel@tonic-gatetypedef fpuenv_t fpuenv32_t;
2387c478bdstevel@tonic-gate#endif	/* _SYSCALL32 */
2397c478bdstevel@tonic-gate
2407c478bdstevel@tonic-gate#elif defined(__sparc)
2417c478bdstevel@tonic-gate
2427c478bdstevel@tonic-gatetypedef struct {	/* fp state structure */
2437c478bdstevel@tonic-gate	greg_t	fsr;
2447c478bdstevel@tonic-gate	greg_t	fpu_en;
2457c478bdstevel@tonic-gate} fpuenv_t;
2467c478bdstevel@tonic-gate
2477c478bdstevel@tonic-gate#ifdef _SYSCALL32
2487c478bdstevel@tonic-gatetypedef struct {
2497c478bdstevel@tonic-gate	greg32_t	fsr;
2507c478bdstevel@tonic-gate	greg32_t	fpu_en;
2517c478bdstevel@tonic-gate} fpuenv32_t;
2527c478bdstevel@tonic-gate#endif	/* _SYSCALL32 */
2537c478bdstevel@tonic-gate
2540ec5755raf#endif	/* __x86 */
2557c478bdstevel@tonic-gate
2560ec5755raf#if defined(__x86)
2577c478bdstevel@tonic-gateextern	void	ht_pause(void);		/* "pause" instruction */
2587c478bdstevel@tonic-gate#define	SMT_PAUSE()	ht_pause()
259b1593d5Jason Beloro#elif defined(SMT_PAUSE_FUNCTION)
260b1593d5Jason Beloroextern	void	SMT_PAUSE_FUNCTION(void);
261b1593d5Jason Beloro#define	SMT_PAUSE()	SMT_PAUSE_FUNCTION()
2627c478bdstevel@tonic-gate#else
263e2c5185Christopher Kiick#define	SMT_PAUSE()	smt_pause()
2640ec5755raf#endif	/* __x86 */
2657c478bdstevel@tonic-gate
2667c478bdstevel@tonic-gate/*
2677c478bdstevel@tonic-gate * Cleanup handler related data.
2687c478bdstevel@tonic-gate * This structure is exported as _cleanup_t in pthread.h.
2697c478bdstevel@tonic-gate * pthread.h exports only the size of this structure, so check
2707c478bdstevel@tonic-gate * _cleanup_t in pthread.h before making any change here.
2717c478bdstevel@tonic-gate */
2727c478bdstevel@tonic-gatetypedef struct __cleanup {
2737c478bdstevel@tonic-gate	struct __cleanup *next;		/* pointer to next handler */
2747c478bdstevel@tonic-gate	caddr_t	fp;			/* current frame pointer */
2757c478bdstevel@tonic-gate	void	(*func)(void *);	/* cleanup handler address */
2767c478bdstevel@tonic-gate	void	*arg;			/* handler's argument */
2777c478bdstevel@tonic-gate} __cleanup_t;
2787c478bdstevel@tonic-gate
2797c478bdstevel@tonic-gate/*
2807c478bdstevel@tonic-gate * Thread-Specific Data (TSD)
2817c478bdstevel@tonic-gate * TSD_NFAST includes the invalid key zero, so there
2827c478bdstevel@tonic-gate * are really only (TSD_NFAST - 1) fast key slots.
2837c478bdstevel@tonic-gate */
2847c478bdstevel@tonic-gatetypedef	void (*PFrV)(void *);
2857c478bdstevel@tonic-gate#define	TSD_UNALLOCATED	((PFrV)1)
2867c478bdstevel@tonic-gate#define	TSD_NFAST	9
2877c478bdstevel@tonic-gate
2887c478bdstevel@tonic-gate/*
2897c478bdstevel@tonic-gate * The tsd union is designed to burn a little memory (9 words) to make
2907c478bdstevel@tonic-gate * lookups blindingly fast.  Note that tsd_nalloc could be placed at the
2917c478bdstevel@tonic-gate * end of the pad region to increase the likelihood that it falls on the
2927c478bdstevel@tonic-gate * same cache line as the data.
2937c478bdstevel@tonic-gate */
2947c478bdstevel@tonic-gatetypedef union tsd {
2957c478bdstevel@tonic-gate	uint_t tsd_nalloc;		/* Amount of allocated storage */
2967c478bdstevel@tonic-gate	void *tsd_pad[TSD_NFAST];
2977c478bdstevel@tonic-gate	void *tsd_data[1];
2987c478bdstevel@tonic-gate} tsd_t;
2997c478bdstevel@tonic-gate
3007c478bdstevel@tonic-gatetypedef struct {
3017c478bdstevel@tonic-gate	mutex_t tsdm_lock;		/* Lock protecting the data */
3027c478bdstevel@tonic-gate	uint_t tsdm_nkeys;		/* Number of allocated keys */
3037c478bdstevel@tonic-gate	uint_t tsdm_nused;		/* Number of used keys */
3047c478bdstevel@tonic-gate	PFrV *tsdm_destro;		/* Per-key destructors */
3057c478bdstevel@tonic-gate	char tsdm_pad[64 -		/* pad to 64 bytes */
3067c478bdstevel@tonic-gate		(sizeof (mutex_t) + 2 * sizeof (uint_t) + sizeof (PFrV *))];
3077c478bdstevel@tonic-gate} tsd_metadata_t;
3087c478bdstevel@tonic-gate
3097c478bdstevel@tonic-gate#ifdef _SYSCALL32
3107c478bdstevel@tonic-gatetypedef union tsd32 {
3117c478bdstevel@tonic-gate	uint_t tsd_nalloc;		/* Amount of allocated storage */
3127c478bdstevel@tonic-gate	caddr32_t tsd_pad[TSD_NFAST];
3137c478bdstevel@tonic-gate	caddr32_t tsd_data[1];
3147c478bdstevel@tonic-gate} tsd32_t;
3157c478bdstevel@tonic-gate
3167c478bdstevel@tonic-gatetypedef struct {
3177c478bdstevel@tonic-gate	mutex_t tsdm_lock;		/* Lock protecting the data */
3187c478bdstevel@tonic-gate	uint_t tsdm_nkeys;		/* Number of allocated keys */
3197c478bdstevel@tonic-gate	uint_t tsdm_nused;		/* Number of used keys */
3207c478bdstevel@tonic-gate	caddr32_t tsdm_destro;		/* Per-key destructors */
3217c478bdstevel@tonic-gate	char tsdm_pad[64 -		/* pad to 64 bytes */
3227c478bdstevel@tonic-gate		(sizeof (mutex_t) + 2 * sizeof (uint_t) + sizeof (caddr32_t))];
3237c478bdstevel@tonic-gate} tsd_metadata32_t;
3247c478bdstevel@tonic-gate#endif	/* _SYSCALL32 */
3257c478bdstevel@tonic-gate
3267c478bdstevel@tonic-gate
3277c478bdstevel@tonic-gate/*
3287c478bdstevel@tonic-gate * Thread-Local Storage (TLS)
3297c478bdstevel@tonic-gate */
3307c478bdstevel@tonic-gatetypedef struct {
3317c478bdstevel@tonic-gate	void		*tls_data;
3327c478bdstevel@tonic-gate	size_t		tls_size;
3337c478bdstevel@tonic-gate} tls_t;
3347c478bdstevel@tonic-gate
3357c478bdstevel@tonic-gatetypedef struct {
3367c478bdstevel@tonic-gate	mutex_t	tls_lock;		/* Lock protecting the data */
3377c478bdstevel@tonic-gate	tls_t	tls_modinfo;		/* Root of all TLS_modinfo data */
3387c478bdstevel@tonic-gate	tls_t	static_tls;		/* Template for static TLS */
3397c478bdstevel@tonic-gate	char	tls_pad[64 -		/* pad to 64 bytes */
3407c478bdstevel@tonic-gate		(sizeof (mutex_t) + 2 * sizeof (tls_t))];
3417c478bdstevel@tonic-gate} tls_metadata_t;
3427c478bdstevel@tonic-gate
3437c478bdstevel@tonic-gate#ifdef _SYSCALL32
3447c478bdstevel@tonic-gatetypedef struct {
3457c478bdstevel@tonic-gate	caddr32_t	tls_data;
3467c478bdstevel@tonic-gate	size32_t	tls_size;
3477c478bdstevel@tonic-gate} tls32_t;
3487c478bdstevel@tonic-gate
3497c478bdstevel@tonic-gatetypedef struct {
3507c478bdstevel@tonic-gate	mutex_t	tls_lock;		/* Lock protecting the data */
3517c478bdstevel@tonic-gate	tls32_t	tls_modinfo;		/* Root of all TLS_modinfo data */
3527c478bdstevel@tonic-gate	tls32_t	static_tls;		/* Template for static TLS */
3537c478bdstevel@tonic-gate	char	tls_pad[64 -		/* pad to 64 bytes */
3547c478bdstevel@tonic-gate		(sizeof (mutex_t) + 2 * sizeof (tls32_t))];
3557c478bdstevel@tonic-gate} tls_metadata32_t;
3567c478bdstevel@tonic-gate#endif	/* _SYSCALL32 */
3577c478bdstevel@tonic-gate
3587c478bdstevel@tonic-gate
3597c478bdstevel@tonic-gate/*
360d4204c8raf * Sleep queue root for USYNC_THREAD condvars and mutexes.
361d4204c8raf * There is a default queue root for each queue head (see below).
362d4204c8raf * Also, each ulwp_t contains a queue root that can be used
363d4204c8raf * when the thread is enqueued on the queue, if necessary
364d4204c8raf * (when more than one wchan hashes to the same queue head).
365d4204c8raf */
366d4204c8raftypedef struct queue_root {
367d4204c8raf	struct queue_root	*qr_next;
368d4204c8raf	struct queue_root	*qr_prev;
369d4204c8raf	struct ulwp		*qr_head;
370d4204c8raf	struct ulwp		*qr_tail;
371d4204c8raf	void			*qr_wchan;
372d4204c8raf	uint32_t		qr_rtcount;
373d4204c8raf	uint32_t		qr_qlen;
374d4204c8raf	uint32_t		qr_qmax;
375d4204c8raf} queue_root_t;
376d4204c8raf
377d4204c8raf#ifdef _SYSCALL32
378d4204c8raftypedef struct queue_root32 {
379d4204c8raf	caddr32_t		qr_next;
380d4204c8raf	caddr32_t		qr_prev;
381d4204c8raf	caddr32_t		qr_head;
382d4204c8raf	caddr32_t		qr_tail;
383d4204c8raf	caddr32_t		qr_wchan;
384d4204c8raf	uint32_t		qr_rtcount;
385d4204c8raf	uint32_t		qr_qlen;
386d4204c8raf	uint32_t		qr_qmax;
387d4204c8raf} queue_root32_t;
388d4204c8raf#endif
389d4204c8raf
390d4204c8raf/*
391d4204c8raf * Sleep queue heads for USYNC_THREAD condvars and mutexes.
392d4204c8raf * The size and alignment is 128 bytes to reduce cache conflicts.
393d4204c8raf * Each queue head points to a list of queue roots, defined above.
394d4204c8raf * Each queue head contains a default queue root for use when only one
395d4204c8raf * is needed.  It is always at the tail of the queue root hash chain.
3967c478bdstevel@tonic-gate */
3977c478bdstevel@tonic-gatetypedef union {
398d4204c8raf	uint64_t		qh_64[16];
3997c478bdstevel@tonic-gate	struct {
4007c478bdstevel@tonic-gate		mutex_t		q_lock;
4017c478bdstevel@tonic-gate		uint8_t		q_qcnt;
402d4204c8raf		uint8_t		q_type;		/* MX or CV */
403d4204c8raf		uint8_t		q_pad1[2];
404d4204c8raf		uint32_t	q_lockcount;
4057c478bdstevel@tonic-gate		uint32_t	q_qlen;
4067c478bdstevel@tonic-gate		uint32_t	q_qmax;
407d4204c8raf		void		*q_wchan;	/* valid only while locked */
408d4204c8raf		struct queue_root *q_root;	/* valid only while locked */
409d4204c8raf		struct queue_root *q_hlist;
410d4204c8raf#if !defined(_LP64)
411d4204c8raf		caddr_t		q_pad2[3];
412d4204c8raf#endif
413d4204c8raf		queue_root_t	q_def_root;
414d4204c8raf		uint32_t	q_hlen;
415d4204c8raf		uint32_t	q_hmax;
4167c478bdstevel@tonic-gate	} qh_qh;
4177c478bdstevel@tonic-gate} queue_head_t;
4187c478bdstevel@tonic-gate
4197c478bdstevel@tonic-gate#define	qh_lock		qh_qh.q_lock
4207c478bdstevel@tonic-gate#define	qh_qcnt		qh_qh.q_qcnt
421d4204c8raf#define	qh_type		qh_qh.q_type
422d4204c8raf#if defined(THREAD_DEBUG)
4237c478bdstevel@tonic-gate#define	qh_lockcount	qh_qh.q_lockcount
4247c478bdstevel@tonic-gate#define	qh_qlen		qh_qh.q_qlen
4257c478bdstevel@tonic-gate#define	qh_qmax		qh_qh.q_qmax
426d4204c8raf#endif
427d4204c8raf#define	qh_wchan	qh_qh.q_wchan
428d4204c8raf#define	qh_root		qh_qh.q_root
429d4204c8raf#define	qh_hlist	qh_qh.q_hlist
430d4204c8raf#define	qh_def_root	qh_qh.q_def_root
431d4204c8raf#define	qh_hlen		qh_qh.q_hlen
432d4204c8raf#define	qh_hmax		qh_qh.q_hmax
433d4204c8raf
434d4204c8raf/* queue types passed to queue_lock() */
4357c478bdstevel@tonic-gate#define	MX	0
4367c478bdstevel@tonic-gate#define	CV	1
437883492draf#define	QHASHSHIFT	9			/* number of hashing bits */
438883492draf#define	QHASHSIZE	(1 << QHASHSHIFT)	/* power of 2 (1<<9 == 512) */
439883492draf#define	QUEUE_HASH(wchan, type)	((uint_t)			\
440883492draf	((((uintptr_t)(wchan) >> 3)				\
441883492draf	^ ((uintptr_t)(wchan) >> (QHASHSHIFT + 3)))		\
4427c478bdstevel@tonic-gate	& (QHASHSIZE - 1)) + (((type) == MX)? 0 : QHASHSIZE))
4437c478bdstevel@tonic-gate
4447c478bdstevel@tonic-gateextern	queue_head_t	*queue_lock(void *, int);
4457c478bdstevel@tonic-gateextern	void		queue_unlock(queue_head_t *);
446d4204c8rafextern	void		enqueue(queue_head_t *, struct ulwp *, int);
447d4204c8rafextern	struct ulwp	*dequeue(queue_head_t *, int *);
448d4204c8rafextern	struct ulwp	**queue_slot(queue_head_t *, struct ulwp **, int *);
449d4204c8rafextern	struct ulwp	*queue_waiter(queue_head_t *);
450d4204c8rafextern	int		dequeue_self(queue_head_t *);
451d4204c8rafextern	void		queue_unlink(queue_head_t *,
45241efec2raf				struct ulwp **, struct ulwp *);
4537c478bdstevel@tonic-gateextern	void		unsleep_self(void);
4547c478bdstevel@tonic-gateextern	void		spin_lock_set(mutex_t *);
4557c478bdstevel@tonic-gateextern	void		spin_lock_clear(mutex_t *);
4567c478bdstevel@tonic-gate
4577c478bdstevel@tonic-gate/*
458d4204c8raf * Scheduling class information structure.
459d4204c8raf */
460d4204c8raftypedef struct {
461d4204c8raf	short		pcc_state;
462d4204c8raf	short		pcc_policy;
463d4204c8raf	pri_t		pcc_primin;
464d4204c8raf	pri_t		pcc_primax;
465d4204c8raf	pcinfo_t	pcc_info;
466d4204c8raf} pcclass_t;
467d4204c8raf
468d4204c8raf/*
4697c478bdstevel@tonic-gate * Memory block for chain of owned ceiling mutexes.
4707c478bdstevel@tonic-gate */
4717c478bdstevel@tonic-gatetypedef struct mxchain {
4727c478bdstevel@tonic-gate	struct mxchain	*mxchain_next;
4737c478bdstevel@tonic-gate	mutex_t		*mxchain_mx;
4747c478bdstevel@tonic-gate} mxchain_t;
4757c478bdstevel@tonic-gate
4767c478bdstevel@tonic-gate/*
4777c478bdstevel@tonic-gate * Pointer to an rwlock that is held for reading.
4787c478bdstevel@tonic-gate * Used in rw_rdlock() to allow a thread that already holds a read
4797c478bdstevel@tonic-gate * lock to acquire another read lock on the same rwlock even if
4807c478bdstevel@tonic-gate * there are writers waiting.  This to avoid deadlock when acquiring
4817c478bdstevel@tonic-gate * a read lock more than once in the presence of pending writers.
4827c478bdstevel@tonic-gate * POSIX mandates this behavior.
4837c478bdstevel@tonic-gate */
4847c478bdstevel@tonic-gatetypedef struct {
4857c478bdstevel@tonic-gate	void	*rd_rwlock;	/* the rwlock held for reading */
4867c478bdstevel@tonic-gate	size_t	rd_count;	/* count of read locks applied */
4877c478bdstevel@tonic-gate} readlock_t;
4887c478bdstevel@tonic-gate
4897c478bdstevel@tonic-gate#ifdef _SYSCALL32
4907c478bdstevel@tonic-gatetypedef struct {
4917c478bdstevel@tonic-gate	caddr32_t	rd_rwlock;
4927c478bdstevel@tonic-gate	size32_t	rd_count;
4937c478bdstevel@tonic-gate} readlock32_t;
4947c478bdstevel@tonic-gate#endif	/* _SYSCALL32 */
4957c478bdstevel@tonic-gate
4967c478bdstevel@tonic-gate/*
4974f364e7Robert Mustacchi * As part of per-thread caching libumem (ptcumem), we add a small amount to the
4984f364e7Robert Mustacchi * thread's uberdata to facilitate it. The tm_roots are the roots of linked
4994f364e7Robert Mustacchi * lists which is used by libumem to chain together allocations. tm_size is used
5004f364e7Robert Mustacchi * to track the total amount of data stored across those linked lists. For more
5014f364e7Robert Mustacchi * information, see libumem's big theory statement.
5024f364e7Robert Mustacchi */
5034f364e7Robert Mustacchi#define	NTMEMBASE	16
5044f364e7Robert Mustacchi
5054f364e7Robert Mustacchitypedef struct {
5064f364e7Robert Mustacchi	size_t		tm_size;
5074f364e7Robert Mustacchi	void		*tm_roots[NTMEMBASE];
5084f364e7Robert Mustacchi} tumem_t;
5094f364e7Robert Mustacchi
5104f364e7Robert Mustacchi#ifdef _SYSCALL32
5114f364e7Robert Mustacchitypedef struct {
5124f364e7Robert Mustacchi	uint32_t	tm_size;
5134f364e7Robert Mustacchi	caddr32_t	tm_roots[NTMEMBASE];
5144f364e7Robert Mustacchi} tumem32_t;
5154f364e7Robert Mustacchi#endif
5164f364e7Robert Mustacchi
5174f364e7Robert Mustacchitypedef void (*tmem_func_t)(void *, int);
5184f364e7Robert Mustacchi
5194f364e7Robert Mustacchi/*
5207c478bdstevel@tonic-gate * Maximum number of read locks allowed for one thread on one rwlock.
5217c478bdstevel@tonic-gate * This could be as large as INT_MAX, but the SUSV3 test suite would
5227c478bdstevel@tonic-gate * take an inordinately long time to complete.  This is big enough.
5237c478bdstevel@tonic-gate */
5247c478bdstevel@tonic-gate#define	READ_LOCK_MAX	100000
5257c478bdstevel@tonic-gate
5267c478bdstevel@tonic-gate#define	ul_tlsent	ul_tls.tls_data	/* array of pointers to dynamic TLS */
5277c478bdstevel@tonic-gate#define	ul_ntlsent	ul_tls.tls_size	/* number of entries in ul_tlsent */
5287c478bdstevel@tonic-gate
5297c478bdstevel@tonic-gate/*
5307c478bdstevel@tonic-gate * Round up an integral value to a multiple of 64
5317c478bdstevel@tonic-gate */
5327c478bdstevel@tonic-gate#define	roundup64(x)	(-(-(x) & -64))
5337c478bdstevel@tonic-gate
5347c478bdstevel@tonic-gate/*
5357c478bdstevel@tonic-gate * NOTE:  Whatever changes are made to ulwp_t must be
5367c478bdstevel@tonic-gate * reflected in $SRC/cmd/mdb/common/modules/libc/libc.c
5377c478bdstevel@tonic-gate *
5387c478bdstevel@tonic-gate * NOTE: ul_self *must* be the first member of ulwp_t on x86
5397c478bdstevel@tonic-gate * Low-level x86 code relies on this.
5407c478bdstevel@tonic-gate */
5417c478bdstevel@tonic-gatetypedef struct ulwp {
5427c478bdstevel@tonic-gate	/*
5437c478bdstevel@tonic-gate	 * These members always need to come first on sparc.
5447c478bdstevel@tonic-gate	 * For dtrace, a ulwp_t must be aligned on a 64-byte boundary.
5457c478bdstevel@tonic-gate	 */
5467c478bdstevel@tonic-gate#if defined(__sparc)
5477c478bdstevel@tonic-gate	uint32_t	ul_dinstr;	/* scratch space for dtrace */
5487c478bdstevel@tonic-gate	uint32_t	ul_padsparc0[15];
5497c478bdstevel@tonic-gate	uint32_t	ul_dsave;	/* dtrace: save %g1, %g0, %sp */
5507c478bdstevel@tonic-gate	uint32_t	ul_drestore;	/* dtrace: restore %g0, %g0, %g0 */
5517c478bdstevel@tonic-gate	uint32_t	ul_dftret;	/* dtrace: return probe fasttrap */
5527c478bdstevel@tonic-gate	uint32_t	ul_dreturn;	/* dtrace: return %o0 */
5537c478bdstevel@tonic-gate#endif
5547c478bdstevel@tonic-gate	struct ulwp	*ul_self;	/* pointer to self */
5557c478bdstevel@tonic-gate#if defined(__i386)
5567c478bdstevel@tonic-gate	uint8_t		ul_dinstr[40];	/* scratch space for dtrace */
5577c478bdstevel@tonic-gate#elif defined(__amd64)
5587c478bdstevel@tonic-gate	uint8_t		ul_dinstr[56];	/* scratch space for dtrace */
5597c478bdstevel@tonic-gate#endif
5607c478bdstevel@tonic-gate	struct uberdata *ul_uberdata;	/* uber (super-global) data */
5617c478bdstevel@tonic-gate	tls_t		ul_tls;		/* dynamic thread-local storage base */
5627c478bdstevel@tonic-gate	struct ulwp	*ul_forw;	/* forw, back all_lwps list, */
5637c478bdstevel@tonic-gate	struct ulwp	*ul_back;	/* protected by link_lock */
5647c478bdstevel@tonic-gate	struct ulwp	*ul_next;	/* list to keep track of stacks */
5657c478bdstevel@tonic-gate	struct ulwp	*ul_hash;	/* hash chain linked list */
5667c478bdstevel@tonic-gate	void		*ul_rval;	/* return value from thr_exit() */
5677c478bdstevel@tonic-gate	caddr_t		ul_stk;		/* mapping base of the stack */
5687c478bdstevel@tonic-gate	size_t		ul_mapsiz;	/* mapping size of the stack */
5697c478bdstevel@tonic-gate	size_t		ul_guardsize;	/* normally _lpagesize */
5707c478bdstevel@tonic-gate	uintptr_t	ul_stktop;	/* broken thr_stksegment() interface */
5717c478bdstevel@tonic-gate	size_t		ul_stksiz;	/* broken thr_stksegment() interface */
5727c478bdstevel@tonic-gate	stack_t		ul_ustack;	/* current stack boundaries */
5737c478bdstevel@tonic-gate	int		ul_ix;		/* hash index */
5747c478bdstevel@tonic-gate	lwpid_t		ul_lwpid;	/* thread id, aka the lwp id */
575d4204c8raf	pri_t		ul_pri;		/* scheduling priority */
576d4204c8raf	pri_t		ul_epri;	/* real-time ceiling priority */
5777c478bdstevel@tonic-gate	char		ul_policy;	/* scheduling policy */
578d4204c8raf	char		ul_cid;		/* scheduling class id */
5797c478bdstevel@tonic-gate	union {
5807c478bdstevel@tonic-gate		struct {
5817c478bdstevel@tonic-gate			char	cursig;	/* deferred signal number */
5827c478bdstevel@tonic-gate			char	pleasestop; /* lwp requested to stop itself */
5837c478bdstevel@tonic-gate		} s;
5847c478bdstevel@tonic-gate		short	curplease;	/* for testing both at once */
5857c478bdstevel@tonic-gate	} ul_cp;
5867c478bdstevel@tonic-gate	char		ul_stop;	/* reason for stopping */
5877c478bdstevel@tonic-gate	char		ul_signalled;	/* this lwp was cond_signal()d */
5887c478bdstevel@tonic-gate	char		ul_dead;	/* this lwp has called thr_exit */
5897c478bdstevel@tonic-gate	char		ul_unwind;	/* posix: unwind C++ stack */
5907c478bdstevel@tonic-gate	char		ul_detached;	/* THR_DETACHED at thread_create() */
5917c478bdstevel@tonic-gate					/* or pthread_detach() was called */
5927c478bdstevel@tonic-gate	char		ul_writer;	/* sleeping in rw_wrlock() */
5937c478bdstevel@tonic-gate	char		ul_stopping;	/* set by curthread: stopping self */
5943470957raf	char		ul_cancel_prologue;	/* for _cancel_prologue() */
5957c478bdstevel@tonic-gate	short		ul_preempt;	/* no_preempt()/preempt() */
5967c478bdstevel@tonic-gate	short		ul_savpreempt;	/* pre-existing preempt value */
5977c478bdstevel@tonic-gate	char		ul_sigsuspend;	/* thread is in sigsuspend/pollsys */
5987c478bdstevel@tonic-gate	char		ul_main;	/* thread is the main thread */
5997c478bdstevel@tonic-gate	char		ul_fork;	/* thread is performing a fork */
6007c478bdstevel@tonic-gate	char		ul_primarymap;	/* primary link-map is initialized */
6017c478bdstevel@tonic-gate	/* per-thread copies of the corresponding global variables */
6025d1dd9araf	uint8_t		ul_max_spinners;	/* thread_max_spinners */
6037c478bdstevel@tonic-gate	char		ul_door_noreserve;	/* thread_door_noreserve */
6047c478bdstevel@tonic-gate	char		ul_queue_fifo;		/* thread_queue_fifo */
6057c478bdstevel@tonic-gate	char		ul_cond_wait_defer;	/* thread_cond_wait_defer */
6067c478bdstevel@tonic-gate	char		ul_error_detection;	/* thread_error_detection */
6077c478bdstevel@tonic-gate	char		ul_async_safe;		/* thread_async_safe */
608d4204c8raf	char		ul_rt;			/* found on an RT queue */
609d4204c8raf	char		ul_rtqueued;		/* was RT when queued */
6107c5714fraf	char		ul_misaligned;		/* thread_locks_misaligned */
6117c5714fraf	char		ul_pad[3];
6127c478bdstevel@tonic-gate	int		ul_adaptive_spin;	/* thread_adaptive_spin */
6137c478bdstevel@tonic-gate	int		ul_queue_spin;		/* thread_queue_spin */
6147c478bdstevel@tonic-gate	volatile int	ul_critical;	/* non-zero == in a critical region */
6157c478bdstevel@tonic-gate	int		ul_sigdefer;	/* non-zero == defer signals */
6167c478bdstevel@tonic-gate	int		ul_vfork;	/* thread is the child of vfork() */
6177c478bdstevel@tonic-gate	int		ul_cancelable;	/* _cancelon()/_canceloff() */
6187c478bdstevel@tonic-gate	char		ul_cancel_pending;  /* pthread_cancel() was called */
6197c478bdstevel@tonic-gate	char		ul_cancel_disabled; /* PTHREAD_CANCEL_DISABLE */
6207c478bdstevel@tonic-gate	char		ul_cancel_async;    /* PTHREAD_CANCEL_ASYNCHRONOUS */
6217c478bdstevel@tonic-gate	char		ul_save_async;	/* saved copy of ul_cancel_async */
6227c478bdstevel@tonic-gate	char		ul_mutator;	/* lwp is a mutator (java interface) */
6237c478bdstevel@tonic-gate	char		ul_created;	/* created suspended */
6247c478bdstevel@tonic-gate	char		ul_replace;	/* replacement; must be free()d */
6257c478bdstevel@tonic-gate	uchar_t		ul_nocancel;	/* cancellation can't happen */
6267c478bdstevel@tonic-gate	int		ul_errno;	/* per-thread errno */
6277c478bdstevel@tonic-gate	int		*ul_errnop;	/* pointer to errno or self->ul_errno */
6287c478bdstevel@tonic-gate	__cleanup_t	*ul_clnup_hdr;	/* head of cleanup handlers list */
629d4204c8raf	uberflags_t	*ul_schedctl_called;	/* ul_schedctl is set up */
630d4204c8raf	volatile sc_shared_t *ul_schedctl;	/* schedctl data */
6317c478bdstevel@tonic-gate	int		ul_bindflags;	/* bind_guard() interface to ld.so.1 */
632a574db8raf	uint_t		ul_libc_locks;	/* count of cancel_safe_mutex_lock()s */
6337c478bdstevel@tonic-gate	tsd_t		*ul_stsd;	/* slow TLS for keys >= TSD_NFAST */
6347c478bdstevel@tonic-gate	void		*ul_ftsd[TSD_NFAST]; /* fast TLS for keys < TSD_NFAST */
6357c478bdstevel@tonic-gate	td_evbuf_t	ul_td_evbuf;	/* event buffer */
6367c478bdstevel@tonic-gate	char		ul_td_events_enable;	/* event mechanism enabled */
6377c478bdstevel@tonic-gate	char		ul_sync_obj_reg;	/* tdb_sync_obj_register() */
6387c478bdstevel@tonic-gate	char		ul_qtype;	/* MX or CV */
6397c478bdstevel@tonic-gate	char		ul_cv_wake;	/* != 0: just wake up, don't requeue */
640e54ab87Roger A. Faulkner	int		ul_rtld;	/* thread is running inside ld.so.1 */
6417c478bdstevel@tonic-gate	int		ul_usropts;	/* flags given to thr_create() */
6427c478bdstevel@tonic-gate	void		*(*ul_startpc)(void *); /* start func (thr_create()) */
6437c478bdstevel@tonic-gate	void		*ul_startarg;	/* argument for start function */
6447c478bdstevel@tonic-gate	void		*ul_wchan;	/* synch object when sleeping */
6457c478bdstevel@tonic-gate	struct ulwp	*ul_link;	/* sleep queue link */
6467c478bdstevel@tonic-gate	queue_head_t	*ul_sleepq;	/* sleep queue thread is waiting on */
6477c478bdstevel@tonic-gate	mutex_t		*ul_cvmutex;	/* mutex dropped when waiting on a cv */
6487c478bdstevel@tonic-gate	mxchain_t	*ul_mxchain;	/* chain of owned ceiling mutexes */
649d4204c8raf	int		ul_save_state;	/* bind_guard() interface to ld.so.1 */
650883492draf	uint_t		ul_rdlockcnt;	/* # entries in ul_readlock array */
651883492draf				/* 0 means there is but a single entry */
652883492draf	union {				/* single entry or pointer to array */
6537c478bdstevel@tonic-gate		readlock_t	single;
6547c478bdstevel@tonic-gate		readlock_t	*array;
6557c478bdstevel@tonic-gate	} ul_readlock;
656883492draf	uint_t		ul_heldlockcnt;	/* # entries in ul_heldlocks array */
657883492draf				/* 0 means there is but a single entry */
658883492draf	union {				/* single entry or pointer to array */
659883492draf		mutex_t		*single;
660883492draf		mutex_t		**array;
661883492draf	} ul_heldlocks;
6627c478bdstevel@tonic-gate	/* PROBE_SUPPORT begin */
6637c478bdstevel@tonic-gate	void		*ul_tpdp;
6647c478bdstevel@tonic-gate	/* PROBE_SUPPORT end */
6657c478bdstevel@tonic-gate	ucontext_t	*ul_siglink;	/* pointer to previous context */
6667c478bdstevel@tonic-gate	uint_t		ul_spin_lock_spin;	/* spin lock statistics */
6677c478bdstevel@tonic-gate	uint_t		ul_spin_lock_spin2;
6687c478bdstevel@tonic-gate	uint_t		ul_spin_lock_sleep;
6697c478bdstevel@tonic-gate	uint_t		ul_spin_lock_wakeup;
670d4204c8raf	queue_root_t	ul_queue_root;	/* root of a sleep queue */
671d4204c8raf	id_t		ul_rtclassid;	/* real-time class id */
672d4204c8raf	uint_t		ul_pilocks;	/* count of PI locks held */
6737c478bdstevel@tonic-gate		/* the following members *must* be last in the structure */
6747c478bdstevel@tonic-gate		/* they are discarded when ulwp is replaced on thr_exit() */
6757c478bdstevel@tonic-gate	sigset_t	ul_sigmask;	/* thread's current signal mask */
6767c478bdstevel@tonic-gate	sigset_t	ul_tmpmask;	/* signal mask for sigsuspend/pollsys */
6777c478bdstevel@tonic-gate	siginfo_t	ul_siginfo;	/* deferred siginfo */
6787c478bdstevel@tonic-gate	mutex_t		ul_spinlock;	/* used when suspending/continuing */
6797c478bdstevel@tonic-gate	fpuenv_t	ul_fpuenv;	/* floating point state */
6807c478bdstevel@tonic-gate	uintptr_t	ul_sp;		/* stack pointer when blocked */
6817c478bdstevel@tonic-gate	void		*ul_ex_unwind;	/* address of _ex_unwind() or -1 */
6827c478bdstevel@tonic-gate#if defined(sparc)
6837c478bdstevel@tonic-gate	void		*ul_unwind_ret;	/* used only by _ex_clnup_handler() */
6847c478bdstevel@tonic-gate#endif
6854f364e7Robert Mustacchi	tumem_t		ul_tmem;	/* used only by umem */
686e56998eRobert Mustacchi	uint_t		ul_ptinherit;	/* pthreads sched inherit value */
6878a7aa2aYuri Pankov	char		ul_ntoabuf[18];	/* thread-specific inet_ntoa buffer */
6887c478bdstevel@tonic-gate} ulwp_t;
6897c478bdstevel@tonic-gate
6907c478bdstevel@tonic-gate#define	ul_cursig	ul_cp.s.cursig		/* deferred signal number */
6917c478bdstevel@tonic-gate#define	ul_pleasestop	ul_cp.s.pleasestop	/* lwp requested to stop */
6927c478bdstevel@tonic-gate#define	ul_curplease	ul_cp.curplease		/* for testing both at once */
6937c478bdstevel@tonic-gate
6947c478bdstevel@tonic-gate/*
6957c478bdstevel@tonic-gate * This is the size of a replacement ulwp, retained only for the benefit
6967c478bdstevel@tonic-gate * of thr_join().  The trailing members are unneeded for this purpose.
6977c478bdstevel@tonic-gate */
6987c478bdstevel@tonic-gate#define	REPLACEMENT_SIZE	((size_t)&((ulwp_t *)NULL)->ul_sigmask)
6997c478bdstevel@tonic-gate
7007c478bdstevel@tonic-gate/*
7017c478bdstevel@tonic-gate * Definitions for static initialization of signal sets,
7027c478bdstevel@tonic-gate * plus some sneaky optimizations in various places.
7037c478bdstevel@tonic-gate */
7047c478bdstevel@tonic-gate
7057c478bdstevel@tonic-gate#define	SIGMASK(sig)	((uint32_t)1 << (((sig) - 1) & (32 - 1)))
7067c478bdstevel@tonic-gate
707bdf0047Roger A. Faulkner#if (MAXSIG > (2 * 32) && MAXSIG <= (3 * 32))
7087c478bdstevel@tonic-gate#define	FILLSET0	0xffffffffu
709bdf0047Roger A. Faulkner#define	FILLSET1	0xffffffffu
710bdf0047Roger A. Faulkner#define	FILLSET2	((1u << (MAXSIG - 64)) - 1)
711bdf0047Roger A. Faulkner#define	FILLSET3	0
7127c478bdstevel@tonic-gate#else
7137c478bdstevel@tonic-gate#error "fix me: MAXSIG out of bounds"
7147c478bdstevel@tonic-gate#endif
7157c478bdstevel@tonic-gate
7167c478bdstevel@tonic-gate#define	CANTMASK0	(SIGMASK(SIGKILL) | SIGMASK(SIGSTOP))
7177c478bdstevel@tonic-gate#define	CANTMASK1	0
718bdf0047Roger A. Faulkner#define	CANTMASK2	0
719bdf0047Roger A. Faulkner#define	CANTMASK3	0
7207c478bdstevel@tonic-gate
7217c478bdstevel@tonic-gate#define	MASKSET0	(FILLSET0 & ~CANTMASK0)
7227c478bdstevel@tonic-gate#define	MASKSET1	(FILLSET1 & ~CANTMASK1)
723bdf0047Roger A. Faulkner#define	MASKSET2	(FILLSET2 & ~CANTMASK2)
724bdf0047Roger A. Faulkner#define	MASKSET3	(FILLSET3 & ~CANTMASK3)
7257c478bdstevel@tonic-gate
726f841f6arafextern	const sigset_t maskset;		/* set of all maskable signals */
7277c478bdstevel@tonic-gate
7287c478bdstevel@tonic-gateextern	int	thread_adaptive_spin;
7297c478bdstevel@tonic-gateextern	uint_t	thread_max_spinners;
7307c478bdstevel@tonic-gateextern	int	thread_queue_spin;
7317c478bdstevel@tonic-gateextern	int	thread_queue_fifo;
7327c478bdstevel@tonic-gateextern	int	thread_queue_dump;
7337c478bdstevel@tonic-gateextern	int	thread_cond_wait_defer;
7347c478bdstevel@tonic-gateextern	int	thread_async_safe;
7357c478bdstevel@tonic-gateextern	int	thread_queue_verify;
7367c478bdstevel@tonic-gate
7377c478bdstevel@tonic-gate/*
7387c478bdstevel@tonic-gate * pthread_atfork() related data, used to store atfork handlers.
7397c478bdstevel@tonic-gate */
7407c478bdstevel@tonic-gatetypedef struct atfork {
7417c478bdstevel@tonic-gate	struct atfork *forw;		/* forward pointer */
7427c478bdstevel@tonic-gate	struct atfork *back;		/* backward pointer */
7437c478bdstevel@tonic-gate	void (*prepare)(void);		/* pre-fork handler */
7447c478bdstevel@tonic-gate	void (*parent)(void);		/* post-fork parent handler */
7457c478bdstevel@tonic-gate	void (*child)(void);		/* post-fork child handler */
7467c478bdstevel@tonic-gate} atfork_t;
7477c478bdstevel@tonic-gate
7487c478bdstevel@tonic-gate/*
74909ce0d4Roger A. Faulkner * Element in the table and in the list of registered process
75009ce0d4Roger A. Faulkner * robust locks.  We keep track of these to make sure that we
75109ce0d4Roger A. Faulkner * only call ___lwp_mutex_register() once for each such lock
75209ce0d4Roger A. Faulkner * after it is first mapped in (or newly mapped in).
753883492draf */
754883492draftypedef struct robust {
75509ce0d4Roger A. Faulkner	struct robust	*robust_next;	/* hash table list */
75609ce0d4Roger A. Faulkner	struct robust	*robust_list;	/* global list */
757883492draf	mutex_t		*robust_lock;
758883492draf} robust_t;
759883492draf
760883492draf/*
76109ce0d4Roger A. Faulkner * Invalid address, used to mark an unused element in the hash table.
76209ce0d4Roger A. Faulkner */
76309ce0d4Roger A. Faulkner#define	INVALID_ADDR	((void *)(uintptr_t)(-1L))
76409ce0d4Roger A. Faulkner
76509ce0d4Roger A. Faulkner/*
766883492draf * Parameters of the lock registration hash table.
767883492draf */
768daa03c9raf#define	LOCKSHIFT	15			/* number of hashing bits */
769daa03c9raf#define	LOCKHASHSZ	(1 << LOCKSHIFT)	/* power of 2 (1<<15 == 32K) */
770883492draf#define	LOCK_HASH(addr)	(uint_t)			\
771883492draf	((((uintptr_t)(addr) >> 3)			\
772883492draf	^ ((uintptr_t)(addr) >> (LOCKSHIFT + 3)))	\
773883492draf	& (LOCKHASHSZ - 1))
774883492draf
775883492draf/*
7767c478bdstevel@tonic-gate * Make our hot locks reside on private cache lines (64 bytes).
7777c478bdstevel@tonic-gate */
7787c478bdstevel@tonic-gatetypedef struct {
7797c478bdstevel@tonic-gate	mutex_t	pad_lock;
7802e14588raf	char	pad_pad[64 - sizeof (mutex_t)];
7817c478bdstevel@tonic-gate} pad_lock_t;
7827c478bdstevel@tonic-gate
7837c478bdstevel@tonic-gate/*
7842e14588raf * Make our semi-hot locks reside on semi-private cache lines (32 bytes).
7852e14588raf */
7862e14588raftypedef struct {
7872e14588raf	mutex_t	pad_lock;
7882e14588raf	char	pad_pad[32 - sizeof (mutex_t)];
7892e14588raf} pad32_lock_t;
7902e14588raf
7912e14588raf/*
7927c478bdstevel@tonic-gate * The threads hash table is used for fast lookup and locking of an active
7937c478bdstevel@tonic-gate * thread structure (ulwp_t) given a thread-id.  It is an N-element array of
7947c478bdstevel@tonic-gate * thr_hash_table_t structures, where N == 1 before the main thread creates
7957c478bdstevel@tonic-gate * the first additional thread and N == 1024 afterwards.  Each element of the
7967c478bdstevel@tonic-gate * table is 64 bytes in size and alignment to reduce cache conflicts.
7977c478bdstevel@tonic-gate */
7987c478bdstevel@tonic-gatetypedef struct {
7997c478bdstevel@tonic-gate	mutex_t	hash_lock;	/* lock per bucket */
8007c478bdstevel@tonic-gate	cond_t	hash_cond;	/* convar per bucket */
8017c478bdstevel@tonic-gate	ulwp_t	*hash_bucket;	/* hash bucket points to the list of ulwps */
8027c478bdstevel@tonic-gate	char	hash_pad[64 -	/* pad out to 64 bytes */
8037c478bdstevel@tonic-gate		(sizeof (mutex_t) + sizeof (cond_t) + sizeof (ulwp_t *))];
8047c478bdstevel@tonic-gate} thr_hash_table_t;
8057c478bdstevel@tonic-gate
8067c478bdstevel@tonic-gate#ifdef _SYSCALL32
8077c478bdstevel@tonic-gatetypedef struct {
8087c478bdstevel@tonic-gate	mutex_t	hash_lock;
8097c478bdstevel@tonic-gate	cond_t	hash_cond;
8107c478bdstevel@tonic-gate	caddr32_t hash_bucket;
8117c478bdstevel@tonic-gate	char	hash_pad[64 -
8127c478bdstevel@tonic-gate		(sizeof (mutex_t) + sizeof (cond_t) + sizeof (caddr32_t))];
8137c478bdstevel@tonic-gate} thr_hash_table32_t;
8147c478bdstevel@tonic-gate#endif	/* _SYSCALL32 */
8157c478bdstevel@tonic-gate
8167c478bdstevel@tonic-gate
8177c478bdstevel@tonic-gate/*
81841efec2raf * siguaction members have 128-byte size and 64-byte alignment.
81941efec2raf * We know that sizeof (struct sigaction) is 32 bytes for both
82041efec2raf * _ILP32 and _LP64 and that sizeof (rwlock_t) is 64 bytes.
8217c478bdstevel@tonic-gate */
8227c478bdstevel@tonic-gatetypedef struct {
82341efec2raf	rwlock_t	sig_lock;
8247c478bdstevel@tonic-gate	struct sigaction sig_uaction;
82541efec2raf	char	sig_pad[128 - sizeof (rwlock_t) - sizeof (struct sigaction)];
8267c478bdstevel@tonic-gate} siguaction_t;
8277c478bdstevel@tonic-gate
8287c478bdstevel@tonic-gate#ifdef _SYSCALL32
8297c478bdstevel@tonic-gatetypedef struct {
83041efec2raf	rwlock_t	sig_lock;
8317c478bdstevel@tonic-gate	struct sigaction32 sig_uaction;
83241efec2raf	char	sig_pad[128 - sizeof (rwlock_t) - sizeof (struct sigaction32)];
8337c478bdstevel@tonic-gate} siguaction32_t;
8347c478bdstevel@tonic-gate#endif	/* _SYSCALL32 */
8357c478bdstevel@tonic-gate
8367c478bdstevel@tonic-gate
8377c478bdstevel@tonic-gate/*
8387c478bdstevel@tonic-gate * Bucket structures, used by lmalloc()/lfree().
8397c478bdstevel@tonic-gate * See port/threads/alloc.c for details.
8407c478bdstevel@tonic-gate * A bucket's size and alignment is 64 bytes.
8417c478bdstevel@tonic-gate */
8427c478bdstevel@tonic-gatetypedef struct {
8437c478bdstevel@tonic-gate	mutex_t	bucket_lock;	/* protects the free list allocations */
8447c478bdstevel@tonic-gate	void	*free_list;	/* LIFO list of blocks to allocate/free */
8457c478bdstevel@tonic-gate	size_t	chunks;		/* number of 64K blocks mmap()ed last time */
8467c478bdstevel@tonic-gate	char	pad64[64 -	/* pad out to 64 bytes */
8477c478bdstevel@tonic-gate		(sizeof (mutex_t) + sizeof (void *) + sizeof (size_t))];
8487c478bdstevel@tonic-gate} bucket_t;
8497c478bdstevel@tonic-gate
8507c478bdstevel@tonic-gate#ifdef _SYSCALL32
8517c478bdstevel@tonic-gatetypedef struct {
8527c478bdstevel@tonic-gate	mutex_t		bucket_lock;
8537c478bdstevel@tonic-gate	caddr32_t	free_list;
8547c478bdstevel@tonic-gate	size32_t	chunks;
8557c478bdstevel@tonic-gate	char	pad64[64 -	/* pad out to 64 bytes */
8567c478bdstevel@tonic-gate		(sizeof (mutex_t) + sizeof (caddr32_t) + sizeof (size32_t))];
8577c478bdstevel@tonic-gate} bucket32_t;
8587c478bdstevel@tonic-gate#endif	/* _SYSCALL32 */
8597c478bdstevel@tonic-gate
8607c478bdstevel@tonic-gate#define	NBUCKETS	10	/* sizes ranging from 64 to 32768 */
8617c478bdstevel@tonic-gate
8627c478bdstevel@tonic-gate
8637c478bdstevel@tonic-gate/*
8647c478bdstevel@tonic-gate * atexit() data structures.
8657c478bdstevel@tonic-gate * See port/gen/atexit.c for details.
8667c478bdstevel@tonic-gate */
8675c069a6Richard Lowetypedef void (*_exithdlr_func_t) (void*);
8687c478bdstevel@tonic-gate
8697c478bdstevel@tonic-gatetypedef struct _exthdlr {
8708a7aa2aYuri Pankov	struct _exthdlr		*next;	/* next in handler list */
8717c478bdstevel@tonic-gate	_exithdlr_func_t	hdlr;	/* handler itself */
8725c069a6Richard Lowe	void			*arg;	/* argument to handler */
8735c069a6Richard Lowe	void			*dso;	/* DSO associated with handler */
8747c478bdstevel@tonic-gate} _exthdlr_t;
8757c478bdstevel@tonic-gate
8767c478bdstevel@tonic-gatetypedef struct {
8777c478bdstevel@tonic-gate	mutex_t		exitfns_lock;
8787c478bdstevel@tonic-gate	_exthdlr_t	*head;
8795c069a6Richard Lowe	/*
8805c069a6Richard Lowe	 * exit_frame_monitor is part of a private contract between libc and
8815c069a6Richard Lowe	 * the Sun C++ runtime.
8825c069a6Richard Lowe	 *
8835c069a6Richard Lowe	 * It should be NULL until exit() is called, and thereafter hold the
8845c069a6Richard Lowe	 * frame pointer of the function implementing our exit processing.
8855c069a6Richard Lowe	 */
8867c478bdstevel@tonic-gate	void		*exit_frame_monitor;
8877c478bdstevel@tonic-gate	char		exit_pad[64 -	/* pad out to 64 bytes */
8887c478bdstevel@tonic-gate		(sizeof (mutex_t) + sizeof (_exthdlr_t *) + sizeof (void *))];
8897c478bdstevel@tonic-gate} atexit_root_t;
8907c478bdstevel@tonic-gate
8917c478bdstevel@tonic-gate#ifdef _SYSCALL32
8927c478bdstevel@tonic-gatetypedef struct {
8937c478bdstevel@tonic-gate	mutex_t		exitfns_lock;
8947c478bdstevel@tonic-gate	caddr32_t	head;
8957c478bdstevel@tonic-gate	caddr32_t	exit_frame_monitor;
8967c478bdstevel@tonic-gate	char		exit_pad[64 -	/* pad out to 64 bytes */
8977c478bdstevel@tonic-gate		(sizeof (mutex_t) + sizeof (caddr32_t) + sizeof (caddr32_t))];
8987c478bdstevel@tonic-gate} atexit_root32_t;
8997c478bdstevel@tonic-gate#endif	/* _SYSCALL32 */
9007c478bdstevel@tonic-gate
901fc2512cRobert Mustacchi/*
902fc2512cRobert Mustacchi * at_quick_exit() and quick_exit() data structures. The ISO/IEC C11 odd
903fc2512cRobert Mustacchi * siblings of atexit()
904fc2512cRobert Mustacchi */
905fc2512cRobert Mustacchitypedef void (*_quick_exithdlr_func_t)(void);
906fc2512cRobert Mustacchi
907fc2512cRobert Mustacchitypedef struct _qexthdlr {
9088a7aa2aYuri Pankov	struct _qexthdlr	*next;	/* next in handler list */
909fc2512cRobert Mustacchi	_quick_exithdlr_func_t	hdlr;	/* handler itself */
910fc2512cRobert Mustacchi} _qexthdlr_t;
911fc2512cRobert Mustacchi
912ff892b7Robert Mustacchi/*
913ff892b7Robert Mustacchi * We add a pad on 32-bit systems to allow us to always have the structure size
914ff892b7Robert Mustacchi * be 32-bytes which helps us deal with the compiler's alignment when building
915ff892b7Robert Mustacchi * in ILP32 / LP64 systems.
916ff892b7Robert Mustacchi */
917fc2512cRobert Mustacchitypedef struct {
918fc2512cRobert Mustacchi	mutex_t		exitfns_lock;
919fc2512cRobert Mustacchi	_qexthdlr_t	*head;
920ff892b7Robert Mustacchi#if !defined(_LP64)
921ff892b7Robert Mustacchi	uint32_t	pad;
922ff892b7Robert Mustacchi#endif
923fc2512cRobert Mustacchi} quickexit_root_t;
924fc2512cRobert Mustacchi
925fc2512cRobert Mustacchi#ifdef _SYSCALL32
926fc2512cRobert Mustacchitypedef struct {
927fc2512cRobert Mustacchi	mutex_t		exitfns_lock;
928fc2512cRobert Mustacchi	caddr32_t	head;
929ff892b7Robert Mustacchi	uint32_t	pad;
930fc2512cRobert Mustacchi} quickexit_root32_t;
931fc2512cRobert Mustacchi#endif /* _SYSCALL32 */
9327c478bdstevel@tonic-gate
9337c478bdstevel@tonic-gate/*
9347c478bdstevel@tonic-gate * This is data that is global to all link maps (uberdata, aka super-global).
935fe4be5aGordon Ross * Note: When changing this, please be sure to keep the 32-bit variant of
936fe4be5aGordon Ross * this in sync.  (see uberdata32_t below)
9377c478bdstevel@tonic-gate */
9387c478bdstevel@tonic-gatetypedef struct uberdata {
9397c478bdstevel@tonic-gate	pad_lock_t	_link_lock;
9408cd4554raf	pad_lock_t	_ld_lock;
9418cd4554raf	pad_lock_t	_fork_lock;
9428cd4554raf	pad_lock_t	_atfork_lock;
94398c1a6braf	pad32_lock_t	_callout_lock;
94498c1a6braf	pad32_lock_t	_tdb_hash_lock;
9457c478bdstevel@tonic-gate	tdb_sync_stats_t tdb_hash_lock_stats;
9467c478bdstevel@tonic-gate	siguaction_t	siguaction[NSIG];
9477c478bdstevel@tonic-gate	bucket_t	bucket[NBUCKETS];
9487c478bdstevel@tonic-gate	atexit_root_t	atexit_root;
949fc2512cRobert Mustacchi	quickexit_root_t quickexit_root;
9507c478bdstevel@tonic-gate	tsd_metadata_t	tsd_metadata;
9517c478bdstevel@tonic-gate	tls_metadata_t	tls_metadata;
9527c478bdstevel@tonic-gate	/*
9537c478bdstevel@tonic-gate	 * Every object before this point has size and alignment of 64 bytes.
9547c478bdstevel@tonic-gate	 * Don't add any other type of data before this point.
9557c478bdstevel@tonic-gate	 */
9567c478bdstevel@tonic-gate	char	primary_map;	/* set when primary link map is initialized */
9577c478bdstevel@tonic-gate	char	bucket_init;	/* set when bucket[NBUCKETS] is initialized */
9587c478bdstevel@tonic-gate	char	pad[2];
9597c478bdstevel@tonic-gate	uberflags_t	uberflags;
9607c478bdstevel@tonic-gate	queue_head_t	*queue_head;
9617c478bdstevel@tonic-gate	thr_hash_table_t *thr_hash_table;
9627c478bdstevel@tonic-gate	uint_t		hash_size;	/* # of entries in thr_hash_table[] */
9637c478bdstevel@tonic-gate	uint_t		hash_mask;	/* hash_size - 1 */
9647c478bdstevel@tonic-gate	ulwp_t	*ulwp_one;	/* main thread */
9657c478bdstevel@tonic-gate	ulwp_t	*all_lwps;	/* circular ul_forw/ul_back list of live lwps */
9667c478bdstevel@tonic-gate	ulwp_t	*all_zombies;	/* circular ul_forw/ul_back list of zombies */
9677c478bdstevel@tonic-gate	int	nthreads;	/* total number of live threads/lwps */
9687c478bdstevel@tonic-gate	int	nzombies;	/* total number of zombie threads */
9697c478bdstevel@tonic-gate	int	ndaemons;	/* total number of THR_DAEMON threads/lwps */
9707c478bdstevel@tonic-gate	pid_t	pid;		/* the current process's pid */
9717c478bdstevel@tonic-gate	void	(*sigacthandler)(int, siginfo_t *, void *);
9727c478bdstevel@tonic-gate	ulwp_t	*lwp_stacks;
9737c478bdstevel@tonic-gate	ulwp_t	*lwp_laststack;
9747c478bdstevel@tonic-gate	int	nfreestack;
9757c478bdstevel@tonic-gate	int	thread_stack_cache;
9767c478bdstevel@tonic-gate	ulwp_t	*ulwp_freelist;
9777c478bdstevel@tonic-gate	ulwp_t	*ulwp_lastfree;
9787c478bdstevel@tonic-gate	ulwp_t	*ulwp_replace_free;
9797c478bdstevel@tonic-gate	ulwp_t	*ulwp_replace_last;
9807c478bdstevel@tonic-gate	atfork_t	*atforklist;	/* circular Q for fork handlers */
981883492draf	robust_t	**robustlocks;	/* table of registered robust locks */
98209ce0d4Roger A. Faulkner	robust_t	*robustlist;	/* list of registered robust locks */
98323a1cceRoger A. Faulkner	char	*progname;	/* the basename of the program, from argv[0] */
9842428aadPatrick Mooney	void	*ub_comm_page;	/* arch-specific comm page of kernel data */
9857c478bdstevel@tonic-gate	struct uberdata **tdb_bootstrap;
9867c478bdstevel@tonic-gate	tdb_t	tdb;		/* thread debug interfaces (for libc_db) */
9877c478bdstevel@tonic-gate} uberdata_t;
9887c478bdstevel@tonic-gate
9897c478bdstevel@tonic-gate#define	link_lock	_link_lock.pad_lock
9908cd4554raf#define	ld_lock		_ld_lock.pad_lock
9917c478bdstevel@tonic-gate#define	fork_lock	_fork_lock.pad_lock
9922e14588raf#define	atfork_lock	_atfork_lock.pad_lock
99398c1a6braf#define	callout_lock	_callout_lock.pad_lock
9947c478bdstevel@tonic-gate#define	tdb_hash_lock	_tdb_hash_lock.pad_lock
9957c478bdstevel@tonic-gate
9967c478bdstevel@tonic-gate#pragma align 64(__uberdata)
9977c478bdstevel@tonic-gateextern	uberdata_t	__uberdata;
9987c478bdstevel@tonic-gateextern	uberdata_t	**__tdb_bootstrap;	/* known to libc_db and mdb */
9997c478bdstevel@tonic-gateextern	int		primary_link_map;
10007c478bdstevel@tonic-gate
10017c478bdstevel@tonic-gate#define	ulwp_mutex(ulwp, udp)	\
10027c478bdstevel@tonic-gate	(&(udp)->thr_hash_table[(ulwp)->ul_ix].hash_lock)
10037c478bdstevel@tonic-gate#define	ulwp_condvar(ulwp, udp)	\
10047c478bdstevel@tonic-gate	(&(udp)->thr_hash_table[(ulwp)->ul_ix].hash_cond)
10057c478bdstevel@tonic-gate
10067c478bdstevel@tonic-gate/*
10077c478bdstevel@tonic-gate * Grab and release the hash table lock for the specified lwp.
10087c478bdstevel@tonic-gate */
10097c478bdstevel@tonic-gate#define	ulwp_lock(ulwp, udp)	lmutex_lock(ulwp_mutex(ulwp, udp))
10107c478bdstevel@tonic-gate#define	ulwp_unlock(ulwp, udp)	lmutex_unlock(ulwp_mutex(ulwp, udp))
10117c478bdstevel@tonic-gate
10127c478bdstevel@tonic-gate#ifdef _SYSCALL32	/* needed by libc_db */
10137c478bdstevel@tonic-gate
10147c478bdstevel@tonic-gatetypedef struct ulwp32 {
10157c478bdstevel@tonic-gate#if defined(__sparc)
10167c478bdstevel@tonic-gate	uint32_t	ul_dinstr;	/* scratch space for dtrace */
10177c478bdstevel@tonic-gate	uint32_t	ul_padsparc0[15];
10187c478bdstevel@tonic-gate	uint32_t	ul_dsave;	/* dtrace: save %g1, %g0, %sp */
10197c478bdstevel@tonic-gate	uint32_t	ul_drestore;	/* dtrace: restore %g0, %g0, %g0 */
10207c478bdstevel@tonic-gate	uint32_t	ul_dftret;	/* dtrace: return probe fasttrap */
10217c478bdstevel@tonic-gate	uint32_t	ul_dreturn;	/* dtrace: return %o0 */
10227c478bdstevel@tonic-gate#endif
10237c478bdstevel@tonic-gate	caddr32_t	ul_self;	/* pointer to self */
10240ec5755raf#if defined(__x86)
10257c478bdstevel@tonic-gate	uint8_t		ul_dinstr[40];	/* scratch space for dtrace */
10267c478bdstevel@tonic-gate#endif
10277c478bdstevel@tonic-gate	caddr32_t	ul_uberdata;	/* uber (super-global) data */
10287c478bdstevel@tonic-gate	tls32_t		ul_tls;		/* dynamic thread-local storage base */
10297c478bdstevel@tonic-gate	caddr32_t	ul_forw;	/* forw, back all_lwps list, */
10307c478bdstevel@tonic-gate	caddr32_t	ul_back;	/* protected by link_lock */
10317c478bdstevel@tonic-gate	caddr32_t	ul_next;	/* list to keep track of stacks */
10327c478bdstevel@tonic-gate	caddr32_t	ul_hash;	/* hash chain linked list */
10337c478bdstevel@tonic-gate	caddr32_t	ul_rval;	/* return value from thr_exit() */
10347c478bdstevel@tonic-gate	caddr32_t	ul_stk;		/* mapping base of the stack */
10357c478bdstevel@tonic-gate	size32_t	ul_mapsiz;	/* mapping size of the stack */
10367c478bdstevel@tonic-gate	size32_t	ul_guardsize;	/* normally _lpagesize */
10377c478bdstevel@tonic-gate	caddr32_t	ul_stktop;	/* broken thr_stksegment() interface */
10387c478bdstevel@tonic-gate	size32_t	ul_stksiz;	/* broken thr_stksegment() interface */
10397c478bdstevel@tonic-gate	stack32_t	ul_ustack;	/* current stack boundaries */
10407c478bdstevel@tonic-gate	int		ul_ix;		/* hash index */
10417c478bdstevel@tonic-gate	lwpid_t		ul_lwpid;	/* thread id, aka the lwp id */
1042d4204c8raf	pri_t		ul_pri;		/* scheduling priority */
1043d4204c8raf	pri_t		ul_epri;	/* real-time ceiling priority */
10447c478bdstevel@tonic-gate	char		ul_policy;	/* scheduling policy */
1045d4204c8raf	char		ul_cid;		/* scheduling class id */
10467c478bdstevel@tonic-gate	union {
10477c478bdstevel@tonic-gate		struct {
10487c478bdstevel@tonic-gate			char	cursig;	/* deferred signal number */
10497c478bdstevel@tonic-gate			char	pleasestop; /* lwp requested to stop itself */
10507c478bdstevel@tonic-gate		} s;
10517c478bdstevel@tonic-gate		short	curplease;	/* for testing both at once */
10527c478bdstevel@tonic-gate	} ul_cp;
10537c478bdstevel@tonic-gate	char		ul_stop;	/* reason for stopping */
10547c478bdstevel@tonic-gate	char		ul_signalled;	/* this lwp was cond_signal()d */
10557c478bdstevel@tonic-gate	char		ul_dead;	/* this lwp has called thr_exit */
10567c478bdstevel@tonic-gate	char		ul_unwind;	/* posix: unwind C++ stack */
10577c478bdstevel@tonic-gate	char		ul_detached;	/* THR_DETACHED at thread_create() */
10587c478bdstevel@tonic-gate					/* or pthread_detach() was called */
10597c478bdstevel@tonic-gate	char		ul_writer;	/* sleeping in rw_wrlock() */
10607c478bdstevel@tonic-gate	char		ul_stopping;	/* set by curthread: stopping self */
10613470957raf	char		ul_cancel_prologue;	/* for _cancel_prologue() */
10627c478bdstevel@tonic-gate	short		ul_preempt;	/* no_preempt()/preempt() */
10637c478bdstevel@tonic-gate	short		ul_savpreempt;	/* pre-existing preempt value */
10647c478bdstevel@tonic-gate	char		ul_sigsuspend;	/* thread is in sigsuspend/pollsys */
10657c478bdstevel@tonic-gate	char		ul_main;	/* thread is the main thread */
10667c478bdstevel@tonic-gate	char		ul_fork;	/* thread is performing a fork */
10677c478bdstevel@tonic-gate	char		ul_primarymap;	/* primary link-map is initialized */
10687c478bdstevel@tonic-gate	/* per-thread copies of the corresponding global variables */
10695d1dd9araf	uint8_t		ul_max_spinners;	/* thread_max_spinners */
10707c478bdstevel@tonic-gate	char		ul_door_noreserve;	/* thread_door_noreserve */
10717c478bdstevel@tonic-gate	char		ul_queue_fifo;		/* thread_queue_fifo */
10727c478bdstevel@tonic-gate	char		ul_cond_wait_defer;	/* thread_cond_wait_defer */
10737c478bdstevel@tonic-gate	char		ul_error_detection;	/* thread_error_detection */
10747c478bdstevel@tonic-gate	char		ul_async_safe;		/* thread_async_safe */
1075d4204c8raf	char		ul_rt;			/* found on an RT queue */
1076d4204c8raf	char		ul_rtqueued;		/* was RT when queued */
10777c5714fraf	char		ul_misaligned;		/* thread_locks_misaligned */
10787c5714fraf	char		ul_pad[3];
10797c478bdstevel@tonic-gate	int		ul_adaptive_spin;	/* thread_adaptive_spin */
10807c478bdstevel@tonic-gate	int		ul_queue_spin;		/* thread_queue_spin */
10817c478bdstevel@tonic-gate	int		ul_critical;	/* non-zero == in a critical region */
10827c478bdstevel@tonic-gate	int		ul_sigdefer;	/* non-zero == defer signals */
10837c478bdstevel@tonic-gate	int		ul_vfork;	/* thread is the child of vfork() */
10847c478bdstevel@tonic-gate	int		ul_cancelable;	/* _cancelon()/_canceloff() */
10857c478bdstevel@tonic-gate	char		ul_cancel_pending;  /* pthread_cancel() was called */
10867c478bdstevel@tonic-gate	char		ul_cancel_disabled; /* PTHREAD_CANCEL_DISABLE */
10877c478bdstevel@tonic-gate	char		ul_cancel_async;    /* PTHREAD_CANCEL_ASYNCHRONOUS */
10887c478bdstevel@tonic-gate	char		ul_save_async;	/* saved copy of ul_cancel_async */
10897c478bdstevel@tonic-gate	char		ul_mutator;	/* lwp is a mutator (java interface) */
10907c478bdstevel@tonic-gate	char		ul_created;	/* created suspended */
10917c478bdstevel@tonic-gate	char		ul_replace;	/* replacement; must be free()d */
10927c478bdstevel@tonic-gate	uchar_t		ul_nocancel;	/* cancellation can't happen */
10937c478bdstevel@tonic-gate	int		ul_errno;	/* per-thread errno */
10947c478bdstevel@tonic-gate	caddr32_t	ul_errnop;	/* pointer to errno or self->ul_errno */
10957c478bdstevel@tonic-gate	caddr32_t	ul_clnup_hdr;	/* head of cleanup handlers list */
10967c478bdstevel@tonic-gate	caddr32_t	ul_schedctl_called; /* ul_schedctl is set up */
10977c478bdstevel@tonic-gate	caddr32_t	ul_schedctl;	/* schedctl data */
10987c478bdstevel@tonic-gate	int		ul_bindflags;	/* bind_guard() interface to ld.so.1 */
1099a574db8raf	uint_t		ul_libc_locks;	/* count of cancel_safe_mutex_lock()s */
11007c478bdstevel@tonic-gate	caddr32_t	ul_stsd;	/* slow TLS for keys >= TSD_NFAST */
11017c478bdstevel@tonic-gate	caddr32_t	ul_ftsd[TSD_NFAST]; /* fast TLS for keys < TSD_NFAST */
11027c478bdstevel@tonic-gate	td_evbuf32_t	ul_td_evbuf;	/* event buffer */
11037c478bdstevel@tonic-gate	char		ul_td_events_enable;	/* event mechanism enabled */
11047c478bdstevel@tonic-gate	char		ul_sync_obj_reg;	/* tdb_sync_obj_register() */
11057c478bdstevel@tonic-gate	char		ul_qtype;	/* MX or CV */
11067c478bdstevel@tonic-gate	char		ul_cv_wake;	/* != 0: just wake up, don't requeue */
1107e54ab87Roger A. Faulkner	int		ul_rtld;	/* thread is running inside ld.so.1 */
11087c478bdstevel@tonic-gate	int		ul_usropts;	/* flags given to thr_create() */
11097c478bdstevel@tonic-gate	caddr32_t	ul_startpc;	/* start func (thr_create()) */
11107c478bdstevel@tonic-gate	caddr32_t	ul_startarg;	/* argument for start function */
11117c478bdstevel@tonic-gate	caddr32_t	ul_wchan;	/* synch object when sleeping */
11127c478bdstevel@tonic-gate	caddr32_t	ul_link;	/* sleep queue link */
11137c478bdstevel@tonic-gate	caddr32_t	ul_sleepq;	/* sleep queue thread is waiting on */
11147c478bdstevel@tonic-gate	caddr32_t	ul_cvmutex;	/* mutex dropped when waiting on a cv */
11157c478bdstevel@tonic-gate	caddr32_t	ul_mxchain;	/* chain of owned ceiling mutexes */
1116d4204c8raf	int		ul_save_state;	/* bind_guard() interface to ld.so.1 */
1117883492draf	uint_t		ul_rdlockcnt;	/* # entries in ul_readlock array */
1118883492draf				/* 0 means there is but a single entry */
1119883492draf	union {				/* single entry or pointer to array */
11207c478bdstevel@tonic-gate		readlock32_t	single;
11217c478bdstevel@tonic-gate		caddr32_t	array;
11227c478bdstevel@tonic-gate	} ul_readlock;
1123883492draf	uint_t		ul_heldlockcnt;	/* # entries in ul_heldlocks array */
1124883492draf				/* 0 means there is but a single entry */
1125883492draf	union {				/* single entry or pointer to array */
1126883492draf		caddr32_t	single;
1127883492draf		caddr32_t	array;
1128883492draf	} ul_heldlocks;
11297c478bdstevel@tonic-gate	/* PROBE_SUPPORT begin */
11307c478bdstevel@tonic-gate	caddr32_t	ul_tpdp;
11317c478bdstevel@tonic-gate	/* PROBE_SUPPORT end */
11327c478bdstevel@tonic-gate	caddr32_t	ul_siglink;	/* pointer to previous context */
11337c478bdstevel@tonic-gate	uint_t		ul_spin_lock_spin;	/* spin lock statistics */
11347c478bdstevel@tonic-gate	uint_t		ul_spin_lock_spin2;
11357c478bdstevel@tonic-gate	uint_t		ul_spin_lock_sleep;
11367c478bdstevel@tonic-gate	uint_t		ul_spin_lock_wakeup;
1137d4204c8raf	queue_root32_t	ul_queue_root;	/* root of a sleep queue */
1138d4204c8raf	id_t		ul_rtclassid;	/* real-time class id */
1139d4204c8raf	uint_t		ul_pilocks;	/* count of PI locks held */
11407c478bdstevel@tonic-gate		/* the following members *must* be last in the structure */
11417c478bdstevel@tonic-gate		/* they are discarded when ulwp is replaced on thr_exit() */
1142bdf0047Roger A. Faulkner	sigset_t	ul_sigmask;	/* thread's current signal mask */
1143bdf0047Roger A. Faulkner	sigset_t	ul_tmpmask;	/* signal mask for sigsuspend/pollsys */
11447c478bdstevel@tonic-gate	siginfo32_t	ul_siginfo;	/* deferred siginfo */
11457c478bdstevel@tonic-gate	mutex_t		ul_spinlock;	/* used when suspending/continuing */
11467c478bdstevel@tonic-gate	fpuenv32_t	ul_fpuenv;	/* floating point state */
11477c478bdstevel@tonic-gate	caddr32_t	ul_sp;		/* stack pointer when blocked */
11487c478bdstevel@tonic-gate#if defined(sparc)
11497c478bdstevel@tonic-gate	caddr32_t	ul_unwind_ret;	/* used only by _ex_clnup_handler() */
11507c478bdstevel@tonic-gate#endif
11514f364e7Robert Mustacchi	tumem32_t	ul_tmem;	/* used only by umem */
11527c478bdstevel@tonic-gate} ulwp32_t;
11537c478bdstevel@tonic-gate
11547c478bdstevel@tonic-gate#define	REPLACEMENT_SIZE32	((size_t)&((ulwp32_t *)NULL)->ul_sigmask)
11557c478bdstevel@tonic-gate
11567c478bdstevel@tonic-gatetypedef struct uberdata32 {
11577c478bdstevel@tonic-gate	pad_lock_t	_link_lock;
11588cd4554raf	pad_lock_t	_ld_lock;
11598cd4554raf	pad_lock_t	_fork_lock;
11608cd4554raf	pad_lock_t	_atfork_lock;
116198c1a6braf	pad32_lock_t	_callout_lock;
116298c1a6braf	pad32_lock_t	_tdb_hash_lock;
11637c478bdstevel@tonic-gate	tdb_sync_stats_t tdb_hash_lock_stats;
11647c478bdstevel@tonic-gate	siguaction32_t	siguaction[NSIG];
11657c478bdstevel@tonic-gate	bucket32_t	bucket[NBUCKETS];
11667c478bdstevel@tonic-gate	atexit_root32_t	atexit_root;
1167fc2512cRobert Mustacchi	quickexit_root32_t quickexit_root;
11687c478bdstevel@tonic-gate	tsd_metadata32_t tsd_metadata;
11697c478bdstevel@tonic-gate	tls_metadata32_t tls_metadata;
11707c478bdstevel@tonic-gate	char		primary_map;
11717c478bdstevel@tonic-gate	char		bucket_init;
11727c478bdstevel@tonic-gate	char		pad[2];
11737c478bdstevel@tonic-gate	uberflags_t	uberflags;
11747c478bdstevel@tonic-gate	caddr32_t	queue_head;
11757c478bdstevel@tonic-gate	caddr32_t	thr_hash_table;
11767c478bdstevel@tonic-gate	uint_t		hash_size;
11777c478bdstevel@tonic-gate	uint_t		hash_mask;
11787c478bdstevel@tonic-gate	caddr32_t	ulwp_one;
11797c478bdstevel@tonic-gate	caddr32_t	all_lwps;
11807c478bdstevel@tonic-gate	caddr32_t	all_zombies;
11817c478bdstevel@tonic-gate	int		nthreads;
11827c478bdstevel@tonic-gate	int		nzombies;
11837c478bdstevel@tonic-gate	int		ndaemons;
11847c478bdstevel@tonic-gate	int		pid;
11857c478bdstevel@tonic-gate	caddr32_t	sigacthandler;
11867c478bdstevel@tonic-gate	caddr32_t	lwp_stacks;
11877c478bdstevel@tonic-gate	caddr32_t	lwp_laststack;
11887c478bdstevel@tonic-gate	int		nfreestack;
11897c478bdstevel@tonic-gate	int		thread_stack_cache;
11907c478bdstevel@tonic-gate	caddr32_t	ulwp_freelist;
11917c478bdstevel@tonic-gate	caddr32_t	ulwp_lastfree;
11927c478bdstevel@tonic-gate	caddr32_t	ulwp_replace_free;
11937c478bdstevel@tonic-gate	caddr32_t	ulwp_replace_last;
11947c478bdstevel@tonic-gate	caddr32_t	atforklist;
1195883492draf	caddr32_t	robustlocks;
119609ce0d4Roger A. Faulkner	caddr32_t	robustlist;
1197fe4be5aGordon Ross	caddr32_t	progname;
11982428aadPatrick Mooney	caddr32_t	ub_comm_page;
11997c478bdstevel@tonic-gate	caddr32_t	tdb_bootstrap;
12007c478bdstevel@tonic-gate	tdb32_t		tdb;
12017c478bdstevel@tonic-gate} uberdata32_t;
12027c478bdstevel@tonic-gate
12037c478bdstevel@tonic-gate#endif	/* _SYSCALL32 */
12047c478bdstevel@tonic-gate
12057c478bdstevel@tonic-gate/* ul_stop values */
12067c478bdstevel@tonic-gate#define	TSTP_REGULAR	0x01	/* Stopped by thr_suspend() */
12077c478bdstevel@tonic-gate#define	TSTP_MUTATOR	0x08	/* stopped by thr_suspend_*mutator*() */
12087c478bdstevel@tonic-gate#define	TSTP_FORK	0x20	/* stopped by suspend_fork() */
12097c478bdstevel@tonic-gate
12107c478bdstevel@tonic-gate/*
12117c478bdstevel@tonic-gate * Implementation-specific attribute types for pthread_mutexattr_init() etc.
12127c478bdstevel@tonic-gate */
12137c478bdstevel@tonic-gate
12147c478bdstevel@tonic-gatetypedef	struct	_cvattr {
12157c478bdstevel@tonic-gate	int	pshared;
12167c478bdstevel@tonic-gate	clockid_t clockid;
12177c478bdstevel@tonic-gate} cvattr_t;
12187c478bdstevel@tonic-gate
12197c478bdstevel@tonic-gatetypedef	struct	_mattr {
12207c478bdstevel@tonic-gate	int	pshared;
12217c478bdstevel@tonic-gate	int	protocol;
12227c478bdstevel@tonic-gate	int	prioceiling;
12237c478bdstevel@tonic-gate	int	type;
12247c478bdstevel@tonic-gate	int	robustness;
12257c478bdstevel@tonic-gate} mattr_t;
12267c478bdstevel@tonic-gate
12277c478bdstevel@tonic-gatetypedef	struct	_thrattr {
12287c478bdstevel@tonic-gate	size_t	stksize;
12297c478bdstevel@tonic-gate	void	*stkaddr;
12307c478bdstevel@tonic-gate	int	detachstate;
12313470957raf	int	daemonstate;
12327c478bdstevel@tonic-gate	int	scope;
12337c478bdstevel@tonic-gate	int	prio;
12347c478bdstevel@tonic-gate	int	policy;
12357c478bdstevel@tonic-gate	int	inherit;
12367c478bdstevel@tonic-gate	size_t	guardsize;
1237ab61854John Levon	char	name[THREAD_NAME_MAX];
12387c478bdstevel@tonic-gate} thrattr_t;
12397c478bdstevel@tonic-gate
12407c478bdstevel@tonic-gatetypedef	struct	_rwlattr {
12417c478bdstevel@tonic-gate	int	pshared;
12427c478bdstevel@tonic-gate} rwlattr_t;
12437c478bdstevel@tonic-gate
12447c478bdstevel@tonic-gate/* _curthread() is inline for speed */
12457c478bdstevel@tonic-gateextern	ulwp_t		*_curthread(void);
12467c478bdstevel@tonic-gate#define	curthread	(_curthread())
12477c478bdstevel@tonic-gate
12487c478bdstevel@tonic-gate/* this version (also inline) can be tested for NULL */
12497c478bdstevel@tonic-gateextern	ulwp_t		*__curthread(void);
12507c478bdstevel@tonic-gate
12517c478bdstevel@tonic-gate/* get the current stack pointer (also inline) */
12527c478bdstevel@tonic-gateextern	greg_t		stkptr(void);
12537c478bdstevel@tonic-gate
12547c478bdstevel@tonic-gate/*
12557c478bdstevel@tonic-gate * Suppress __attribute__((...)) if we are not compiling with gcc
12567c478bdstevel@tonic-gate */
12577c478bdstevel@tonic-gate#if !defined(__GNUC__)
12587c478bdstevel@tonic-gate#define	__attribute__(string)
12597c478bdstevel@tonic-gate#endif
12607c478bdstevel@tonic-gate
1261d4204c8raf/* Fetch the dispatch (kernel) priority of a thread */
1262d4204c8raf#define	real_priority(ulwp)	\
1263d4204c8raf	((ulwp)->ul_schedctl? (ulwp)->ul_schedctl->sc_priority : 0)
1264d4204c8raf
12657c478bdstevel@tonic-gate/*
12667c478bdstevel@tonic-gate * Implementation functions.  Not visible outside of the library itself.
12677c478bdstevel@tonic-gate */
1268f841f6arafextern	int	__nanosleep(const timespec_t *, timespec_t *);
12697c478bdstevel@tonic-gateextern	void	getgregs(ulwp_t *, gregset_t);
12707c478bdstevel@tonic-gateextern	void	setgregs(ulwp_t *, gregset_t);
12717c478bdstevel@tonic-gateextern	void	thr_panic(const char *);
12727c478bdstevel@tonic-gate#pragma rarely_called(thr_panic)
12730d045c0Robert Mustacchiextern	void	mutex_panic(mutex_t *, const char *);
12740d045c0Robert Mustacchi#pragma rarely_called(mutex_panic)
12757c478bdstevel@tonic-gateextern	ulwp_t	*find_lwp(thread_t);
12767c478bdstevel@tonic-gateextern	void	finish_init(void);
1277d4204c8rafextern	void	update_sched(ulwp_t *);
12787c478bdstevel@tonic-gateextern	void	queue_alloc(void);
12794f364e7Robert Mustacchiextern	void	tmem_exit(void);
12807c478bdstevel@tonic-gateextern	void	tsd_exit(void);
12817c478bdstevel@tonic-gateextern	void	tsd_free(ulwp_t *);
12827c478bdstevel@tonic-gateextern	void	tls_setup(void);
12837c478bdstevel@tonic-gateextern	void	tls_exit(void);
12847c478bdstevel@tonic-gateextern	void	tls_free(ulwp_t *);
12857c478bdstevel@tonic-gateextern	void	rwl_free(ulwp_t *);
1286883492drafextern	void	heldlock_exit(void);
1287883492drafextern	void	heldlock_free(ulwp_t *);
12887c478bdstevel@tonic-gateextern	void	sigacthandler(int, siginfo_t *, void *);
12897c478bdstevel@tonic-gateextern	void	signal_init(void);
12907c478bdstevel@tonic-gateextern	int	sigequalset(const sigset_t *, const sigset_t *);
12917c478bdstevel@tonic-gateextern	void	mutex_setup(void);
12927c478bdstevel@tonic-gateextern	void	take_deferred_signal(int);
1293373d25aRoger A. Faulknerextern	void	*setup_top_frame(void *, size_t, ulwp_t *);
12947c478bdstevel@tonic-gateextern	int	setup_context(ucontext_t *, void *(*func)(ulwp_t *),
12957c478bdstevel@tonic-gate			ulwp_t *ulwp, caddr_t stk, size_t stksize);
12967c478bdstevel@tonic-gateextern	volatile sc_shared_t *setup_schedctl(void);
12977c478bdstevel@tonic-gateextern	void	*lmalloc(size_t);
12987c478bdstevel@tonic-gateextern	void	lfree(void *, size_t);
12997c478bdstevel@tonic-gateextern	void	*libc_malloc(size_t);
13007c478bdstevel@tonic-gateextern	void	*libc_realloc(void *, size_t);
13017c478bdstevel@tonic-gateextern	void	libc_free(void *);
13027c478bdstevel@tonic-gateextern	char	*libc_strdup(const char *);
13037c478bdstevel@tonic-gateextern	void	ultos(uint64_t, int, char *);
13047c478bdstevel@tonic-gateextern	void	lock_error(const mutex_t *, const char *, void *, const char *);
13057c478bdstevel@tonic-gateextern	void	rwlock_error(const rwlock_t *, const char *, const char *);
13067c478bdstevel@tonic-gateextern	void	thread_error(const char *);
13077c478bdstevel@tonic-gateextern	void	grab_assert_lock(void);
13087c478bdstevel@tonic-gateextern	void	dump_queue_statistics(void);
13097c478bdstevel@tonic-gateextern	void	collect_queue_statistics(void);
13107c478bdstevel@tonic-gateextern	void	record_spin_locks(ulwp_t *);
1311883492drafextern	void	remember_lock(mutex_t *);
1312883492drafextern	void	forget_lock(mutex_t *);
1313883492drafextern	void	register_lock(mutex_t *);
1314c242ec1Roger A. Faulknerextern	void	unregister_locks(void);
13157c478bdstevel@tonic-gate#if defined(__sparc)
13167c478bdstevel@tonic-gateextern	void	_flush_windows(void);
13177c478bdstevel@tonic-gate#else
13187c478bdstevel@tonic-gate#define	_flush_windows()
13197c478bdstevel@tonic-gate#endif
13207c478bdstevel@tonic-gateextern	void	set_curthread(void *);
13217c478bdstevel@tonic-gate
132241efec2raf/*
1323883492draf * Utility function used when waking up many threads (more than MAXLWPS)
1324883492draf * all at once.  See mutex_wakeup_all(), cond_broadcast(), and rw_unlock().
132541efec2raf */
132641efec2raf#define	MAXLWPS	128	/* max remembered lwpids before overflow */
132741efec2raf#define	NEWLWPS	2048	/* max remembered lwpids at first overflow */
132841efec2rafextern	lwpid_t	*alloc_lwpids(lwpid_t *, int *, int *);
132941efec2raf
13307c478bdstevel@tonic-gate/* enter a critical section */
13317c478bdstevel@tonic-gate#define	enter_critical(self)	(self->ul_critical++)
13327c478bdstevel@tonic-gate
13337c478bdstevel@tonic-gate/* exit a critical section, take deferred actions if necessary */
13347c478bdstevel@tonic-gateextern	void	do_exit_critical(void);
13357c478bdstevel@tonic-gate#define	exit_critical(self)					\
13367c478bdstevel@tonic-gate	(void) (self->ul_critical--,				\
13377c478bdstevel@tonic-gate	    ((self->ul_curplease && self->ul_critical == 0)?	\
13387c478bdstevel@tonic-gate	    (do_exit_critical(), 0) : 0))
13397c478bdstevel@tonic-gate
13407c478bdstevel@tonic-gate/*
13417c478bdstevel@tonic-gate * Like enter_critical()/exit_critical() but just for deferring signals.
13427c478bdstevel@tonic-gate * Unlike enter_critical()/exit_critical(), ul_sigdefer may be set while
13437c478bdstevel@tonic-gate * calling application functions like constructors and destructors.
13447c478bdstevel@tonic-gate * Care must be taken if the application function attempts to set
13457c478bdstevel@tonic-gate * the signal mask while a deferred signal is present; the setting
13467c478bdstevel@tonic-gate * of the signal mask must also be deferred.
13477c478bdstevel@tonic-gate */
13487c478bdstevel@tonic-gate#define	sigoff(self)	(self->ul_sigdefer++)
1349e54ab87Roger A. Faulkner#define	sigon(self)						\
1350e54ab87Roger A. Faulkner	(void) ((--self->ul_sigdefer == 0 &&			\
1351e54ab87Roger A. Faulkner	    self->ul_curplease && self->ul_critical == 0)?	\
1352e54ab87Roger A. Faulkner	    (do_exit_critical(), 0) : 0)
13537c478bdstevel@tonic-gate
13547c478bdstevel@tonic-gate/* these are exported functions */
13557c478bdstevel@tonic-gateextern	void	_sigoff(void);
13567c478bdstevel@tonic-gateextern	void	_sigon(void);
13577c478bdstevel@tonic-gate
13587c478bdstevel@tonic-gate#define	sigorset(s1, s2)				\
13597c478bdstevel@tonic-gate	(((s1)->__sigbits[0] |= (s2)->__sigbits[0]),	\
13607c478bdstevel@tonic-gate	((s1)->__sigbits[1] |= (s2)->__sigbits[1]),	\
13617c478bdstevel@tonic-gate	((s1)->__sigbits[2] |= (s2)->__sigbits[2]),	\
13627c478bdstevel@tonic-gate	((s1)->__sigbits[3] |= (s2)->__sigbits[3]))
13637c478bdstevel@tonic-gate
13647c478bdstevel@tonic-gate#define	sigandset(s1, s2)				\
13657c478bdstevel@tonic-gate	(((s1)->__sigbits[0] &= (s2)->__sigbits[0]),	\
13667c478bdstevel@tonic-gate	((s1)->__sigbits[1] &= (s2)->__sigbits[1]),	\
13677c478bdstevel@tonic-gate	((s1)->__sigbits[2] &= (s2)->__sigbits[2]),	\
13687c478bdstevel@tonic-gate	((s1)->__sigbits[3] &= (s2)->__sigbits[3]))
13697c478bdstevel@tonic-gate
13707c478bdstevel@tonic-gate#define	sigdiffset(s1, s2)				\
13717c478bdstevel@tonic-gate	(((s1)->__sigbits[0] &= ~(s2)->__sigbits[0]),	\
13727c478bdstevel@tonic-gate	((s1)->__sigbits[1] &= ~(s2)->__sigbits[1]),	\
13737c478bdstevel@tonic-gate	((s1)->__sigbits[2] &= ~(s2)->__sigbits[2]),	\
13747c478bdstevel@tonic-gate	((s1)->__sigbits[3] &= ~(s2)->__sigbits[3]))
13757c478bdstevel@tonic-gate
13767c478bdstevel@tonic-gate#define	delete_reserved_signals(s)			\
13777c478bdstevel@tonic-gate	(((s)->__sigbits[0] &= MASKSET0),		\
13787c478bdstevel@tonic-gate	((s)->__sigbits[1] &= (MASKSET1 & ~SIGMASK(SIGCANCEL))),\
1379bdf0047Roger A. Faulkner	((s)->__sigbits[2] &= MASKSET2),		\
1380bdf0047Roger A. Faulkner	((s)->__sigbits[3] &= MASKSET3))
13817c478bdstevel@tonic-gate
13827c478bdstevel@tonic-gateextern	void	block_all_signals(ulwp_t *self);
13837c478bdstevel@tonic-gate
13847c478bdstevel@tonic-gate/*
13857c478bdstevel@tonic-gate * When restoring the signal mask after having previously called
13867c478bdstevel@tonic-gate * block_all_signals(), if we have a deferred signal present then
13877c478bdstevel@tonic-gate * do nothing other than ASSERT() that we are in a critical region.
13887c478bdstevel@tonic-gate * The signal mask will be set when we emerge from the critical region
13897c478bdstevel@tonic-gate * and call take_deferred_signal().  There is no race condition here
13907c478bdstevel@tonic-gate * because the kernel currently has all signals blocked for this thread.
13917c478bdstevel@tonic-gate */
13927c478bdstevel@tonic-gate#define	restore_signals(self)						\
13937c478bdstevel@tonic-gate	((void) ((self)->ul_cursig?					\
13947c478bdstevel@tonic-gate	(ASSERT((self)->ul_critical + (self)->ul_sigdefer != 0), 0) :	\
1395bdf0047Roger A. Faulkner	__lwp_sigmask(SIG_SETMASK, &(self)->ul_sigmask)))
13967c478bdstevel@tonic-gate
1397a574db8rafextern	void	set_cancel_pending_flag(ulwp_t *, int);
1398a574db8rafextern	void	set_cancel_eintr_flag(ulwp_t *);
13997c478bdstevel@tonic-gateextern	void	set_parking_flag(ulwp_t *, int);
1400a574db8rafextern	int	cancel_active(void);
14017c478bdstevel@tonic-gate
14027257d1brafextern	void	*_thrp_setup(ulwp_t *);
14037c478bdstevel@tonic-gateextern	void	_fpinherit(ulwp_t *);
14047c478bdstevel@tonic-gateextern	void	_lwp_start(void);
14057c478bdstevel@tonic-gateextern	void	_lwp_terminate(void);
14067c478bdstevel@tonic-gateextern	void	lmutex_lock(mutex_t *);
1407f841f6arafextern	void	lmutex_unlock(mutex_t *);
140841efec2rafextern	void	lrw_rdlock(rwlock_t *);
140941efec2rafextern	void	lrw_wrlock(rwlock_t *);
141041efec2rafextern	void	lrw_unlock(rwlock_t *);
1411f841f6arafextern	void	sig_mutex_lock(mutex_t *);
1412f841f6arafextern	void	sig_mutex_unlock(mutex_t *);
1413f841f6arafextern	int	sig_mutex_trylock(mutex_t *);
1414f841f6arafextern	int	sig_cond_wait(cond_t *, mutex_t *);
1415f841f6arafextern	int	sig_cond_reltimedwait(cond_t *, mutex_t *, const timespec_t *);
1416a574db8rafextern	void	cancel_safe_mutex_lock(mutex_t *);
1417a574db8rafextern	void	cancel_safe_mutex_unlock(mutex_t *);
1418a574db8rafextern	int	cancel_safe_mutex_trylock(mutex_t *);
14197c478bdstevel@tonic-gateextern	void	_prefork_handler(void);
14207c478bdstevel@tonic-gateextern	void	_postfork_parent_handler(void);
14217c478bdstevel@tonic-gateextern	void	_postfork_child_handler(void);
1422f841f6arafextern	void	postfork1_child(void);
1423f841f6arafextern	void	postfork1_child_aio(void);
1424f841f6arafextern	void	postfork1_child_sigev_aio(void);
1425f841f6arafextern	void	postfork1_child_sigev_mq(void);
1426f841f6arafextern	void	postfork1_child_sigev_timer(void);
1427f841f6arafextern	void	postfork1_child_tpool(void);
14282e14588rafextern	void	fork_lock_enter(void);
14297c478bdstevel@tonic-gateextern	void	fork_lock_exit(void);
14307c478bdstevel@tonic-gateextern	void	suspend_fork(void);
14317c478bdstevel@tonic-gateextern	void	continue_fork(int);
14327c478bdstevel@tonic-gateextern	void	do_sigcancel(void);
1433f841f6arafextern	void	setup_cancelsig(int);
1434f841f6arafextern	void	init_sigev_thread(void);
1435f841f6arafextern	void	init_aio(void);
143623a1cceRoger A. Faulknerextern	void	init_progname(void);
14377c478bdstevel@tonic-gateextern	void	_cancelon(void);
14387c478bdstevel@tonic-gateextern	void	_canceloff(void);
14397c478bdstevel@tonic-gateextern	void	_canceloff_nocancel(void);
1440f841f6arafextern	void	_cancel_prologue(void);
1441f841f6arafextern	void	_cancel_epilogue(void);
14427c478bdstevel@tonic-gateextern	void	no_preempt(ulwp_t *);
14437c478bdstevel@tonic-gateextern	void	preempt(ulwp_t *);
14447c478bdstevel@tonic-gateextern	void	_thrp_unwind(void *);
14457c478bdstevel@tonic-gate
1446657b1f3rafextern	pid_t	__forkx(int);
1447657b1f3rafextern	pid_t	__forkallx(int);
14488fd04b8Roger A. Faulknerextern	int	__open(const char *, int, mode_t);
14498fd04b8Roger A. Faulknerextern	int	__open64(const char *, int, mode_t);
14508fd04b8Roger A. Faulknerextern	int	__openat(int, const char *, int, mode_t);
14518fd04b8Roger A. Faulknerextern	int	__openat64(int, const char *, int, mode_t);
14528cd4554rafextern	int	__close(int);
1453a574db8rafextern	ssize_t	__read(int, void *, size_t);
1454a574db8rafextern	ssize_t	__write(int, const void *, size_t);
14558cd4554rafextern	int	__fcntl(int, int, ...);
14567c478bdstevel@tonic-gateextern	int	__lwp_continue(lwpid_t);
14577c478bdstevel@tonic-gateextern	int	__lwp_create(ucontext_t *, uint_t, lwpid_t *);
14587c478bdstevel@tonic-gateextern	int	___lwp_suspend(lwpid_t);
14597c478bdstevel@tonic-gateextern	int	lwp_wait(lwpid_t, lwpid_t *);
14607c478bdstevel@tonic-gateextern	int	__lwp_wait(lwpid_t, lwpid_t *);
14617c478bdstevel@tonic-gateextern	int	__lwp_detach(lwpid_t);
14627c478bdstevel@tonic-gateextern	sc_shared_t *__schedctl(void);
14637c478bdstevel@tonic-gate
14647c478bdstevel@tonic-gate/* actual system call traps */
14658cd4554rafextern	int	__setcontext(const ucontext_t *);
14668cd4554rafextern	int	__getcontext(ucontext_t *);
14677c478bdstevel@tonic-gateextern	int	__clock_gettime(clockid_t, timespec_t *);
14687c478bdstevel@tonic-gateextern	void	abstime_to_reltime(clockid_t, const timespec_t *, timespec_t *);