xref: /illumos-gate/usr/src/lib/libc/inc/thr_inlines.h (revision d3b5f563)
10ec57554Sraf /*
20ec57554Sraf  * CDDL HEADER START
30ec57554Sraf  *
40ec57554Sraf  * The contents of this file are subject to the terms of the
50ec57554Sraf  * Common Development and Distribution License (the "License").
60ec57554Sraf  * You may not use this file except in compliance with the License.
70ec57554Sraf  *
80ec57554Sraf  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
90ec57554Sraf  * or http://www.opensolaris.org/os/licensing.
100ec57554Sraf  * See the License for the specific language governing permissions
110ec57554Sraf  * and limitations under the License.
120ec57554Sraf  *
130ec57554Sraf  * When distributing Covered Code, include this CDDL HEADER in each
140ec57554Sraf  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
150ec57554Sraf  * If applicable, add the following below this CDDL HEADER, with the
160ec57554Sraf  * fields enclosed by brackets "[]" replaced with your own identifying
170ec57554Sraf  * information: Portions Copyright [yyyy] [name of copyright owner]
180ec57554Sraf  *
190ec57554Sraf  * CDDL HEADER END
200ec57554Sraf  */
220ec57554Sraf /*
23ae115bc7Smrj  * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
240ec57554Sraf  * Use is subject to license terms.
250ec57554Sraf  */
27*d3b5f563SJohn Levon /*
28*d3b5f563SJohn Levon  * Copyright 2019 Joyent, Inc.
29*d3b5f563SJohn Levon  */
30*d3b5f563SJohn Levon 
310ec57554Sraf #ifndef _THR_INLINES_H
320ec57554Sraf #define	_THR_INLINES_H
346a3e8e86SRichard Lowe #include <sys/ccompile.h>
360ec57554Sraf #if !defined(__lint) && defined(__GNUC__)
380ec57554Sraf /* inlines for gcc */
406a3e8e86SRichard Lowe /*
416a3e8e86SRichard Lowe  * ON-usable GCC 4.x emits register pseudo-ops declaring %g7 as ignored, rather
426a3e8e86SRichard Lowe  * than scratch, GCC 3 does the reverse.  All uses, both ones it generated
436a3e8e86SRichard Lowe  * (_curthread) and ones it didn't (__curthread) must agree.
446a3e8e86SRichard Lowe  */
456a3e8e86SRichard Lowe #if __GNUC__ > 3
466a3e8e86SRichard Lowe #define	SPARC_REG_SPEC	"#ignore"
476a3e8e86SRichard Lowe #else
486a3e8e86SRichard Lowe #define	SPARC_REG_SPEC	"#scratch"
496a3e8e86SRichard Lowe #endif
506a3e8e86SRichard Lowe 
516a3e8e86SRichard Lowe extern __GNU_INLINE ulwp_t *
_curthread(void)520ec57554Sraf _curthread(void)
530ec57554Sraf {
540ec57554Sraf #if defined(__amd64)
550ec57554Sraf 	ulwp_t *__value;
560ec57554Sraf 	__asm__ __volatile__("movq %%fs:0, %0" : "=r" (__value));
570ec57554Sraf #elif defined(__i386)
580ec57554Sraf 	ulwp_t *__value;
590ec57554Sraf 	__asm__ __volatile__("movl %%gs:0, %0" : "=r" (__value));
600ec57554Sraf #elif defined(__sparc)
610ec57554Sraf 	register ulwp_t *__value __asm__("g7");
620ec57554Sraf #else
630ec57554Sraf #error	"port me"
640ec57554Sraf #endif
650ec57554Sraf 	return (__value);
660ec57554Sraf }
686a3e8e86SRichard Lowe extern __GNU_INLINE ulwp_t *
__curthread(void)690ec57554Sraf __curthread(void)
700ec57554Sraf {
710ec57554Sraf 	ulwp_t *__value;
720ec57554Sraf 	__asm__ __volatile__(
730ec57554Sraf #if defined(__amd64)
746a3e8e86SRichard Lowe 	    "movq %%fs:0, %0\n\t"
750ec57554Sraf #elif defined(__i386)
766a3e8e86SRichard Lowe 	    "movl %%gs:0, %0\n\t"
770ec57554Sraf #elif defined(__sparcv9)
786a3e8e86SRichard Lowe 	    ".register %%g7, " SPARC_REG_SPEC "\n\t"
796a3e8e86SRichard Lowe 	    "ldx [%%g7 + 80], %0\n\t"
800ec57554Sraf #elif defined(__sparc)
816a3e8e86SRichard Lowe 	    ".register %%g7, " SPARC_REG_SPEC "\n\t"
826a3e8e86SRichard Lowe 	    "ld [%%g7 + 80], %0\n\t"
830ec57554Sraf #else
840ec57554Sraf #error	"port me"
850ec57554Sraf #endif
866a3e8e86SRichard Lowe 	    : "=r" (__value));
870ec57554Sraf 	return (__value);
880ec57554Sraf }
906a3e8e86SRichard Lowe extern __GNU_INLINE greg_t
stkptr(void)910ec57554Sraf stkptr(void)
920ec57554Sraf {
930ec57554Sraf #if defined(__amd64)
940ec57554Sraf 	register greg_t __value __asm__("rsp");
950ec57554Sraf #elif defined(__i386)
960ec57554Sraf 	register greg_t __value __asm__("esp");
970ec57554Sraf #elif defined(__sparc)
980ec57554Sraf 	register greg_t __value __asm__("sp");
990ec57554Sraf #else
1000ec57554Sraf #error	"port me"
1010ec57554Sraf #endif
1020ec57554Sraf 	return (__value);
1030ec57554Sraf }
1056a3e8e86SRichard Lowe extern __GNU_INLINE hrtime_t
gethrtime(void)1060ec57554Sraf gethrtime(void)		/* note: caller-saved registers are trashed */
1070ec57554Sraf {
1080ec57554Sraf #if defined(__amd64)
1090ec57554Sraf 	hrtime_t __value;
1100ec57554Sraf 	__asm__ __volatile__(
1116a3e8e86SRichard Lowe 	    "movl $3, %%eax\n\t"
1126a3e8e86SRichard Lowe 	    "int $0xd2"
1136a3e8e86SRichard Lowe 	    : "=a" (__value)
1146a3e8e86SRichard Lowe 	    : : "rcx", "rdx", "rsi", "rdi", "r8", "r9", "r10", "r11", "cc");
1150ec57554Sraf #elif defined(__i386)
1160ec57554Sraf 	hrtime_t __value;
1170ec57554Sraf 	__asm__ __volatile__(
1186a3e8e86SRichard Lowe 	    "movl $3, %%eax\n\t"
1196a3e8e86SRichard Lowe 	    "int $0xd2"
1206a3e8e86SRichard Lowe 	    : "=A" (__value)
1216a3e8e86SRichard Lowe 	    : : "ecx", "cc");
1220ec57554Sraf #elif defined(__sparcv9)
1230ec57554Sraf 	register hrtime_t __value __asm__("o0");
1240ec57554Sraf 	__asm__ __volatile__(
1256a3e8e86SRichard Lowe 	    "ta 0x24\n\t"
1266a3e8e86SRichard Lowe 	    "sllx %%o0, 32, %0\n\t"
1276a3e8e86SRichard Lowe 	    "or %%o1, %0, %0"
1286a3e8e86SRichard Lowe 	    : "=r" (__value)
1296a3e8e86SRichard Lowe 	    : : "o1", "o2", "o3", "o4", "o5", "cc");
1300ec57554Sraf #elif defined(__sparc)
1310ec57554Sraf 	register hrtime_t __value __asm__("o0");
1320ec57554Sraf 	__asm__ __volatile__(
1336a3e8e86SRichard Lowe 	    "ta 0x24"
1346a3e8e86SRichard Lowe 	    : "=r" (__value)
1356a3e8e86SRichard Lowe 	    : : "o2", "o3", "o4", "o5", "cc");
1360ec57554Sraf #else
1370ec57554Sraf #error	"port me"
1380ec57554Sraf #endif
1390ec57554Sraf 	return (__value);
1400ec57554Sraf }
1426a3e8e86SRichard Lowe extern __GNU_INLINE int
set_lock_byte(volatile uint8_t * __lockp)1430ec57554Sraf set_lock_byte(volatile uint8_t *__lockp)
1440ec57554Sraf {
145*d3b5f563SJohn Levon 	int __value = 0;
1460ec57554Sraf #if defined(__x86)
1470ec57554Sraf 	__asm__ __volatile__(
1486a3e8e86SRichard Lowe 	    "movl $1, %0\n\t"
1496a3e8e86SRichard Lowe 	    "xchgb %%dl, %1"
1506a3e8e86SRichard Lowe 	    : "+d" (__value), "+m" (*__lockp));
1510ec57554Sraf #elif defined(__sparc)
1520ec57554Sraf 	__asm__ __volatile__(
1536a3e8e86SRichard Lowe 	    "ldstub %1, %0\n\t"
1546a3e8e86SRichard Lowe 	    "membar #LoadLoad"
1556a3e8e86SRichard Lowe 	    : "=r" (__value), "+m" (*__lockp));
1560ec57554Sraf #else
1570ec57554Sraf #error	"port me"
1580ec57554Sraf #endif
1590ec57554Sraf 	return (__value);
1600ec57554Sraf }
1626a3e8e86SRichard Lowe extern __GNU_INLINE uint32_t
atomic_swap_32(volatile uint32_t * __memory,uint32_t __value)16341efec22Sraf atomic_swap_32(volatile uint32_t *__memory, uint32_t __value)
1640ec57554Sraf {
1650ec57554Sraf #if defined(__x86)
1660ec57554Sraf 	__asm__ __volatile__(
1676a3e8e86SRichard Lowe 	    "xchgl %0, %1"
1686a3e8e86SRichard Lowe 	    : "+q" (__value), "+m" (*__memory));
1690ec57554Sraf 	return (__value);
1700ec57554Sraf #elif defined(__sparc)
1710ec57554Sraf 	uint32_t __tmp1, __tmp2;
1720ec57554Sraf 	__asm__ __volatile__(
1736a3e8e86SRichard Lowe 	    "ld [%3], %0\n\t"
1746a3e8e86SRichard Lowe 	    "1:\n\t"
1756a3e8e86SRichard Lowe 	    "mov %4, %1\n\t"
1766a3e8e86SRichard Lowe 	    "cas [%3], %0, %1\n\t"
1776a3e8e86SRichard Lowe 	    "cmp %0, %1\n\t"
1786a3e8e86SRichard Lowe 	    "bne,a,pn %%icc, 1b\n\t"
1796a3e8e86SRichard Lowe 	    "  mov %1, %0"
1806a3e8e86SRichard Lowe 	    : "=&r" (__tmp1), "=&r" (__tmp2), "=m" (*__memory)
1816a3e8e86SRichard Lowe 	    : "r" (__memory), "r" (__value)
1826a3e8e86SRichard Lowe 	    : "cc");
1830ec57554Sraf 	return (__tmp2);
1840ec57554Sraf #else
1850ec57554Sraf #error	"port me"
1860ec57554Sraf #endif
1870ec57554Sraf }
1896a3e8e86SRichard Lowe extern __GNU_INLINE uint32_t
atomic_cas_32(volatile uint32_t * __memory,uint32_t __cmp,uint32_t __newvalue)19041efec22Sraf atomic_cas_32(volatile uint32_t *__memory, uint32_t __cmp, uint32_t __newvalue)
1910ec57554Sraf {
1920ec57554Sraf 	uint32_t __oldvalue;
1930ec57554Sraf #if defined(__x86)
1940ec57554Sraf 	__asm__ __volatile__(
1956a3e8e86SRichard Lowe 	    "lock; cmpxchgl %3, %0"
1966a3e8e86SRichard Lowe 	    : "=m" (*__memory), "=a" (__oldvalue)
1976a3e8e86SRichard Lowe 	    : "a" (__cmp), "r" (__newvalue));
1980ec57554Sraf #elif defined(__sparc)
1990ec57554Sraf 	__asm__ __volatile__(
2006a3e8e86SRichard Lowe 	    "cas [%2], %3, %1"
2016a3e8e86SRichard Lowe 	    : "=m" (*__memory), "=&r" (__oldvalue)
2026a3e8e86SRichard Lowe 	    : "r" (__memory), "r" (__cmp), "1" (__newvalue));
2030ec57554Sraf #else
2040ec57554Sraf #error	"port me"
2050ec57554Sraf #endif
2060ec57554Sraf 	return (__oldvalue);
2070ec57554Sraf }
2096a3e8e86SRichard Lowe extern __GNU_INLINE void
atomic_inc_32(volatile uint32_t * __memory)21041efec22Sraf atomic_inc_32(volatile uint32_t *__memory)
2110ec57554Sraf {
2120ec57554Sraf #if defined(__x86)
2130ec57554Sraf 	__asm__ __volatile__(
2146a3e8e86SRichard Lowe 	    "lock; incl %0"
2156a3e8e86SRichard Lowe 	    : "+m" (*__memory));
2160ec57554Sraf #elif defined(__sparc)
2170ec57554Sraf 	uint32_t __tmp1, __tmp2;
2180ec57554Sraf 	__asm__ __volatile__(
2196a3e8e86SRichard Lowe 	    "ld [%3], %0\n\t"
2206a3e8e86SRichard Lowe 	    "1:\n\t"
2216a3e8e86SRichard Lowe 	    "add %0, 1, %1\n\t"
2226a3e8e86SRichard Lowe 	    "cas [%3], %0, %1\n\t"
2236a3e8e86SRichard Lowe 	    "cmp %0, %1\n\t"
2246a3e8e86SRichard Lowe 	    "bne,a,pn %%icc, 1b\n\t"
2256a3e8e86SRichard Lowe 	    "  mov %1, %0"
2266a3e8e86SRichard Lowe 	    : "=&r" (__tmp1), "=&r" (__tmp2), "=m" (*__memory)
2276a3e8e86SRichard Lowe 	    : "r" (__memory)
2286a3e8e86SRichard Lowe 	    : "cc");
2290ec57554Sraf #else
2300ec57554Sraf #error	"port me"
2310ec57554Sraf #endif
2320ec57554Sraf }
2346a3e8e86SRichard Lowe extern __GNU_INLINE void
atomic_dec_32(volatile uint32_t * __memory)23541efec22Sraf atomic_dec_32(volatile uint32_t *__memory)
2360ec57554Sraf {
2370ec57554Sraf #if defined(__x86)
2380ec57554Sraf 	__asm__ __volatile__(
2396a3e8e86SRichard Lowe 	    "lock; decl %0"
2406a3e8e86SRichard Lowe 	    : "+m" (*__memory));
2410ec57554Sraf #elif defined(__sparc)
2420ec57554Sraf 	uint32_t __tmp1, __tmp2;
2430ec57554Sraf 	__asm__ __volatile__(
2446a3e8e86SRichard Lowe 	    "ld [%3], %0\n\t"
2456a3e8e86SRichard Lowe 	    "1:\n\t"
2466a3e8e86SRichard Lowe 	    "sub %0, 1, %1\n\t"
2476a3e8e86SRichard Lowe 	    "cas [%3], %0, %1\n\t"
2486a3e8e86SRichard Lowe 	    "cmp %0, %1\n\t"
2496a3e8e86SRichard Lowe 	    "bne,a,pn %%icc, 1b\n\t"
2506a3e8e86SRichard Lowe 	    "  mov %1, %0"
2516a3e8e86SRichard Lowe 	    : "=&r" (__tmp1), "=&r" (__tmp2), "=m" (*__memory)
2526a3e8e86SRichard Lowe 	    : "r" (__memory)
2536a3e8e86SRichard Lowe 	    : "cc");
2540ec57554Sraf #else
2550ec57554Sraf #error	"port me"
2560ec57554Sraf #endif
2570ec57554Sraf }
2596a3e8e86SRichard Lowe extern __GNU_INLINE void
atomic_and_32(volatile uint32_t * __memory,uint32_t __bits)26041efec22Sraf atomic_and_32(volatile uint32_t *__memory, uint32_t __bits)
26141efec22Sraf {
26241efec22Sraf #if defined(__x86)
26341efec22Sraf 	__asm__ __volatile__(
2646a3e8e86SRichard Lowe 	    "lock; andl %1, %0"
2656a3e8e86SRichard Lowe 	    : "+m" (*__memory)
2666a3e8e86SRichard Lowe 	    : "r" (__bits));
26741efec22Sraf #elif defined(__sparc)
26841efec22Sraf 	uint32_t __tmp1, __tmp2;
26941efec22Sraf 	__asm__ __volatile__(
2706a3e8e86SRichard Lowe 	    "ld [%3], %0\n\t"
2716a3e8e86SRichard Lowe 	    "1:\n\t"
2726a3e8e86SRichard Lowe 	    "and %0, %4, %1\n\t"
2736a3e8e86SRichard Lowe 	    "cas [%3], %0, %1\n\t"
2746a3e8e86SRichard Lowe 	    "cmp %0, %1\n\t"
2756a3e8e86SRichard Lowe 	    "bne,a,pn %%icc, 1b\n\t"
2766a3e8e86SRichard Lowe 	    "  mov %1, %0"
2776a3e8e86SRichard Lowe 	    : "=&r" (__tmp1), "=&r" (__tmp2), "=m" (*__memory)
2786a3e8e86SRichard Lowe 	    : "r" (__memory), "r" (__bits)
2796a3e8e86SRichard Lowe 	    : "cc");
28041efec22Sraf #else
28141efec22Sraf #error	"port me"
28241efec22Sraf #endif
28341efec22Sraf }
2856a3e8e86SRichard Lowe extern __GNU_INLINE void
atomic_or_32(volatile uint32_t * __memory,uint32_t __bits)28641efec22Sraf atomic_or_32(volatile uint32_t *__memory, uint32_t __bits)
28741efec22Sraf {
28841efec22Sraf #if defined(__x86)
28941efec22Sraf 	__asm__ __volatile__(
2906a3e8e86SRichard Lowe 	    "lock; orl %1, %0"
2916a3e8e86SRichard Lowe 	    : "+m" (*__memory)
2926a3e8e86SRichard Lowe 	    : "r" (__bits));
29341efec22Sraf #elif defined(__sparc)
29441efec22Sraf 	uint32_t __tmp1, __tmp2;
29541efec22Sraf 	__asm__ __volatile__(
2966a3e8e86SRichard Lowe 	    "ld [%3], %0\n\t"
2976a3e8e86SRichard Lowe 	    "1:\n\t"
2986a3e8e86SRichard Lowe 	    "or %0, %4, %1\n\t"
2996a3e8e86SRichard Lowe 	    "cas [%3], %0, %1\n\t"
3006a3e8e86SRichard Lowe 	    "cmp %0, %1\n\t"
3016a3e8e86SRichard Lowe 	    "bne,a,pn %%icc, 1b\n\t"
3026a3e8e86SRichard Lowe 	    "  mov %1, %0"
3036a3e8e86SRichard Lowe 	    : "=&r" (__tmp1), "=&r" (__tmp2), "=m" (*__memory)
3046a3e8e86SRichard Lowe 	    : "r" (__memory), "r" (__bits)
3056a3e8e86SRichard Lowe 	    : "cc");
30641efec22Sraf #else
30741efec22Sraf #error	"port me"
30841efec22Sraf #endif
30941efec22Sraf }
3110ec57554Sraf #if defined(__sparc)	/* only needed on sparc */
3136a3e8e86SRichard Lowe extern __GNU_INLINE ulong_t
caller(void)3140ec57554Sraf caller(void)
3150ec57554Sraf {
3160ec57554Sraf 	register ulong_t __value __asm__("i7");
3170ec57554Sraf 	return (__value);
3180ec57554Sraf }
3206a3e8e86SRichard Lowe extern __GNU_INLINE ulong_t
getfp(void)3210ec57554Sraf getfp(void)
3220ec57554Sraf {
3230ec57554Sraf 	register ulong_t __value __asm__("fp");
3240ec57554Sraf 	return (__value);
3250ec57554Sraf }
3270ec57554Sraf #endif	/* __sparc */
3290ec57554Sraf #if defined(__x86)	/* only needed on x86 */
3316a3e8e86SRichard Lowe extern __GNU_INLINE void
ht_pause(void)3320ec57554Sraf ht_pause(void)
3330ec57554Sraf {
3340ec57554Sraf 	__asm__ __volatile__("rep; nop");
3350ec57554Sraf }
3370ec57554Sraf #endif	/* __x86 */
3390ec57554Sraf #endif	/* !__lint && __GNUC__ */
3410ec57554Sraf #endif	/* _THR_INLINES_H */