27c478bdstevel@tonic-gate * CDDL HEADER START
37c478bdstevel@tonic-gate *
47c478bdstevel@tonic-gate * The contents of this file are subject to the terms of the
5dfb96a4ab * Common Development and Distribution License (the "License").
6dfb96a4ab * You may not use this file except in compliance with the License.
77c478bdstevel@tonic-gate *
87c478bdstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bdstevel@tonic-gate * or http://www.opensolaris.org/os/licensing.
107c478bdstevel@tonic-gate * See the License for the specific language governing permissions
117c478bdstevel@tonic-gate * and limitations under the License.
127c478bdstevel@tonic-gate *
137c478bdstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each
147c478bdstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bdstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the
167c478bdstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying
177c478bdstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bdstevel@tonic-gate *
197c478bdstevel@tonic-gate * CDDL HEADER END
207c478bdstevel@tonic-gate */
23c38855fShesha Sreenivasamurthy * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
247c478bdstevel@tonic-gate * Use is subject to license terms.
257c478bdstevel@tonic-gate */
279a70fc3Mark J. Nelson	.file	"atomic.s"
297c478bdstevel@tonic-gate#include <sys/asm_linkage.h>
32895ca17ae * ATOMIC_BO_ENABLE_SHIFT can be selectively defined by processors
33895ca17ae * to enable exponential backoff. No definition means backoff is
34895ca17ae * not desired i.e. backoff should be disabled.
35895ca17ae * By default, the shift value is used to generate a power of 2
36895ca17ae * value for backoff limit. In the kernel, processors scale this
37895ca17ae * shift value with the number of online cpus.
38895ca17ae */
407c478bdstevel@tonic-gate#if defined(_KERNEL)
417c478bdstevel@tonic-gate	/*
426ed9368Josef 'Jeff' Sipek	 * Legacy kernel interfaces; they will go away the moment our closed
436ed9368Josef 'Jeff' Sipek	 * bins no longer require them.
447c478bdstevel@tonic-gate	 */
457c478bdstevel@tonic-gate	ANSI_PRAGMA_WEAK2(cas8,atomic_cas_8,function)
467c478bdstevel@tonic-gate	ANSI_PRAGMA_WEAK2(cas32,atomic_cas_32,function)
477c478bdstevel@tonic-gate	ANSI_PRAGMA_WEAK2(cas64,atomic_cas_64,function)
487c478bdstevel@tonic-gate	ANSI_PRAGMA_WEAK2(caslong,atomic_cas_ulong,function)
497c478bdstevel@tonic-gate	ANSI_PRAGMA_WEAK2(casptr,atomic_cas_ptr,function)
507c478bdstevel@tonic-gate	ANSI_PRAGMA_WEAK2(atomic_and_long,atomic_and_ulong,function)
517c478bdstevel@tonic-gate	ANSI_PRAGMA_WEAK2(atomic_or_long,atomic_or_ulong,function)
527c478bdstevel@tonic-gate	ANSI_PRAGMA_WEAK2(swapl,atomic_swap_32,function)
54895ca17ae#ifdef ATOMIC_BO_ENABLE_SHIFT
56895ca17ae#if !defined(lint)
57895ca17ae	.weak   cpu_atomic_delay
58895ca17ae	.type   cpu_atomic_delay, #function
59895ca17ae#endif  /* lint */
62895ca17ae * For the kernel, invoke processor specific delay routine to perform
63895ca17ae * low-impact spin delay. The value of ATOMIC_BO_ENABLE_SHIFT is tuned
64895ca17ae * with respect to the specific spin delay implementation.
65895ca17ae */
66895ca17ae#define	DELAY_SPIN(label, tmp1, tmp2)					\
67895ca17ae	/*								; \
68895ca17ae	 * Define a pragma weak reference to a cpu specific		; \
69895ca17ae	 * delay routine for atomic backoff. For CPUs that		; \
70895ca17ae	 * have no such delay routine defined, the delay becomes	; \
71895ca17ae	 * just a simple tight loop.					; \
72895ca17ae	 *								; \
73895ca17ae	 * tmp1 = holds CPU specific delay routine			; \
74895ca17ae	 * tmp2 = holds atomic routine's callee return address		; \
75895ca17ae	 */								; \
76895ca17ae	sethi	%hi(cpu_atomic_delay), tmp1				; \
77895ca17ae	or	tmp1, %lo(cpu_atomic_delay), tmp1			; \
78895ca17aelabel/**/0:								; \
79895ca17ae	brz,pn	tmp1, label/**/1					; \
80895ca17ae	mov	%o7, tmp2						; \
81895ca17ae	jmpl	tmp1, %o7	/* call CPU specific delay routine */	; \
82895ca17ae	  nop			/* delay slot : do nothing */		; \
83895ca17ae	mov	tmp2, %o7	/* restore callee's return address */	; \
87895ca17ae * For the kernel, we take into consideration of cas failures
88895ca17ae * and also scale the backoff limit w.r.t. the number of cpus.
89895ca17ae * For cas failures, we reset the backoff value to 1 if the cas
90895ca17ae * failures exceed or equal to the number of online cpus. This
91895ca17ae * will enforce some degree of fairness and prevent starvation.
92895ca17ae * We also scale/normalize the processor provided specific
93895ca17ae * ATOMIC_BO_ENABLE_SHIFT w.r.t. the number of online cpus to
94895ca17ae * obtain the actual final limit to use.
95895ca17ae */
96895ca17ae#define ATOMIC_BACKOFF_CPU(val, limit, ncpu, cas_cnt, label)		\
97895ca17ae	brnz,pt	ncpu, label/**/0					; \
98895ca17ae	  inc	cas_cnt							; \
99895ca17ae	sethi	%hi(ncpus_online), ncpu					; \
100895ca17ae	ld	[ncpu + %lo(ncpus_online)], ncpu			; \
101895ca17aelabel/**/0:								; \
102895ca17ae	cmp	cas_cnt, ncpu						; \
103895ca17ae	blu,pt	%xcc, label/**/1					; \
104895ca17ae	  sllx	ncpu, ATOMIC_BO_ENABLE_SHIFT, limit			; \
105895ca17ae	mov	%g0, cas_cnt						; \
106895ca17ae	mov	1, val							; \
108895ca17ae#endif	/* ATOMIC_BO_ENABLE_SHIFT */
110895ca17ae#else	/* _KERNEL */
113895ca17ae * ATOMIC_BO_ENABLE_SHIFT may be enabled/defined here for generic
114895ca17ae * libc atomics. None for now.
115895ca17ae */
116895ca17ae#ifdef ATOMIC_BO_ENABLE_SHIFT
117895ca17ae#define	DELAY_SPIN(label, tmp1, tmp2)	\
120895ca17ae#define ATOMIC_BACKOFF_CPU(val, limit, ncpu, cas_cnt, label)  \
121895ca17ae	set	1 << ATOMIC_BO_ENABLE_SHIFT, limit
122895ca17ae#endif	/* ATOMIC_BO_ENABLE_SHIFT */
123895ca17ae#endif	/* _KERNEL */
125895ca17ae#ifdef ATOMIC_BO_ENABLE_SHIFT
127895ca17ae * ATOMIC_BACKOFF_INIT macro for initialization.
128895ca17ae * backoff val is initialized to 1.
129895ca17ae * ncpu is initialized to 0
130895ca17ae * The cas_cnt counts the cas instruction failure and is
131895ca17ae * initialized to 0.
132895ca17ae */
133895ca17ae#define ATOMIC_BACKOFF_INIT(val, ncpu, cas_cnt)	\
134895ca17ae	mov	1, val				; \
135895ca17ae	mov	%g0, ncpu			; \
136895ca17ae	mov	%g0, cas_cnt
138895ca17ae#define ATOMIC_BACKOFF_BRANCH(cr, backoff, loop) \
139895ca17ae	bne,a,pn cr, backoff
142895ca17ae * Main ATOMIC_BACKOFF_BACKOFF macro for backoff.
143895ca17ae */
144895ca17ae#define ATOMIC_BACKOFF_BACKOFF(val, limit, ncpu, cas_cnt, label, retlabel) \
145895ca17ae	ATOMIC_BACKOFF_CPU(val, limit, ncpu, cas_cnt, label/**/_0)	; \
146895ca17ae	cmp	val, limit						; \
147895ca17ae	blu,a,pt %xcc, label/**/_1					; \
148895ca17ae	  mov	val, limit						; \
149895ca17aelabel/**/_1:								; \
150895ca17ae	mov	limit, val						; \
151895ca17ae	DELAY_SPIN(label/**/_2, %g2, %g3)				; \
152895ca17ae	deccc	limit							; \
153895ca17ae	bgu,pn	%xcc, label/**/_20 /* branch to middle of DELAY_SPIN */	; \
154895ca17ae	  nop								; \
155895ca17ae	ba	retlabel						; \
156c38855fShesha Sreenivasamurthy	sllx	val, 1, val
157c38855fShesha Sreenivasamurthy
158895ca17ae#else	/* ATOMIC_BO_ENABLE_SHIFT */
159895ca17ae#define ATOMIC_BACKOFF_INIT(val, ncpu, cas_cnt)
161895ca17ae#define ATOMIC_BACKOFF_BRANCH(cr, backoff, loop) \
162895ca17ae	bne,a,pn cr, loop
164895ca17ae#define ATOMIC_BACKOFF_BACKOFF(val, limit, ncpu, cas_cnt, label, retlabel)
165895ca17ae#endif	/* ATOMIC_BO_ENABLE_SHIFT */
167dfb96a4ab	/*
168dfb96a4ab	 * NOTE: If atomic_inc_8 and atomic_inc_8_nv are ever
169dfb96a4ab	 * separated, you need to also edit the libc sparcv9 platform
170dfb96a4ab	 * specific mapfile and remove the NODYNSORT attribute
171dfb96a4ab	 * from atomic_inc_8_nv.
172dfb96a4ab	 */
1737c478bdstevel@tonic-gate	ENTRY(atomic_inc_8)
1747c478bdstevel@tonic-gate	ALTENTRY(atomic_inc_8_nv)
1757c478bdstevel@tonic-gate	ALTENTRY(atomic_inc_uchar)
1767c478bdstevel@tonic-gate	ALTENTRY(atomic_inc_uchar_nv)
1777c478bdstevel@tonic-gate	ba	add_8
1787c478bdstevel@tonic-gate	  add	%g0, 1, %o1
1797c478bdstevel@tonic-gate	SET_SIZE(atomic_inc_uchar_nv)
1807c478bdstevel@tonic-gate	SET_SIZE(atomic_inc_uchar)
1817c478bdstevel@tonic-gate	SET_SIZE(atomic_inc_8_nv)
1827c478bdstevel@tonic-gate	SET_SIZE(atomic_inc_8)
184dfb96a4ab	/*
185dfb96a4ab	 * NOTE: If atomic_dec_8 and atomic_dec_8_nv are ever
186dfb96a4ab	 * separated, you need to also edit the libc sparcv9 platform
187dfb96a4ab	 * specific mapfile and remove the NODYNSORT attribute
188dfb96a4ab	 * from atomic_dec_8_nv.
189dfb96a4ab	 */
1907c478bdstevel@tonic-gate	ENTRY(atomic_dec_8)
1917c478bdstevel@tonic-gate	ALTENTRY(atomic_dec_8_nv)
1927c478bdstevel@tonic-gate	ALTENTRY(atomic_dec_uchar)
1937c478bdstevel@tonic-gate	ALTENTRY(atomic_dec_uchar_nv)
1947c478bdstevel@tonic-gate	ba	add_8
1957c478bdstevel@tonic-gate	  sub	%g0, 1, %o1
1967c478bdstevel@tonic-gate	SET_SIZE(atomic_dec_uchar_nv)
1977c478bdstevel@tonic-gate	SET_SIZE(atomic_dec_uchar)
1987c478bdstevel@tonic-gate	SET_SIZE(atomic_dec_8_nv)
1997c478bdstevel@tonic-gate	SET_SIZE(atomic_dec_8)
201dfb96a4ab	/*
202dfb96a4ab	 * NOTE: If atomic_add_8 and atomic_add_8_nv are ever
203dfb96a4ab	 * separated, you need to also edit the libc sparcv9 platform
204dfb96a4ab	 * specific mapfile and remove the NODYNSORT attribute
205dfb96a4ab	 * from atomic_add_8_nv.
206dfb96a4ab	 */
2077c478bdstevel@tonic-gate	ENTRY(atomic_add_8)
2087c478bdstevel@tonic-gate	ALTENTRY(atomic_add_8_nv)
2097c478bdstevel@tonic-gate	ALTENTRY(atomic_add_char)
2107c478bdstevel@tonic-gate	ALTENTRY(atomic_add_char_nv)
2127c478bdstevel@tonic-gate	and	%o0, 0x3, %o4		! %o4 = byte offset, left-to-right
2137c478bdstevel@tonic-gate	xor	%o4, 0x3, %g1		! %g1 = byte offset, right-to-left
2147c478bdstevel@tonic-gate	sll	%g1, 3, %g1		! %g1 = bit offset, right-to-left
2157c478bdstevel@tonic-gate	set	0xff, %o3		! %o3 = mask
2167c478bdstevel@tonic-gate	sll	%o3, %g1, %o3		! %o3 = shifted to bit offset
2177c478bdstevel@tonic-gate	sll	%o1, %g1, %o1		! %o1 = shifted to bit offset
2187c478bdstevel@tonic-gate	and	%o1, %o3, %o1		! %o1 = single byte value
2197c478bdstevel@tonic-gate	andn	%o0, 0x3, %o0		! %o0 = word address
2207c478bdstevel@tonic-gate	ld	[%o0], %o2		! read old value
2227c478bdstevel@tonic-gate	add	%o2, %o1, %o5		! add value to the old value
2237c478bdstevel@tonic-gate	and	%o5, %o3, %o5		! clear other bits
2247c478bdstevel@tonic-gate	andn	%o2, %o3, %o4		! clear target bits
2257c478bdstevel@tonic-gate	or	%o4, %o5, %o5		! insert the new value
2267c478bdstevel@tonic-gate	cas	[%o0], %o2, %o5
2277c478bdstevel@tonic-gate	cmp	%o2, %o5
2287c478bdstevel@tonic-gate	bne,a,pn %icc, 1b
2297c478bdstevel@tonic-gate	  mov	%o5, %o2		! %o2 = old value
2307c478bdstevel@tonic-gate	add	%o2, %o1, %o5
2317c478bdstevel@tonic-gate	and	%o5, %o3, %o5
2327c478bdstevel@tonic-gate	retl
2337c478bdstevel@tonic-gate	srl	%o5, %g1, %o0		! %o0 = new value
2347c478bdstevel@tonic-gate	SET_SIZE(atomic_add_char_nv)
2357c478bdstevel@tonic-gate	SET_SIZE(atomic_add_char)
2367c478bdstevel@tonic-gate	SET_SIZE(atomic_add_8_nv)
2377c478bdstevel@tonic-gate	SET_SIZE(atomic_add_8)
239dfb96a4ab	/*
240dfb96a4ab	 * NOTE: If atomic_inc_16 and atomic_inc_16_nv are ever
241dfb96a4ab	 * separated, you need to also edit the libc sparcv9 platform
242dfb96a4ab	 * specific mapfile and remove the NODYNSORT attribute
243dfb96a4ab	 * from atomic_inc_16_nv.
244dfb96a4ab	 */
2457c478bdstevel@tonic-gate	ENTRY(atomic_inc_16)
2467c478bdstevel@tonic-gate	ALTENTRY(atomic_inc_16_nv)
2477c478bdstevel@tonic-gate	ALTENTRY(atomic_inc_ushort)
2487c478bdstevel@tonic-gate	ALTENTRY(atomic_inc_ushort_nv)
2497c478bdstevel@tonic-gate	ba	add_16
2507c478bdstevel@tonic-gate	  add	%g0, 1, %o1
2517c478bdstevel@tonic-gate	SET_SIZE(atomic_inc_ushort_nv)
2527c478bdstevel@tonic-gate	SET_SIZE(atomic_inc_ushort)
2537c478bdstevel@tonic-gate	SET_SIZE(atomic_inc_16_nv)
2547c478bdstevel@tonic-gate	SET_SIZE(atomic_inc_16)
256dfb96a4ab	/*
257dfb96a4ab	 * NOTE: If atomic_dec_16 and atomic_dec_16_nv are ever
258dfb96a4ab	 * separated, you need to also edit the libc sparcv9 platform
259dfb96a4ab	 * specific mapfile and remove the NODYNSORT attribute
260dfb96a4ab	 * from atomic_dec_16_nv.
261dfb96a4ab	 */
2627c478bdstevel@tonic-gate	ENTRY(atomic_dec_16)
2637c478bdstevel@tonic-gate	ALTENTRY(atomic_dec_16_nv)
2647c478bdstevel@tonic-gate	ALTENTRY(atomic_dec_ushort)
2657c478bdstevel@tonic-gate	ALTENTRY(atomic_dec_ushort_nv)
2667c478bdstevel@tonic-gate	ba	add_16
2677c478bdstevel@tonic-gate	  sub	%g0, 1, %o1
2687c478bdstevel@tonic-gate	SET_SIZE(atomic_dec_ushort_nv)
2697c478bdstevel@tonic-gate	SET_SIZE(atomic_dec_ushort)
2707c478bdstevel@tonic-gate	SET_SIZE(atomic_dec_16_nv)
2717c478bdstevel@tonic-gate	SET_SIZE(atomic_dec_16)
273dfb96a4ab	/*
274dfb96a4ab	 * NOTE: If atomic_add_16 and atomic_add_16_nv are ever
275dfb96a4ab	 * separated, you need to also edit the libc sparcv9 platform
276dfb96a4ab	 * specific mapfile and remove the NODYNSORT attribute
277dfb96a4ab	 * from atomic_add_16_nv.
278dfb96a4ab	 */
2797c478bdstevel@tonic-gate	ENTRY(atomic_add_16)
2807c478bdstevel@tonic-gate	ALTENTRY(atomic_add_16_nv)
2817c478bdstevel@tonic-gate	ALTENTRY(atomic_add_short)
2827c478bdstevel@tonic-gate	ALTENTRY(atomic_add_short_nv)
2847c478bdstevel@tonic-gate	and	%o0, 0x2, %o4		! %o4 = byte offset, left-to-right
2857c478bdstevel@tonic-gate	xor	%o4, 0x2, %g1		! %g1 = byte offset, right-to-left
2867c478bdstevel@tonic-gate	sll	%o4, 3, %o4		! %o4 = bit offset, left-to-right
2877c478bdstevel@tonic-gate	sll	%g1, 3, %g1		! %g1 = bit offset, right-to-left
2887c478bdstevel@tonic-gate	sethi	%hi(0xffff0000), %o3	! %o3 = mask
2897c478bdstevel@tonic-gate	srl	%o3, %o4, %o3		! %o3 = shifted to bit offset
2907c478bdstevel@tonic-gate	sll	%o1, %g1, %o1		! %o1 = shifted to bit offset
2917c478bdstevel@tonic-gate	and	%o1, %o3, %o1		! %o1 = single short value
2927c478bdstevel@tonic-gate	andn	%o0, 0x2, %o0		! %o0 = word address
2937c478bdstevel@tonic-gate	! if low-order bit is 1, we will properly get an alignment fault here
2947c478bdstevel@tonic-gate	ld	[%o0], %o2		! read old value
2967c478bdstevel@tonic-gate	add	%o1, %o2, %o5		! add value to the old value
2977c478bdstevel@tonic-gate	and	%o5, %o3, %o5		! clear other bits
2987c478bdstevel@tonic-gate	andn	%o2, %o3, %o4		! clear target bits
2997c478bdstevel@tonic-gate	or	%o4, %o5, %o5		! insert the new value
3007c478bdstevel@tonic-gate	cas	[%o0], %o2, %o5
3017c478bdstevel@tonic-gate	cmp	%o2, %o5
3027c478bdstevel@tonic-gate	bne,a,pn %icc, 1b
3037c478bdstevel@tonic-gate	  mov	%o5, %o2		! %o2 = old value
3047c478bdstevel@tonic-gate	add	%o1, %o2, %o5
3057c478bdstevel@tonic-gate	and	%o5, %o3, %o5
3067c478bdstevel@tonic-gate	retl
3077c478bdstevel@tonic-gate	srl	%o5, %g1, %o0		! %o0 = new value
3087c478bdstevel@tonic-gate	SET_SIZE(atomic_add_short_nv)
3097c478bdstevel@tonic-gate	SET_SIZE(atomic_add_short)
3107c478bdstevel@tonic-gate	SET_SIZE(atomic_add_16_nv)
3117c478bdstevel@tonic-gate	SET_SIZE(atomic_add_16)
313dfb96a4ab	/*
314dfb96a4ab	 * NOTE: If atomic_inc_32 and atomic_inc_32_nv are ever
315dfb96a4ab	 * separated, you need to also edit the libc sparcv9 platform
316dfb96a4ab	 * specific mapfile and remove the NODYNSORT attribute
317dfb96a4ab	 * from atomic_inc_32_nv.
318dfb96a4ab	 */
3197c478bdstevel@tonic-gate	ENTRY(atomic_inc_32)
3207c478bdstevel@tonic-gate	ALTENTRY(atomic_inc_32_nv)
3217c478bdstevel@tonic-gate	ALTENTRY(atomic_inc_uint)
3227c478bdstevel@tonic-gate	ALTENTRY(atomic_inc_uint_nv)
3237c478bdstevel@tonic-gate	ba	add_32
3247c478bdstevel@tonic-gate	  add	%g0, 1, %o1
3257c478bdstevel@tonic-gate	SET_SIZE(atomic_inc_uint_nv)
3267c478bdstevel@tonic-gate	SET_SIZE(atomic_inc_uint)
3277c478bdstevel@tonic-gate	SET_SIZE(atomic_inc_32_nv)
3287c478bdstevel@tonic-gate	SET_SIZE(atomic_inc_32)
330dfb96a4ab	/*
331dfb96a4ab	 * NOTE: If atomic_dec_32 and atomic_dec_32_nv are ever
332dfb96a4ab	 * separated, you need to also edit the libc sparcv9 platform
333dfb96a4ab	 * specific mapfile and remove the NODYNSORT attribute
334dfb96a4ab	 * from atomic_dec_32_nv.
335dfb96a4ab	 */
3367c478bdstevel@tonic-gate	ENTRY(atomic_dec_32)
3377c478bdstevel@tonic-gate	ALTENTRY(atomic_dec_32_nv)
3387c478bdstevel@tonic-gate	ALTENTRY(atomic_dec_uint)
3397c478bdstevel@tonic-gate	ALTENTRY(atomic_dec_uint_nv)
3407c478bdstevel@tonic-gate	ba	add_32
3417c478bdstevel@tonic-gate	  sub	%g0, 1, %o1
3427c478bdstevel@tonic-gate	SET_SIZE(atomic_dec_uint_nv)
3437c478bdstevel@tonic-gate	SET_SIZE(atomic_dec_uint)
3447c478bdstevel@tonic-gate	SET_SIZE(atomic_dec_32_nv)
3457c478bdstevel@tonic-gate	SET_SIZE(atomic_dec_32)
347dfb96a4ab	/*
348dfb96a4ab	 * NOTE: If atomic_add_32 and atomic_add_32_nv are ever
349dfb96a4ab	 * separated, you need to also edit the libc sparcv9 platform
350dfb96a4ab	 * specific mapfile and remove the NODYNSORT attribute
351dfb96a4ab	 * from atomic_add_32_nv.
352dfb96a4ab	 */
3537c478bdstevel@tonic-gate	ENTRY(atomic_add_32)
3547c478bdstevel@tonic-gate	ALTENTRY(atomic_add_32_nv)
3557c478bdstevel@tonic-gate	ALTENTRY(atomic_add_int)
3567c478bdstevel@tonic-gate	ALTENTRY(atomic_add_int_nv)
358895ca17ae	ATOMIC_BACKOFF_INIT(%o4, %g4, %g5)
3607c478bdstevel@tonic-gate	ld	[%o0], %o2
3627c478bdstevel@tonic-gate	add	%o2, %o1, %o3
3637c478bdstevel@tonic-gate	cas	[%o0], %o2, %o3
3647c478bdstevel@tonic-gate	cmp	%o2, %o3
365895ca17ae	ATOMIC_BACKOFF_BRANCH(%icc, 2f, 1b)
3667c478bdstevel@tonic-gate	  mov	%o3, %o2
3677c478bdstevel@tonic-gate	retl
3687c478bdstevel@tonic-gate	add	%o2, %o1, %o0		! return new value
370895ca17ae	ATOMIC_BACKOFF_BACKOFF(%o4, %o5, %g4, %g5, add32, 0b)
3717c478bdstevel@tonic-gate	SET_SIZE(atomic_add_int_nv)
3727c478bdstevel@tonic-gate	SET_SIZE(atomic_add_int)
3737c478bdstevel@tonic-gate	SET_SIZE(atomic_add_32_nv)
3747c478bdstevel@tonic-gate	SET_SIZE(atomic_add_32)
376dfb96a4ab	/*
377dfb96a4ab	 * NOTE: If atomic_inc_64 and atomic_inc_64_nv are ever
378dfb96a4ab	 * separated, you need to also edit the libc sparcv9 platform
379dfb96a4ab	 * specific mapfile and remove the NODYNSORT attribute
380dfb96a4ab	 * from atomic_inc_64_nv.
381dfb96a4ab	 */
3827c478bdstevel@tonic-gate	ENTRY(atomic_inc_64)
3837c478bdstevel@tonic-gate	ALTENTRY(atomic_inc_64_nv)
3847c478bdstevel@tonic-gate	ALTENTRY(atomic_inc_ulong)
3857c478bdstevel@tonic-gate	ALTENTRY(atomic_inc_ulong_nv)
3867c478bdstevel@tonic-gate	ba	add_64
3877c478bdstevel@tonic-gate	  add	%g0, 1, %o1
3887c478bdstevel@tonic-gate	SET_SIZE(atomic_inc_ulong_nv)
3897c478bdstevel@tonic-gate	SET_SIZE(atomic_inc_ulong)
3907c478bdstevel@tonic-gate	SET_SIZE(atomic_inc_64_nv)
3917c478bdstevel@tonic-gate	SET_SIZE(atomic_inc_64)
393dfb96a4ab	/*
394dfb96a4ab	 * NOTE: If atomic_dec_64 and atomic_dec_64_nv are ever
395dfb96a4ab	 * separated, you need to also edit the libc sparcv9 platform
396dfb96a4ab	 * specific mapfile and remove the NODYNSORT attribute
397dfb96a4ab	 * from atomic_dec_64_nv.
398dfb96a4ab	 */
3997c478bdstevel@tonic-gate	ENTRY(atomic_dec_64)
4007c478bdstevel@tonic-gate	ALTENTRY(atomic_dec_64_nv)
4017c478bdstevel@tonic-gate	ALTENTRY(atomic_dec_ulong)
4027c478bdstevel@tonic-gate	ALTENTRY(atomic_dec_ulong_nv)
4037c478bdstevel@tonic-gate	ba	add_64
4047c478bdstevel@tonic-gate	  sub	%g0, 1, %o1
4057c478bdstevel@tonic-gate	SET_SIZE(atomic_dec_ulong_nv)
4067c478bdstevel@tonic-gate	SET_SIZE(atomic_dec_ulong)
4077c478bdstevel@tonic-gate	SET_SIZE(atomic_dec_64_nv)
4087c478bdstevel@tonic-gate	SET_SIZE(atomic_dec_64)
410dfb96a4ab	/*
411dfb96a4ab	 * NOTE: If atomic_add_64 and atomic_add_64_nv are ever
412dfb96a4ab	 * separated, you need to also edit the libc sparcv9 platform
413dfb96a4ab	 * specific mapfile and remove the NODYNSORT attribute
414dfb96a4ab	 * from atomic_add_64_nv.
415dfb96a4ab	 */
4167c478bdstevel@tonic-gate	ENTRY(atomic_add_64)
4177c478bdstevel@tonic-gate	ALTENTRY(atomic_add_64_nv)
4187c478bdstevel@tonic-gate	ALTENTRY(atomic_add_ptr)
4197c478bdstevel@tonic-gate	ALTENTRY(atomic_add_ptr_nv)
4207c478bdstevel@tonic-gate	ALTENTRY(atomic_add_long)
4217c478bdstevel@tonic-gate	ALTENTRY(atomic_add_long_nv)
423895ca17ae	ATOMIC_BACKOFF_INIT(%o4, %g4, %g5)
4257c478bdstevel@tonic-gate	ldx	[%o0], %o2
4277c478bdstevel@tonic-gate	add	%o2, %o1, %o3
4287c478bdstevel@tonic-gate	casx	[%o0], %o2, %o3
4297c478bdstevel@tonic-gate	cmp	%o2, %o3
430895ca17ae	ATOMIC_BACKOFF_BRANCH(%xcc, 2f, 1b)
4317c478bdstevel@tonic-gate	  mov	%o3, %o2
4327c478bdstevel@tonic-gate	retl
4337c478bdstevel@tonic-gate	add	%o2, %o1, %o0		! return new value
435895ca17ae	ATOMIC_BACKOFF_BACKOFF(%o4, %o5, %g4, %g5, add64, 0b)
4367c478bdstevel@tonic-gate	SET_SIZE(atomic_add_long_nv)
4377c478bdstevel@tonic-gate	SET_SIZE(atomic_add_long)
4387c478bdstevel@tonic-gate	SET_SIZE(atomic_add_ptr_nv)
4397c478bdstevel@tonic-gate	SET_SIZE(atomic_add_ptr)
4407c478bdstevel@tonic-gate	SET_SIZE(atomic_add_64_nv)
4417c478bdstevel@tonic-gate	SET_SIZE(atomic_add_64)
443dfb96a4ab	/*
444dfb96a4ab	 * NOTE: If atomic_or_8 and atomic_or_8_nv are ever
445dfb96a4ab	 * separated, you need to also edit the libc sparcv9 platform
446dfb96a4ab	 * specific mapfile and remove the NODYNSORT attribute
447dfb96a4ab	 * from atomic_or_8_nv.
448dfb96a4ab	 */
4497c478bdstevel@tonic-gate	ENTRY(atomic_or_8)
4507c478bdstevel@tonic-gate	ALTENTRY(atomic_or_8_nv)
4517c478bdstevel@tonic-gate	ALTENTRY(atomic_or_uchar)
4527c478bdstevel@tonic-gate	ALTENTRY(atomic_or_uchar_nv)
4537c478bdstevel@tonic-gate	and	%o0, 0x3, %o4		! %o4 = byte offset, left-to-right
4547c478bdstevel@tonic-gate	xor	%o4, 0x3, %g1		! %g1 = byte offset, right-to-left
4557c478bdstevel@tonic-gate	sll	%g1, 3, %g1		! %g1 = bit offset, right-to-left
4567c478bdstevel@tonic-gate	set	0xff, %o3		! %o3 = mask
4577c478bdstevel@tonic-gate	sll	%o3, %g1, %o3		! %o3 = shifted to bit offset
4587c478bdstevel@tonic-gate	sll	%o1, %g1, %o1		! %o1 = shifted to bit offset
4597c478bdstevel@tonic-gate	and	%o1, %o3, %o1		! %o1 = single byte value
4607c478bdstevel@tonic-gate	andn	%o0, 0x3, %o0		! %o0 = word address
4617c478bdstevel@tonic-gate	ld	[%o0], %o2		! read old value
4637c478bdstevel@tonic-gate	or	%o2, %o1, %o5		! or in the new value
4647c478bdstevel@tonic-gate	cas	[%o0], %o2, %o5
4657c478bdstevel@tonic-gate	cmp	%o2, %o5
4667c478bdstevel@tonic-gate	bne,a,pn %icc, 1b
4677c478bdstevel@tonic-gate	  mov	%o5, %o2		! %o2 = old value
4687c478bdstevel@tonic-gate	or	%o2, %o1, %o5
4697c478bdstevel@tonic-gate	and	%o5, %o3, %o5
4707c478bdstevel@tonic-gate	retl
4717c478bdstevel@tonic-gate	srl	%o5, %g1, %o0		! %o0 = new value
4727c478bdstevel@tonic-gate	SET_SIZE(atomic_or_uchar_nv)
4737c478bdstevel@tonic-gate	SET_SIZE(atomic_or_uchar)
4747c478bdstevel@tonic-gate	SET_SIZE(atomic_or_8_nv)
4757c478bdstevel@tonic-gate	SET_SIZE(atomic_or_8)
477dfb96a4ab	/*
478dfb96a4ab	 * NOTE: If atomic_or_16 and atomic_or_16_nv are ever
479dfb96a4ab	 * separated, you need to also edit the libc sparcv9 platform
480dfb96a4ab	 * specific mapfile and remove the NODYNSORT attribute
481dfb96a4ab	 * from atomic_or_16_nv.
482dfb96a4ab	 */
4837c478bdstevel@tonic-gate	ENTRY(atomic_or_16)
4847c478bdstevel@tonic-gate	ALTENTRY(atomic_or_16_nv)
4857c478bdstevel@tonic-gate	ALTENTRY(atomic_or_ushort)
4867c478bdstevel@tonic-gate	ALTENTRY(atomic_or_ushort_nv)
4877c478bdstevel@tonic-gate	and	%o0, 0x2, %o4		! %o4 = byte offset, left-to-right
4887c478bdstevel@tonic-gate	xor	%o4, 0x2, %g1		! %g1 = byte offset, right-to-left
4897c478bdstevel@tonic-gate	sll	%o4, 3, %o4		! %o4 = bit offset, left-to-right
4907c478bdstevel@tonic-gate	sll	%g1, 3, %g1		! %g1 = bit offset, right-to-left
4917c478bdstevel@tonic-gate	sethi	%hi(0xffff0000), %o3	! %o3 = mask
4927c478bdstevel@tonic-gate	srl	%o3, %o4, %o3		! %o3 = shifted to bit offset
4937c478bdstevel@tonic-gate	sll	%o1, %g1, %o1		! %o1 = shifted to bit offset
4947c478bdstevel@tonic-gate	and	%o1, %o3, %o1		! %o1 = single short value
4957c478bdstevel@tonic-gate	andn	%o0, 0x2, %o0		! %o0 = word address
4967c478bdstevel@tonic-gate	! if low-order bit is 1, we will properly get an alignment fault here
4977c478bdstevel@tonic-gate	ld	[%o0], %o2		! read old value
4997c478bdstevel@tonic-gate	or	%o2, %o1, %o5		! or in the new value
5007c478bdstevel@tonic-gate	cas	[%o0], %o2, %o5
5017c478bdstevel@tonic-gate	cmp	%o2, %o5
5027c478bdstevel@tonic-gate	bne,a,pn %icc, 1b
5037c478bdstevel@tonic-gate	  mov	%o5, %o2		! %o2 = old value
5047c478bdstevel@tonic-gate	or	%o2, %o1, %o5		! or in the new value
5057c478bdstevel@tonic-gate	and	%o5, %o3, %o5
5067c478bdstevel@tonic-gate	retl
5077c478bdstevel@tonic-gate	srl	%o5, %g1, %o0		! %o0 = new value
5087c478bdstevel@tonic-gate	SET_SIZE(atomic_or_ushort_nv)
5097c478bdstevel@tonic-gate	SET_SIZE(atomic_or_ushort)
5107c478bdstevel@tonic-gate	SET_SIZE(atomic_or_16_nv)
5117c478bdstevel@tonic-gate	SET_SIZE(atomic_or_16)
513dfb96a4ab	/*
514dfb96a4ab	 * NOTE: If atomic_or_32 and atomic_or_32_nv are ever
515dfb96a4ab	 * separated, you need to also edit the libc sparcv9 platform
516dfb96a4ab	 * specific mapfile and remove the NODYNSORT attribute
517dfb96a4ab	 * from atomic_or_32_nv.
518dfb96a4ab	 */
5197c478bdstevel@tonic-gate	ENTRY(atomic_or_32)
5207c478bdstevel@tonic-gate	ALTENTRY(atomic_or_32_nv)
5217c478bdstevel@tonic-gate	ALTENTRY(atomic_or_uint)
5227c478bdstevel@tonic-gate	ALTENTRY(atomic_or_uint_nv)
523895ca17ae	ATOMIC_BACKOFF_INIT(%o4, %g4, %g5)
5257c478bdstevel@tonic-gate	ld	[%o0], %o2
5277c478bdstevel@tonic-gate	or	%o2, %o1, %o3
5287c478bdstevel@tonic-gate	cas	[%o0], %o2, %o3
5297c478bdstevel@tonic-gate	cmp	%o2, %o3
530895ca17ae	ATOMIC_BACKOFF_BRANCH(%icc, 2f, 1b)
5317c478bdstevel@tonic-gate	  mov	%o3, %o2
5327c478bdstevel@tonic-gate	retl
5337c478bdstevel@tonic-gate	or	%o2, %o1, %o0		! return new value
535895ca17ae	ATOMIC_BACKOFF_BACKOFF(%o4, %o5, %g4, %g5, or32, 0b)
5367c478bdstevel@tonic-gate	SET_SIZE(atomic_or_uint_nv)
5377c478bdstevel@tonic-gate	SET_SIZE(atomic_or_uint)
5387c478bdstevel@tonic-gate	SET_SIZE(atomic_or_32_nv)
5397c478bdstevel@tonic-gate	SET_SIZE(atomic_or_32)
541dfb96a4ab	/*
542dfb96a4ab	 * NOTE: If atomic_or_64 and atomic_or_64_nv are ever
543dfb96a4ab	 * separated, you need to also edit the libc sparcv9 platform
544dfb96a4ab	 * specific mapfile and remove the NODYNSORT attribute
545dfb96a4ab	 * from atomic_or_64_nv.
546dfb96a4ab	 */
5477c478bdstevel@tonic-gate	ENTRY(atomic_or_64)
5487c478bdstevel@tonic-gate	ALTENTRY(atomic_or_64_nv)
5497c478bdstevel@tonic-gate	ALTENTRY(atomic_or_ulong)
5507c478bdstevel@tonic-gate	ALTENTRY(atomic_or_ulong_nv)
551895ca17ae	ATOMIC_BACKOFF_INIT(%o4, %g4, %g5)
5537c478bdstevel@tonic-gate	ldx	[%o0], %o2
5557c478bdstevel@tonic-gate	or	%o2, %o1, %o3
5567c478bdstevel@tonic-gate	casx	[%o0], %o2, %o3
5577c478bdstevel@tonic-gate	cmp	%o2, %o3
558895ca17ae	ATOMIC_BACKOFF_BRANCH(%xcc, 2f, 1b)
5597c478bdstevel@tonic-gate	  mov	%o3, %o2
5607c478bdstevel@tonic-gate	retl
5617c478bdstevel@tonic-gate	or	%o2, %o1, %o0		! return new value
563895ca17ae	ATOMIC_BACKOFF_BACKOFF(%o4, %o5, %g4, %g5, or64, 0b)
5647c478bdstevel@tonic-gate	SET_SIZE(atomic_or_ulong_nv)
5657c478bdstevel@tonic-gate	SET_SIZE(atomic_or_ulong)
5667c478bdstevel@tonic-gate	SET_SIZE(atomic_or_64_nv)
5677c478bdstevel@tonic-gate	SET_SIZE(atomic_or_64)
569dfb96a4ab	/*
570dfb96a4ab	 * NOTE: If atomic_and_8 and atomic_and_8_nv are ever
571dfb96a4ab	 * separated, you need to also edit the libc sparcv9 platform
572dfb96a4ab	 * specific mapfile and remove the NODYNSORT attribute
573dfb96a4ab	 * from atomic_and_8_nv.
574dfb96a4ab	 */
5757c478bdstevel@tonic-gate	ENTRY(atomic_and_8)
5767c478bdstevel@tonic-gate	ALTENTRY(atomic_and_8_nv)
5777c478bdstevel@tonic-gate	ALTENTRY(atomic_and_uchar)
5787c478bdstevel@tonic-gate	ALTENTRY(atomic_and_uchar_nv)
5797c478bdstevel@tonic-gate	and	%o0, 0x3, %o4		! %o4 = byte offset, left-to-right
5807c478bdstevel@tonic-gate	xor	%o4, 0x3, %g1		! %g1 = byte offset, right-to-left
5817c478bdstevel@tonic-gate	sll	%g1, 3, %g1		! %g1 = bit offset, right-to-left
5827c478bdstevel@tonic-gate	set	0xff, %o3		! %o3 = mask
5837c478bdstevel@tonic-gate	sll	%o3, %g1, %o3		! %o3 = shifted to bit offset
5847c478bdstevel@tonic-gate	sll	%o1, %g1, %o1		! %o1 = shifted to bit offset
5857c478bdstevel@tonic-gate	orn	%o1, %o3, %o1		! all ones in other bytes
5867c478bdstevel@tonic-gate	andn	%o0, 0x3, %o0		! %o0 = word address
5877c478bdstevel@tonic-gate	ld	[%o0], %o2		! read old value
5897c478bdstevel@tonic-gate	and	%o2, %o1, %o5		! and in the new value
5907c478bdstevel@tonic-gate	cas	[%o0], %o2, %o5
5917c478bdstevel@tonic-gate	cmp	%o2, %o5
5927c478bdstevel@tonic-gate	bne,a,pn %icc, 1b
5937c478bdstevel@tonic-gate	  mov	%o5, %o2		! %o2 = old value
5947c478bdstevel@tonic-gate	and	%o2, %o1, %o5
5957c478bdstevel@tonic-gate	and	%o5, %o3, %o5
5967c478bdstevel@tonic-gate	retl
5977c478bdstevel@tonic-gate	srl	%o5, %g1, %o0		! %o0 = new value
5987c478bdstevel@tonic-gate	SET_SIZE(atomic_and_uchar_nv)
5997c478bdstevel@tonic-gate	SET_SIZE(atomic_and_uchar)
6007c478bdstevel@tonic-gate	SET_SIZE(atomic_and_8_nv)
6017c478bdstevel@tonic-gate	SET_SIZE(atomic_and_8)
603dfb96a4ab	/*
604dfb96a4ab	 * NOTE: If atomic_and_16 and atomic_and_16_nv are ever
605dfb96a4ab	 * separated, you need to also edit the libc sparcv9 platform
606dfb96a4ab	 * specific mapfile and remove the NODYNSORT attribute
607dfb96a4ab	 * from atomic_and_16_nv.
608dfb96a4ab	 */
6097c478bdstevel@tonic-gate	ENTRY(atomic_and_16)
6107c478bdstevel@tonic-gate	ALTENTRY(atomic_and_16_nv)
6117c478bdstevel@tonic-gate	ALTENTRY(atomic_and_ushort)
6127c478bdstevel@tonic-gate	ALTENTRY(atomic_and_ushort_nv)
6137c478bdstevel@tonic-gate	and	%o0, 0x2, %o4		! %o4 = byte offset, left-to-right
6147c478bdstevel@tonic-gate	xor	%o4, 0x2, %g1		! %g1 = byte offset, right-to-left
6157c478bdstevel@tonic-gate	sll	%o4, 3, %o4		! %o4 = bit offset, left-to-right
6167c478bdstevel@tonic-gate	sll	%g1, 3, %g1		! %g1 = bit offset, right-to-left
6177c478bdstevel@tonic-gate	sethi	%hi(0xffff0000), %o3	! %o3 = mask
6187c478bdstevel@tonic-gate	srl	%o3, %o4, %o3		! %o3 = shifted to bit offset
6197c478bdstevel@tonic-gate	sll	%o1, %g1, %o1		! %o1 = shifted to bit offset
6207c478bdstevel@tonic-gate	orn	%o1, %o3, %o1		! all ones in the other half
6217c478bdstevel@tonic-gate	andn	%o0, 0x2, %o0		! %o0 = word address
6227c478bdstevel@tonic-gate	! if low-order bit is 1, we will properly get an alignment fault here
6237c478bdstevel@tonic-gate	ld	[%o0], %o2		! read old value
6257c478bdstevel@tonic-gate	and	%o2, %o1, %o5		! and in the new value
6267c478bdstevel@tonic-gate	cas	[%o0], %o2, %o5
6277c478bdstevel@tonic-gate	cmp	%o2, %o5
6287c478bdstevel@tonic-gate	bne,a,pn %icc, 1b
6297c478bdstevel@tonic-gate	  mov	%o5, %o2		! %o2 = old value
6307c478bdstevel@tonic-gate	and	%o2, %o1, %o5
6317c478bdstevel@tonic-gate	and	%o5, %o3, %o5
6327c478bdstevel@tonic-gate	retl
6337c478bdstevel@tonic-gate	srl	%o5, %g1, %o0		! %o0 = new value
6347c478bdstevel@tonic-gate	SET_SIZE(atomic_and_ushort_nv)
6357c478bdstevel@tonic-gate	SET_SIZE(atomic_and_ushort)
6367c478bdstevel@tonic-gate	SET_SIZE(atomic_and_16_nv)
6377c478bdstevel@tonic-gate	SET_SIZE(atomic_and_16)
639dfb96a4ab	/*
640dfb96a4ab	 * NOTE: If atomic_and_32 and atomic_and_32_nv are ever
641dfb96a4ab	 * separated, you need to also edit the libc sparcv9 platform
642dfb96a4ab	 * specific mapfile and remove the NODYNSORT attribute
643dfb96a4ab	 * from atomic_and_32_nv.
644dfb96a4ab	 */
6457c478bdstevel@tonic-gate	ENTRY(atomic_and_32)
6467c478bdstevel@tonic-gate	ALTENTRY(atomic_and_32_nv)
6477c478bdstevel@tonic-gate	ALTENTRY(atomic_and_uint)
6487c478bdstevel@tonic-gate	ALTENTRY(atomic_and_uint_nv)
649895ca17ae	ATOMIC_BACKOFF_INIT(%o4, %g4, %g5)
6517c478bdstevel@tonic-gate	ld	[%o0], %o2
6537c478bdstevel@tonic-gate	and	%o2, %o1, %o3
6547c478bdstevel@tonic-gate	cas	[%o0], %o2, %o3
6557c478bdstevel@tonic-gate	cmp	%o2, %o3
656895ca17ae	ATOMIC_BACKOFF_BRANCH(%icc, 2f, 1b)
6577c478bdstevel@tonic-gate	  mov	%o3, %o2
6587c478bdstevel@tonic-gate	retl
6597c478bdstevel@tonic-gate	and	%o2, %o1, %o0		! return new value
661895ca17ae	ATOMIC_BACKOFF_BACKOFF(%o4, %o5, %g4, %g5, and32, 0b)
6627c478bdstevel@tonic-gate	SET_SIZE(atomic_and_uint_nv)
6637c478bdstevel@tonic-gate	SET_SIZE(atomic_and_uint)
6647c478bdstevel@tonic-gate	SET_SIZE(atomic_and_32_nv)
6657c478bdstevel@tonic-gate	SET_SIZE(atomic_and_32)
667dfb96a4ab	/*
668dfb96a4ab	 * NOTE: If atomic_and_64 and atomic_and_64_nv are ever
669dfb96a4ab	 * separated, you need to also edit the libc sparcv9 platform
670dfb96a4ab	 * specific mapfile and remove the NODYNSORT attribute
671dfb96a4ab	 * from atomic_and_64_nv.
672dfb96a4ab	 */
6737c478bdstevel@tonic-gate	ENTRY(atomic_and_64)
6747c478bdstevel@tonic-gate	ALTENTRY(atomic_and_64_nv)
6757c478bdstevel@tonic-gate	ALTENTRY(atomic_and_ulong)
6767c478bdstevel@tonic-gate	ALTENTRY(atomic_and_ulong_nv)
677895ca17ae	ATOMIC_BACKOFF_INIT(%o4, %g4, %g5)
6797c478bdstevel@tonic-gate	ldx	[%o0], %o2
6817c478bdstevel@tonic-gate	and	%o2, %o1, %o3
6827c478bdstevel@tonic-gate	casx	[%o0], %o2, %o3
6837c478bdstevel@tonic-gate	cmp	%o2, %o3
684895ca17ae	ATOMIC_BACKOFF_BRANCH(%xcc, 2f, 1b)
6857c478bdstevel@tonic-gate	  mov	%o3, %o2
6867c478bdstevel@tonic-gate	retl
6877c478bdstevel@tonic-gate	and	%o2, %o1, %o0		! return new value
689895ca17ae	ATOMIC_BACKOFF_BACKOFF(%o4, %o5, %g4, %g5, and64, 0b)
6907c478bdstevel@tonic-gate	SET_SIZE(atomic_and_ulong_nv)
6917c478bdstevel@tonic-gate	SET_SIZE(atomic_and_ulong)
6927c478bdstevel@tonic-gate	SET_SIZE(atomic_and_64_nv)
6937c478bdstevel@tonic-gate	SET_SIZE(atomic_and_64)
6957c478bdstevel@tonic-gate	ENTRY(atomic_cas_8)
6967c478bdstevel@tonic-gate	ALTENTRY(atomic_cas_uchar)
6977c478bdstevel@tonic-gate	and	%o0, 0x3, %o4		! %o4 = byte offset, left-to-right
6987c478bdstevel@tonic-gate	xor	%o4, 0x3, %g1		! %g1 = byte offset, right-to-left
6997c478bdstevel@tonic-gate	sll	%g1, 3, %g1		! %g1 = bit offset, right-to-left
7007c478bdstevel@tonic-gate	set	0xff, %o3		! %o3 = mask
7017c478bdstevel@tonic-gate	sll	%o3, %g1, %o3		! %o3 = shifted to bit offset
7027c478bdstevel@tonic-gate	sll	%o1, %g1, %o1		! %o1 = shifted to bit offset
7037c478bdstevel@tonic-gate	and	%o1, %o3, %o1		! %o1 = single byte value
7047c478bdstevel@tonic-gate	sll	%o2, %g1, %o2		! %o2 = shifted to bit offset
7057c478bdstevel@tonic-gate	and	%o2, %o3, %o2		! %o2 = single byte value
7067c478bdstevel@tonic-gate	andn	%o0, 0x3, %o0		! %o0 = word address
7077c478bdstevel@tonic-gate	ld	[%o0], %o4		! read old value
7097c478bdstevel@tonic-gate	andn	%o4, %o3, %o4		! clear target bits
7107c478bdstevel@tonic-gate	or	%o4, %o2, %o5		! insert the new value
7117c478bdstevel@tonic-gate	or	%o4, %o1, %o4		! insert the comparison value
7127c478bdstevel@tonic-gate	cas	[%o0], %o4, %o5
7137c478bdstevel@tonic-gate	cmp	%o4, %o5		! did we succeed?
7147c478bdstevel@tonic-gate	be,pt	%icc, 2f
7157c478bdstevel@tonic-gate	  and	%o5, %o3, %o4		! isolate the old value
7167c478bdstevel@tonic-gate	cmp	%o1, %o4		! should we have succeeded?
7177c478bdstevel@tonic-gate	be,a,pt	%icc, 1b		! yes, try again
7187c478bdstevel@tonic-gate	  mov	%o5, %o4		! %o4 = old value
7207c478bdstevel@tonic-gate	retl
7217c478bdstevel@tonic-gate	srl	%o4, %g1, %o0		! %o0 = old value
7227c478bdstevel@tonic-gate	SET_SIZE(atomic_cas_uchar)
7237c478bdstevel@tonic-gate	SET_SIZE(atomic_cas_8)
7257c478bdstevel@tonic-gate	ENTRY(atomic_cas_16)
7267c478bdstevel@tonic-gate	ALTENTRY(atomic_cas_ushort)
7277c478bdstevel@tonic-gate	and	%o0, 0x2, %o4		! %o4 = byte offset, left-to-right
7287c478bdstevel@tonic-gate	xor	%o4, 0x2, %g1		! %g1 = byte offset, right-to-left
7297c478bdstevel@tonic-gate	sll	%o4, 3, %o4		! %o4 = bit offset, left-to-right
7307c478bdstevel@tonic-gate	sll	%g1, 3, %g1		! %g1 = bit offset, right-to-left
7317c478bdstevel@tonic-gate	sethi	%hi(0xffff0000), %o3	! %o3 = mask
7327c478bdstevel@tonic-gate	srl	%o3, %o4, %o3		! %o3 = shifted to bit offset
7337c478bdstevel@tonic-gate	sll	%o1, %g1, %o1		! %o1 = shifted to bit offset
7347c478bdstevel@tonic-gate	and	%o1, %o3, %o1		! %o1 = single short value
7357c478bdstevel@tonic-gate	sll	%o2, %g1, %o2		! %o2 = shifted to bit offset
7367c478bdstevel@tonic-gate	and	%o2, %o3, %o2		! %o2 = single short value
7377c478bdstevel@tonic-gate	andn	%o0, 0x2, %o0		! %o0 = word address
7387c478bdstevel@tonic-gate	! if low-order bit is 1, we will properly get an alignment fault here
7397c478bdstevel@tonic-gate	ld	[%o0], %o4		! read old value
7417c478bdstevel@tonic-gate	andn	%o4, %o3, %o4		! clear target bits
7427c478bdstevel@tonic-gate	or	%o4, %o2, %o5		! insert the new value
7437c478bdstevel@tonic-gate	or	%o4, %o1, %o4		! insert the comparison value
7447c478bdstevel@tonic-gate	cas	[%o0], %o4, %o5
7457c478bdstevel@tonic-gate	cmp	%o4, %o5		! did we succeed?
7467c478bdstevel@tonic-gate	be,pt	%icc, 2f
7477c478bdstevel@tonic-gate	  and	%o5, %o3, %o4		! isolate the old value
7487c478bdstevel@tonic-gate	cmp	%o1, %o4		! should we have succeeded?
7497c478bdstevel@tonic-gate	be,a,pt	%icc, 1b		! yes, try again
7507c478bdstevel@tonic-gate	  mov	%o5, %o4		! %o4 = old value
7527c478bdstevel@tonic-gate	retl
7537c478bdstevel@tonic-gate	srl	%o4, %g1, %o0		! %o0 = old value
7547c478bdstevel@tonic-gate	SET_SIZE(atomic_cas_ushort)
7557c478bdstevel@tonic-gate	SET_SIZE(atomic_cas_16)
7577c478bdstevel@tonic-gate	ENTRY(atomic_cas_32)
7587c478bdstevel@tonic-gate	ALTENTRY(atomic_cas_uint)
7597c478bdstevel@tonic-gate	cas	[%o0], %o1, %o2
7607c478bdstevel@tonic-gate	retl
7617c478bdstevel@tonic-gate	mov	%o2, %o0
7627c478bdstevel@tonic-gate	SET_SIZE(atomic_cas_uint)
7637c478bdstevel@tonic-gate	SET_SIZE(atomic_cas_32)
7657c478bdstevel@tonic-gate	ENTRY(atomic_cas_64)
7667c478bdstevel@tonic-gate	ALTENTRY(atomic_cas_ptr)
7677c478bdstevel@tonic-gate	ALTENTRY(atomic_cas_ulong)
7687c478bdstevel@tonic-gate	casx	[%o0], %o1, %o2
7697c478bdstevel@tonic-gate	retl
7707c478bdstevel@tonic-gate	mov	%o2, %o0
7717c478bdstevel@tonic-gate	SET_SIZE(atomic_cas_ulong)
7727c478bdstevel@tonic-gate	SET_SIZE(atomic_cas_ptr)
7737c478bdstevel@tonic-gate	SET_SIZE(atomic_cas_64)
7757c478bdstevel@tonic-gate	ENTRY(atomic_swap_8)
7767c478bdstevel@tonic-gate	ALTENTRY(atomic_swap_uchar)
7777c478bdstevel@tonic-gate	and	%o0, 0x3, %o4		! %o4 = byte offset, left-to-right
7787c478bdstevel@tonic-gate	xor	%o4, 0x3, %g1		! %g1 = byte offset, right-to-left
7797c478bdstevel@tonic-gate	sll	%g1, 3, %g1		! %g1 = bit offset, right-to-left
7807c478bdstevel@tonic-gate	set	0xff, %o3		! %o3 = mask
7817c478bdstevel@tonic-gate	sll	%o3, %g1, %o3		! %o3 = shifted to bit offset
7827c478bdstevel@tonic-gate	sll	%o1, %g1, %o1		! %o1 = shifted to bit offset
7837c478bdstevel@tonic-gate	and	%o1, %o3, %o1		! %o1 = single byte value
7847c478bdstevel@tonic-gate	andn	%o0, 0x3, %o0		! %o0 = word address
7857c478bdstevel@tonic-gate	ld	[%o0], %o2		! read old value
7877c478bdstevel@tonic-gate	andn	%o2, %o3, %o5		! clear target bits
7887c478bdstevel@tonic-gate	or	%o5, %o1, %o5		! insert the new value
7897c478bdstevel@tonic-gate	cas	[%o0], %o2, %o5
7907c478bdstevel@tonic-gate	cmp	%o2, %o5
7917c478bdstevel@tonic-gate	bne,a,pn %icc, 1b
7927c478bdstevel@tonic-gate	  mov	%o5, %o2		! %o2 = old value
7937c478bdstevel@tonic-gate	and	%o5, %o3, %o5
7947c478bdstevel@tonic-gate	retl
7957c478bdstevel@tonic-gate	srl	%o5, %g1, %o0		! %o0 = old value
7967c478bdstevel@tonic-gate	SET_SIZE(atomic_swap_uchar)
7977c478bdstevel@tonic-gate	SET_SIZE(atomic_swap_8)
7997c478bdstevel@tonic-gate	ENTRY(atomic_swap_16)
8007c478bdstevel@tonic-gate	ALTENTRY(atomic_swap_ushort)
8017c478bdstevel@tonic-gate	and	%o0, 0x2, %o4		! %o4 = byte offset, left-to-right
8027c478bdstevel@tonic-gate	xor	%o4, 0x2, %g1		! %g1 = byte offset, right-to-left
8037c478bdstevel@tonic-gate	sll	%o4, 3, %o4		! %o4 = bit offset, left-to-right
8047c478bdstevel@tonic-gate	sll	%g1, 3, %g1		! %g1 = bit offset, right-to-left
8057c478bdstevel@tonic-gate	sethi	%hi(0xffff0000), %o3	! %o3 = mask
8067c478bdstevel@tonic-gate	srl	%o3, %o4, %o3		! %o3 = shifted to bit offset
8077c478bdstevel@tonic-gate	sll	%o1, %g1, %o1		! %o1 = shifted to bit offset
8087c478bdstevel@tonic-gate	and	%o1, %o3, %o1		! %o1 = single short value
8097c478bdstevel@tonic-gate	andn	%o0, 0x2, %o0		! %o0 = word address
8107c478bdstevel@tonic-gate	! if low-order bit is 1, we will properly get an alignment fault here
8117c478bdstevel@tonic-gate	ld	[%o0], %o2		! read old value
8137c478bdstevel@tonic-gate	andn	%o2, %o3, %o5		! clear target bits
8147c478bdstevel@tonic-gate	or	%o5, %o1, %o5		! insert the new value
8157c478bdstevel@tonic-gate	cas	[%o0], %o2, %o5
8167c478bdstevel@tonic-gate	cmp	%o2, %o5
8177c478bdstevel@tonic-gate	bne,a,pn %icc, 1b
8187c478bdstevel@tonic-gate	  mov	%o5, %o2		! %o2 = old value
8197c478bdstevel@tonic-gate	and	%o5, %o3, %o5
8207c478bdstevel@tonic-gate	retl
8217c478bdstevel@tonic-gate	srl	%o5, %g1, %o0		! %o0 = old value
8227c478bdstevel@tonic-gate	SET_SIZE(atomic_swap_ushort)
8237c478bdstevel@tonic-gate	SET_SIZE(atomic_swap_16)
8257c478bdstevel@tonic-gate	ENTRY(atomic_swap_32)
8267c478bdstevel@tonic-gate	ALTENTRY(atomic_swap_uint)
827895ca17ae	ATOMIC_BACKOFF_INIT(%o4, %g4, %g5)
8297c478bdstevel@tonic-gate	ld	[%o0], %o2
8317c478bdstevel@tonic-gate	mov	%o1, %o3
8327c478bdstevel@tonic-gate	cas	[%o0], %o2, %o3
8337c478bdstevel@tonic-gate	cmp	%o2, %o3
834895ca17ae	ATOMIC_BACKOFF_BRANCH(%icc, 2f, 1b)
8357c478bdstevel@tonic-gate	  mov	%o3, %o2
8367c478bdstevel@tonic-gate	retl
8377c478bdstevel@tonic-gate	mov	%o3, %o0
839895ca17ae	ATOMIC_BACKOFF_BACKOFF(%o4, %o5, %g4, %g5, swap32, 0b)
8407c478bdstevel@tonic-gate	SET_SIZE(atomic_swap_uint)
8417c478bdstevel@tonic-gate	SET_SIZE(atomic_swap_32)
8437c478bdstevel@tonic-gate	ENTRY(atomic_swap_64)
8447c478bdstevel@tonic-gate	ALTENTRY(atomic_swap_ptr)
8457c478bdstevel@tonic-gate	ALTENTRY(atomic_swap_ulong)
846895ca17ae	ATOMIC_BACKOFF_INIT(%o4, %g4, %g5)
8487c478bdstevel@tonic-gate	ldx	[%o0], %o2
8507c478bdstevel@tonic-gate	mov	%o1, %o3
8517c478bdstevel@tonic-gate	casx	[%o0], %o2, %o3
8527c478bdstevel@tonic-gate	cmp	%o2, %o3
853895ca17ae	ATOMIC_BACKOFF_BRANCH(%xcc, 2f, 1b)
8547c478bdstevel@tonic-gate	  mov	%o3, %o2
8557c478bdstevel@tonic-gate	retl
8567c478bdstevel@tonic-gate	mov	%o3, %o0
858895ca17ae	ATOMIC_BACKOFF_BACKOFF(%o4, %o5, %g4, %g5, swap64, 0b)
8597c478bdstevel@tonic-gate	SET_SIZE(atomic_swap_ulong)
8607c478bdstevel@tonic-gate	SET_SIZE(atomic_swap_ptr)
8617c478bdstevel@tonic-gate	SET_SIZE(atomic_swap_64)
8637c478bdstevel@tonic-gate	ENTRY(atomic_set_long_excl)
864895ca17ae	ATOMIC_BACKOFF_INIT(%o5, %g4, %g5)
8657c478bdstevel@tonic-gate	mov	1, %o3
8667c478bdstevel@tonic-gate	slln	%o3, %o1, %o3
8687c478bdstevel@tonic-gate	ldn	[%o0], %o2
8707c478bdstevel@tonic-gate	andcc	%o2, %o3, %g0		! test if the bit is set
8717c478bdstevel@tonic-gate	bnz,a,pn %ncc, 2f		! if so, then fail out
8727c478bdstevel@tonic-gate	  mov	-1, %o0
8737c478bdstevel@tonic-gate	or	%o2, %o3, %o4		! set the bit, and try to commit it
8747c478bdstevel@tonic-gate	casn	[%o0], %o2, %o4
8757c478bdstevel@tonic-gate	cmp	%o2, %o4
876895ca17ae	ATOMIC_BACKOFF_BRANCH(%ncc, 5f, 1b)
8777c478bdstevel@tonic-gate	  mov	%o4, %o2
8787c478bdstevel@tonic-gate	mov	%g0, %o0
8807c478bdstevel@tonic-gate	retl
8817c478bdstevel@tonic-gate	nop
883895ca17ae	ATOMIC_BACKOFF_BACKOFF(%o5, %g1, %g4, %g5, setlongexcl, 0b)
8847c478bdstevel@tonic-gate	SET_SIZE(atomic_set_long_excl)
8867c478bdstevel@tonic-gate	ENTRY(atomic_clear_long_excl)
887895ca17ae	ATOMIC_BACKOFF_INIT(%o5, %g4, %g5)
8887c478bdstevel@tonic-gate	mov	1, %o3
8897c478bdstevel@tonic-gate	slln	%o3, %o1, %o3
8917c478bdstevel@tonic-gate	ldn	[%o0], %o2
8937c478bdstevel@tonic-gate	andncc	%o3, %o2, %g0		! test if the bit is clear
8947c478bdstevel@tonic-gate	bnz,a,pn %ncc, 2f		! if so, then fail out
8957c478bdstevel@tonic-gate	  mov	-1, %o0
8967c478bdstevel@tonic-gate	andn	%o2, %o3, %o4		! clear the bit, and try to commit it
8977c478bdstevel@tonic-gate	casn	[%o0], %o2, %o4
8987c478bdstevel@tonic-gate	cmp	%o2, %o4
899895ca17ae	ATOMIC_BACKOFF_BRANCH(%ncc, 5f, 1b)
9007c478bdstevel@tonic-gate	  mov	%o4, %o2
9017c478bdstevel@tonic-gate	mov	%g0, %o0
9037c478bdstevel@tonic-gate	retl
9047c478bdstevel@tonic-gate	nop
906895ca17ae	ATOMIC_BACKOFF_BACKOFF(%o5, %g1, %g4, %g5, clrlongexcl, 0b)
9077c478bdstevel@tonic-gate	SET_SIZE(atomic_clear_long_excl)
9097c478bdstevel@tonic-gate#if !defined(_KERNEL)
9117c478bdstevel@tonic-gate	/*
9127c478bdstevel@tonic-gate	 * Spitfires and Blackbirds have a problem with membars in the
9137c478bdstevel@tonic-gate	 * delay slot (SF_ERRATA_51).  For safety's sake, we assume
9147c478bdstevel@tonic-gate	 * that the whole world needs the workaround.
9157c478bdstevel@tonic-gate	 */
9167c478bdstevel@tonic-gate	ENTRY(membar_enter)
9177c478bdstevel@tonic-gate	membar	#StoreLoad|#StoreStore
9187c478bdstevel@tonic-gate	retl
9197c478bdstevel@tonic-gate	nop
9207c478bdstevel@tonic-gate	SET_SIZE(membar_enter)
9227c478bdstevel@tonic-gate	ENTRY(membar_exit)
9237c478bdstevel@tonic-gate	membar	#LoadStore|#StoreStore
9247c478bdstevel@tonic-gate	retl
9257c478bdstevel@tonic-gate	nop
9267c478bdstevel@tonic-gate	SET_SIZE(membar_exit)
9287c478bdstevel@tonic-gate	ENTRY(membar_producer)
9297c478bdstevel@tonic-gate	membar	#StoreStore
9307c478bdstevel@tonic-gate	retl
9317c478bdstevel@tonic-gate	nop
9327c478bdstevel@tonic-gate	SET_SIZE(membar_producer)
9347c478bdstevel@tonic-gate	ENTRY(membar_consumer)
9357c478bdstevel@tonic-gate	membar	#LoadLoad
9367c478bdstevel@tonic-gate	retl
9377c478bdstevel@tonic-gate	nop
9387c478bdstevel@tonic-gate	SET_SIZE(membar_consumer)
9407c478bdstevel@tonic-gate#endif	/* !_KERNEL */