1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22/*
23 * Copyright 2014 Garrett D'Amore <garrett@damore.org>
24 *
25 * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
26 * Use is subject to license terms.
27 */
28
29#ifndef	_SYS_ATOMIC_H
30#define	_SYS_ATOMIC_H
31
32#include <sys/types.h>
33#include <sys/inttypes.h>
34
35#ifdef	__cplusplus
36extern "C" {
37#endif
38
39#if defined(_KERNEL) && defined(__GNUC__) && defined(_ASM_INLINES) && \
40	(defined(__i386) || defined(__amd64))
41#include <asm/atomic.h>
42#endif
43
44/*
45 * Increment target.
46 */
47extern void atomic_inc_8(volatile uint8_t *);
48extern void atomic_inc_uchar(volatile uchar_t *);
49extern void atomic_inc_16(volatile uint16_t *);
50extern void atomic_inc_ushort(volatile ushort_t *);
51extern void atomic_inc_32(volatile uint32_t *);
52extern void atomic_inc_uint(volatile uint_t *);
53extern void atomic_inc_ulong(volatile ulong_t *);
54#if defined(_KERNEL) || defined(_INT64_TYPE)
55extern void atomic_inc_64(volatile uint64_t *);
56
57/*
58 * Decrement target
59 */
60extern void atomic_dec_8(volatile uint8_t *);
61extern void atomic_dec_uchar(volatile uchar_t *);
62extern void atomic_dec_16(volatile uint16_t *);
63extern void atomic_dec_ushort(volatile ushort_t *);
64extern void atomic_dec_32(volatile uint32_t *);
65extern void atomic_dec_uint(volatile uint_t *);
66extern void atomic_dec_ulong(volatile ulong_t *);
67#if defined(_KERNEL) || defined(_INT64_TYPE)
68extern void atomic_dec_64(volatile uint64_t *);
69#endif
70
71/*
72 * Add delta to target
73 */
74extern void atomic_add_8(volatile uint8_t *, int8_t);
75extern void atomic_add_char(volatile uchar_t *, signed char);
76extern void atomic_add_16(volatile uint16_t *, int16_t);
77extern void atomic_add_short(volatile ushort_t *, short);
78extern void atomic_add_32(volatile uint32_t *, int32_t);
79extern void atomic_add_int(volatile uint_t *, int);
80extern void atomic_add_ptr(volatile void *, ssize_t);
81extern void atomic_add_long(volatile ulong_t *, long);
82#if defined(_KERNEL) || defined(_INT64_TYPE)
83extern void atomic_add_64(volatile uint64_t *, int64_t);
84#endif
85
86/*
87 * logical OR bits with target
88 */
89extern void atomic_or_8(volatile uint8_t *, uint8_t);
90extern void atomic_or_uchar(volatile uchar_t *, uchar_t);
91extern void atomic_or_16(volatile uint16_t *, uint16_t);
92extern void atomic_or_ushort(volatile ushort_t *, ushort_t);
93extern void atomic_or_32(volatile uint32_t *, uint32_t);
94extern void atomic_or_uint(volatile uint_t *, uint_t);
95extern void atomic_or_ulong(volatile ulong_t *, ulong_t);
96#if defined(_KERNEL) || defined(_INT64_TYPE)
97extern void atomic_or_64(volatile uint64_t *, uint64_t);
98#endif
99
100/*
101 * logical AND bits with target
102 */
103extern void atomic_and_8(volatile uint8_t *, uint8_t);
104extern void atomic_and_uchar(volatile uchar_t *, uchar_t);
105extern void atomic_and_16(volatile uint16_t *, uint16_t);
106extern void atomic_and_ushort(volatile ushort_t *, ushort_t);
107extern void atomic_and_32(volatile uint32_t *, uint32_t);
108extern void atomic_and_uint(volatile uint_t *, uint_t);
109extern void atomic_and_ulong(volatile ulong_t *, ulong_t);
110#if defined(_KERNEL) || defined(_INT64_TYPE)
111extern void atomic_and_64(volatile uint64_t *, uint64_t);
112#endif
113
114/*
115 * As above, but return the new value.  Note that these _nv() variants are
116 * substantially more expensive on some platforms than the no-return-value
117 * versions above, so don't use them unless you really need to know the
118 * new value *atomically* (e.g. when decrementing a reference count and
119 * checking whether it went to zero).
120 */
121
122/*
123 * Increment target and return new value.
124 */
125extern uint8_t atomic_inc_8_nv(volatile uint8_t *);
126extern uchar_t atomic_inc_uchar_nv(volatile uchar_t *);
127extern uint16_t atomic_inc_16_nv(volatile uint16_t *);
128extern ushort_t atomic_inc_ushort_nv(volatile ushort_t *);
129extern uint32_t atomic_inc_32_nv(volatile uint32_t *);
130extern uint_t atomic_inc_uint_nv(volatile uint_t *);
131extern ulong_t atomic_inc_ulong_nv(volatile ulong_t *);
132#if defined(_KERNEL) || defined(_INT64_TYPE)
133extern uint64_t atomic_inc_64_nv(volatile uint64_t *);
134#endif
135
136/*
137 * Decrement target and return new value.
138 */
139extern uint8_t atomic_dec_8_nv(volatile uint8_t *);
140extern uchar_t atomic_dec_uchar_nv(volatile uchar_t *);
141extern uint16_t atomic_dec_16_nv(volatile uint16_t *);
142extern ushort_t atomic_dec_ushort_nv(volatile ushort_t *);
143extern uint32_t atomic_dec_32_nv(volatile uint32_t *);
144extern uint_t atomic_dec_uint_nv(volatile uint_t *);
145extern ulong_t atomic_dec_ulong_nv(volatile ulong_t *);
146#if defined(_KERNEL) || defined(_INT64_TYPE)
147extern uint64_t atomic_dec_64_nv(volatile uint64_t *);
148#endif
149
150/*
151 * Add delta to target
152 */
153extern uint8_t atomic_add_8_nv(volatile uint8_t *, int8_t);
154extern uchar_t atomic_add_char_nv(volatile uchar_t *, signed char);
155extern uint16_t atomic_add_16_nv(volatile uint16_t *, int16_t);
156extern ushort_t atomic_add_short_nv(volatile ushort_t *, short);
157extern uint32_t atomic_add_32_nv(volatile uint32_t *, int32_t);
158extern uint_t atomic_add_int_nv(volatile uint_t *, int);
159extern void *atomic_add_ptr_nv(volatile void *, ssize_t);
160extern ulong_t atomic_add_long_nv(volatile ulong_t *, long);
161#if defined(_KERNEL) || defined(_INT64_TYPE)
162extern uint64_t atomic_add_64_nv(volatile uint64_t *, int64_t);
163#endif
164
165/*
166 * logical OR bits with target and return new value.
167 */
168extern uint8_t atomic_or_8_nv(volatile uint8_t *, uint8_t);
169extern uchar_t atomic_or_uchar_nv(volatile uchar_t *, uchar_t);
170extern uint16_t atomic_or_16_nv(volatile uint16_t *, uint16_t);
171extern ushort_t atomic_or_ushort_nv(volatile ushort_t *, ushort_t);
172extern uint32_t atomic_or_32_nv(volatile uint32_t *, uint32_t);
173extern uint_t atomic_or_uint_nv(volatile uint_t *, uint_t);
174extern ulong_t atomic_or_ulong_nv(volatile ulong_t *, ulong_t);
175#if defined(_KERNEL) || defined(_INT64_TYPE)
176extern uint64_t atomic_or_64_nv(volatile uint64_t *, uint64_t);
177#endif
178
179/*
180 * logical AND bits with target and return new value.
181 */
182extern uint8_t atomic_and_8_nv(volatile uint8_t *, uint8_t);
183extern uchar_t atomic_and_uchar_nv(volatile uchar_t *, uchar_t);
184extern uint16_t atomic_and_16_nv(volatile uint16_t *, uint16_t);
185extern ushort_t atomic_and_ushort_nv(volatile ushort_t *, ushort_t);
186extern uint32_t atomic_and_32_nv(volatile uint32_t *, uint32_t);
187extern uint_t atomic_and_uint_nv(volatile uint_t *, uint_t);
188extern ulong_t atomic_and_ulong_nv(volatile ulong_t *, ulong_t);
189#if defined(_KERNEL) || defined(_INT64_TYPE)
190extern uint64_t atomic_and_64_nv(volatile uint64_t *, uint64_t);
191#endif
192
193/*
194 * If *arg1 == arg2, set *arg1 = arg3; return old value
195 */
196extern uint8_t atomic_cas_8(volatile uint8_t *, uint8_t, uint8_t);
197extern uchar_t atomic_cas_uchar(volatile uchar_t *, uchar_t, uchar_t);
198extern uint16_t atomic_cas_16(volatile uint16_t *, uint16_t, uint16_t);
199extern ushort_t atomic_cas_ushort(volatile ushort_t *, ushort_t, ushort_t);
200extern uint32_t atomic_cas_32(volatile uint32_t *, uint32_t, uint32_t);
201extern uint_t atomic_cas_uint(volatile uint_t *, uint_t, uint_t);
202extern void *atomic_cas_ptr(volatile void *, void *, void *);
203extern ulong_t atomic_cas_ulong(volatile ulong_t *, ulong_t, ulong_t);
204#if defined(_KERNEL) || defined(_INT64_TYPE)
205extern uint64_t atomic_cas_64(volatile uint64_t *, uint64_t, uint64_t);
206#endif
207
208/*
209 * Swap target and return old value
210 */
211extern uint8_t atomic_swap_8(volatile uint8_t *, uint8_t);
212extern uchar_t atomic_swap_uchar(volatile uchar_t *, uchar_t);
213extern uint16_t atomic_swap_16(volatile uint16_t *, uint16_t);
214extern ushort_t atomic_swap_ushort(volatile ushort_t *, ushort_t);
215extern uint32_t atomic_swap_32(volatile uint32_t *, uint32_t);
216extern uint_t atomic_swap_uint(volatile uint_t *, uint_t);
217extern void *atomic_swap_ptr(volatile void *, void *);
218extern ulong_t atomic_swap_ulong(volatile ulong_t *, ulong_t);
219#if defined(_KERNEL) || defined(_INT64_TYPE)
220extern uint64_t atomic_swap_64(volatile uint64_t *, uint64_t);
221#endif
222
223/*
224 * Perform an exclusive atomic bit set/clear on a target.
225 * Returns 0 if bit was sucessfully set/cleared, or -1
226 * if the bit was already set/cleared.
227 */
228extern int atomic_set_long_excl(volatile ulong_t *, uint_t);
229extern int atomic_clear_long_excl(volatile ulong_t *, uint_t);
230
231/*
232 * Generic memory barrier used during lock entry, placed after the
233 * memory operation that acquires the lock to guarantee that the lock
234 * protects its data.  No stores from after the memory barrier will
235 * reach visibility, and no loads from after the barrier will be
236 * resolved, before the lock acquisition reaches global visibility.
237 */
238extern void membar_enter(void);
239
240/*
241 * Generic memory barrier used during lock exit, placed before the
242 * memory operation that releases the lock to guarantee that the lock
243 * protects its data.  All loads and stores issued before the barrier
244 * will be resolved before the subsequent lock update reaches visibility.
245 */
246extern void membar_exit(void);
247
248/*
249 * Arrange that all stores issued before this point in the code reach
250 * global visibility before any stores that follow; useful in producer
251 * modules that update a data item, then set a flag that it is available.
252 * The memory barrier guarantees that the available flag is not visible
253 * earlier than the updated data, i.e. it imposes store ordering.
254 */
255extern void membar_producer(void);
256
257/*
258 * Arrange that all loads issued before this point in the code are
259 * completed before any subsequent loads; useful in consumer modules
260 * that check to see if data is available and read the data.
261 * The memory barrier guarantees that the data is not sampled until
262 * after the available flag has been seen, i.e. it imposes load ordering.
263 */
264extern void membar_consumer(void);
265#endif
266
267#if defined(_KERNEL)
268
269#if defined(_LP64) || defined(_ILP32)
270#define	atomic_add_ip		atomic_add_long
271#define	atomic_add_ip_nv	atomic_add_long_nv
272#define	casip			atomic_cas_ulong
273#endif
274
275#if defined(__sparc)
276extern uint8_t ldstub(uint8_t *);
277#endif
278
279#endif	/* _KERNEL */
280
281#ifdef	__cplusplus
282}
283#endif
284
285#endif	/* _SYS_ATOMIC_H */
286