1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License, Version 1.0 only
6 * (the "License").  You may not use this file except in compliance
7 * with the License.
8 *
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or http://www.opensolaris.org/os/licensing.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
13 *
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 */
22/*
23 * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
24 * Use is subject to license terms.
25 * Copyright 2014 Nexenta Systems, Inc.  All rights reserved.
26 * Copyright 2015 Josef 'Jeff' Sipek <jeffpc@josefsipek.net>
27 */
28
29#ifndef _ASM_ATOMIC_H
30#define	_ASM_ATOMIC_H
31
32#include <sys/ccompile.h>
33#include <sys/types.h>
34
35#ifdef	__cplusplus
36extern "C" {
37#endif
38
39#if !defined(__lint) && defined(__GNUC__)
40
41/* BEGIN CSTYLED */
42/*
43 * This file contains a number of static inline functions implementing
44 * various atomic variable functions.  Note that these are *not* all of the
45 * atomic_* functions as defined in usr/src/uts/common/sys/atomic.h.  All
46 * possible atomic_* functions are implemented in usr/src/common/atomic in
47 * pure assembly.  In the absence of an identically named function in this
48 * header file, any use of the function will result in the compiler emitting
49 * a function call as usual.  On the other hand, if an identically named
50 * function exists in this header as a static inline, the compiler will
51 * inline its contents and the linker never sees the symbol reference.  We
52 * use this to avoid implementing some of the more complex and less used
53 * functions and instead falling back to function calls.  Note that in some
54 * cases (e.g., atomic_inc_64) we implement a static inline only on AMD64
55 * but not i386.
56 */
57
58/*
59 * Instruction suffixes for various operand sizes (assuming AMD64)
60 */
61#define	SUF_8		"b"
62#define	SUF_16		"w"
63#define	SUF_32		"l"
64#define	SUF_64		"q"
65
66#if defined(__amd64)
67#define	SUF_LONG	SUF_64
68#define	SUF_PTR		SUF_64
69#define	__ATOMIC_OP64(...)	__ATOMIC_OPXX(__VA_ARGS__)
70#elif defined(__i386)
71#define	SUF_LONG	SUF_32
72#define	SUF_PTR		SUF_32
73#define	__ATOMIC_OP64(...)
74#else
75#error "port me"
76#endif
77
78#if defined(__amd64) || defined(__i386)
79
80#define	__ATOMIC_OPXX(fxn, type, op)					\
81extern __GNU_INLINE void						\
82fxn(volatile type *target)						\
83{									\
84	__asm__ __volatile__(						\
85	    "lock; " op " %0"						\
86	    : "+m" (*target)						\
87	    : /* no inputs */						\
88	    : "cc");							\
89}
90
91__ATOMIC_OPXX(atomic_inc_8,      uint8_t,  "inc" SUF_8)
92__ATOMIC_OPXX(atomic_inc_16,     uint16_t, "inc" SUF_16)
93__ATOMIC_OPXX(atomic_inc_32,     uint32_t, "inc" SUF_32)
94__ATOMIC_OP64(atomic_inc_64,     uint64_t, "inc" SUF_64)
95__ATOMIC_OPXX(atomic_inc_uchar,  uchar_t,  "inc" SUF_8)
96__ATOMIC_OPXX(atomic_inc_ushort, ushort_t, "inc" SUF_16)
97__ATOMIC_OPXX(atomic_inc_uint,   uint_t,   "inc" SUF_32)
98__ATOMIC_OPXX(atomic_inc_ulong,  ulong_t,  "inc" SUF_LONG)
99
100__ATOMIC_OPXX(atomic_dec_8,      uint8_t,  "dec" SUF_8)
101__ATOMIC_OPXX(atomic_dec_16,     uint16_t, "dec" SUF_16)
102__ATOMIC_OPXX(atomic_dec_32,     uint32_t, "dec" SUF_32)
103__ATOMIC_OP64(atomic_dec_64,     uint64_t, "dec" SUF_64)
104__ATOMIC_OPXX(atomic_dec_uchar,  uchar_t,  "dec" SUF_8)
105__ATOMIC_OPXX(atomic_dec_ushort, ushort_t, "dec" SUF_16)
106__ATOMIC_OPXX(atomic_dec_uint,   uint_t,   "dec" SUF_32)
107__ATOMIC_OPXX(atomic_dec_ulong,  ulong_t,  "dec" SUF_LONG)
108
109#undef __ATOMIC_OPXX
110
111#define	__ATOMIC_OPXX(fxn, type1, type2, op, reg)			\
112extern __GNU_INLINE void						\
113fxn(volatile type1 *target, type2 delta)				\
114{									\
115	__asm__ __volatile__(						\
116	    "lock; " op " %1,%0"					\
117	    : "+m" (*target)						\
118	    : "i" reg (delta)						\
119	    : "cc");							\
120}
121
122__ATOMIC_OPXX(atomic_add_8,     uint8_t,  int8_t,      "add" SUF_8,    "q")
123__ATOMIC_OPXX(atomic_add_16,    uint16_t, int16_t,     "add" SUF_16,   "r")
124__ATOMIC_OPXX(atomic_add_32,    uint32_t, int32_t,     "add" SUF_32,   "r")
125__ATOMIC_OP64(atomic_add_64,    uint64_t, int64_t,     "add" SUF_64,   "r")
126__ATOMIC_OPXX(atomic_add_char,  uchar_t,  signed char, "add" SUF_8,    "q")
127__ATOMIC_OPXX(atomic_add_short, ushort_t, short,       "add" SUF_16,   "r")
128__ATOMIC_OPXX(atomic_add_int,   uint_t,   int,         "add" SUF_32,   "r")
129__ATOMIC_OPXX(atomic_add_long,  ulong_t,  long,        "add" SUF_LONG, "r")
130
131/*
132 * We don't use the above macro here because atomic_add_ptr has an
133 * inconsistent type.  The first argument should really be a 'volatile void
134 * **'.
135 */
136extern __GNU_INLINE void
137atomic_add_ptr(volatile void *target, ssize_t delta)
138{
139	volatile void **tmp = (volatile void **)target;
140
141	__asm__ __volatile__(
142	    "lock; add" SUF_PTR " %1,%0"
143	    : "+m" (*tmp)
144	    : "ir" (delta)
145	    : "cc");
146}
147
148__ATOMIC_OPXX(atomic_or_8,       uint8_t,  uint8_t,  "or" SUF_8,    "q")
149__ATOMIC_OPXX(atomic_or_16,      uint16_t, uint16_t, "or" SUF_16,   "r")
150__ATOMIC_OPXX(atomic_or_32,      uint32_t, uint32_t, "or" SUF_32,   "r")
151__ATOMIC_OP64(atomic_or_64,      uint64_t, uint64_t, "or" SUF_64,   "r")
152__ATOMIC_OPXX(atomic_or_uchar,   uchar_t,  uchar_t,  "or" SUF_8,    "q")
153__ATOMIC_OPXX(atomic_or_ushort,  ushort_t, ushort_t, "or" SUF_16,   "r")
154__ATOMIC_OPXX(atomic_or_uint,    uint_t,   uint_t,   "or" SUF_32,   "r")
155__ATOMIC_OPXX(atomic_or_ulong,   ulong_t,  ulong_t,  "or" SUF_LONG, "r")
156
157__ATOMIC_OPXX(atomic_and_8,      uint8_t,  uint8_t,  "and" SUF_8,    "q")
158__ATOMIC_OPXX(atomic_and_16,     uint16_t, uint16_t, "and" SUF_16,   "r")
159__ATOMIC_OPXX(atomic_and_32,     uint32_t, uint32_t, "and" SUF_32,   "r")
160__ATOMIC_OP64(atomic_and_64,     uint64_t, uint64_t, "and" SUF_64,   "r")
161__ATOMIC_OPXX(atomic_and_uchar,  uchar_t,  uchar_t,  "and" SUF_8,    "q")
162__ATOMIC_OPXX(atomic_and_ushort, ushort_t, ushort_t, "and" SUF_16,   "r")
163__ATOMIC_OPXX(atomic_and_uint,   uint_t,   uint_t,   "and" SUF_32,   "r")
164__ATOMIC_OPXX(atomic_and_ulong,  ulong_t,  ulong_t,  "and" SUF_LONG, "r")
165
166#undef __ATOMIC_OPXX
167
168#define	__ATOMIC_OPXX(fxn, type, op, reg)				\
169extern __GNU_INLINE type						\
170fxn(volatile type *target, type cmp, type new)				\
171{									\
172	type ret;							\
173	__asm__ __volatile__(						\
174	    "lock; " op " %2,%0"					\
175	    : "+m" (*target), "=a" (ret)				\
176	    : reg (new), "1" (cmp)					\
177	    : "cc");							\
178	return (ret);							\
179}
180
181__ATOMIC_OPXX(atomic_cas_8,      uint8_t,  "cmpxchg" SUF_8,    "q")
182__ATOMIC_OPXX(atomic_cas_16,     uint16_t, "cmpxchg" SUF_16,   "r")
183__ATOMIC_OPXX(atomic_cas_32,     uint32_t, "cmpxchg" SUF_32,   "r")
184__ATOMIC_OP64(atomic_cas_64,     uint64_t, "cmpxchg" SUF_64,   "r")
185__ATOMIC_OPXX(atomic_cas_uchar,  uchar_t,  "cmpxchg" SUF_8,    "q")
186__ATOMIC_OPXX(atomic_cas_ushort, ushort_t, "cmpxchg" SUF_16,   "r")
187__ATOMIC_OPXX(atomic_cas_uint,   uint_t,   "cmpxchg" SUF_32,   "r")
188__ATOMIC_OPXX(atomic_cas_ulong,  ulong_t,  "cmpxchg" SUF_LONG, "r")
189
190#undef __ATOMIC_OPXX
191
192/*
193 * We don't use the above macro here because atomic_cas_ptr has an
194 * inconsistent type.  The first argument should really be a 'volatile void
195 * **'.
196 */
197extern __GNU_INLINE void *
198atomic_cas_ptr(volatile void *target, void *cmp, void *new)
199{
200	volatile void **tmp = (volatile void **)target;
201	void *ret;
202
203	__asm__ __volatile__(
204	    "lock; cmpxchg" SUF_PTR " %2,%0"
205	    : "+m" (*tmp), "=a" (ret)
206	    : "r" (new), "1" (cmp)
207	    : "cc");
208
209	return (ret);
210}
211
212#define	__ATOMIC_OPXX(fxn, type, op, reg)				\
213extern __GNU_INLINE type						\
214fxn(volatile type *target, type val)					\
215{									\
216	__asm__ __volatile__(						\
217	    op " %1,%0"							\
218	    : "+m" (*target), "+" reg (val));				\
219	return (val);							\
220}
221
222__ATOMIC_OPXX(atomic_swap_8,      uint8_t,  "xchg" SUF_8,    "q")
223__ATOMIC_OPXX(atomic_swap_16,     uint16_t, "xchg" SUF_16,   "r")
224__ATOMIC_OPXX(atomic_swap_32,     uint32_t, "xchg" SUF_32,   "r")
225__ATOMIC_OP64(atomic_swap_64,     uint64_t, "xchg" SUF_64,   "r")
226__ATOMIC_OPXX(atomic_swap_uchar,  uchar_t,  "xchg" SUF_8,    "q")
227__ATOMIC_OPXX(atomic_swap_ushort, ushort_t, "xchg" SUF_16,   "r")
228__ATOMIC_OPXX(atomic_swap_uint,   uint_t,   "xchg" SUF_32,   "r")
229__ATOMIC_OPXX(atomic_swap_ulong,  ulong_t,  "xchg" SUF_LONG, "r")
230
231#undef __ATOMIC_OPXX
232
233/*
234 * We don't use the above macro here because atomic_swap_ptr has an
235 * inconsistent type.  The first argument should really be a 'volatile void
236 * **'.
237 */
238extern __GNU_INLINE void *
239atomic_swap_ptr(volatile void *target, void *val)
240{
241	volatile void **tmp = (volatile void **)target;
242
243	__asm__ __volatile__(
244	    "xchg" SUF_PTR " %1,%0"
245	    : "+m" (*tmp), "+r" (val));
246
247	return (val);
248}
249
250#else
251#error	"port me"
252#endif
253
254#undef SUF_8
255#undef SUF_16
256#undef SUF_32
257#undef SUF_64
258#undef SUF_LONG
259#undef SUF_PTR
260
261#undef __ATOMIC_OP64
262
263/* END CSTYLED */
264
265#endif /* !__lint && __GNUC__ */
266
267#ifdef __cplusplus
268}
269#endif
270
271#endif	/* _ASM_ATOMIC_H */
272