1 /*
2  * This file and its contents are supplied under the terms of the
3  * Common Development and Distribution License ("CDDL"), version 1.0.
4  * You may only use this file in accordance with the terms of version
5  * 1.0 of the CDDL.
6  *
7  * A full copy of the text of the CDDL should have accompanied this
8  * source.  A copy of the CDDL is also available via the Internet at
9  * http://www.illumos.org/license/CDDL.
10  */
11 
12 /*
13  * Copyright 2014 Pluribus Networks Inc.
14  * Copyright 2019 Joyent, Inc.
15  */
16 
17 #ifndef _COMPAT_FREEBSD_AMD64_MACHINE_ATOMIC_H_
18 #define	_COMPAT_FREEBSD_AMD64_MACHINE_ATOMIC_H_
19 
20 static __inline u_int
atomic_load_acq_short(volatile u_short * p)21 atomic_load_acq_short(volatile u_short *p)
22 {
23 	u_short res;
24 
25 	res = *p;
26 	__asm volatile("" : : : "memory");
27 
28 	return (res);
29 }
30 
31 static __inline u_int
atomic_load_acq_int(volatile u_int * p)32 atomic_load_acq_int(volatile u_int *p)
33 {
34 	u_int res;
35 
36 	res = *p;
37 	__asm volatile("" : : : "memory");
38 
39 	return (res);
40 }
41 
42 static __inline u_long
atomic_load_acq_long(volatile u_long * p)43 atomic_load_acq_long(volatile u_long *p)
44 {
45 	u_long res;
46 
47 	res = *p;
48 	__asm volatile("" : : : "memory");
49 
50 	return (res);
51 }
52 
53 static __inline void
atomic_store_rel_int(volatile u_int * p,u_int v)54 atomic_store_rel_int(volatile u_int *p, u_int v)
55 {
56 	__asm volatile("" : : : "memory");
57 	*p = v;
58 }
59 
60 static __inline void
atomic_store_rel_long(volatile u_long * p,u_long v)61 atomic_store_rel_long(volatile u_long *p, u_long v)
62 {
63 	__asm volatile("" : : : "memory");
64 	*p = v;
65 }
66 
67 /*
68  * Atomic compare and set.
69  *
70  * if (*dst == expect) *dst = src (all 32 bit words)
71  *
72  * Returns 0 on failure, non-zero on success
73  */
74 static __inline int
atomic_cmpset_int(volatile u_int * dst,u_int expect,u_int src)75 atomic_cmpset_int(volatile u_int *dst, u_int expect, u_int src)
76 {
77 	u_char res;
78 
79 	__asm __volatile(
80 	"	lock ;			"
81 	"	cmpxchgl %3,%1 ;	"
82 	"       sete	%0 ;		"
83 	"# atomic_cmpset_int"
84 	: "=q" (res),			/* 0 */
85 	  "+m" (*dst),			/* 1 */
86 	  "+a" (expect)			/* 2 */
87 	: "r" (src)			/* 3 */
88 	: "memory", "cc");
89 	return (res);
90 }
91 
92 static __inline int
atomic_cmpset_long(volatile u_long * dst,u_long expect,u_long src)93 atomic_cmpset_long(volatile u_long *dst, u_long expect, u_long src)
94 {
95 	u_char res;
96 
97 	__asm __volatile(
98 	"	lock ;			"
99 	"	cmpxchgq %3,%1 ;	"
100 	"       sete	%0 ;		"
101 	"# atomic_cmpset_long"
102 	: "=q" (res),			/* 0 */
103 	  "+m" (*dst),			/* 1 */
104 	  "+a" (expect)			/* 2 */
105 	: "r" (src)			/* 3 */
106 	: "memory", "cc");
107 	return (res);
108 }
109 
110 static __inline int
atomic_testandset_int(volatile u_int * p,u_int v)111 atomic_testandset_int(volatile u_int *p, u_int v)
112 {
113 	u_char res;
114 
115 	__asm __volatile(
116 	"	lock ;			"
117 	"	btsl	%2,%1 ;		"
118 	"	setc	%0 ;		"
119 	"# atomic_testandset_int"
120 	: "=q" (res),		/* 0 */
121 	"+m" (*p)		/* 1 */
122 	: "Ir" (v & 0x1f)	/* 2 */
123 	: "cc");
124 	return (res);
125 }
126 
127 /*
128  * Atomically add the value of v to the integer pointed to by p and return
129  * the previous value of *p.
130  */
131 static __inline u_int
atomic_fetchadd_int(volatile u_int * p,u_int v)132 atomic_fetchadd_int(volatile u_int *p, u_int v)
133 {
134 
135 	__asm __volatile(
136 	"	lock ;			"
137 	"	xaddl	%0, %1 ;	"
138 	"# atomic_fetchadd_int"
139 	: "+r" (v),			/* 0 (result) */
140 	  "=m" (*p)			/* 1 */
141 	: "m" (*p)			/* 2 */
142 	: "cc");
143 	return (v);
144 }
145 
146 static __inline void
atomic_set_int(volatile u_int * p,u_int v)147 atomic_set_int(volatile u_int *p, u_int v)
148 {
149 	__asm volatile(
150 	"lock ; " "orl %1,%0"
151 	: "=m" (*p)
152 	: "ir" (v), "m" (*p)
153 	: "cc");
154 }
155 
156 static __inline void
atomic_clear_int(volatile u_int * p,u_int v)157 atomic_clear_int(volatile u_int *p, u_int v)
158 {
159 	__asm volatile(
160 	"lock ; " "andl %1,%0"
161 	: "=m" (*p)
162 	: "ir" (~v), "m" (*p)
163 	: "cc");
164 }
165 
166 static __inline void
atomic_subtract_int(volatile u_int * p,u_int v)167 atomic_subtract_int(volatile u_int *p, u_int v)
168 {
169 	__asm volatile(
170 	"lock ; " "subl %1,%0"
171 	: "=m" (*p)
172 	: "ir" (v), "m" (*p)
173 	: "cc");
174 }
175 
176 static __inline void
atomic_set_long(volatile u_long * p,u_long v)177 atomic_set_long(volatile u_long *p, u_long v)
178 {
179 	__asm volatile(
180 	"lock ; " "orq %1,%0"
181 	: "+m" (*p)
182 	: "ir" (v)
183 	: "cc");
184 }
185 
186 static __inline void
atomic_clear_long(volatile u_long * p,u_long v)187 atomic_clear_long(volatile u_long *p, u_long v)
188 {
189 	__asm volatile("lock ; " "andq %1,%0"
190 	: "+m" (*p)
191 	: "ir" (~v)
192 	: "cc");
193 }
194 
195 static __inline u_int
atomic_swap_int(volatile u_int * p,u_int v)196 atomic_swap_int(volatile u_int *p, u_int v)
197 {
198 
199 	__asm __volatile(
200 	"	xchgl	%1,%0 ;		"
201 	"# atomic_swap_int"
202 	: "+r" (v),			/* 0 */
203 	  "+m" (*p));			/* 1 */
204 	return (v);
205 }
206 
207 static __inline u_long
atomic_swap_long(volatile u_long * p,u_long v)208 atomic_swap_long(volatile u_long *p, u_long v)
209 {
210 
211 	__asm __volatile(
212 	"	xchgq	%1,%0 ;		"
213 	"# atomic_swap_long"
214 	: "+r" (v),			/* 0 */
215 	  "+m" (*p));			/* 1 */
216 	return (v);
217 }
218 
219 
220 #define	atomic_store_short(p, v)	\
221 	    (*(volatile u_short *)(p) = (u_short)(v))
222 #define	atomic_store_int(p, v)		\
223 	    (*(volatile u_int *)(p) = (u_int)(v))
224 
225 
226 #define	atomic_readandclear_int(p)	atomic_swap_int(p, 0)
227 #define	atomic_readandclear_long(p)	atomic_swap_long(p, 0)
228 
229 /* Operations on 32-bit double words. */
230 #define	atomic_load_acq_32	atomic_load_acq_int
231 #define	atomic_store_rel_32	atomic_store_rel_int
232 #define	atomic_cmpset_32	atomic_cmpset_int
233 
234 /* Operations on 64-bit quad words. */
235 #define	atomic_cmpset_64	atomic_cmpset_long
236 #define	atomic_readandclear_64	atomic_readandclear_long
237 
238 /* Operations on pointers. */
239 #define	atomic_cmpset_ptr	atomic_cmpset_long
240 
241 /* Needed for the membar functions */
242 #include_next <sys/atomic.h>
243 
244 static __inline void
atomic_thread_fence_rel(void)245 atomic_thread_fence_rel(void)
246 {
247 	/* Equivalent to their __compiler_membar() */
248 	__asm __volatile(" " : : : "memory");
249 }
250 
251 static __inline void
atomic_thread_fence_seq_cst(void)252 atomic_thread_fence_seq_cst(void)
253 {
254 	/* Equivalent to their !KERNEL storeload_barrer() */
255 	__asm __volatile("lock; addl $0,-8(%%rsp)" : : : "memory", "cc");
256 }
257 
258 #define	mb()			membar_enter()
259 #define	rmb()			membar_consumer()
260 #define	wmb()			membar_producer()
261 
262 #endif	/* _COMPAT_FREEBSD_AMD64_MACHINE_ATOMIC_H_ */
263