1 /*
2 * This file and its contents are supplied under the terms of the
3 * Common Development and Distribution License ("CDDL"), version 1.0.
4 * You may only use this file in accordance with the terms of version
5 * 1.0 of the CDDL.
6 *
7 * A full copy of the text of the CDDL should have accompanied this
8 * source. A copy of the CDDL is also available via the Internet at
9 * http://www.illumos.org/license/CDDL.
10 */
11
12 /*
13 * Copyright 2020 Joyent, Inc.
14 */
15
16 #ifndef _SIMD_H
17 #define _SIMD_H
18
19 #if defined(__amd64__) || defined(__i386__)
20
21 #define kfpu_initialize(tsk) do {} while (0)
22 #define kfpu_init() (0)
23 #define kfpu_fini() do {} while (0)
24
25 #ifdef _KERNEL
26 #include <sys/x86_archext.h>
27 #include <sys/archsystm.h>
28 #include <sys/kfpu.h>
29 #include <sys/proc.h>
30 #include <sys/disp.h>
31 #include <sys/cpuvar.h>
32
33 static inline int
kfpu_allowed(void)34 kfpu_allowed(void)
35 {
36 extern int zfs_fpu_enabled;
37
38 return (zfs_fpu_enabled != 0 ? 1 : 0);
39 }
40
41 static inline void
kfpu_begin(void)42 kfpu_begin(void)
43 {
44 if (curthread->t_lwp != NULL && (curthread->t_procp->p_flag & SSYS)) {
45 kernel_fpu_begin(NULL, KFPU_USE_LWP);
46 } else {
47 kpreempt_disable();
48 kernel_fpu_begin(NULL, KFPU_NO_STATE);
49 }
50 }
51
52 static inline void
kfpu_end(void)53 kfpu_end(void)
54 {
55 if (curthread->t_lwp != NULL && (curthread->t_procp->p_flag & SSYS)) {
56 kernel_fpu_end(NULL, KFPU_USE_LWP);
57 } else {
58 kernel_fpu_end(NULL, KFPU_NO_STATE);
59 kpreempt_enable();
60 }
61 }
62
63 /*
64 * Check if various vector instruction sets are available.
65 */
66
67 static inline boolean_t
zfs_sse_available(void)68 zfs_sse_available(void)
69 {
70 return (is_x86_feature(x86_featureset, X86FSET_SSE));
71 }
72
73 static inline boolean_t
zfs_sse2_available(void)74 zfs_sse2_available(void)
75 {
76 return (is_x86_feature(x86_featureset, X86FSET_SSE2));
77 }
78
79 static inline boolean_t
zfs_sse3_available(void)80 zfs_sse3_available(void)
81 {
82 return (is_x86_feature(x86_featureset, X86FSET_SSE3));
83 }
84
85 static inline boolean_t
zfs_ssse3_available(void)86 zfs_ssse3_available(void)
87 {
88 return (is_x86_feature(x86_featureset, X86FSET_SSSE3));
89 }
90
91 static inline boolean_t
zfs_avx_available(void)92 zfs_avx_available(void)
93 {
94 return (is_x86_feature(x86_featureset, X86FSET_AVX));
95 }
96
97 static inline boolean_t
zfs_avx2_available(void)98 zfs_avx2_available(void)
99 {
100 return (is_x86_feature(x86_featureset, X86FSET_AVX2));
101 }
102
103 static inline boolean_t
zfs_avx512f_available(void)104 zfs_avx512f_available(void)
105 {
106 return (is_x86_feature(x86_featureset, X86FSET_AVX512F));
107 }
108
109 static inline boolean_t
zfs_avx512bw_available(void)110 zfs_avx512bw_available(void)
111 {
112 return (is_x86_feature(x86_featureset, X86FSET_AVX512BW));
113 }
114
115 #else /* ! _KERNEL */
116
117 #include <sys/auxv.h>
118 #include <sys/auxv_386.h>
119
120 #define kfpu_allowed() 1
121 #define kfpu_begin() do {} while (0)
122 #define kfpu_end() do {} while (0)
123
124 /*
125 * User-level check if various vector instruction sets are available.
126 */
127
128 static inline boolean_t
zfs_sse_available(void)129 zfs_sse_available(void)
130 {
131 uint32_t u = 0;
132
133 (void) getisax(&u, 1);
134 return ((u & AV_386_SSE) != 0);
135 }
136
137 static inline boolean_t
zfs_sse2_available(void)138 zfs_sse2_available(void)
139 {
140 uint32_t u = 0;
141
142 (void) getisax(&u, 1);
143 return ((u & AV_386_SSE2) != 0);
144 }
145
146 static inline boolean_t
zfs_sse3_available(void)147 zfs_sse3_available(void)
148 {
149 uint32_t u = 0;
150
151 (void) getisax(&u, 1);
152 return ((u & AV_386_SSE3) != 0);
153 }
154
155 static inline boolean_t
zfs_ssse3_available(void)156 zfs_ssse3_available(void)
157 {
158 uint32_t u = 0;
159
160 (void) getisax(&u, 1);
161 return ((u & AV_386_SSSE3) != 0);
162 }
163
164 static inline boolean_t
zfs_avx_available(void)165 zfs_avx_available(void)
166 {
167 uint_t u = 0;
168
169 (void) getisax(&u, 1);
170 return ((u & AV_386_AVX) != 0);
171 }
172
173 static inline boolean_t
zfs_avx2_available(void)174 zfs_avx2_available(void)
175 {
176 uint32_t u[2] = { 0 };
177
178 (void) getisax((uint32_t *)&u, 2);
179 return ((u[1] & AV_386_2_AVX2) != 0);
180 }
181
182 static inline boolean_t
zfs_avx512f_available(void)183 zfs_avx512f_available(void)
184 {
185 uint32_t u[2] = { 0 };
186
187 (void) getisax((uint32_t *)&u, 2);
188 return ((u[1] & AV_386_2_AVX512F) != 0);
189 }
190
191 static inline boolean_t
zfs_avx512bw_available(void)192 zfs_avx512bw_available(void)
193 {
194 uint32_t u[2] = { 0 };
195
196 (void) getisax((uint32_t *)&u, 2);
197 return ((u[1] & AV_386_2_AVX512BW) != 0);
198 }
199
200 #endif /* _KERNEL */
201
202
203 #else
204
205 /* Non-x86 CPUs currently always disallow kernel FPU support */
206 #define kfpu_allowed() 0
207 #define kfpu_initialize(tsk) do {} while (0)
208 #define kfpu_begin() do {} while (0)
209 #define kfpu_end() do {} while (0)
210 #define kfpu_init() (0)
211 #define kfpu_fini() do {} while (0)
212 #endif
213
214 #endif /* _SIMD_H */
215