1e86372a0SGvozden Neskovic /*
2e86372a0SGvozden Neskovic * This file and its contents are supplied under the terms of the
3e86372a0SGvozden Neskovic * Common Development and Distribution License ("CDDL"), version 1.0.
4e86372a0SGvozden Neskovic * You may only use this file in accordance with the terms of version
5e86372a0SGvozden Neskovic * 1.0 of the CDDL.
6e86372a0SGvozden Neskovic *
7e86372a0SGvozden Neskovic * A full copy of the text of the CDDL should have accompanied this
8e86372a0SGvozden Neskovic * source. A copy of the CDDL is also available via the Internet at
9e86372a0SGvozden Neskovic * http://www.illumos.org/license/CDDL.
10e86372a0SGvozden Neskovic */
11e86372a0SGvozden Neskovic
12e86372a0SGvozden Neskovic /*
13e86372a0SGvozden Neskovic * Copyright 2020 Joyent, Inc.
14e86372a0SGvozden Neskovic */
15e86372a0SGvozden Neskovic
16e86372a0SGvozden Neskovic #ifndef _SIMD_H
17e86372a0SGvozden Neskovic #define _SIMD_H
18e86372a0SGvozden Neskovic
19e86372a0SGvozden Neskovic #if defined(__amd64__) || defined(__i386__)
20e86372a0SGvozden Neskovic
21e86372a0SGvozden Neskovic #define kfpu_initialize(tsk) do {} while (0)
22e86372a0SGvozden Neskovic #define kfpu_init() (0)
23e86372a0SGvozden Neskovic #define kfpu_fini() do {} while (0)
24e86372a0SGvozden Neskovic
25f91a4547SGvozden Neskovic #ifdef _KERNEL
26f91a4547SGvozden Neskovic #include <sys/x86_archext.h>
27f91a4547SGvozden Neskovic #include <sys/archsystm.h>
28f91a4547SGvozden Neskovic #include <sys/kfpu.h>
29f91a4547SGvozden Neskovic #include <sys/proc.h>
30f91a4547SGvozden Neskovic #include <sys/disp.h>
31f91a4547SGvozden Neskovic #include <sys/cpuvar.h>
32f91a4547SGvozden Neskovic
33fd440315SJoshua M. Clulow static inline int
kfpu_allowed(void)34fd440315SJoshua M. Clulow kfpu_allowed(void)
35fd440315SJoshua M. Clulow {
36fd440315SJoshua M. Clulow extern int zfs_fpu_enabled;
37fd440315SJoshua M. Clulow
38fd440315SJoshua M. Clulow return (zfs_fpu_enabled != 0 ? 1 : 0);
39fd440315SJoshua M. Clulow }
40fd440315SJoshua M. Clulow
41f91a4547SGvozden Neskovic static inline void
kfpu_begin(void)42f91a4547SGvozden Neskovic kfpu_begin(void)
43f91a4547SGvozden Neskovic {
44f91a4547SGvozden Neskovic if (curthread->t_lwp != NULL && (curthread->t_procp->p_flag & SSYS)) {
45f91a4547SGvozden Neskovic kernel_fpu_begin(NULL, KFPU_USE_LWP);
46f91a4547SGvozden Neskovic } else {
47f91a4547SGvozden Neskovic kpreempt_disable();
48f91a4547SGvozden Neskovic kernel_fpu_begin(NULL, KFPU_NO_STATE);
49f91a4547SGvozden Neskovic }
50f91a4547SGvozden Neskovic }
51f91a4547SGvozden Neskovic
52f91a4547SGvozden Neskovic static inline void
kfpu_end(void)53f91a4547SGvozden Neskovic kfpu_end(void)
54f91a4547SGvozden Neskovic {
55f91a4547SGvozden Neskovic if (curthread->t_lwp != NULL && (curthread->t_procp->p_flag & SSYS)) {
56f91a4547SGvozden Neskovic kernel_fpu_end(NULL, KFPU_USE_LWP);
57f91a4547SGvozden Neskovic } else {
58f91a4547SGvozden Neskovic kernel_fpu_end(NULL, KFPU_NO_STATE);
59f91a4547SGvozden Neskovic kpreempt_enable();
60f91a4547SGvozden Neskovic }
61f91a4547SGvozden Neskovic }
62f91a4547SGvozden Neskovic
63f91a4547SGvozden Neskovic /*
64f91a4547SGvozden Neskovic * Check if various vector instruction sets are available.
65f91a4547SGvozden Neskovic */
66f91a4547SGvozden Neskovic
67f91a4547SGvozden Neskovic static inline boolean_t
zfs_sse_available(void)68f91a4547SGvozden Neskovic zfs_sse_available(void)
69f91a4547SGvozden Neskovic {
70f91a4547SGvozden Neskovic return (is_x86_feature(x86_featureset, X86FSET_SSE));
71f91a4547SGvozden Neskovic }
72f91a4547SGvozden Neskovic
73f91a4547SGvozden Neskovic static inline boolean_t
zfs_sse2_available(void)74f91a4547SGvozden Neskovic zfs_sse2_available(void)
75f91a4547SGvozden Neskovic {
76f91a4547SGvozden Neskovic return (is_x86_feature(x86_featureset, X86FSET_SSE2));
77f91a4547SGvozden Neskovic }
78f91a4547SGvozden Neskovic
79f91a4547SGvozden Neskovic static inline boolean_t
zfs_sse3_available(void)80f91a4547SGvozden Neskovic zfs_sse3_available(void)
81f91a4547SGvozden Neskovic {
82f91a4547SGvozden Neskovic return (is_x86_feature(x86_featureset, X86FSET_SSE3));
83f91a4547SGvozden Neskovic }
84f91a4547SGvozden Neskovic
85f91a4547SGvozden Neskovic static inline boolean_t
zfs_ssse3_available(void)86f91a4547SGvozden Neskovic zfs_ssse3_available(void)
87f91a4547SGvozden Neskovic {
88f91a4547SGvozden Neskovic return (is_x86_feature(x86_featureset, X86FSET_SSSE3));
89f91a4547SGvozden Neskovic }
90f91a4547SGvozden Neskovic
91f91a4547SGvozden Neskovic static inline boolean_t
zfs_avx_available(void)92f91a4547SGvozden Neskovic zfs_avx_available(void)
93f91a4547SGvozden Neskovic {
94f91a4547SGvozden Neskovic return (is_x86_feature(x86_featureset, X86FSET_AVX));
95f91a4547SGvozden Neskovic }
96f91a4547SGvozden Neskovic
97f91a4547SGvozden Neskovic static inline boolean_t
zfs_avx2_available(void)98f91a4547SGvozden Neskovic zfs_avx2_available(void)
99f91a4547SGvozden Neskovic {
100f91a4547SGvozden Neskovic return (is_x86_feature(x86_featureset, X86FSET_AVX2));
101f91a4547SGvozden Neskovic }
102f91a4547SGvozden Neskovic
103*0886dcadSAndy Fiddaman static inline boolean_t
zfs_avx512f_available(void)104*0886dcadSAndy Fiddaman zfs_avx512f_available(void)
105*0886dcadSAndy Fiddaman {
106*0886dcadSAndy Fiddaman return (is_x86_feature(x86_featureset, X86FSET_AVX512F));
107*0886dcadSAndy Fiddaman }
108*0886dcadSAndy Fiddaman
109*0886dcadSAndy Fiddaman static inline boolean_t
zfs_avx512bw_available(void)110*0886dcadSAndy Fiddaman zfs_avx512bw_available(void)
111*0886dcadSAndy Fiddaman {
112*0886dcadSAndy Fiddaman return (is_x86_feature(x86_featureset, X86FSET_AVX512BW));
113*0886dcadSAndy Fiddaman }
114*0886dcadSAndy Fiddaman
115f91a4547SGvozden Neskovic #else /* ! _KERNEL */
116f91a4547SGvozden Neskovic
117f91a4547SGvozden Neskovic #include <sys/auxv.h>
118f91a4547SGvozden Neskovic #include <sys/auxv_386.h>
119f91a4547SGvozden Neskovic
120fd440315SJoshua M. Clulow #define kfpu_allowed() 1
121f91a4547SGvozden Neskovic #define kfpu_begin() do {} while (0)
122f91a4547SGvozden Neskovic #define kfpu_end() do {} while (0)
123f91a4547SGvozden Neskovic
124f91a4547SGvozden Neskovic /*
125f91a4547SGvozden Neskovic * User-level check if various vector instruction sets are available.
126f91a4547SGvozden Neskovic */
127f91a4547SGvozden Neskovic
128f91a4547SGvozden Neskovic static inline boolean_t
zfs_sse_available(void)129f91a4547SGvozden Neskovic zfs_sse_available(void)
130f91a4547SGvozden Neskovic {
131f91a4547SGvozden Neskovic uint32_t u = 0;
132f91a4547SGvozden Neskovic
133f91a4547SGvozden Neskovic (void) getisax(&u, 1);
134f91a4547SGvozden Neskovic return ((u & AV_386_SSE) != 0);
135f91a4547SGvozden Neskovic }
136f91a4547SGvozden Neskovic
137f91a4547SGvozden Neskovic static inline boolean_t
zfs_sse2_available(void)138f91a4547SGvozden Neskovic zfs_sse2_available(void)
139f91a4547SGvozden Neskovic {
140f91a4547SGvozden Neskovic uint32_t u = 0;
141f91a4547SGvozden Neskovic
142f91a4547SGvozden Neskovic (void) getisax(&u, 1);
143f91a4547SGvozden Neskovic return ((u & AV_386_SSE2) != 0);
144f91a4547SGvozden Neskovic }
145f91a4547SGvozden Neskovic
146f91a4547SGvozden Neskovic static inline boolean_t
zfs_sse3_available(void)147f91a4547SGvozden Neskovic zfs_sse3_available(void)
148f91a4547SGvozden Neskovic {
149f91a4547SGvozden Neskovic uint32_t u = 0;
150f91a4547SGvozden Neskovic
151f91a4547SGvozden Neskovic (void) getisax(&u, 1);
152f91a4547SGvozden Neskovic return ((u & AV_386_SSE3) != 0);
153f91a4547SGvozden Neskovic }
154f91a4547SGvozden Neskovic
155f91a4547SGvozden Neskovic static inline boolean_t
zfs_ssse3_available(void)156f91a4547SGvozden Neskovic zfs_ssse3_available(void)
157f91a4547SGvozden Neskovic {
158f91a4547SGvozden Neskovic uint32_t u = 0;
159f91a4547SGvozden Neskovic
160f91a4547SGvozden Neskovic (void) getisax(&u, 1);
161f91a4547SGvozden Neskovic return ((u & AV_386_SSSE3) != 0);
162f91a4547SGvozden Neskovic }
163f91a4547SGvozden Neskovic
164f91a4547SGvozden Neskovic static inline boolean_t
zfs_avx_available(void)165f91a4547SGvozden Neskovic zfs_avx_available(void)
166f91a4547SGvozden Neskovic {
167f91a4547SGvozden Neskovic uint_t u = 0;
168f91a4547SGvozden Neskovic
169f91a4547SGvozden Neskovic (void) getisax(&u, 1);
170f91a4547SGvozden Neskovic return ((u & AV_386_AVX) != 0);
171f91a4547SGvozden Neskovic }
172f91a4547SGvozden Neskovic
173f91a4547SGvozden Neskovic static inline boolean_t
zfs_avx2_available(void)174f91a4547SGvozden Neskovic zfs_avx2_available(void)
175f91a4547SGvozden Neskovic {
176f91a4547SGvozden Neskovic uint32_t u[2] = { 0 };
177f91a4547SGvozden Neskovic
178f91a4547SGvozden Neskovic (void) getisax((uint32_t *)&u, 2);
179f91a4547SGvozden Neskovic return ((u[1] & AV_386_2_AVX2) != 0);
180f91a4547SGvozden Neskovic }
181f91a4547SGvozden Neskovic
182*0886dcadSAndy Fiddaman static inline boolean_t
zfs_avx512f_available(void)183*0886dcadSAndy Fiddaman zfs_avx512f_available(void)
184*0886dcadSAndy Fiddaman {
185*0886dcadSAndy Fiddaman uint32_t u[2] = { 0 };
186*0886dcadSAndy Fiddaman
187*0886dcadSAndy Fiddaman (void) getisax((uint32_t *)&u, 2);
188*0886dcadSAndy Fiddaman return ((u[1] & AV_386_2_AVX512F) != 0);
189*0886dcadSAndy Fiddaman }
190*0886dcadSAndy Fiddaman
191*0886dcadSAndy Fiddaman static inline boolean_t
zfs_avx512bw_available(void)192*0886dcadSAndy Fiddaman zfs_avx512bw_available(void)
193*0886dcadSAndy Fiddaman {
194*0886dcadSAndy Fiddaman uint32_t u[2] = { 0 };
195*0886dcadSAndy Fiddaman
196*0886dcadSAndy Fiddaman (void) getisax((uint32_t *)&u, 2);
197*0886dcadSAndy Fiddaman return ((u[1] & AV_386_2_AVX512BW) != 0);
198*0886dcadSAndy Fiddaman }
199*0886dcadSAndy Fiddaman
200f91a4547SGvozden Neskovic #endif /* _KERNEL */
201f91a4547SGvozden Neskovic
202f91a4547SGvozden Neskovic
203e86372a0SGvozden Neskovic #else
204e86372a0SGvozden Neskovic
205e86372a0SGvozden Neskovic /* Non-x86 CPUs currently always disallow kernel FPU support */
206e86372a0SGvozden Neskovic #define kfpu_allowed() 0
207e86372a0SGvozden Neskovic #define kfpu_initialize(tsk) do {} while (0)
208e86372a0SGvozden Neskovic #define kfpu_begin() do {} while (0)
209e86372a0SGvozden Neskovic #define kfpu_end() do {} while (0)
210e86372a0SGvozden Neskovic #define kfpu_init() (0)
211e86372a0SGvozden Neskovic #define kfpu_fini() do {} while (0)
212e86372a0SGvozden Neskovic #endif
213e86372a0SGvozden Neskovic
214e86372a0SGvozden Neskovic #endif /* _SIMD_H */
215