1 /*
2 * This file and its contents are supplied under the terms of the
3 * Common Development and Distribution License ("CDDL"), version 1.0.
4 * You may only use this file in accordance with the terms of version
5 * 1.0 of the CDDL.
6 *
7 * A full copy of the text of the CDDL should have accompanied this
8 * source. A copy of the CDDL is also available via the Internet at
9 * http://www.illumos.org/license/CDDL.
10 */
11
12 /*
13 * Copyright 2020 Joyent, Inc.
14 */
15
16 #ifndef _SIMD_H
17 #define _SIMD_H
18
19 #if defined(__amd64__) || defined(__i386__)
20
21 #define kfpu_initialize(tsk) do {} while (0)
22 #define kfpu_init() (0)
23 #define kfpu_fini() do {} while (0)
24
25 #ifdef _KERNEL
26 #include <sys/x86_archext.h>
27 #include <sys/archsystm.h>
28 #include <sys/kfpu.h>
29 #include <sys/proc.h>
30 #include <sys/disp.h>
31 #include <sys/cpuvar.h>
32
33 static inline int
kfpu_allowed(void)34 kfpu_allowed(void)
35 {
36 extern int zfs_fpu_enabled;
37
38 return (zfs_fpu_enabled != 0 ? 1 : 0);
39 }
40
41 static inline void
kfpu_begin(void)42 kfpu_begin(void)
43 {
44 if (curthread->t_lwp != NULL && (curthread->t_procp->p_flag & SSYS)) {
45 kernel_fpu_begin(NULL, KFPU_USE_LWP);
46 } else {
47 kpreempt_disable();
48 kernel_fpu_begin(NULL, KFPU_NO_STATE);
49 }
50 }
51
52 static inline void
kfpu_end(void)53 kfpu_end(void)
54 {
55 if (curthread->t_lwp != NULL && (curthread->t_procp->p_flag & SSYS)) {
56 kernel_fpu_end(NULL, KFPU_USE_LWP);
57 } else {
58 kernel_fpu_end(NULL, KFPU_NO_STATE);
59 kpreempt_enable();
60 }
61 }
62
63 /*
64 * Check if various vector instruction sets are available.
65 */
66
67 static inline boolean_t
zfs_sse_available(void)68 zfs_sse_available(void)
69 {
70 return (is_x86_feature(x86_featureset, X86FSET_SSE));
71 }
72
73 static inline boolean_t
zfs_sse2_available(void)74 zfs_sse2_available(void)
75 {
76 return (is_x86_feature(x86_featureset, X86FSET_SSE2));
77 }
78
79 static inline boolean_t
zfs_sse3_available(void)80 zfs_sse3_available(void)
81 {
82 return (is_x86_feature(x86_featureset, X86FSET_SSE3));
83 }
84
85 static inline boolean_t
zfs_ssse3_available(void)86 zfs_ssse3_available(void)
87 {
88 return (is_x86_feature(x86_featureset, X86FSET_SSSE3));
89 }
90
91 static inline boolean_t
zfs_avx_available(void)92 zfs_avx_available(void)
93 {
94 return (is_x86_feature(x86_featureset, X86FSET_AVX));
95 }
96
97 static inline boolean_t
zfs_avx2_available(void)98 zfs_avx2_available(void)
99 {
100 return (is_x86_feature(x86_featureset, X86FSET_AVX2));
101 }
102
103 #else /* ! _KERNEL */
104
105 #include <sys/auxv.h>
106 #include <sys/auxv_386.h>
107
108 #define kfpu_allowed() 1
109 #define kfpu_begin() do {} while (0)
110 #define kfpu_end() do {} while (0)
111
112 /*
113 * User-level check if various vector instruction sets are available.
114 */
115
116 static inline boolean_t
zfs_sse_available(void)117 zfs_sse_available(void)
118 {
119 uint32_t u = 0;
120
121 (void) getisax(&u, 1);
122 return ((u & AV_386_SSE) != 0);
123 }
124
125 static inline boolean_t
zfs_sse2_available(void)126 zfs_sse2_available(void)
127 {
128 uint32_t u = 0;
129
130 (void) getisax(&u, 1);
131 return ((u & AV_386_SSE2) != 0);
132 }
133
134 static inline boolean_t
zfs_sse3_available(void)135 zfs_sse3_available(void)
136 {
137 uint32_t u = 0;
138
139 (void) getisax(&u, 1);
140 return ((u & AV_386_SSE3) != 0);
141 }
142
143 static inline boolean_t
zfs_ssse3_available(void)144 zfs_ssse3_available(void)
145 {
146 uint32_t u = 0;
147
148 (void) getisax(&u, 1);
149 return ((u & AV_386_SSSE3) != 0);
150 }
151
152 static inline boolean_t
zfs_avx_available(void)153 zfs_avx_available(void)
154 {
155 uint_t u = 0;
156
157 (void) getisax(&u, 1);
158 return ((u & AV_386_AVX) != 0);
159 }
160
161 static inline boolean_t
zfs_avx2_available(void)162 zfs_avx2_available(void)
163 {
164 uint32_t u[2] = { 0 };
165
166 (void) getisax((uint32_t *)&u, 2);
167 return ((u[1] & AV_386_2_AVX2) != 0);
168 }
169
170 #endif /* _KERNEL */
171
172
173 #else
174
175 /* Non-x86 CPUs currently always disallow kernel FPU support */
176 #define kfpu_allowed() 0
177 #define kfpu_initialize(tsk) do {} while (0)
178 #define kfpu_begin() do {} while (0)
179 #define kfpu_end() do {} while (0)
180 #define kfpu_init() (0)
181 #define kfpu_fini() do {} while (0)
182 #endif
183
184 #endif /* _SIMD_H */
185