1199767f8SToomas Soome /*-
2199767f8SToomas Soome * Copyright (c) 2003 Peter Wemm.
3199767f8SToomas Soome * Copyright (c) 1993 The Regents of the University of California.
4199767f8SToomas Soome * All rights reserved.
5199767f8SToomas Soome *
6199767f8SToomas Soome * Redistribution and use in source and binary forms, with or without
7199767f8SToomas Soome * modification, are permitted provided that the following conditions
8199767f8SToomas Soome * are met:
9199767f8SToomas Soome * 1. Redistributions of source code must retain the above copyright
10199767f8SToomas Soome * notice, this list of conditions and the following disclaimer.
11199767f8SToomas Soome * 2. Redistributions in binary form must reproduce the above copyright
12199767f8SToomas Soome * notice, this list of conditions and the following disclaimer in the
13199767f8SToomas Soome * documentation and/or other materials provided with the distribution.
14*9c8f3233SToomas Soome * 3. Neither the name of the University nor the names of its contributors
15199767f8SToomas Soome * may be used to endorse or promote products derived from this software
16199767f8SToomas Soome * without specific prior written permission.
17199767f8SToomas Soome *
18199767f8SToomas Soome * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
19199767f8SToomas Soome * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20199767f8SToomas Soome * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21199767f8SToomas Soome * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
22199767f8SToomas Soome * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23199767f8SToomas Soome * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24199767f8SToomas Soome * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25199767f8SToomas Soome * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26199767f8SToomas Soome * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27199767f8SToomas Soome * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28199767f8SToomas Soome * SUCH DAMAGE.
29199767f8SToomas Soome *
30199767f8SToomas Soome * $FreeBSD$
31199767f8SToomas Soome */
32199767f8SToomas Soome
33199767f8SToomas Soome /*
34199767f8SToomas Soome * Functions to provide access to special i386 instructions.
35199767f8SToomas Soome * This in included in sys/systm.h, and that file should be
36199767f8SToomas Soome * used in preference to this.
37199767f8SToomas Soome */
38199767f8SToomas Soome
39199767f8SToomas Soome #ifndef _MACHINE_CPUFUNC_H_
40199767f8SToomas Soome #define _MACHINE_CPUFUNC_H_
41199767f8SToomas Soome
42199767f8SToomas Soome #ifndef _SYS_CDEFS_H_
43199767f8SToomas Soome #error this file needs sys/cdefs.h as a prerequisite
44199767f8SToomas Soome #endif
45199767f8SToomas Soome
46199767f8SToomas Soome struct region_descriptor;
47199767f8SToomas Soome
48199767f8SToomas Soome #define readb(va) (*(volatile uint8_t *) (va))
49199767f8SToomas Soome #define readw(va) (*(volatile uint16_t *) (va))
50199767f8SToomas Soome #define readl(va) (*(volatile uint32_t *) (va))
51199767f8SToomas Soome #define readq(va) (*(volatile uint64_t *) (va))
52199767f8SToomas Soome
53199767f8SToomas Soome #define writeb(va, d) (*(volatile uint8_t *) (va) = (d))
54199767f8SToomas Soome #define writew(va, d) (*(volatile uint16_t *) (va) = (d))
55199767f8SToomas Soome #define writel(va, d) (*(volatile uint32_t *) (va) = (d))
56199767f8SToomas Soome #define writeq(va, d) (*(volatile uint64_t *) (va) = (d))
57199767f8SToomas Soome
58199767f8SToomas Soome #if defined(__GNUCLIKE_ASM) && defined(__CC_SUPPORTS___INLINE)
59199767f8SToomas Soome
60199767f8SToomas Soome static __inline void
breakpoint(void)61199767f8SToomas Soome breakpoint(void)
62199767f8SToomas Soome {
63199767f8SToomas Soome __asm __volatile("int $3");
64199767f8SToomas Soome }
65199767f8SToomas Soome
66199767f8SToomas Soome static __inline u_int
bsfl(u_int mask)67199767f8SToomas Soome bsfl(u_int mask)
68199767f8SToomas Soome {
69199767f8SToomas Soome u_int result;
70199767f8SToomas Soome
71199767f8SToomas Soome __asm __volatile("bsfl %1,%0" : "=r" (result) : "rm" (mask));
72199767f8SToomas Soome return (result);
73199767f8SToomas Soome }
74199767f8SToomas Soome
75199767f8SToomas Soome static __inline u_long
bsfq(u_long mask)76199767f8SToomas Soome bsfq(u_long mask)
77199767f8SToomas Soome {
78199767f8SToomas Soome u_long result;
79199767f8SToomas Soome
80199767f8SToomas Soome __asm __volatile("bsfq %1,%0" : "=r" (result) : "rm" (mask));
81199767f8SToomas Soome return (result);
82199767f8SToomas Soome }
83199767f8SToomas Soome
84199767f8SToomas Soome static __inline u_int
bsrl(u_int mask)85199767f8SToomas Soome bsrl(u_int mask)
86199767f8SToomas Soome {
87199767f8SToomas Soome u_int result;
88199767f8SToomas Soome
89199767f8SToomas Soome __asm __volatile("bsrl %1,%0" : "=r" (result) : "rm" (mask));
90199767f8SToomas Soome return (result);
91199767f8SToomas Soome }
92199767f8SToomas Soome
93199767f8SToomas Soome static __inline u_long
bsrq(u_long mask)94199767f8SToomas Soome bsrq(u_long mask)
95199767f8SToomas Soome {
96199767f8SToomas Soome u_long result;
97199767f8SToomas Soome
98199767f8SToomas Soome __asm __volatile("bsrq %1,%0" : "=r" (result) : "rm" (mask));
99199767f8SToomas Soome return (result);
100199767f8SToomas Soome }
101199767f8SToomas Soome
102199767f8SToomas Soome static __inline void
clflush(u_long addr)103199767f8SToomas Soome clflush(u_long addr)
104199767f8SToomas Soome {
105199767f8SToomas Soome
106199767f8SToomas Soome __asm __volatile("clflush %0" : : "m" (*(char *)addr));
107199767f8SToomas Soome }
108199767f8SToomas Soome
109199767f8SToomas Soome static __inline void
clflushopt(u_long addr)110199767f8SToomas Soome clflushopt(u_long addr)
111199767f8SToomas Soome {
112199767f8SToomas Soome
113199767f8SToomas Soome __asm __volatile(".byte 0x66;clflush %0" : : "m" (*(char *)addr));
114199767f8SToomas Soome }
115199767f8SToomas Soome
116199767f8SToomas Soome static __inline void
clts(void)117199767f8SToomas Soome clts(void)
118199767f8SToomas Soome {
119199767f8SToomas Soome
120199767f8SToomas Soome __asm __volatile("clts");
121199767f8SToomas Soome }
122199767f8SToomas Soome
123199767f8SToomas Soome static __inline void
disable_intr(void)124199767f8SToomas Soome disable_intr(void)
125199767f8SToomas Soome {
126199767f8SToomas Soome __asm __volatile("cli" : : : "memory");
127199767f8SToomas Soome }
128199767f8SToomas Soome
129199767f8SToomas Soome static __inline void
do_cpuid(u_int ax,u_int * p)130199767f8SToomas Soome do_cpuid(u_int ax, u_int *p)
131199767f8SToomas Soome {
132199767f8SToomas Soome __asm __volatile("cpuid"
133199767f8SToomas Soome : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3])
134199767f8SToomas Soome : "0" (ax));
135199767f8SToomas Soome }
136199767f8SToomas Soome
137199767f8SToomas Soome static __inline void
cpuid_count(u_int ax,u_int cx,u_int * p)138199767f8SToomas Soome cpuid_count(u_int ax, u_int cx, u_int *p)
139199767f8SToomas Soome {
140199767f8SToomas Soome __asm __volatile("cpuid"
141199767f8SToomas Soome : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3])
142199767f8SToomas Soome : "0" (ax), "c" (cx));
143199767f8SToomas Soome }
144199767f8SToomas Soome
145199767f8SToomas Soome static __inline void
enable_intr(void)146199767f8SToomas Soome enable_intr(void)
147199767f8SToomas Soome {
148199767f8SToomas Soome __asm __volatile("sti");
149199767f8SToomas Soome }
150199767f8SToomas Soome
151199767f8SToomas Soome #ifdef _KERNEL
152199767f8SToomas Soome
153199767f8SToomas Soome #define HAVE_INLINE_FFS
154199767f8SToomas Soome #define ffs(x) __builtin_ffs(x)
155199767f8SToomas Soome
156199767f8SToomas Soome #define HAVE_INLINE_FFSL
157199767f8SToomas Soome
158199767f8SToomas Soome static __inline int
ffsl(long mask)159199767f8SToomas Soome ffsl(long mask)
160199767f8SToomas Soome {
161199767f8SToomas Soome return (mask == 0 ? mask : (int)bsfq((u_long)mask) + 1);
162199767f8SToomas Soome }
163199767f8SToomas Soome
164199767f8SToomas Soome #define HAVE_INLINE_FFSLL
165199767f8SToomas Soome
166199767f8SToomas Soome static __inline int
ffsll(long long mask)167199767f8SToomas Soome ffsll(long long mask)
168199767f8SToomas Soome {
169199767f8SToomas Soome return (ffsl((long)mask));
170199767f8SToomas Soome }
171199767f8SToomas Soome
172199767f8SToomas Soome #define HAVE_INLINE_FLS
173199767f8SToomas Soome
174199767f8SToomas Soome static __inline int
fls(int mask)175199767f8SToomas Soome fls(int mask)
176199767f8SToomas Soome {
177199767f8SToomas Soome return (mask == 0 ? mask : (int)bsrl((u_int)mask) + 1);
178199767f8SToomas Soome }
179199767f8SToomas Soome
180199767f8SToomas Soome #define HAVE_INLINE_FLSL
181199767f8SToomas Soome
182199767f8SToomas Soome static __inline int
flsl(long mask)183199767f8SToomas Soome flsl(long mask)
184199767f8SToomas Soome {
185199767f8SToomas Soome return (mask == 0 ? mask : (int)bsrq((u_long)mask) + 1);
186199767f8SToomas Soome }
187199767f8SToomas Soome
188199767f8SToomas Soome #define HAVE_INLINE_FLSLL
189199767f8SToomas Soome
190199767f8SToomas Soome static __inline int
flsll(long long mask)191199767f8SToomas Soome flsll(long long mask)
192199767f8SToomas Soome {
193199767f8SToomas Soome return (flsl((long)mask));
194199767f8SToomas Soome }
195199767f8SToomas Soome
196199767f8SToomas Soome #endif /* _KERNEL */
197199767f8SToomas Soome
198199767f8SToomas Soome static __inline void
halt(void)199199767f8SToomas Soome halt(void)
200199767f8SToomas Soome {
201199767f8SToomas Soome __asm __volatile("hlt");
202199767f8SToomas Soome }
203199767f8SToomas Soome
204199767f8SToomas Soome static __inline u_char
inb(u_int port)205199767f8SToomas Soome inb(u_int port)
206199767f8SToomas Soome {
207199767f8SToomas Soome u_char data;
208199767f8SToomas Soome
209199767f8SToomas Soome __asm __volatile("inb %w1, %0" : "=a" (data) : "Nd" (port));
210199767f8SToomas Soome return (data);
211199767f8SToomas Soome }
212199767f8SToomas Soome
213199767f8SToomas Soome static __inline u_int
inl(u_int port)214199767f8SToomas Soome inl(u_int port)
215199767f8SToomas Soome {
216199767f8SToomas Soome u_int data;
217199767f8SToomas Soome
218199767f8SToomas Soome __asm __volatile("inl %w1, %0" : "=a" (data) : "Nd" (port));
219199767f8SToomas Soome return (data);
220199767f8SToomas Soome }
221199767f8SToomas Soome
222199767f8SToomas Soome static __inline void
insb(u_int port,void * addr,size_t count)223199767f8SToomas Soome insb(u_int port, void *addr, size_t count)
224199767f8SToomas Soome {
225199767f8SToomas Soome __asm __volatile("cld; rep; insb"
226199767f8SToomas Soome : "+D" (addr), "+c" (count)
227199767f8SToomas Soome : "d" (port)
228199767f8SToomas Soome : "memory");
229199767f8SToomas Soome }
230199767f8SToomas Soome
231199767f8SToomas Soome static __inline void
insw(u_int port,void * addr,size_t count)232199767f8SToomas Soome insw(u_int port, void *addr, size_t count)
233199767f8SToomas Soome {
234199767f8SToomas Soome __asm __volatile("cld; rep; insw"
235199767f8SToomas Soome : "+D" (addr), "+c" (count)
236199767f8SToomas Soome : "d" (port)
237199767f8SToomas Soome : "memory");
238199767f8SToomas Soome }
239199767f8SToomas Soome
240199767f8SToomas Soome static __inline void
insl(u_int port,void * addr,size_t count)241199767f8SToomas Soome insl(u_int port, void *addr, size_t count)
242199767f8SToomas Soome {
243199767f8SToomas Soome __asm __volatile("cld; rep; insl"
244199767f8SToomas Soome : "+D" (addr), "+c" (count)
245199767f8SToomas Soome : "d" (port)
246199767f8SToomas Soome : "memory");
247199767f8SToomas Soome }
248199767f8SToomas Soome
249199767f8SToomas Soome static __inline void
invd(void)250199767f8SToomas Soome invd(void)
251199767f8SToomas Soome {
252199767f8SToomas Soome __asm __volatile("invd");
253199767f8SToomas Soome }
254199767f8SToomas Soome
255199767f8SToomas Soome static __inline u_short
inw(u_int port)256199767f8SToomas Soome inw(u_int port)
257199767f8SToomas Soome {
258199767f8SToomas Soome u_short data;
259199767f8SToomas Soome
260199767f8SToomas Soome __asm __volatile("inw %w1, %0" : "=a" (data) : "Nd" (port));
261199767f8SToomas Soome return (data);
262199767f8SToomas Soome }
263199767f8SToomas Soome
264199767f8SToomas Soome static __inline void
outb(u_int port,u_char data)265199767f8SToomas Soome outb(u_int port, u_char data)
266199767f8SToomas Soome {
267199767f8SToomas Soome __asm __volatile("outb %0, %w1" : : "a" (data), "Nd" (port));
268199767f8SToomas Soome }
269199767f8SToomas Soome
270199767f8SToomas Soome static __inline void
outl(u_int port,u_int data)271199767f8SToomas Soome outl(u_int port, u_int data)
272199767f8SToomas Soome {
273199767f8SToomas Soome __asm __volatile("outl %0, %w1" : : "a" (data), "Nd" (port));
274199767f8SToomas Soome }
275199767f8SToomas Soome
276199767f8SToomas Soome static __inline void
outsb(u_int port,const void * addr,size_t count)277199767f8SToomas Soome outsb(u_int port, const void *addr, size_t count)
278199767f8SToomas Soome {
279199767f8SToomas Soome __asm __volatile("cld; rep; outsb"
280199767f8SToomas Soome : "+S" (addr), "+c" (count)
281199767f8SToomas Soome : "d" (port));
282199767f8SToomas Soome }
283199767f8SToomas Soome
284199767f8SToomas Soome static __inline void
outsw(u_int port,const void * addr,size_t count)285199767f8SToomas Soome outsw(u_int port, const void *addr, size_t count)
286199767f8SToomas Soome {
287199767f8SToomas Soome __asm __volatile("cld; rep; outsw"
288199767f8SToomas Soome : "+S" (addr), "+c" (count)
289199767f8SToomas Soome : "d" (port));
290199767f8SToomas Soome }
291199767f8SToomas Soome
292199767f8SToomas Soome static __inline void
outsl(u_int port,const void * addr,size_t count)293199767f8SToomas Soome outsl(u_int port, const void *addr, size_t count)
294199767f8SToomas Soome {
295199767f8SToomas Soome __asm __volatile("cld; rep; outsl"
296199767f8SToomas Soome : "+S" (addr), "+c" (count)
297199767f8SToomas Soome : "d" (port));
298199767f8SToomas Soome }
299199767f8SToomas Soome
300199767f8SToomas Soome static __inline void
outw(u_int port,u_short data)301199767f8SToomas Soome outw(u_int port, u_short data)
302199767f8SToomas Soome {
303199767f8SToomas Soome __asm __volatile("outw %0, %w1" : : "a" (data), "Nd" (port));
304199767f8SToomas Soome }
305199767f8SToomas Soome
306199767f8SToomas Soome static __inline u_long
popcntq(u_long mask)307199767f8SToomas Soome popcntq(u_long mask)
308199767f8SToomas Soome {
309199767f8SToomas Soome u_long result;
310199767f8SToomas Soome
311199767f8SToomas Soome __asm __volatile("popcntq %1,%0" : "=r" (result) : "rm" (mask));
312199767f8SToomas Soome return (result);
313199767f8SToomas Soome }
314199767f8SToomas Soome
315199767f8SToomas Soome static __inline void
lfence(void)316199767f8SToomas Soome lfence(void)
317199767f8SToomas Soome {
318199767f8SToomas Soome
319199767f8SToomas Soome __asm __volatile("lfence" : : : "memory");
320199767f8SToomas Soome }
321199767f8SToomas Soome
322199767f8SToomas Soome static __inline void
mfence(void)323199767f8SToomas Soome mfence(void)
324199767f8SToomas Soome {
325199767f8SToomas Soome
326199767f8SToomas Soome __asm __volatile("mfence" : : : "memory");
327199767f8SToomas Soome }
328199767f8SToomas Soome
329*9c8f3233SToomas Soome static __inline void
sfence(void)330*9c8f3233SToomas Soome sfence(void)
331*9c8f3233SToomas Soome {
332*9c8f3233SToomas Soome
333*9c8f3233SToomas Soome __asm __volatile("sfence" : : : "memory");
334*9c8f3233SToomas Soome }
335*9c8f3233SToomas Soome
336199767f8SToomas Soome static __inline void
ia32_pause(void)337199767f8SToomas Soome ia32_pause(void)
338199767f8SToomas Soome {
339199767f8SToomas Soome __asm __volatile("pause");
340199767f8SToomas Soome }
341199767f8SToomas Soome
342199767f8SToomas Soome static __inline u_long
read_rflags(void)343199767f8SToomas Soome read_rflags(void)
344199767f8SToomas Soome {
345199767f8SToomas Soome u_long rf;
346199767f8SToomas Soome
347199767f8SToomas Soome __asm __volatile("pushfq; popq %0" : "=r" (rf));
348199767f8SToomas Soome return (rf);
349199767f8SToomas Soome }
350199767f8SToomas Soome
351199767f8SToomas Soome static __inline uint64_t
rdmsr(u_int msr)352199767f8SToomas Soome rdmsr(u_int msr)
353199767f8SToomas Soome {
354199767f8SToomas Soome uint32_t low, high;
355199767f8SToomas Soome
356199767f8SToomas Soome __asm __volatile("rdmsr" : "=a" (low), "=d" (high) : "c" (msr));
357199767f8SToomas Soome return (low | ((uint64_t)high << 32));
358199767f8SToomas Soome }
359199767f8SToomas Soome
360199767f8SToomas Soome static __inline uint32_t
rdmsr32(u_int msr)361199767f8SToomas Soome rdmsr32(u_int msr)
362199767f8SToomas Soome {
363199767f8SToomas Soome uint32_t low;
364199767f8SToomas Soome
365199767f8SToomas Soome __asm __volatile("rdmsr" : "=a" (low) : "c" (msr) : "rdx");
366199767f8SToomas Soome return (low);
367199767f8SToomas Soome }
368199767f8SToomas Soome
369199767f8SToomas Soome static __inline uint64_t
rdpmc(u_int pmc)370199767f8SToomas Soome rdpmc(u_int pmc)
371199767f8SToomas Soome {
372199767f8SToomas Soome uint32_t low, high;
373199767f8SToomas Soome
374199767f8SToomas Soome __asm __volatile("rdpmc" : "=a" (low), "=d" (high) : "c" (pmc));
375199767f8SToomas Soome return (low | ((uint64_t)high << 32));
376199767f8SToomas Soome }
377199767f8SToomas Soome
378199767f8SToomas Soome static __inline uint64_t
rdtsc(void)379199767f8SToomas Soome rdtsc(void)
380199767f8SToomas Soome {
381199767f8SToomas Soome uint32_t low, high;
382199767f8SToomas Soome
383199767f8SToomas Soome __asm __volatile("rdtsc" : "=a" (low), "=d" (high));
384199767f8SToomas Soome return (low | ((uint64_t)high << 32));
385199767f8SToomas Soome }
386199767f8SToomas Soome
387199767f8SToomas Soome static __inline uint32_t
rdtsc32(void)388199767f8SToomas Soome rdtsc32(void)
389199767f8SToomas Soome {
390199767f8SToomas Soome uint32_t rv;
391199767f8SToomas Soome
392199767f8SToomas Soome __asm __volatile("rdtsc" : "=a" (rv) : : "edx");
393199767f8SToomas Soome return (rv);
394199767f8SToomas Soome }
395199767f8SToomas Soome
396199767f8SToomas Soome static __inline void
wbinvd(void)397199767f8SToomas Soome wbinvd(void)
398199767f8SToomas Soome {
399199767f8SToomas Soome __asm __volatile("wbinvd");
400199767f8SToomas Soome }
401199767f8SToomas Soome
402199767f8SToomas Soome static __inline void
write_rflags(u_long rf)403199767f8SToomas Soome write_rflags(u_long rf)
404199767f8SToomas Soome {
405199767f8SToomas Soome __asm __volatile("pushq %0; popfq" : : "r" (rf));
406199767f8SToomas Soome }
407199767f8SToomas Soome
408199767f8SToomas Soome static __inline void
wrmsr(u_int msr,uint64_t newval)409199767f8SToomas Soome wrmsr(u_int msr, uint64_t newval)
410199767f8SToomas Soome {
411199767f8SToomas Soome uint32_t low, high;
412199767f8SToomas Soome
413199767f8SToomas Soome low = newval;
414199767f8SToomas Soome high = newval >> 32;
415199767f8SToomas Soome __asm __volatile("wrmsr" : : "a" (low), "d" (high), "c" (msr));
416199767f8SToomas Soome }
417199767f8SToomas Soome
418199767f8SToomas Soome static __inline void
load_cr0(u_long data)419199767f8SToomas Soome load_cr0(u_long data)
420199767f8SToomas Soome {
421199767f8SToomas Soome
422199767f8SToomas Soome __asm __volatile("movq %0,%%cr0" : : "r" (data));
423199767f8SToomas Soome }
424199767f8SToomas Soome
425199767f8SToomas Soome static __inline u_long
rcr0(void)426199767f8SToomas Soome rcr0(void)
427199767f8SToomas Soome {
428199767f8SToomas Soome u_long data;
429199767f8SToomas Soome
430199767f8SToomas Soome __asm __volatile("movq %%cr0,%0" : "=r" (data));
431199767f8SToomas Soome return (data);
432199767f8SToomas Soome }
433199767f8SToomas Soome
434199767f8SToomas Soome static __inline u_long
rcr2(void)435199767f8SToomas Soome rcr2(void)
436199767f8SToomas Soome {
437199767f8SToomas Soome u_long data;
438199767f8SToomas Soome
439199767f8SToomas Soome __asm __volatile("movq %%cr2,%0" : "=r" (data));
440199767f8SToomas Soome return (data);
441199767f8SToomas Soome }
442199767f8SToomas Soome
443199767f8SToomas Soome static __inline void
load_cr3(u_long data)444199767f8SToomas Soome load_cr3(u_long data)
445199767f8SToomas Soome {
446199767f8SToomas Soome
447199767f8SToomas Soome __asm __volatile("movq %0,%%cr3" : : "r" (data) : "memory");
448199767f8SToomas Soome }
449199767f8SToomas Soome
450199767f8SToomas Soome static __inline u_long
rcr3(void)451199767f8SToomas Soome rcr3(void)
452199767f8SToomas Soome {
453199767f8SToomas Soome u_long data;
454199767f8SToomas Soome
455199767f8SToomas Soome __asm __volatile("movq %%cr3,%0" : "=r" (data));
456199767f8SToomas Soome return (data);
457199767f8SToomas Soome }
458199767f8SToomas Soome
459199767f8SToomas Soome static __inline void
load_cr4(u_long data)460199767f8SToomas Soome load_cr4(u_long data)
461199767f8SToomas Soome {
462199767f8SToomas Soome __asm __volatile("movq %0,%%cr4" : : "r" (data));
463199767f8SToomas Soome }
464199767f8SToomas Soome
465199767f8SToomas Soome static __inline u_long
rcr4(void)466199767f8SToomas Soome rcr4(void)
467199767f8SToomas Soome {
468199767f8SToomas Soome u_long data;
469199767f8SToomas Soome
470199767f8SToomas Soome __asm __volatile("movq %%cr4,%0" : "=r" (data));
471199767f8SToomas Soome return (data);
472199767f8SToomas Soome }
473199767f8SToomas Soome
474199767f8SToomas Soome static __inline u_long
rxcr(u_int reg)475199767f8SToomas Soome rxcr(u_int reg)
476199767f8SToomas Soome {
477199767f8SToomas Soome u_int low, high;
478199767f8SToomas Soome
479199767f8SToomas Soome __asm __volatile("xgetbv" : "=a" (low), "=d" (high) : "c" (reg));
480199767f8SToomas Soome return (low | ((uint64_t)high << 32));
481199767f8SToomas Soome }
482199767f8SToomas Soome
483199767f8SToomas Soome static __inline void
load_xcr(u_int reg,u_long val)484199767f8SToomas Soome load_xcr(u_int reg, u_long val)
485199767f8SToomas Soome {
486199767f8SToomas Soome u_int low, high;
487199767f8SToomas Soome
488199767f8SToomas Soome low = val;
489199767f8SToomas Soome high = val >> 32;
490199767f8SToomas Soome __asm __volatile("xsetbv" : : "c" (reg), "a" (low), "d" (high));
491199767f8SToomas Soome }
492199767f8SToomas Soome
493199767f8SToomas Soome /*
494199767f8SToomas Soome * Global TLB flush (except for thise for pages marked PG_G)
495199767f8SToomas Soome */
496199767f8SToomas Soome static __inline void
invltlb(void)497199767f8SToomas Soome invltlb(void)
498199767f8SToomas Soome {
499199767f8SToomas Soome
500199767f8SToomas Soome load_cr3(rcr3());
501199767f8SToomas Soome }
502199767f8SToomas Soome
503199767f8SToomas Soome #ifndef CR4_PGE
504199767f8SToomas Soome #define CR4_PGE 0x00000080 /* Page global enable */
505199767f8SToomas Soome #endif
506199767f8SToomas Soome
507199767f8SToomas Soome /*
508199767f8SToomas Soome * Perform the guaranteed invalidation of all TLB entries. This
509199767f8SToomas Soome * includes the global entries, and entries in all PCIDs, not only the
510199767f8SToomas Soome * current context. The function works both on non-PCID CPUs and CPUs
511199767f8SToomas Soome * with the PCID turned off or on. See IA-32 SDM Vol. 3a 4.10.4.1
512199767f8SToomas Soome * Operations that Invalidate TLBs and Paging-Structure Caches.
513199767f8SToomas Soome */
514199767f8SToomas Soome static __inline void
invltlb_glob(void)515199767f8SToomas Soome invltlb_glob(void)
516199767f8SToomas Soome {
517199767f8SToomas Soome uint64_t cr4;
518199767f8SToomas Soome
519199767f8SToomas Soome cr4 = rcr4();
520199767f8SToomas Soome load_cr4(cr4 & ~CR4_PGE);
521199767f8SToomas Soome /*
522199767f8SToomas Soome * Although preemption at this point could be detrimental to
523199767f8SToomas Soome * performance, it would not lead to an error. PG_G is simply
524199767f8SToomas Soome * ignored if CR4.PGE is clear. Moreover, in case this block
525199767f8SToomas Soome * is re-entered, the load_cr4() either above or below will
526199767f8SToomas Soome * modify CR4.PGE flushing the TLB.
527199767f8SToomas Soome */
528199767f8SToomas Soome load_cr4(cr4 | CR4_PGE);
529199767f8SToomas Soome }
530199767f8SToomas Soome
531199767f8SToomas Soome /*
532199767f8SToomas Soome * TLB flush for an individual page (even if it has PG_G).
533199767f8SToomas Soome * Only works on 486+ CPUs (i386 does not have PG_G).
534199767f8SToomas Soome */
535199767f8SToomas Soome static __inline void
invlpg(u_long addr)536199767f8SToomas Soome invlpg(u_long addr)
537199767f8SToomas Soome {
538199767f8SToomas Soome
539199767f8SToomas Soome __asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory");
540199767f8SToomas Soome }
541199767f8SToomas Soome
542199767f8SToomas Soome #define INVPCID_ADDR 0
543199767f8SToomas Soome #define INVPCID_CTX 1
544199767f8SToomas Soome #define INVPCID_CTXGLOB 2
545199767f8SToomas Soome #define INVPCID_ALLCTX 3
546199767f8SToomas Soome
547199767f8SToomas Soome struct invpcid_descr {
548199767f8SToomas Soome uint64_t pcid:12 __packed;
549199767f8SToomas Soome uint64_t pad:52 __packed;
550199767f8SToomas Soome uint64_t addr;
551199767f8SToomas Soome } __packed;
552199767f8SToomas Soome
553199767f8SToomas Soome static __inline void
invpcid(struct invpcid_descr * d,int type)554199767f8SToomas Soome invpcid(struct invpcid_descr *d, int type)
555199767f8SToomas Soome {
556199767f8SToomas Soome
557199767f8SToomas Soome __asm __volatile("invpcid (%0),%1"
558199767f8SToomas Soome : : "r" (d), "r" ((u_long)type) : "memory");
559199767f8SToomas Soome }
560199767f8SToomas Soome
561199767f8SToomas Soome static __inline u_short
rfs(void)562199767f8SToomas Soome rfs(void)
563199767f8SToomas Soome {
564199767f8SToomas Soome u_short sel;
565199767f8SToomas Soome __asm __volatile("movw %%fs,%0" : "=rm" (sel));
566199767f8SToomas Soome return (sel);
567199767f8SToomas Soome }
568199767f8SToomas Soome
569199767f8SToomas Soome static __inline u_short
rgs(void)570199767f8SToomas Soome rgs(void)
571199767f8SToomas Soome {
572199767f8SToomas Soome u_short sel;
573199767f8SToomas Soome __asm __volatile("movw %%gs,%0" : "=rm" (sel));
574199767f8SToomas Soome return (sel);
575199767f8SToomas Soome }
576199767f8SToomas Soome
577199767f8SToomas Soome static __inline u_short
rss(void)578199767f8SToomas Soome rss(void)
579199767f8SToomas Soome {
580199767f8SToomas Soome u_short sel;
581199767f8SToomas Soome __asm __volatile("movw %%ss,%0" : "=rm" (sel));
582199767f8SToomas Soome return (sel);
583199767f8SToomas Soome }
584199767f8SToomas Soome
585199767f8SToomas Soome static __inline void
load_ds(u_short sel)586199767f8SToomas Soome load_ds(u_short sel)
587199767f8SToomas Soome {
588199767f8SToomas Soome __asm __volatile("movw %0,%%ds" : : "rm" (sel));
589199767f8SToomas Soome }
590199767f8SToomas Soome
591199767f8SToomas Soome static __inline void
load_es(u_short sel)592199767f8SToomas Soome load_es(u_short sel)
593199767f8SToomas Soome {
594199767f8SToomas Soome __asm __volatile("movw %0,%%es" : : "rm" (sel));
595199767f8SToomas Soome }
596199767f8SToomas Soome
597199767f8SToomas Soome static __inline void
cpu_monitor(const void * addr,u_long extensions,u_int hints)598199767f8SToomas Soome cpu_monitor(const void *addr, u_long extensions, u_int hints)
599199767f8SToomas Soome {
600199767f8SToomas Soome
601199767f8SToomas Soome __asm __volatile("monitor"
602199767f8SToomas Soome : : "a" (addr), "c" (extensions), "d" (hints));
603199767f8SToomas Soome }
604199767f8SToomas Soome
605199767f8SToomas Soome static __inline void
cpu_mwait(u_long extensions,u_int hints)606199767f8SToomas Soome cpu_mwait(u_long extensions, u_int hints)
607199767f8SToomas Soome {
608199767f8SToomas Soome
609199767f8SToomas Soome __asm __volatile("mwait" : : "a" (hints), "c" (extensions));
610199767f8SToomas Soome }
611199767f8SToomas Soome
612199767f8SToomas Soome #ifdef _KERNEL
613199767f8SToomas Soome /* This is defined in <machine/specialreg.h> but is too painful to get to */
614199767f8SToomas Soome #ifndef MSR_FSBASE
615199767f8SToomas Soome #define MSR_FSBASE 0xc0000100
616199767f8SToomas Soome #endif
617199767f8SToomas Soome static __inline void
load_fs(u_short sel)618199767f8SToomas Soome load_fs(u_short sel)
619199767f8SToomas Soome {
620199767f8SToomas Soome /* Preserve the fsbase value across the selector load */
621199767f8SToomas Soome __asm __volatile("rdmsr; movw %0,%%fs; wrmsr"
622199767f8SToomas Soome : : "rm" (sel), "c" (MSR_FSBASE) : "eax", "edx");
623199767f8SToomas Soome }
624199767f8SToomas Soome
625199767f8SToomas Soome #ifndef MSR_GSBASE
626199767f8SToomas Soome #define MSR_GSBASE 0xc0000101
627199767f8SToomas Soome #endif
628199767f8SToomas Soome static __inline void
load_gs(u_short sel)629199767f8SToomas Soome load_gs(u_short sel)
630199767f8SToomas Soome {
631199767f8SToomas Soome /*
632199767f8SToomas Soome * Preserve the gsbase value across the selector load.
633199767f8SToomas Soome * Note that we have to disable interrupts because the gsbase
634199767f8SToomas Soome * being trashed happens to be the kernel gsbase at the time.
635199767f8SToomas Soome */
636199767f8SToomas Soome __asm __volatile("pushfq; cli; rdmsr; movw %0,%%gs; wrmsr; popfq"
637199767f8SToomas Soome : : "rm" (sel), "c" (MSR_GSBASE) : "eax", "edx");
638199767f8SToomas Soome }
639199767f8SToomas Soome #else
640199767f8SToomas Soome /* Usable by userland */
641199767f8SToomas Soome static __inline void
load_fs(u_short sel)642199767f8SToomas Soome load_fs(u_short sel)
643199767f8SToomas Soome {
644199767f8SToomas Soome __asm __volatile("movw %0,%%fs" : : "rm" (sel));
645199767f8SToomas Soome }
646199767f8SToomas Soome
647199767f8SToomas Soome static __inline void
load_gs(u_short sel)648199767f8SToomas Soome load_gs(u_short sel)
649199767f8SToomas Soome {
650199767f8SToomas Soome __asm __volatile("movw %0,%%gs" : : "rm" (sel));
651199767f8SToomas Soome }
652199767f8SToomas Soome #endif
653199767f8SToomas Soome
654*9c8f3233SToomas Soome static __inline void
bare_lgdt(struct region_descriptor * addr)655*9c8f3233SToomas Soome bare_lgdt(struct region_descriptor *addr)
656*9c8f3233SToomas Soome {
657*9c8f3233SToomas Soome __asm __volatile("lgdt (%0)" : : "r" (addr));
658*9c8f3233SToomas Soome }
659*9c8f3233SToomas Soome
660*9c8f3233SToomas Soome static __inline void
sgdt(struct region_descriptor * addr)661*9c8f3233SToomas Soome sgdt(struct region_descriptor *addr)
662*9c8f3233SToomas Soome {
663*9c8f3233SToomas Soome char *loc;
664*9c8f3233SToomas Soome
665*9c8f3233SToomas Soome loc = (char *)addr;
666*9c8f3233SToomas Soome __asm __volatile("sgdt %0" : "=m" (*loc) : : "memory");
667*9c8f3233SToomas Soome }
668*9c8f3233SToomas Soome
669199767f8SToomas Soome static __inline void
lidt(struct region_descriptor * addr)670199767f8SToomas Soome lidt(struct region_descriptor *addr)
671199767f8SToomas Soome {
672199767f8SToomas Soome __asm __volatile("lidt (%0)" : : "r" (addr));
673199767f8SToomas Soome }
674199767f8SToomas Soome
675*9c8f3233SToomas Soome static __inline void
sidt(struct region_descriptor * addr)676*9c8f3233SToomas Soome sidt(struct region_descriptor *addr)
677*9c8f3233SToomas Soome {
678*9c8f3233SToomas Soome char *loc;
679*9c8f3233SToomas Soome
680*9c8f3233SToomas Soome loc = (char *)addr;
681*9c8f3233SToomas Soome __asm __volatile("sidt %0" : "=m" (*loc) : : "memory");
682*9c8f3233SToomas Soome }
683*9c8f3233SToomas Soome
684199767f8SToomas Soome static __inline void
lldt(u_short sel)685199767f8SToomas Soome lldt(u_short sel)
686199767f8SToomas Soome {
687199767f8SToomas Soome __asm __volatile("lldt %0" : : "r" (sel));
688199767f8SToomas Soome }
689199767f8SToomas Soome
690199767f8SToomas Soome static __inline void
ltr(u_short sel)691199767f8SToomas Soome ltr(u_short sel)
692199767f8SToomas Soome {
693199767f8SToomas Soome __asm __volatile("ltr %0" : : "r" (sel));
694199767f8SToomas Soome }
695199767f8SToomas Soome
696*9c8f3233SToomas Soome static __inline uint32_t
read_tr(void)697*9c8f3233SToomas Soome read_tr(void)
698*9c8f3233SToomas Soome {
699*9c8f3233SToomas Soome u_short sel;
700*9c8f3233SToomas Soome
701*9c8f3233SToomas Soome __asm __volatile("str %0" : "=r" (sel));
702*9c8f3233SToomas Soome return (sel);
703*9c8f3233SToomas Soome }
704*9c8f3233SToomas Soome
705199767f8SToomas Soome static __inline uint64_t
rdr0(void)706199767f8SToomas Soome rdr0(void)
707199767f8SToomas Soome {
708199767f8SToomas Soome uint64_t data;
709199767f8SToomas Soome __asm __volatile("movq %%dr0,%0" : "=r" (data));
710199767f8SToomas Soome return (data);
711199767f8SToomas Soome }
712199767f8SToomas Soome
713199767f8SToomas Soome static __inline void
load_dr0(uint64_t dr0)714199767f8SToomas Soome load_dr0(uint64_t dr0)
715199767f8SToomas Soome {
716199767f8SToomas Soome __asm __volatile("movq %0,%%dr0" : : "r" (dr0));
717199767f8SToomas Soome }
718199767f8SToomas Soome
719199767f8SToomas Soome static __inline uint64_t
rdr1(void)720199767f8SToomas Soome rdr1(void)
721199767f8SToomas Soome {
722199767f8SToomas Soome uint64_t data;
723199767f8SToomas Soome __asm __volatile("movq %%dr1,%0" : "=r" (data));
724199767f8SToomas Soome return (data);
725199767f8SToomas Soome }
726199767f8SToomas Soome
727199767f8SToomas Soome static __inline void
load_dr1(uint64_t dr1)728199767f8SToomas Soome load_dr1(uint64_t dr1)
729199767f8SToomas Soome {
730199767f8SToomas Soome __asm __volatile("movq %0,%%dr1" : : "r" (dr1));
731199767f8SToomas Soome }
732199767f8SToomas Soome
733199767f8SToomas Soome static __inline uint64_t
rdr2(void)734199767f8SToomas Soome rdr2(void)
735199767f8SToomas Soome {
736199767f8SToomas Soome uint64_t data;
737199767f8SToomas Soome __asm __volatile("movq %%dr2,%0" : "=r" (data));
738199767f8SToomas Soome return (data);
739199767f8SToomas Soome }
740199767f8SToomas Soome
741199767f8SToomas Soome static __inline void
load_dr2(uint64_t dr2)742199767f8SToomas Soome load_dr2(uint64_t dr2)
743199767f8SToomas Soome {
744199767f8SToomas Soome __asm __volatile("movq %0,%%dr2" : : "r" (dr2));
745199767f8SToomas Soome }
746199767f8SToomas Soome
747199767f8SToomas Soome static __inline uint64_t
rdr3(void)748199767f8SToomas Soome rdr3(void)
749199767f8SToomas Soome {
750199767f8SToomas Soome uint64_t data;
751199767f8SToomas Soome __asm __volatile("movq %%dr3,%0" : "=r" (data));
752199767f8SToomas Soome return (data);
753199767f8SToomas Soome }
754199767f8SToomas Soome
755199767f8SToomas Soome static __inline void
load_dr3(uint64_t dr3)756199767f8SToomas Soome load_dr3(uint64_t dr3)
757199767f8SToomas Soome {
758199767f8SToomas Soome __asm __volatile("movq %0,%%dr3" : : "r" (dr3));
759199767f8SToomas Soome }
760199767f8SToomas Soome
761199767f8SToomas Soome static __inline uint64_t
rdr6(void)762199767f8SToomas Soome rdr6(void)
763199767f8SToomas Soome {
764199767f8SToomas Soome uint64_t data;
765199767f8SToomas Soome __asm __volatile("movq %%dr6,%0" : "=r" (data));
766199767f8SToomas Soome return (data);
767199767f8SToomas Soome }
768199767f8SToomas Soome
769199767f8SToomas Soome static __inline void
load_dr6(uint64_t dr6)770199767f8SToomas Soome load_dr6(uint64_t dr6)
771199767f8SToomas Soome {
772199767f8SToomas Soome __asm __volatile("movq %0,%%dr6" : : "r" (dr6));
773199767f8SToomas Soome }
774199767f8SToomas Soome
775199767f8SToomas Soome static __inline uint64_t
rdr7(void)776199767f8SToomas Soome rdr7(void)
777199767f8SToomas Soome {
778199767f8SToomas Soome uint64_t data;
779199767f8SToomas Soome __asm __volatile("movq %%dr7,%0" : "=r" (data));
780199767f8SToomas Soome return (data);
781199767f8SToomas Soome }
782199767f8SToomas Soome
783199767f8SToomas Soome static __inline void
load_dr7(uint64_t dr7)784199767f8SToomas Soome load_dr7(uint64_t dr7)
785199767f8SToomas Soome {
786199767f8SToomas Soome __asm __volatile("movq %0,%%dr7" : : "r" (dr7));
787199767f8SToomas Soome }
788199767f8SToomas Soome
789199767f8SToomas Soome static __inline register_t
intr_disable(void)790199767f8SToomas Soome intr_disable(void)
791199767f8SToomas Soome {
792199767f8SToomas Soome register_t rflags;
793199767f8SToomas Soome
794199767f8SToomas Soome rflags = read_rflags();
795199767f8SToomas Soome disable_intr();
796199767f8SToomas Soome return (rflags);
797199767f8SToomas Soome }
798199767f8SToomas Soome
799199767f8SToomas Soome static __inline void
intr_restore(register_t rflags)800199767f8SToomas Soome intr_restore(register_t rflags)
801199767f8SToomas Soome {
802199767f8SToomas Soome write_rflags(rflags);
803199767f8SToomas Soome }
804199767f8SToomas Soome
805199767f8SToomas Soome #else /* !(__GNUCLIKE_ASM && __CC_SUPPORTS___INLINE) */
806199767f8SToomas Soome
807199767f8SToomas Soome int breakpoint(void);
808199767f8SToomas Soome u_int bsfl(u_int mask);
809199767f8SToomas Soome u_int bsrl(u_int mask);
810199767f8SToomas Soome void clflush(u_long addr);
811199767f8SToomas Soome void clts(void);
812199767f8SToomas Soome void cpuid_count(u_int ax, u_int cx, u_int *p);
813199767f8SToomas Soome void disable_intr(void);
814199767f8SToomas Soome void do_cpuid(u_int ax, u_int *p);
815199767f8SToomas Soome void enable_intr(void);
816199767f8SToomas Soome void halt(void);
817199767f8SToomas Soome void ia32_pause(void);
818199767f8SToomas Soome u_char inb(u_int port);
819199767f8SToomas Soome u_int inl(u_int port);
820199767f8SToomas Soome void insb(u_int port, void *addr, size_t count);
821199767f8SToomas Soome void insl(u_int port, void *addr, size_t count);
822199767f8SToomas Soome void insw(u_int port, void *addr, size_t count);
823199767f8SToomas Soome register_t intr_disable(void);
824199767f8SToomas Soome void intr_restore(register_t rf);
825199767f8SToomas Soome void invd(void);
826199767f8SToomas Soome void invlpg(u_int addr);
827199767f8SToomas Soome void invltlb(void);
828199767f8SToomas Soome u_short inw(u_int port);
829199767f8SToomas Soome void lidt(struct region_descriptor *addr);
830199767f8SToomas Soome void lldt(u_short sel);
831199767f8SToomas Soome void load_cr0(u_long cr0);
832199767f8SToomas Soome void load_cr3(u_long cr3);
833199767f8SToomas Soome void load_cr4(u_long cr4);
834199767f8SToomas Soome void load_dr0(uint64_t dr0);
835199767f8SToomas Soome void load_dr1(uint64_t dr1);
836199767f8SToomas Soome void load_dr2(uint64_t dr2);
837199767f8SToomas Soome void load_dr3(uint64_t dr3);
838199767f8SToomas Soome void load_dr6(uint64_t dr6);
839199767f8SToomas Soome void load_dr7(uint64_t dr7);
840199767f8SToomas Soome void load_fs(u_short sel);
841199767f8SToomas Soome void load_gs(u_short sel);
842199767f8SToomas Soome void ltr(u_short sel);
843199767f8SToomas Soome void outb(u_int port, u_char data);
844199767f8SToomas Soome void outl(u_int port, u_int data);
845199767f8SToomas Soome void outsb(u_int port, const void *addr, size_t count);
846199767f8SToomas Soome void outsl(u_int port, const void *addr, size_t count);
847199767f8SToomas Soome void outsw(u_int port, const void *addr, size_t count);
848199767f8SToomas Soome void outw(u_int port, u_short data);
849199767f8SToomas Soome u_long rcr0(void);
850199767f8SToomas Soome u_long rcr2(void);
851199767f8SToomas Soome u_long rcr3(void);
852199767f8SToomas Soome u_long rcr4(void);
853199767f8SToomas Soome uint64_t rdmsr(u_int msr);
854199767f8SToomas Soome uint32_t rdmsr32(u_int msr);
855199767f8SToomas Soome uint64_t rdpmc(u_int pmc);
856199767f8SToomas Soome uint64_t rdr0(void);
857199767f8SToomas Soome uint64_t rdr1(void);
858199767f8SToomas Soome uint64_t rdr2(void);
859199767f8SToomas Soome uint64_t rdr3(void);
860199767f8SToomas Soome uint64_t rdr6(void);
861199767f8SToomas Soome uint64_t rdr7(void);
862199767f8SToomas Soome uint64_t rdtsc(void);
863199767f8SToomas Soome u_long read_rflags(void);
864199767f8SToomas Soome u_int rfs(void);
865199767f8SToomas Soome u_int rgs(void);
866199767f8SToomas Soome void wbinvd(void);
867199767f8SToomas Soome void write_rflags(u_int rf);
868199767f8SToomas Soome void wrmsr(u_int msr, uint64_t newval);
869199767f8SToomas Soome
870199767f8SToomas Soome #endif /* __GNUCLIKE_ASM && __CC_SUPPORTS___INLINE */
871199767f8SToomas Soome
872199767f8SToomas Soome void reset_dbregs(void);
873199767f8SToomas Soome
874199767f8SToomas Soome #ifdef _KERNEL
875199767f8SToomas Soome int rdmsr_safe(u_int msr, uint64_t *val);
876199767f8SToomas Soome int wrmsr_safe(u_int msr, uint64_t newval);
877199767f8SToomas Soome #endif
878199767f8SToomas Soome
879199767f8SToomas Soome #endif /* !_MACHINE_CPUFUNC_H_ */
880