1/*-
2 * Copyright (c) 2003 Peter Wemm.
3 * Copyright (c) 1993 The Regents of the University of California.
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 *    notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 *    notice, this list of conditions and the following disclaimer in the
13 *    documentation and/or other materials provided with the distribution.
14 * 3. Neither the name of the University nor the names of its contributors
15 *    may be used to endorse or promote products derived from this software
16 *    without specific prior written permission.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 *
30 * $FreeBSD$
31 */
32
33/*
34 * Functions to provide access to special i386 instructions.
35 * This in included in sys/systm.h, and that file should be
36 * used in preference to this.
37 */
38
39#ifndef _MACHINE_CPUFUNC_H_
40#define	_MACHINE_CPUFUNC_H_
41
42#ifndef _SYS_CDEFS_H_
43#error this file needs sys/cdefs.h as a prerequisite
44#endif
45
46struct region_descriptor;
47
48#define readb(va)	(*(volatile uint8_t *) (va))
49#define readw(va)	(*(volatile uint16_t *) (va))
50#define readl(va)	(*(volatile uint32_t *) (va))
51#define readq(va)	(*(volatile uint64_t *) (va))
52
53#define writeb(va, d)	(*(volatile uint8_t *) (va) = (d))
54#define writew(va, d)	(*(volatile uint16_t *) (va) = (d))
55#define writel(va, d)	(*(volatile uint32_t *) (va) = (d))
56#define writeq(va, d)	(*(volatile uint64_t *) (va) = (d))
57
58#if defined(__GNUCLIKE_ASM) && defined(__CC_SUPPORTS___INLINE)
59
60static __inline void
61breakpoint(void)
62{
63	__asm __volatile("int $3");
64}
65
66static __inline u_int
67bsfl(u_int mask)
68{
69	u_int	result;
70
71	__asm __volatile("bsfl %1,%0" : "=r" (result) : "rm" (mask));
72	return (result);
73}
74
75static __inline u_long
76bsfq(u_long mask)
77{
78	u_long	result;
79
80	__asm __volatile("bsfq %1,%0" : "=r" (result) : "rm" (mask));
81	return (result);
82}
83
84static __inline u_int
85bsrl(u_int mask)
86{
87	u_int	result;
88
89	__asm __volatile("bsrl %1,%0" : "=r" (result) : "rm" (mask));
90	return (result);
91}
92
93static __inline u_long
94bsrq(u_long mask)
95{
96	u_long	result;
97
98	__asm __volatile("bsrq %1,%0" : "=r" (result) : "rm" (mask));
99	return (result);
100}
101
102static __inline void
103clflush(u_long addr)
104{
105
106	__asm __volatile("clflush %0" : : "m" (*(char *)addr));
107}
108
109static __inline void
110clflushopt(u_long addr)
111{
112
113	__asm __volatile(".byte 0x66;clflush %0" : : "m" (*(char *)addr));
114}
115
116static __inline void
117clts(void)
118{
119
120	__asm __volatile("clts");
121}
122
123static __inline void
124disable_intr(void)
125{
126	__asm __volatile("cli" : : : "memory");
127}
128
129static __inline void
130do_cpuid(u_int ax, u_int *p)
131{
132	__asm __volatile("cpuid"
133			 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3])
134			 :  "0" (ax));
135}
136
137static __inline void
138cpuid_count(u_int ax, u_int cx, u_int *p)
139{
140	__asm __volatile("cpuid"
141			 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3])
142			 :  "0" (ax), "c" (cx));
143}
144
145static __inline void
146enable_intr(void)
147{
148	__asm __volatile("sti");
149}
150
151#ifdef _KERNEL
152
153#define	HAVE_INLINE_FFS
154#define        ffs(x)  __builtin_ffs(x)
155
156#define	HAVE_INLINE_FFSL
157
158static __inline int
159ffsl(long mask)
160{
161	return (mask == 0 ? mask : (int)bsfq((u_long)mask) + 1);
162}
163
164#define	HAVE_INLINE_FFSLL
165
166static __inline int
167ffsll(long long mask)
168{
169	return (ffsl((long)mask));
170}
171
172#define	HAVE_INLINE_FLS
173
174static __inline int
175fls(int mask)
176{
177	return (mask == 0 ? mask : (int)bsrl((u_int)mask) + 1);
178}
179
180#define	HAVE_INLINE_FLSL
181
182static __inline int
183flsl(long mask)
184{
185	return (mask == 0 ? mask : (int)bsrq((u_long)mask) + 1);
186}
187
188#define	HAVE_INLINE_FLSLL
189
190static __inline int
191flsll(long long mask)
192{
193	return (flsl((long)mask));
194}
195
196#endif /* _KERNEL */
197
198static __inline void
199halt(void)
200{
201	__asm __volatile("hlt");
202}
203
204static __inline u_char
205inb(u_int port)
206{
207	u_char	data;
208
209	__asm __volatile("inb %w1, %0" : "=a" (data) : "Nd" (port));
210	return (data);
211}
212
213static __inline u_int
214inl(u_int port)
215{
216	u_int	data;
217
218	__asm __volatile("inl %w1, %0" : "=a" (data) : "Nd" (port));
219	return (data);
220}
221
222static __inline void
223insb(u_int port, void *addr, size_t count)
224{
225	__asm __volatile("cld; rep; insb"
226			 : "+D" (addr), "+c" (count)
227			 : "d" (port)
228			 : "memory");
229}
230
231static __inline void
232insw(u_int port, void *addr, size_t count)
233{
234	__asm __volatile("cld; rep; insw"
235			 : "+D" (addr), "+c" (count)
236			 : "d" (port)
237			 : "memory");
238}
239
240static __inline void
241insl(u_int port, void *addr, size_t count)
242{
243	__asm __volatile("cld; rep; insl"
244			 : "+D" (addr), "+c" (count)
245			 : "d" (port)
246			 : "memory");
247}
248
249static __inline void
250invd(void)
251{
252	__asm __volatile("invd");
253}
254
255static __inline u_short
256inw(u_int port)
257{
258	u_short	data;
259
260	__asm __volatile("inw %w1, %0" : "=a" (data) : "Nd" (port));
261	return (data);
262}
263
264static __inline void
265outb(u_int port, u_char data)
266{
267	__asm __volatile("outb %0, %w1" : : "a" (data), "Nd" (port));
268}
269
270static __inline void
271outl(u_int port, u_int data)
272{
273	__asm __volatile("outl %0, %w1" : : "a" (data), "Nd" (port));
274}
275
276static __inline void
277outsb(u_int port, const void *addr, size_t count)
278{
279	__asm __volatile("cld; rep; outsb"
280			 : "+S" (addr), "+c" (count)
281			 : "d" (port));
282}
283
284static __inline void
285outsw(u_int port, const void *addr, size_t count)
286{
287	__asm __volatile("cld; rep; outsw"
288			 : "+S" (addr), "+c" (count)
289			 : "d" (port));
290}
291
292static __inline void
293outsl(u_int port, const void *addr, size_t count)
294{
295	__asm __volatile("cld; rep; outsl"
296			 : "+S" (addr), "+c" (count)
297			 : "d" (port));
298}
299
300static __inline void
301outw(u_int port, u_short data)
302{
303	__asm __volatile("outw %0, %w1" : : "a" (data), "Nd" (port));
304}
305
306static __inline u_long
307popcntq(u_long mask)
308{
309	u_long result;
310
311	__asm __volatile("popcntq %1,%0" : "=r" (result) : "rm" (mask));
312	return (result);
313}
314
315static __inline void
316lfence(void)
317{
318
319	__asm __volatile("lfence" : : : "memory");
320}
321
322static __inline void
323mfence(void)
324{
325
326	__asm __volatile("mfence" : : : "memory");
327}
328
329static __inline void
330sfence(void)
331{
332
333	__asm __volatile("sfence" : : : "memory");
334}
335
336static __inline void
337ia32_pause(void)
338{
339	__asm __volatile("pause");
340}
341
342static __inline u_long
343read_rflags(void)
344{
345	u_long	rf;
346
347	__asm __volatile("pushfq; popq %0" : "=r" (rf));
348	return (rf);
349}
350
351static __inline uint64_t
352rdmsr(u_int msr)
353{
354	uint32_t low, high;
355
356	__asm __volatile("rdmsr" : "=a" (low), "=d" (high) : "c" (msr));
357	return (low | ((uint64_t)high << 32));
358}
359
360static __inline uint32_t
361rdmsr32(u_int msr)
362{
363	uint32_t low;
364
365	__asm __volatile("rdmsr" : "=a" (low) : "c" (msr) : "rdx");
366	return (low);
367}
368
369static __inline uint64_t
370rdpmc(u_int pmc)
371{
372	uint32_t low, high;
373
374	__asm __volatile("rdpmc" : "=a" (low), "=d" (high) : "c" (pmc));
375	return (low | ((uint64_t)high << 32));
376}
377
378static __inline uint64_t
379rdtsc(void)
380{
381	uint32_t low, high;
382
383	__asm __volatile("rdtsc" : "=a" (low), "=d" (high));
384	return (low | ((uint64_t)high << 32));
385}
386
387static __inline uint32_t
388rdtsc32(void)
389{
390	uint32_t rv;
391
392	__asm __volatile("rdtsc" : "=a" (rv) : : "edx");
393	return (rv);
394}
395
396static __inline void
397wbinvd(void)
398{
399	__asm __volatile("wbinvd");
400}
401
402static __inline void
403write_rflags(u_long rf)
404{
405	__asm __volatile("pushq %0;  popfq" : : "r" (rf));
406}
407
408static __inline void
409wrmsr(u_int msr, uint64_t newval)
410{
411	uint32_t low, high;
412
413	low = newval;
414	high = newval >> 32;
415	__asm __volatile("wrmsr" : : "a" (low), "d" (high), "c" (msr));
416}
417
418static __inline void
419load_cr0(u_long data)
420{
421
422	__asm __volatile("movq %0,%%cr0" : : "r" (data));
423}
424
425static __inline u_long
426rcr0(void)
427{
428	u_long	data;
429
430	__asm __volatile("movq %%cr0,%0" : "=r" (data));
431	return (data);
432}
433
434static __inline u_long
435rcr2(void)
436{
437	u_long	data;
438
439	__asm __volatile("movq %%cr2,%0" : "=r" (data));
440	return (data);
441}
442
443static __inline void
444load_cr3(u_long data)
445{
446
447	__asm __volatile("movq %0,%%cr3" : : "r" (data) : "memory");
448}
449
450static __inline u_long
451rcr3(void)
452{
453	u_long	data;
454
455	__asm __volatile("movq %%cr3,%0" : "=r" (data));
456	return (data);
457}
458
459static __inline void
460load_cr4(u_long data)
461{
462	__asm __volatile("movq %0,%%cr4" : : "r" (data));
463}
464
465static __inline u_long
466rcr4(void)
467{
468	u_long	data;
469
470	__asm __volatile("movq %%cr4,%0" : "=r" (data));
471	return (data);
472}
473
474static __inline u_long
475rxcr(u_int reg)
476{
477	u_int low, high;
478
479	__asm __volatile("xgetbv" : "=a" (low), "=d" (high) : "c" (reg));
480	return (low | ((uint64_t)high << 32));
481}
482
483static __inline void
484load_xcr(u_int reg, u_long val)
485{
486	u_int low, high;
487
488	low = val;
489	high = val >> 32;
490	__asm __volatile("xsetbv" : : "c" (reg), "a" (low), "d" (high));
491}
492
493/*
494 * Global TLB flush (except for thise for pages marked PG_G)
495 */
496static __inline void
497invltlb(void)
498{
499
500	load_cr3(rcr3());
501}
502
503#ifndef CR4_PGE
504#define	CR4_PGE	0x00000080	/* Page global enable */
505#endif
506
507/*
508 * Perform the guaranteed invalidation of all TLB entries.  This
509 * includes the global entries, and entries in all PCIDs, not only the
510 * current context.  The function works both on non-PCID CPUs and CPUs
511 * with the PCID turned off or on.  See IA-32 SDM Vol. 3a 4.10.4.1
512 * Operations that Invalidate TLBs and Paging-Structure Caches.
513 */
514static __inline void
515invltlb_glob(void)
516{
517	uint64_t cr4;
518
519	cr4 = rcr4();
520	load_cr4(cr4 & ~CR4_PGE);
521	/*
522	 * Although preemption at this point could be detrimental to
523	 * performance, it would not lead to an error.  PG_G is simply
524	 * ignored if CR4.PGE is clear.  Moreover, in case this block
525	 * is re-entered, the load_cr4() either above or below will
526	 * modify CR4.PGE flushing the TLB.
527	 */
528	load_cr4(cr4 | CR4_PGE);
529}
530
531/*
532 * TLB flush for an individual page (even if it has PG_G).
533 * Only works on 486+ CPUs (i386 does not have PG_G).
534 */
535static __inline void
536invlpg(u_long addr)
537{
538
539	__asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory");
540}
541
542#define	INVPCID_ADDR	0
543#define	INVPCID_CTX	1
544#define	INVPCID_CTXGLOB	2
545#define	INVPCID_ALLCTX	3
546
547struct invpcid_descr {
548	uint64_t	pcid:12 __packed;
549	uint64_t	pad:52 __packed;
550	uint64_t	addr;
551} __packed;
552
553static __inline void
554invpcid(struct invpcid_descr *d, int type)
555{
556
557	__asm __volatile("invpcid (%0),%1"
558	    : : "r" (d), "r" ((u_long)type) : "memory");
559}
560
561static __inline u_short
562rfs(void)
563{
564	u_short sel;
565	__asm __volatile("movw %%fs,%0" : "=rm" (sel));
566	return (sel);
567}
568
569static __inline u_short
570rgs(void)
571{
572	u_short sel;
573	__asm __volatile("movw %%gs,%0" : "=rm" (sel));
574	return (sel);
575}
576
577static __inline u_short
578rss(void)
579{
580	u_short sel;
581	__asm __volatile("movw %%ss,%0" : "=rm" (sel));
582	return (sel);
583}
584
585static __inline void
586load_ds(u_short sel)
587{
588	__asm __volatile("movw %0,%%ds" : : "rm" (sel));
589}
590
591static __inline void
592load_es(u_short sel)
593{
594	__asm __volatile("movw %0,%%es" : : "rm" (sel));
595}
596
597static __inline void
598cpu_monitor(const void *addr, u_long extensions, u_int hints)
599{
600
601	__asm __volatile("monitor"
602	    : : "a" (addr), "c" (extensions), "d" (hints));
603}
604
605static __inline void
606cpu_mwait(u_long extensions, u_int hints)
607{
608
609	__asm __volatile("mwait" : : "a" (hints), "c" (extensions));
610}
611
612#ifdef _KERNEL
613/* This is defined in <machine/specialreg.h> but is too painful to get to */
614#ifndef	MSR_FSBASE
615#define	MSR_FSBASE	0xc0000100
616#endif
617static __inline void
618load_fs(u_short sel)
619{
620	/* Preserve the fsbase value across the selector load */
621	__asm __volatile("rdmsr; movw %0,%%fs; wrmsr"
622	    : : "rm" (sel), "c" (MSR_FSBASE) : "eax", "edx");
623}
624
625#ifndef	MSR_GSBASE
626#define	MSR_GSBASE	0xc0000101
627#endif
628static __inline void
629load_gs(u_short sel)
630{
631	/*
632	 * Preserve the gsbase value across the selector load.
633	 * Note that we have to disable interrupts because the gsbase
634	 * being trashed happens to be the kernel gsbase at the time.
635	 */
636	__asm __volatile("pushfq; cli; rdmsr; movw %0,%%gs; wrmsr; popfq"
637	    : : "rm" (sel), "c" (MSR_GSBASE) : "eax", "edx");
638}
639#else
640/* Usable by userland */
641static __inline void
642load_fs(u_short sel)
643{
644	__asm __volatile("movw %0,%%fs" : : "rm" (sel));
645}
646
647static __inline void
648load_gs(u_short sel)
649{
650	__asm __volatile("movw %0,%%gs" : : "rm" (sel));
651}
652#endif
653
654static __inline void
655bare_lgdt(struct region_descriptor *addr)
656{
657	__asm __volatile("lgdt (%0)" : : "r" (addr));
658}
659
660static __inline void
661sgdt(struct region_descriptor *addr)
662{
663	char *loc;
664
665	loc = (char *)addr;
666	__asm __volatile("sgdt %0" : "=m" (*loc) : : "memory");
667}
668
669static __inline void
670lidt(struct region_descriptor *addr)
671{
672	__asm __volatile("lidt (%0)" : : "r" (addr));
673}
674
675static __inline void
676sidt(struct region_descriptor *addr)
677{
678	char *loc;
679
680	loc = (char *)addr;
681	__asm __volatile("sidt %0" : "=m" (*loc) : : "memory");
682}
683
684static __inline void
685lldt(u_short sel)
686{
687	__asm __volatile("lldt %0" : : "r" (sel));
688}
689
690static __inline void
691ltr(u_short sel)
692{
693	__asm __volatile("ltr %0" : : "r" (sel));
694}
695
696static __inline uint32_t
697read_tr(void)
698{
699	u_short sel;
700
701	__asm __volatile("str %0" : "=r" (sel));
702	return (sel);
703}
704
705static __inline uint64_t
706rdr0(void)
707{
708	uint64_t data;
709	__asm __volatile("movq %%dr0,%0" : "=r" (data));
710	return (data);
711}
712
713static __inline void
714load_dr0(uint64_t dr0)
715{
716	__asm __volatile("movq %0,%%dr0" : : "r" (dr0));
717}
718
719static __inline uint64_t
720rdr1(void)
721{
722	uint64_t data;
723	__asm __volatile("movq %%dr1,%0" : "=r" (data));
724	return (data);
725}
726
727static __inline void
728load_dr1(uint64_t dr1)
729{
730	__asm __volatile("movq %0,%%dr1" : : "r" (dr1));
731}
732
733static __inline uint64_t
734rdr2(void)
735{
736	uint64_t data;
737	__asm __volatile("movq %%dr2,%0" : "=r" (data));
738	return (data);
739}
740
741static __inline void
742load_dr2(uint64_t dr2)
743{
744	__asm __volatile("movq %0,%%dr2" : : "r" (dr2));
745}
746
747static __inline uint64_t
748rdr3(void)
749{
750	uint64_t data;
751	__asm __volatile("movq %%dr3,%0" : "=r" (data));
752	return (data);
753}
754
755static __inline void
756load_dr3(uint64_t dr3)
757{
758	__asm __volatile("movq %0,%%dr3" : : "r" (dr3));
759}
760
761static __inline uint64_t
762rdr6(void)
763{
764	uint64_t data;
765	__asm __volatile("movq %%dr6,%0" : "=r" (data));
766	return (data);
767}
768
769static __inline void
770load_dr6(uint64_t dr6)
771{
772	__asm __volatile("movq %0,%%dr6" : : "r" (dr6));
773}
774
775static __inline uint64_t
776rdr7(void)
777{
778	uint64_t data;
779	__asm __volatile("movq %%dr7,%0" : "=r" (data));
780	return (data);
781}
782
783static __inline void
784load_dr7(uint64_t dr7)
785{
786	__asm __volatile("movq %0,%%dr7" : : "r" (dr7));
787}
788
789static __inline register_t
790intr_disable(void)
791{
792	register_t rflags;
793
794	rflags = read_rflags();
795	disable_intr();
796	return (rflags);
797}
798
799static __inline void
800intr_restore(register_t rflags)
801{
802	write_rflags(rflags);
803}
804
805#else /* !(__GNUCLIKE_ASM && __CC_SUPPORTS___INLINE) */
806
807int	breakpoint(void);
808u_int	bsfl(u_int mask);
809u_int	bsrl(u_int mask);
810void	clflush(u_long addr);
811void	clts(void);
812void	cpuid_count(u_int ax, u_int cx, u_int *p);
813void	disable_intr(void);
814void	do_cpuid(u_int ax, u_int *p);
815void	enable_intr(void);
816void	halt(void);
817void	ia32_pause(void);
818u_char	inb(u_int port);
819u_int	inl(u_int port);
820void	insb(u_int port, void *addr, size_t count);
821void	insl(u_int port, void *addr, size_t count);
822void	insw(u_int port, void *addr, size_t count);
823register_t	intr_disable(void);
824void	intr_restore(register_t rf);
825void	invd(void);
826void	invlpg(u_int addr);
827void	invltlb(void);
828u_short	inw(u_int port);
829void	lidt(struct region_descriptor *addr);
830void	lldt(u_short sel);
831void	load_cr0(u_long cr0);
832void	load_cr3(u_long cr3);
833void	load_cr4(u_long cr4);
834void	load_dr0(uint64_t dr0);
835void	load_dr1(uint64_t dr1);
836void	load_dr2(uint64_t dr2);
837void	load_dr3(uint64_t dr3);
838void	load_dr6(uint64_t dr6);
839void	load_dr7(uint64_t dr7);
840void	load_fs(u_short sel);
841void	load_gs(u_short sel);
842void	ltr(u_short sel);
843void	outb(u_int port, u_char data);
844void	outl(u_int port, u_int data);
845void	outsb(u_int port, const void *addr, size_t count);
846void	outsl(u_int port, const void *addr, size_t count);
847void	outsw(u_int port, const void *addr, size_t count);
848void	outw(u_int port, u_short data);
849u_long	rcr0(void);
850u_long	rcr2(void);
851u_long	rcr3(void);
852u_long	rcr4(void);
853uint64_t rdmsr(u_int msr);
854uint32_t rdmsr32(u_int msr);
855uint64_t rdpmc(u_int pmc);
856uint64_t rdr0(void);
857uint64_t rdr1(void);
858uint64_t rdr2(void);
859uint64_t rdr3(void);
860uint64_t rdr6(void);
861uint64_t rdr7(void);
862uint64_t rdtsc(void);
863u_long	read_rflags(void);
864u_int	rfs(void);
865u_int	rgs(void);
866void	wbinvd(void);
867void	write_rflags(u_int rf);
868void	wrmsr(u_int msr, uint64_t newval);
869
870#endif	/* __GNUCLIKE_ASM && __CC_SUPPORTS___INLINE */
871
872void	reset_dbregs(void);
873
874#ifdef _KERNEL
875int	rdmsr_safe(u_int msr, uint64_t *val);
876int	wrmsr_safe(u_int msr, uint64_t newval);
877#endif
878
879#endif /* !_MACHINE_CPUFUNC_H_ */
880