1/*-
2 * Copyright (c) 1993 The Regents of the University of California.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 *    notice, this list of conditions and the following disclaimer in the
12 *    documentation and/or other materials provided with the distribution.
13 * 4. Neither the name of the University nor the names of its contributors
14 *    may be used to endorse or promote products derived from this software
15 *    without specific prior written permission.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 *
29 * $FreeBSD$
30 */
31
32/*
33 * Functions to provide access to special i386 instructions.
34 * This in included in sys/systm.h, and that file should be
35 * used in preference to this.
36 */
37
38#ifndef _MACHINE_CPUFUNC_H_
39#define	_MACHINE_CPUFUNC_H_
40
41#ifndef _SYS_CDEFS_H_
42#error this file needs sys/cdefs.h as a prerequisite
43#endif
44
45struct region_descriptor;
46
47#define readb(va)	(*(volatile uint8_t *) (va))
48#define readw(va)	(*(volatile uint16_t *) (va))
49#define readl(va)	(*(volatile uint32_t *) (va))
50
51#define writeb(va, d)	(*(volatile uint8_t *) (va) = (d))
52#define writew(va, d)	(*(volatile uint16_t *) (va) = (d))
53#define writel(va, d)	(*(volatile uint32_t *) (va) = (d))
54
55#if defined(__GNUCLIKE_ASM) && defined(__CC_SUPPORTS___INLINE)
56
57static __inline void
58breakpoint(void)
59{
60	__asm __volatile("int $3");
61}
62
63static __inline u_int
64bsfl(u_int mask)
65{
66	u_int	result;
67
68	__asm("bsfl %1,%0" : "=r" (result) : "rm" (mask) : "cc");
69	return (result);
70}
71
72static __inline u_int
73bsrl(u_int mask)
74{
75	u_int	result;
76
77	__asm("bsrl %1,%0" : "=r" (result) : "rm" (mask) : "cc");
78	return (result);
79}
80
81static __inline void
82clflush(u_long addr)
83{
84
85	__asm __volatile("clflush %0" : : "m" (*(char *)addr));
86}
87
88static __inline void
89clflushopt(u_long addr)
90{
91
92	__asm __volatile(".byte 0x66;clflush %0" : : "m" (*(char *)addr));
93}
94
95static __inline void
96clts(void)
97{
98
99	__asm __volatile("clts");
100}
101
102static __inline void
103disable_intr(void)
104{
105
106	__asm __volatile("cli" : : : "memory");
107}
108
109static __inline void
110do_cpuid(u_int ax, u_int *p)
111{
112	__asm __volatile("pushl %%ebx      \n\t"
113			 "cpuid            \n\t"
114			 "movl %%ebx, %1   \n\t"
115			 "popl %%ebx       \n\t"
116			 : "=a" (p[0]), "=m" (p[1]), "=c" (p[2]), "=d" (p[3])
117			 : "0" (ax));
118}
119
120static __inline void
121cpuid_count(u_int ax, u_int cx, u_int *p)
122{
123	__asm __volatile("cpuid"
124			 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3])
125			 :  "0" (ax), "c" (cx));
126}
127
128static __inline void
129enable_intr(void)
130{
131
132	__asm __volatile("sti");
133}
134
135static __inline void
136cpu_monitor(const void *addr, u_long extensions, u_int hints)
137{
138
139	__asm __volatile("monitor"
140	    : : "a" (addr), "c" (extensions), "d" (hints));
141}
142
143static __inline void
144cpu_mwait(u_long extensions, u_int hints)
145{
146
147	__asm __volatile("mwait" : : "a" (hints), "c" (extensions));
148}
149
150static __inline void
151lfence(void)
152{
153
154	__asm __volatile("lfence" : : : "memory");
155}
156
157static __inline void
158mfence(void)
159{
160
161	__asm __volatile("mfence" : : : "memory");
162}
163
164#ifdef _KERNEL
165
166#define	HAVE_INLINE_FFS
167
168static __inline int
169ffs(int mask)
170{
171	/*
172	 * Note that gcc-2's builtin ffs would be used if we didn't declare
173	 * this inline or turn off the builtin.  The builtin is faster but
174	 * broken in gcc-2.4.5 and slower but working in gcc-2.5 and later
175	 * versions.
176	 */
177	 return (mask == 0 ? mask : (int)bsfl((u_int)mask) + 1);
178}
179
180#define	HAVE_INLINE_FFSL
181
182static __inline int
183ffsl(long mask)
184{
185	return (ffs((int)mask));
186}
187
188#define	HAVE_INLINE_FLS
189
190static __inline int
191fls(int mask)
192{
193	return (mask == 0 ? mask : (int)bsrl((u_int)mask) + 1);
194}
195
196#define	HAVE_INLINE_FLSL
197
198static __inline int
199flsl(long mask)
200{
201	return (fls((int)mask));
202}
203
204#endif /* _KERNEL */
205
206static __inline void
207halt(void)
208{
209	__asm __volatile("hlt");
210}
211
212static __inline u_char
213inb(u_int port)
214{
215	u_char	data;
216
217	__asm __volatile("inb %w1, %0" : "=a" (data) : "Nd" (port));
218	return (data);
219}
220
221static __inline u_int
222inl(u_int port)
223{
224	u_int	data;
225
226	__asm __volatile("inl %w1, %0" : "=a" (data) : "Nd" (port));
227	return (data);
228}
229
230static __inline void
231insb(u_int port, void *addr, size_t count)
232{
233	__asm __volatile("cld; rep; insb"
234			 : "+D" (addr), "+c" (count)
235			 : "d" (port)
236			 : "memory");
237}
238
239static __inline void
240insw(u_int port, void *addr, size_t count)
241{
242	__asm __volatile("cld; rep; insw"
243			 : "+D" (addr), "+c" (count)
244			 : "d" (port)
245			 : "memory");
246}
247
248static __inline void
249insl(u_int port, void *addr, size_t count)
250{
251	__asm __volatile("cld; rep; insl"
252			 : "+D" (addr), "+c" (count)
253			 : "d" (port)
254			 : "memory");
255}
256
257static __inline void
258invd(void)
259{
260	__asm __volatile("invd");
261}
262
263static __inline u_short
264inw(u_int port)
265{
266	u_short	data;
267
268	__asm __volatile("inw %w1, %0" : "=a" (data) : "Nd" (port));
269	return (data);
270}
271
272static __inline void
273outb(u_int port, u_char data)
274{
275	__asm __volatile("outb %0, %w1" : : "a" (data), "Nd" (port));
276}
277
278static __inline void
279outl(u_int port, u_int data)
280{
281	__asm __volatile("outl %0, %w1" : : "a" (data), "Nd" (port));
282}
283
284static __inline void
285outsb(u_int port, const void *addr, size_t count)
286{
287	__asm __volatile("cld; rep; outsb"
288			 : "+S" (addr), "+c" (count)
289			 : "d" (port));
290}
291
292static __inline void
293outsw(u_int port, const void *addr, size_t count)
294{
295	__asm __volatile("cld; rep; outsw"
296			 : "+S" (addr), "+c" (count)
297			 : "d" (port));
298}
299
300static __inline void
301outsl(u_int port, const void *addr, size_t count)
302{
303	__asm __volatile("cld; rep; outsl"
304			 : "+S" (addr), "+c" (count)
305			 : "d" (port));
306}
307
308static __inline void
309outw(u_int port, u_short data)
310{
311	__asm __volatile("outw %0, %w1" : : "a" (data), "Nd" (port));
312}
313
314static __inline void
315ia32_pause(void)
316{
317	__asm __volatile("pause");
318}
319
320static __inline u_int
321read_eflags(void)
322{
323	u_int	ef;
324
325	__asm __volatile("pushfl; popl %0" : "=r" (ef));
326	return (ef);
327}
328
329static __inline uint64_t
330rdmsr(u_int msr)
331{
332	uint64_t rv;
333
334	__asm __volatile("rdmsr" : "=A" (rv) : "c" (msr));
335	return (rv);
336}
337
338static __inline uint32_t
339rdmsr32(u_int msr)
340{
341	uint32_t low;
342
343	__asm __volatile("rdmsr" : "=a" (low) : "c" (msr) : "edx");
344	return (low);
345}
346
347static __inline uint64_t
348rdpmc(u_int pmc)
349{
350	uint64_t rv;
351
352	__asm __volatile("rdpmc" : "=A" (rv) : "c" (pmc));
353	return (rv);
354}
355
356static __inline uint64_t
357rdtsc(void)
358{
359	uint64_t rv;
360
361	__asm __volatile("rdtsc" : "=A" (rv));
362	return (rv);
363}
364
365static __inline uint32_t
366rdtsc32(void)
367{
368	uint32_t rv;
369
370	__asm __volatile("rdtsc" : "=a" (rv) : : "edx");
371	return (rv);
372}
373
374static __inline void
375wbinvd(void)
376{
377	__asm __volatile("wbinvd");
378}
379
380static __inline void
381write_eflags(u_int ef)
382{
383	__asm __volatile("pushl %0; popfl" : : "r" (ef));
384}
385
386static __inline void
387wrmsr(u_int msr, uint64_t newval)
388{
389	__asm __volatile("wrmsr" : : "A" (newval), "c" (msr));
390}
391
392static __inline void
393load_cr0(u_int data)
394{
395
396	__asm __volatile("movl %0,%%cr0" : : "r" (data));
397}
398
399static __inline u_int
400rcr0(void)
401{
402	u_int	data;
403
404	__asm __volatile("movl %%cr0,%0" : "=r" (data));
405	return (data);
406}
407
408static __inline u_int
409rcr2(void)
410{
411	u_int	data;
412
413	__asm __volatile("movl %%cr2,%0" : "=r" (data));
414	return (data);
415}
416
417static __inline void
418load_cr3(u_int data)
419{
420
421	__asm __volatile("movl %0,%%cr3" : : "r" (data) : "memory");
422}
423
424static __inline u_int
425rcr3(void)
426{
427	u_int	data;
428
429	__asm __volatile("movl %%cr3,%0" : "=r" (data));
430	return (data);
431}
432
433static __inline void
434load_cr4(u_int data)
435{
436	__asm __volatile("movl %0,%%cr4" : : "r" (data));
437}
438
439static __inline u_int
440rcr4(void)
441{
442	u_int	data;
443
444	__asm __volatile("movl %%cr4,%0" : "=r" (data));
445	return (data);
446}
447
448static __inline uint64_t
449rxcr(u_int reg)
450{
451	u_int low, high;
452
453	__asm __volatile("xgetbv" : "=a" (low), "=d" (high) : "c" (reg));
454	return (low | ((uint64_t)high << 32));
455}
456
457static __inline void
458load_xcr(u_int reg, uint64_t val)
459{
460	u_int low, high;
461
462	low = val;
463	high = val >> 32;
464	__asm __volatile("xsetbv" : : "c" (reg), "a" (low), "d" (high));
465}
466
467/*
468 * Global TLB flush (except for thise for pages marked PG_G)
469 */
470static __inline void
471invltlb(void)
472{
473
474	load_cr3(rcr3());
475}
476
477/*
478 * TLB flush for an individual page (even if it has PG_G).
479 * Only works on 486+ CPUs (i386 does not have PG_G).
480 */
481static __inline void
482invlpg(u_int addr)
483{
484
485	__asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory");
486}
487
488static __inline u_short
489rfs(void)
490{
491	u_short sel;
492	__asm __volatile("movw %%fs,%0" : "=rm" (sel));
493	return (sel);
494}
495
496static __inline uint64_t
497rgdt(void)
498{
499	uint64_t gdtr;
500	__asm __volatile("sgdt %0" : "=m" (gdtr));
501	return (gdtr);
502}
503
504static __inline u_short
505rgs(void)
506{
507	u_short sel;
508	__asm __volatile("movw %%gs,%0" : "=rm" (sel));
509	return (sel);
510}
511
512static __inline uint64_t
513ridt(void)
514{
515	uint64_t idtr;
516	__asm __volatile("sidt %0" : "=m" (idtr));
517	return (idtr);
518}
519
520static __inline u_short
521rldt(void)
522{
523	u_short ldtr;
524	__asm __volatile("sldt %0" : "=g" (ldtr));
525	return (ldtr);
526}
527
528static __inline u_short
529rss(void)
530{
531	u_short sel;
532	__asm __volatile("movw %%ss,%0" : "=rm" (sel));
533	return (sel);
534}
535
536static __inline u_short
537rtr(void)
538{
539	u_short tr;
540	__asm __volatile("str %0" : "=g" (tr));
541	return (tr);
542}
543
544static __inline void
545load_fs(u_short sel)
546{
547	__asm __volatile("movw %0,%%fs" : : "rm" (sel));
548}
549
550static __inline void
551load_gs(u_short sel)
552{
553	__asm __volatile("movw %0,%%gs" : : "rm" (sel));
554}
555
556static __inline void
557lidt(struct region_descriptor *addr)
558{
559	__asm __volatile("lidt (%0)" : : "r" (addr));
560}
561
562static __inline void
563lldt(u_short sel)
564{
565	__asm __volatile("lldt %0" : : "r" (sel));
566}
567
568static __inline void
569ltr(u_short sel)
570{
571	__asm __volatile("ltr %0" : : "r" (sel));
572}
573
574static __inline u_int
575rdr0(void)
576{
577	u_int	data;
578	__asm __volatile("movl %%dr0,%0" : "=r" (data));
579	return (data);
580}
581
582static __inline void
583load_dr0(u_int dr0)
584{
585	__asm __volatile("movl %0,%%dr0" : : "r" (dr0));
586}
587
588static __inline u_int
589rdr1(void)
590{
591	u_int	data;
592	__asm __volatile("movl %%dr1,%0" : "=r" (data));
593	return (data);
594}
595
596static __inline void
597load_dr1(u_int dr1)
598{
599	__asm __volatile("movl %0,%%dr1" : : "r" (dr1));
600}
601
602static __inline u_int
603rdr2(void)
604{
605	u_int	data;
606	__asm __volatile("movl %%dr2,%0" : "=r" (data));
607	return (data);
608}
609
610static __inline void
611load_dr2(u_int dr2)
612{
613	__asm __volatile("movl %0,%%dr2" : : "r" (dr2));
614}
615
616static __inline u_int
617rdr3(void)
618{
619	u_int	data;
620	__asm __volatile("movl %%dr3,%0" : "=r" (data));
621	return (data);
622}
623
624static __inline void
625load_dr3(u_int dr3)
626{
627	__asm __volatile("movl %0,%%dr3" : : "r" (dr3));
628}
629
630static __inline u_int
631rdr4(void)
632{
633	u_int	data;
634	__asm __volatile("movl %%dr4,%0" : "=r" (data));
635	return (data);
636}
637
638static __inline void
639load_dr4(u_int dr4)
640{
641	__asm __volatile("movl %0,%%dr4" : : "r" (dr4));
642}
643
644static __inline u_int
645rdr5(void)
646{
647	u_int	data;
648	__asm __volatile("movl %%dr5,%0" : "=r" (data));
649	return (data);
650}
651
652static __inline void
653load_dr5(u_int dr5)
654{
655	__asm __volatile("movl %0,%%dr5" : : "r" (dr5));
656}
657
658static __inline u_int
659rdr6(void)
660{
661	u_int	data;
662	__asm __volatile("movl %%dr6,%0" : "=r" (data));
663	return (data);
664}
665
666static __inline void
667load_dr6(u_int dr6)
668{
669	__asm __volatile("movl %0,%%dr6" : : "r" (dr6));
670}
671
672static __inline u_int
673rdr7(void)
674{
675	u_int	data;
676	__asm __volatile("movl %%dr7,%0" : "=r" (data));
677	return (data);
678}
679
680static __inline void
681load_dr7(u_int dr7)
682{
683	__asm __volatile("movl %0,%%dr7" : : "r" (dr7));
684}
685
686static __inline u_char
687read_cyrix_reg(u_char reg)
688{
689	outb(0x22, reg);
690	return inb(0x23);
691}
692
693static __inline void
694write_cyrix_reg(u_char reg, u_char data)
695{
696	outb(0x22, reg);
697	outb(0x23, data);
698}
699
700static __inline register_t
701intr_disable(void)
702{
703	register_t eflags;
704
705	eflags = read_eflags();
706	disable_intr();
707	return (eflags);
708}
709
710static __inline void
711intr_restore(register_t eflags)
712{
713	write_eflags(eflags);
714}
715
716#else /* !(__GNUCLIKE_ASM && __CC_SUPPORTS___INLINE) */
717
718int	breakpoint(void);
719u_int	bsfl(u_int mask);
720u_int	bsrl(u_int mask);
721void	clflush(u_long addr);
722void	clts(void);
723void	cpuid_count(u_int ax, u_int cx, u_int *p);
724void	disable_intr(void);
725void	do_cpuid(u_int ax, u_int *p);
726void	enable_intr(void);
727void	halt(void);
728void	ia32_pause(void);
729u_char	inb(u_int port);
730u_int	inl(u_int port);
731void	insb(u_int port, void *addr, size_t count);
732void	insl(u_int port, void *addr, size_t count);
733void	insw(u_int port, void *addr, size_t count);
734register_t	intr_disable(void);
735void	intr_restore(register_t ef);
736void	invd(void);
737void	invlpg(u_int addr);
738void	invltlb(void);
739u_short	inw(u_int port);
740void	lidt(struct region_descriptor *addr);
741void	lldt(u_short sel);
742void	load_cr0(u_int cr0);
743void	load_cr3(u_int cr3);
744void	load_cr4(u_int cr4);
745void	load_dr0(u_int dr0);
746void	load_dr1(u_int dr1);
747void	load_dr2(u_int dr2);
748void	load_dr3(u_int dr3);
749void	load_dr4(u_int dr4);
750void	load_dr5(u_int dr5);
751void	load_dr6(u_int dr6);
752void	load_dr7(u_int dr7);
753void	load_fs(u_short sel);
754void	load_gs(u_short sel);
755void	ltr(u_short sel);
756void	outb(u_int port, u_char data);
757void	outl(u_int port, u_int data);
758void	outsb(u_int port, const void *addr, size_t count);
759void	outsl(u_int port, const void *addr, size_t count);
760void	outsw(u_int port, const void *addr, size_t count);
761void	outw(u_int port, u_short data);
762u_int	rcr0(void);
763u_int	rcr2(void);
764u_int	rcr3(void);
765u_int	rcr4(void);
766uint64_t rdmsr(u_int msr);
767uint64_t rdpmc(u_int pmc);
768u_int	rdr0(void);
769u_int	rdr1(void);
770u_int	rdr2(void);
771u_int	rdr3(void);
772u_int	rdr4(void);
773u_int	rdr5(void);
774u_int	rdr6(void);
775u_int	rdr7(void);
776uint64_t rdtsc(void);
777u_char	read_cyrix_reg(u_char reg);
778u_int	read_eflags(void);
779u_int	rfs(void);
780uint64_t rgdt(void);
781u_int	rgs(void);
782uint64_t ridt(void);
783u_short	rldt(void);
784u_short	rtr(void);
785void	wbinvd(void);
786void	write_cyrix_reg(u_char reg, u_char data);
787void	write_eflags(u_int ef);
788void	wrmsr(u_int msr, uint64_t newval);
789
790#endif	/* __GNUCLIKE_ASM && __CC_SUPPORTS___INLINE */
791
792void    reset_dbregs(void);
793
794#ifdef _KERNEL
795int	rdmsr_safe(u_int msr, uint64_t *val);
796int	wrmsr_safe(u_int msr, uint64_t newval);
797#endif
798
799#endif /* !_MACHINE_CPUFUNC_H_ */
800