1 /*-
2 * Copyright (c) 1993 The Regents of the University of California.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 * 4. Neither the name of the University nor the names of its contributors
14 * may be used to endorse or promote products derived from this software
15 * without specific prior written permission.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 *
29 * $FreeBSD$
30 */
31
32 /*
33 * Functions to provide access to special i386 instructions.
34 * This in included in sys/systm.h, and that file should be
35 * used in preference to this.
36 */
37
38 #ifndef _MACHINE_CPUFUNC_H_
39 #define _MACHINE_CPUFUNC_H_
40
41 #ifndef _SYS_CDEFS_H_
42 #error this file needs sys/cdefs.h as a prerequisite
43 #endif
44
45 struct region_descriptor;
46
47 #define readb(va) (*(volatile uint8_t *) (va))
48 #define readw(va) (*(volatile uint16_t *) (va))
49 #define readl(va) (*(volatile uint32_t *) (va))
50
51 #define writeb(va, d) (*(volatile uint8_t *) (va) = (d))
52 #define writew(va, d) (*(volatile uint16_t *) (va) = (d))
53 #define writel(va, d) (*(volatile uint32_t *) (va) = (d))
54
55 #if defined(__GNUCLIKE_ASM) && defined(__CC_SUPPORTS___INLINE)
56
57 static __inline void
breakpoint(void)58 breakpoint(void)
59 {
60 __asm __volatile("int $3");
61 }
62
63 static __inline u_int
bsfl(u_int mask)64 bsfl(u_int mask)
65 {
66 u_int result;
67
68 __asm("bsfl %1,%0" : "=r" (result) : "rm" (mask) : "cc");
69 return (result);
70 }
71
72 static __inline u_int
bsrl(u_int mask)73 bsrl(u_int mask)
74 {
75 u_int result;
76
77 __asm("bsrl %1,%0" : "=r" (result) : "rm" (mask) : "cc");
78 return (result);
79 }
80
81 static __inline void
clflush(u_long addr)82 clflush(u_long addr)
83 {
84
85 __asm __volatile("clflush %0" : : "m" (*(char *)addr));
86 }
87
88 static __inline void
clflushopt(u_long addr)89 clflushopt(u_long addr)
90 {
91
92 __asm __volatile(".byte 0x66;clflush %0" : : "m" (*(char *)addr));
93 }
94
95 static __inline void
clts(void)96 clts(void)
97 {
98
99 __asm __volatile("clts");
100 }
101
102 static __inline void
disable_intr(void)103 disable_intr(void)
104 {
105
106 __asm __volatile("cli" : : : "memory");
107 }
108
109 static __inline void
do_cpuid(u_int ax,u_int * p)110 do_cpuid(u_int ax, u_int *p)
111 {
112 __asm __volatile("pushl %%ebx \n\t"
113 "cpuid \n\t"
114 "movl %%ebx, %1 \n\t"
115 "popl %%ebx \n\t"
116 : "=a" (p[0]), "=DS" (p[1]), "=c" (p[2]), "=d" (p[3])
117 : "0" (ax));
118 }
119
120 static __inline void
cpuid_count(u_int ax,u_int cx,u_int * p)121 cpuid_count(u_int ax, u_int cx, u_int *p)
122 {
123 __asm __volatile("cpuid"
124 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3])
125 : "0" (ax), "c" (cx));
126 }
127
128 static __inline void
enable_intr(void)129 enable_intr(void)
130 {
131
132 __asm __volatile("sti");
133 }
134
135 static __inline void
cpu_monitor(const void * addr,u_long extensions,u_int hints)136 cpu_monitor(const void *addr, u_long extensions, u_int hints)
137 {
138
139 __asm __volatile("monitor"
140 : : "a" (addr), "c" (extensions), "d" (hints));
141 }
142
143 static __inline void
cpu_mwait(u_long extensions,u_int hints)144 cpu_mwait(u_long extensions, u_int hints)
145 {
146
147 __asm __volatile("mwait" : : "a" (hints), "c" (extensions));
148 }
149
150 static __inline void
lfence(void)151 lfence(void)
152 {
153
154 __asm __volatile("lfence" : : : "memory");
155 }
156
157 static __inline void
mfence(void)158 mfence(void)
159 {
160
161 __asm __volatile("mfence" : : : "memory");
162 }
163
164 #ifdef _KERNEL
165
166 #define HAVE_INLINE_FFS
167
168 static __inline int
ffs(int mask)169 ffs(int mask)
170 {
171 /*
172 * Note that gcc-2's builtin ffs would be used if we didn't declare
173 * this inline or turn off the builtin. The builtin is faster but
174 * broken in gcc-2.4.5 and slower but working in gcc-2.5 and later
175 * versions.
176 */
177 return (mask == 0 ? mask : (int)bsfl((u_int)mask) + 1);
178 }
179
180 #define HAVE_INLINE_FFSL
181
182 static __inline int
ffsl(long mask)183 ffsl(long mask)
184 {
185 return (ffs((int)mask));
186 }
187
188 #define HAVE_INLINE_FLS
189
190 static __inline int
fls(int mask)191 fls(int mask)
192 {
193 return (mask == 0 ? mask : (int)bsrl((u_int)mask) + 1);
194 }
195
196 #define HAVE_INLINE_FLSL
197
198 static __inline int
flsl(long mask)199 flsl(long mask)
200 {
201 return (fls((int)mask));
202 }
203
204 #endif /* _KERNEL */
205
206 static __inline void
halt(void)207 halt(void)
208 {
209 __asm __volatile("hlt");
210 }
211
212 static __inline u_char
inb(u_int port)213 inb(u_int port)
214 {
215 u_char data;
216
217 __asm __volatile("inb %w1, %0" : "=a" (data) : "Nd" (port));
218 return (data);
219 }
220
221 static __inline u_int
inl(u_int port)222 inl(u_int port)
223 {
224 u_int data;
225
226 __asm __volatile("inl %w1, %0" : "=a" (data) : "Nd" (port));
227 return (data);
228 }
229
230 static __inline void
insb(u_int port,void * addr,size_t count)231 insb(u_int port, void *addr, size_t count)
232 {
233 __asm __volatile("cld; rep; insb"
234 : "+D" (addr), "+c" (count)
235 : "d" (port)
236 : "memory");
237 }
238
239 static __inline void
insw(u_int port,void * addr,size_t count)240 insw(u_int port, void *addr, size_t count)
241 {
242 __asm __volatile("cld; rep; insw"
243 : "+D" (addr), "+c" (count)
244 : "d" (port)
245 : "memory");
246 }
247
248 static __inline void
insl(u_int port,void * addr,size_t count)249 insl(u_int port, void *addr, size_t count)
250 {
251 __asm __volatile("cld; rep; insl"
252 : "+D" (addr), "+c" (count)
253 : "d" (port)
254 : "memory");
255 }
256
257 static __inline void
invd(void)258 invd(void)
259 {
260 __asm __volatile("invd");
261 }
262
263 static __inline u_short
inw(u_int port)264 inw(u_int port)
265 {
266 u_short data;
267
268 __asm __volatile("inw %w1, %0" : "=a" (data) : "Nd" (port));
269 return (data);
270 }
271
272 static __inline void
outb(u_int port,u_char data)273 outb(u_int port, u_char data)
274 {
275 __asm __volatile("outb %0, %w1" : : "a" (data), "Nd" (port));
276 }
277
278 static __inline void
outl(u_int port,u_int data)279 outl(u_int port, u_int data)
280 {
281 __asm __volatile("outl %0, %w1" : : "a" (data), "Nd" (port));
282 }
283
284 static __inline void
outsb(u_int port,const void * addr,size_t count)285 outsb(u_int port, const void *addr, size_t count)
286 {
287 __asm __volatile("cld; rep; outsb"
288 : "+S" (addr), "+c" (count)
289 : "d" (port));
290 }
291
292 static __inline void
outsw(u_int port,const void * addr,size_t count)293 outsw(u_int port, const void *addr, size_t count)
294 {
295 __asm __volatile("cld; rep; outsw"
296 : "+S" (addr), "+c" (count)
297 : "d" (port));
298 }
299
300 static __inline void
outsl(u_int port,const void * addr,size_t count)301 outsl(u_int port, const void *addr, size_t count)
302 {
303 __asm __volatile("cld; rep; outsl"
304 : "+S" (addr), "+c" (count)
305 : "d" (port));
306 }
307
308 static __inline void
outw(u_int port,u_short data)309 outw(u_int port, u_short data)
310 {
311 __asm __volatile("outw %0, %w1" : : "a" (data), "Nd" (port));
312 }
313
314 static __inline void
ia32_pause(void)315 ia32_pause(void)
316 {
317 __asm __volatile("pause");
318 }
319
320 static __inline u_int
read_eflags(void)321 read_eflags(void)
322 {
323 u_int ef;
324
325 __asm __volatile("pushfl; popl %0" : "=r" (ef));
326 return (ef);
327 }
328
329 static __inline uint64_t
rdmsr(u_int msr)330 rdmsr(u_int msr)
331 {
332 uint64_t rv;
333
334 __asm __volatile("rdmsr" : "=A" (rv) : "c" (msr));
335 return (rv);
336 }
337
338 static __inline uint32_t
rdmsr32(u_int msr)339 rdmsr32(u_int msr)
340 {
341 uint32_t low;
342
343 __asm __volatile("rdmsr" : "=a" (low) : "c" (msr) : "edx");
344 return (low);
345 }
346
347 static __inline uint64_t
rdpmc(u_int pmc)348 rdpmc(u_int pmc)
349 {
350 uint64_t rv;
351
352 __asm __volatile("rdpmc" : "=A" (rv) : "c" (pmc));
353 return (rv);
354 }
355
356 static __inline uint64_t
rdtsc(void)357 rdtsc(void)
358 {
359 uint64_t rv;
360
361 __asm __volatile("rdtsc" : "=A" (rv));
362 return (rv);
363 }
364
365 static __inline uint32_t
rdtsc32(void)366 rdtsc32(void)
367 {
368 uint32_t rv;
369
370 __asm __volatile("rdtsc" : "=a" (rv) : : "edx");
371 return (rv);
372 }
373
374 static __inline void
wbinvd(void)375 wbinvd(void)
376 {
377 __asm __volatile("wbinvd");
378 }
379
380 static __inline void
write_eflags(u_int ef)381 write_eflags(u_int ef)
382 {
383 __asm __volatile("pushl %0; popfl" : : "r" (ef));
384 }
385
386 static __inline void
wrmsr(u_int msr,uint64_t newval)387 wrmsr(u_int msr, uint64_t newval)
388 {
389 __asm __volatile("wrmsr" : : "A" (newval), "c" (msr));
390 }
391
392 static __inline void
load_cr0(u_int data)393 load_cr0(u_int data)
394 {
395
396 __asm __volatile("movl %0,%%cr0" : : "r" (data));
397 }
398
399 static __inline u_int
rcr0(void)400 rcr0(void)
401 {
402 u_int data;
403
404 __asm __volatile("movl %%cr0,%0" : "=r" (data));
405 return (data);
406 }
407
408 static __inline u_int
rcr2(void)409 rcr2(void)
410 {
411 u_int data;
412
413 __asm __volatile("movl %%cr2,%0" : "=r" (data));
414 return (data);
415 }
416
417 static __inline void
load_cr3(u_int data)418 load_cr3(u_int data)
419 {
420
421 __asm __volatile("movl %0,%%cr3" : : "r" (data) : "memory");
422 }
423
424 static __inline u_int
rcr3(void)425 rcr3(void)
426 {
427 u_int data;
428
429 __asm __volatile("movl %%cr3,%0" : "=r" (data));
430 return (data);
431 }
432
433 static __inline void
load_cr4(u_int data)434 load_cr4(u_int data)
435 {
436 __asm __volatile("movl %0,%%cr4" : : "r" (data));
437 }
438
439 static __inline u_int
rcr4(void)440 rcr4(void)
441 {
442 u_int data;
443
444 __asm __volatile("movl %%cr4,%0" : "=r" (data));
445 return (data);
446 }
447
448 static __inline uint64_t
rxcr(u_int reg)449 rxcr(u_int reg)
450 {
451 u_int low, high;
452
453 __asm __volatile("xgetbv" : "=a" (low), "=d" (high) : "c" (reg));
454 return (low | ((uint64_t)high << 32));
455 }
456
457 static __inline void
load_xcr(u_int reg,uint64_t val)458 load_xcr(u_int reg, uint64_t val)
459 {
460 u_int low, high;
461
462 low = val;
463 high = val >> 32;
464 __asm __volatile("xsetbv" : : "c" (reg), "a" (low), "d" (high));
465 }
466
467 /*
468 * Global TLB flush (except for thise for pages marked PG_G)
469 */
470 static __inline void
invltlb(void)471 invltlb(void)
472 {
473
474 load_cr3(rcr3());
475 }
476
477 /*
478 * TLB flush for an individual page (even if it has PG_G).
479 * Only works on 486+ CPUs (i386 does not have PG_G).
480 */
481 static __inline void
invlpg(u_int addr)482 invlpg(u_int addr)
483 {
484
485 __asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory");
486 }
487
488 static __inline u_short
rfs(void)489 rfs(void)
490 {
491 u_short sel;
492 __asm __volatile("movw %%fs,%0" : "=rm" (sel));
493 return (sel);
494 }
495
496 static __inline uint64_t
rgdt(void)497 rgdt(void)
498 {
499 uint64_t gdtr;
500 __asm __volatile("sgdt %0" : "=m" (gdtr));
501 return (gdtr);
502 }
503
504 static __inline u_short
rgs(void)505 rgs(void)
506 {
507 u_short sel;
508 __asm __volatile("movw %%gs,%0" : "=rm" (sel));
509 return (sel);
510 }
511
512 static __inline uint64_t
ridt(void)513 ridt(void)
514 {
515 uint64_t idtr;
516 __asm __volatile("sidt %0" : "=m" (idtr));
517 return (idtr);
518 }
519
520 static __inline u_short
rldt(void)521 rldt(void)
522 {
523 u_short ldtr;
524 __asm __volatile("sldt %0" : "=g" (ldtr));
525 return (ldtr);
526 }
527
528 static __inline u_short
rss(void)529 rss(void)
530 {
531 u_short sel;
532 __asm __volatile("movw %%ss,%0" : "=rm" (sel));
533 return (sel);
534 }
535
536 static __inline u_short
rtr(void)537 rtr(void)
538 {
539 u_short tr;
540 __asm __volatile("str %0" : "=g" (tr));
541 return (tr);
542 }
543
544 static __inline void
load_fs(u_short sel)545 load_fs(u_short sel)
546 {
547 __asm __volatile("movw %0,%%fs" : : "rm" (sel));
548 }
549
550 static __inline void
load_gs(u_short sel)551 load_gs(u_short sel)
552 {
553 __asm __volatile("movw %0,%%gs" : : "rm" (sel));
554 }
555
556 static __inline void
lidt(struct region_descriptor * addr)557 lidt(struct region_descriptor *addr)
558 {
559 __asm __volatile("lidt (%0)" : : "r" (addr));
560 }
561
562 static __inline void
lldt(u_short sel)563 lldt(u_short sel)
564 {
565 __asm __volatile("lldt %0" : : "r" (sel));
566 }
567
568 static __inline void
ltr(u_short sel)569 ltr(u_short sel)
570 {
571 __asm __volatile("ltr %0" : : "r" (sel));
572 }
573
574 static __inline u_int
rdr0(void)575 rdr0(void)
576 {
577 u_int data;
578 __asm __volatile("movl %%dr0,%0" : "=r" (data));
579 return (data);
580 }
581
582 static __inline void
load_dr0(u_int dr0)583 load_dr0(u_int dr0)
584 {
585 __asm __volatile("movl %0,%%dr0" : : "r" (dr0));
586 }
587
588 static __inline u_int
rdr1(void)589 rdr1(void)
590 {
591 u_int data;
592 __asm __volatile("movl %%dr1,%0" : "=r" (data));
593 return (data);
594 }
595
596 static __inline void
load_dr1(u_int dr1)597 load_dr1(u_int dr1)
598 {
599 __asm __volatile("movl %0,%%dr1" : : "r" (dr1));
600 }
601
602 static __inline u_int
rdr2(void)603 rdr2(void)
604 {
605 u_int data;
606 __asm __volatile("movl %%dr2,%0" : "=r" (data));
607 return (data);
608 }
609
610 static __inline void
load_dr2(u_int dr2)611 load_dr2(u_int dr2)
612 {
613 __asm __volatile("movl %0,%%dr2" : : "r" (dr2));
614 }
615
616 static __inline u_int
rdr3(void)617 rdr3(void)
618 {
619 u_int data;
620 __asm __volatile("movl %%dr3,%0" : "=r" (data));
621 return (data);
622 }
623
624 static __inline void
load_dr3(u_int dr3)625 load_dr3(u_int dr3)
626 {
627 __asm __volatile("movl %0,%%dr3" : : "r" (dr3));
628 }
629
630 static __inline u_int
rdr4(void)631 rdr4(void)
632 {
633 u_int data;
634 __asm __volatile("movl %%dr4,%0" : "=r" (data));
635 return (data);
636 }
637
638 static __inline void
load_dr4(u_int dr4)639 load_dr4(u_int dr4)
640 {
641 __asm __volatile("movl %0,%%dr4" : : "r" (dr4));
642 }
643
644 static __inline u_int
rdr5(void)645 rdr5(void)
646 {
647 u_int data;
648 __asm __volatile("movl %%dr5,%0" : "=r" (data));
649 return (data);
650 }
651
652 static __inline void
load_dr5(u_int dr5)653 load_dr5(u_int dr5)
654 {
655 __asm __volatile("movl %0,%%dr5" : : "r" (dr5));
656 }
657
658 static __inline u_int
rdr6(void)659 rdr6(void)
660 {
661 u_int data;
662 __asm __volatile("movl %%dr6,%0" : "=r" (data));
663 return (data);
664 }
665
666 static __inline void
load_dr6(u_int dr6)667 load_dr6(u_int dr6)
668 {
669 __asm __volatile("movl %0,%%dr6" : : "r" (dr6));
670 }
671
672 static __inline u_int
rdr7(void)673 rdr7(void)
674 {
675 u_int data;
676 __asm __volatile("movl %%dr7,%0" : "=r" (data));
677 return (data);
678 }
679
680 static __inline void
load_dr7(u_int dr7)681 load_dr7(u_int dr7)
682 {
683 __asm __volatile("movl %0,%%dr7" : : "r" (dr7));
684 }
685
686 static __inline u_char
read_cyrix_reg(u_char reg)687 read_cyrix_reg(u_char reg)
688 {
689 outb(0x22, reg);
690 return inb(0x23);
691 }
692
693 static __inline void
write_cyrix_reg(u_char reg,u_char data)694 write_cyrix_reg(u_char reg, u_char data)
695 {
696 outb(0x22, reg);
697 outb(0x23, data);
698 }
699
700 static __inline register_t
intr_disable(void)701 intr_disable(void)
702 {
703 register_t eflags;
704
705 eflags = read_eflags();
706 disable_intr();
707 return (eflags);
708 }
709
710 static __inline void
intr_restore(register_t eflags)711 intr_restore(register_t eflags)
712 {
713 write_eflags(eflags);
714 }
715
716 #else /* !(__GNUCLIKE_ASM && __CC_SUPPORTS___INLINE) */
717
718 int breakpoint(void);
719 u_int bsfl(u_int mask);
720 u_int bsrl(u_int mask);
721 void clflush(u_long addr);
722 void clts(void);
723 void cpuid_count(u_int ax, u_int cx, u_int *p);
724 void disable_intr(void);
725 void do_cpuid(u_int ax, u_int *p);
726 void enable_intr(void);
727 void halt(void);
728 void ia32_pause(void);
729 u_char inb(u_int port);
730 u_int inl(u_int port);
731 void insb(u_int port, void *addr, size_t count);
732 void insl(u_int port, void *addr, size_t count);
733 void insw(u_int port, void *addr, size_t count);
734 register_t intr_disable(void);
735 void intr_restore(register_t ef);
736 void invd(void);
737 void invlpg(u_int addr);
738 void invltlb(void);
739 u_short inw(u_int port);
740 void lidt(struct region_descriptor *addr);
741 void lldt(u_short sel);
742 void load_cr0(u_int cr0);
743 void load_cr3(u_int cr3);
744 void load_cr4(u_int cr4);
745 void load_dr0(u_int dr0);
746 void load_dr1(u_int dr1);
747 void load_dr2(u_int dr2);
748 void load_dr3(u_int dr3);
749 void load_dr4(u_int dr4);
750 void load_dr5(u_int dr5);
751 void load_dr6(u_int dr6);
752 void load_dr7(u_int dr7);
753 void load_fs(u_short sel);
754 void load_gs(u_short sel);
755 void ltr(u_short sel);
756 void outb(u_int port, u_char data);
757 void outl(u_int port, u_int data);
758 void outsb(u_int port, const void *addr, size_t count);
759 void outsl(u_int port, const void *addr, size_t count);
760 void outsw(u_int port, const void *addr, size_t count);
761 void outw(u_int port, u_short data);
762 u_int rcr0(void);
763 u_int rcr2(void);
764 u_int rcr3(void);
765 u_int rcr4(void);
766 uint64_t rdmsr(u_int msr);
767 uint64_t rdpmc(u_int pmc);
768 u_int rdr0(void);
769 u_int rdr1(void);
770 u_int rdr2(void);
771 u_int rdr3(void);
772 u_int rdr4(void);
773 u_int rdr5(void);
774 u_int rdr6(void);
775 u_int rdr7(void);
776 uint64_t rdtsc(void);
777 u_char read_cyrix_reg(u_char reg);
778 u_int read_eflags(void);
779 u_int rfs(void);
780 uint64_t rgdt(void);
781 u_int rgs(void);
782 uint64_t ridt(void);
783 u_short rldt(void);
784 u_short rtr(void);
785 void wbinvd(void);
786 void write_cyrix_reg(u_char reg, u_char data);
787 void write_eflags(u_int ef);
788 void wrmsr(u_int msr, uint64_t newval);
789
790 #endif /* __GNUCLIKE_ASM && __CC_SUPPORTS___INLINE */
791
792 void reset_dbregs(void);
793
794 #ifdef _KERNEL
795 int rdmsr_safe(u_int msr, uint64_t *val);
796 int wrmsr_safe(u_int msr, uint64_t newval);
797 #endif
798
799 #endif /* !_MACHINE_CPUFUNC_H_ */
800