xref: /illumos-gate/usr/src/uts/intel/ml/exception.S (revision 5d9d9091)
1/*
2 * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2013, 2014 by Delphix. All rights reserved.
4 * Copyright 2019 Joyent, Inc.
5 */
6
7/*
8 * Copyright (c) 1989, 1990 William F. Jolitz.
9 * Copyright (c) 1990 The Regents of the University of California.
10 * All rights reserved.
11 *
12 * Redistribution and use in source and binary forms, with or without
13 * modification, are permitted provided that the following conditions
14 * are met:
15 * 1. Redistributions of source code must retain the above copyright
16 *    notice, this list of conditions and the following disclaimer.
17 * 2. Redistributions in binary form must reproduce the above copyright
18 *    notice, this list of conditions and the following disclaimer in the
19 *    documentation and/or other materials provided with the distribution.
20 * 3. All advertising materials mentioning features or use of this software
21 *    must display the following acknowledgement:
22 *	This product includes software developed by the University of
23 *	California, Berkeley and its contributors.
24 * 4. Neither the name of the University nor the names of its contributors
25 *    may be used to endorse or promote products derived from this software
26 *    without specific prior written permission.
27 *
28 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
29 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
30 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
31 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
32 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
33 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
34 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
35 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
36 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
37 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
38 * SUCH DAMAGE.
39 *
40 * $FreeBSD: src/sys/amd64/amd64/exception.S,v 1.113 2003/10/15 02:04:52 peter Exp $
41 */
42
43#include <sys/asm_linkage.h>
44#include <sys/asm_misc.h>
45#include <sys/trap.h>
46#include <sys/psw.h>
47#include <sys/regset.h>
48#include <sys/privregs.h>
49#include <sys/dtrace.h>
50#include <sys/x86_archext.h>
51#include <sys/traptrace.h>
52#include <sys/machparam.h>
53
54#include "assym.h"
55
56/*
57 * push $0 on stack for traps that do not
58 * generate an error code. This is so the rest
59 * of the kernel can expect a consistent stack
60 * from from any exception.
61 *
62 * Note that for all exceptions for amd64
63 * %r11 and %rcx are on the stack. Just pop
64 * them back into their appropriate registers and let
65 * it get saved as is running native.
66 */
67
68#if defined(__xpv)
69
70#define	NPTRAP_NOERR(trapno)	\
71	pushq	$0;		\
72	pushq	$trapno
73
74#define	TRAP_NOERR(trapno)	\
75	XPV_TRAP_POP;		\
76	NPTRAP_NOERR(trapno)
77
78/*
79 * error code already pushed by hw
80 * onto stack.
81 */
82#define	TRAP_ERR(trapno)	\
83	XPV_TRAP_POP;		\
84	pushq	$trapno
85
86#else /* __xpv */
87
88#define	TRAP_NOERR(trapno)	\
89	push	$0;		\
90	push	$trapno
91
92#define	NPTRAP_NOERR(trapno) TRAP_NOERR(trapno)
93
94/*
95 * error code already pushed by hw
96 * onto stack.
97 */
98#define	TRAP_ERR(trapno)	\
99	push	$trapno
100
101#endif	/* __xpv */
102
103	/*
104	 * These are the stacks used on cpu0 for taking double faults,
105	 * NMIs and MCEs.
106	 *
107	 * We define them here instead of in a C file so that we can page-align
108	 * them (gcc won't do that in a .c file).
109	 */
110	.data
111	DGDEF3(dblfault_stack0, DEFAULTSTKSZ, MMU_PAGESIZE)
112	.fill	DEFAULTSTKSZ, 1, 0
113	DGDEF3(nmi_stack0, DEFAULTSTKSZ, MMU_PAGESIZE)
114	.fill	DEFAULTSTKSZ, 1, 0
115	DGDEF3(mce_stack0, DEFAULTSTKSZ, MMU_PAGESIZE)
116	.fill	DEFAULTSTKSZ, 1, 0
117
118	/*
119	 * #DE
120	 */
121	ENTRY_NP(div0trap)
122	TRAP_NOERR(T_ZERODIV)	/* $0 */
123	jmp	cmntrap
124	SET_SIZE(div0trap)
125
126	/*
127	 * #DB
128	 *
129	 * Fetch %dr6 and clear it, handing off the value to the
130	 * cmntrap code in %r15/%esi
131	 */
132	ENTRY_NP(dbgtrap)
133	TRAP_NOERR(T_SGLSTP)	/* $1 */
134
135#if !defined(__xpv)		/* no sysenter support yet */
136	/*
137	 * If we get here as a result of single-stepping a sysenter
138	 * instruction, we suddenly find ourselves taking a #db
139	 * in kernel mode -before- we've swapgs'ed.  So before we can
140	 * take the trap, we do the swapgs here, and fix the return
141	 * %rip in trap() so that we return immediately after the
142	 * swapgs in the sysenter handler to avoid doing the swapgs again.
143	 *
144	 * Nobody said that the design of sysenter was particularly
145	 * elegant, did they?
146	 */
147
148	pushq	%r11
149
150	/*
151	 * At this point the stack looks like this:
152	 *
153	 * (high address)	r_ss
154	 *			r_rsp
155	 *			r_rfl
156	 *			r_cs
157	 *			r_rip		<-- %rsp + 24
158	 *			r_err		<-- %rsp + 16
159	 *			r_trapno	<-- %rsp + 8
160	 * (low address)	%r11		<-- %rsp
161	 */
162	leaq	sys_sysenter(%rip), %r11
163	cmpq	%r11, 24(%rsp)	/* Compare to saved r_rip on the stack */
164	je	1f
165	leaq	brand_sys_sysenter(%rip), %r11
166	cmpq	%r11, 24(%rsp)	/* Compare to saved r_rip on the stack */
167	je	1f
168	leaq	tr_sys_sysenter(%rip), %r11
169	cmpq	%r11, 24(%rsp)
170	je	1f
171	leaq	tr_brand_sys_sysenter(%rip), %r11
172	cmpq	%r11, 24(%rsp)
173	jne	2f
1741:	swapgs
1752:	lfence /* swapgs mitigation */
176	popq	%r11
177#endif	/* !__xpv */
178
179	INTR_PUSH
180#if defined(__xpv)
181	movl	$6, %edi
182	call	kdi_dreg_get
183	movq	%rax, %r15		/* %db6 -> %r15 */
184	movl	$6, %edi
185	movl	$0, %esi
186	call	kdi_dreg_set		/* 0 -> %db6 */
187#else
188	movq	%db6, %r15
189	xorl	%eax, %eax
190	movq	%rax, %db6
191#endif
192
193	jmp	cmntrap_pushed
194	SET_SIZE(dbgtrap)
195
196#if !defined(__xpv)
197
198/*
199 * Macro to set the gsbase or kgsbase to the address of the struct cpu
200 * for this processor.  If we came from userland, set kgsbase else
201 * set gsbase.  We find the proper cpu struct by looping through
202 * the cpu structs for all processors till we find a match for the gdt
203 * of the trapping processor.  The stack is expected to be pointing at
204 * the standard regs pushed by hardware on a trap (plus error code and trapno).
205 *
206 * It's ok for us to clobber gsbase here (and possibly end up with both gsbase
207 * and kgsbase set to the same value) because we're not going back the normal
208 * way out of here (via IRET). Where we're going, we don't need no user %gs.
209 */
210#define	SET_CPU_GSBASE							\
211	subq	$REGOFF_TRAPNO, %rsp;	/* save regs */			\
212	movq	%rax, REGOFF_RAX(%rsp);					\
213	movq	%rbx, REGOFF_RBX(%rsp);					\
214	movq	%rcx, REGOFF_RCX(%rsp);					\
215	movq	%rdx, REGOFF_RDX(%rsp);					\
216	movq	%rbp, REGOFF_RBP(%rsp);					\
217	movq	%rsp, %rbp;						\
218	subq	$16, %rsp;		/* space for gdt */		\
219	sgdt	6(%rsp);						\
220	movq	8(%rsp), %rcx;		/* %rcx has gdt to match */	\
221	xorl	%ebx, %ebx;		/* loop index */		\
222	leaq	cpu(%rip), %rdx;	/* cpu pointer array */		\
2231:									\
224	movq	(%rdx, %rbx, CLONGSIZE), %rax;	/* get cpu[i] */	\
225	cmpq	$0x0, %rax;		/* cpu[i] == NULL ? */		\
226	je	2f;			/* yes, continue */		\
227	cmpq	%rcx, CPU_GDT(%rax);	/* gdt == cpu[i]->cpu_gdt ? */	\
228	je	3f;			/* yes, go set gsbase */	\
2292:									\
230	incl	%ebx;			/* i++ */			\
231	cmpl	$NCPU, %ebx;		/* i < NCPU ? */		\
232	jb	1b;			/* yes, loop */			\
233/* XXX BIG trouble if we fall thru here.  We didn't find a gdt match */	\
2343:									\
235	movl	$MSR_AMD_KGSBASE, %ecx;					\
236	cmpw	$KCS_SEL, REGOFF_CS(%rbp); /* trap from kernel? */	\
237	jne	4f;			/* no, go set KGSBASE */	\
238	movl	$MSR_AMD_GSBASE, %ecx;	/* yes, set GSBASE */		\
239        mfence;				/* OPTERON_ERRATUM_88 */	\
2404:									\
241	movq	%rax, %rdx;		/* write base register */	\
242	shrq	$32, %rdx;						\
243	wrmsr;								\
244	movq	REGOFF_RDX(%rbp), %rdx;	/* restore regs */		\
245	movq	REGOFF_RCX(%rbp), %rcx;					\
246	movq	REGOFF_RBX(%rbp), %rbx;					\
247	movq	REGOFF_RAX(%rbp), %rax;					\
248	movq	%rbp, %rsp;						\
249	movq	REGOFF_RBP(%rsp), %rbp;					\
250	addq	$REGOFF_TRAPNO, %rsp	/* pop stack */
251
252#else	/* __xpv */
253
254#define	SET_CPU_GSBASE	/* noop on the hypervisor */
255
256#endif	/* __xpv */
257
258
259	/*
260	 * #NMI
261	 *
262	 * XXPV: See 6532669.
263	 */
264	ENTRY_NP(nmiint)
265	TRAP_NOERR(T_NMIFLT)	/* $2 */
266
267	SET_CPU_GSBASE
268
269	/*
270	 * Save all registers and setup segment registers
271	 * with kernel selectors.
272	 */
273	INTR_PUSH
274	INTGATE_INIT_KERNEL_FLAGS
275
276	TRACE_PTR(%r12, %rax, %eax, %rdx, $TT_TRAP)
277	TRACE_REGS(%r12, %rsp, %rax, %rbx)
278	TRACE_STAMP(%r12)
279
280	movq	%rsp, %rbp
281
282	movq	%rbp, %rdi
283	call	av_dispatch_nmivect
284
285	INTR_POP
286	call	x86_md_clear
287	jmp	tr_iret_auto
288	/*NOTREACHED*/
289	SET_SIZE(nmiint)
290
291	/*
292	 * #BP
293	 */
294	ENTRY_NP(brktrap)
295	XPV_TRAP_POP
296	cmpw	$KCS_SEL, 8(%rsp)
297	jne	bp_user
298
299	/*
300	 * This is a breakpoint in the kernel -- it is very likely that this
301	 * is DTrace-induced.  To unify DTrace handling, we spoof this as an
302	 * invalid opcode (#UD) fault.  Note that #BP is a trap, not a fault --
303	 * we must decrement the trapping %rip to make it appear as a fault.
304	 * We then push a non-zero error code to indicate that this is coming
305	 * from #BP.
306	 */
307	decq	(%rsp)
308	push	$1			/* error code -- non-zero for #BP */
309	jmp	ud_kernel
310
311bp_user:
312
313	NPTRAP_NOERR(T_BPTFLT)	/* $3 */
314	jmp	dtrace_trap
315
316	SET_SIZE(brktrap)
317
318	/*
319	 * #OF
320	 */
321	ENTRY_NP(ovflotrap)
322	TRAP_NOERR(T_OVFLW)	/* $4 */
323	jmp	cmntrap
324	SET_SIZE(ovflotrap)
325
326	/*
327	 * #BR
328	 */
329	ENTRY_NP(boundstrap)
330	TRAP_NOERR(T_BOUNDFLT)	/* $5 */
331	jmp	cmntrap
332	SET_SIZE(boundstrap)
333
334	ENTRY_NP(invoptrap)
335
336	XPV_TRAP_POP
337
338	cmpw	$KCS_SEL, 8(%rsp)
339	jne	ud_user
340
341#if defined(__xpv)
342	movb	$0, 12(%rsp)		/* clear saved upcall_mask from %cs */
343#endif
344	push	$0			/* error code -- zero for #UD */
345ud_kernel:
346	push	$0xdddd			/* a dummy trap number */
347	INTR_PUSH
348	movq	REGOFF_RIP(%rsp), %rdi
349	movq	REGOFF_RSP(%rsp), %rsi
350	movq	REGOFF_RAX(%rsp), %rdx
351	pushq	(%rsi)
352	movq	%rsp, %rsi
353	subq	$8, %rsp
354	call	dtrace_invop
355	ALTENTRY(dtrace_invop_callsite)
356	addq	$16, %rsp
357	cmpl	$DTRACE_INVOP_PUSHL_EBP, %eax
358	je	ud_push
359	cmpl	$DTRACE_INVOP_LEAVE, %eax
360	je	ud_leave
361	cmpl	$DTRACE_INVOP_NOP, %eax
362	je	ud_nop
363	cmpl	$DTRACE_INVOP_RET, %eax
364	je	ud_ret
365	jmp	ud_trap
366
367ud_push:
368	/*
369	 * We must emulate a "pushq %rbp".  To do this, we pull the stack
370	 * down 8 bytes, and then store the base pointer.
371	 */
372	INTR_POP
373	subq	$16, %rsp		/* make room for %rbp */
374	pushq	%rax			/* push temp */
375	movq	24(%rsp), %rax		/* load calling RIP */
376	addq	$1, %rax		/* increment over trapping instr */
377	movq	%rax, 8(%rsp)		/* store calling RIP */
378	movq	32(%rsp), %rax		/* load calling CS */
379	movq	%rax, 16(%rsp)		/* store calling CS */
380	movq	40(%rsp), %rax		/* load calling RFLAGS */
381	movq	%rax, 24(%rsp)		/* store calling RFLAGS */
382	movq	48(%rsp), %rax		/* load calling RSP */
383	subq	$8, %rax		/* make room for %rbp */
384	movq	%rax, 32(%rsp)		/* store calling RSP */
385	movq	56(%rsp), %rax		/* load calling SS */
386	movq	%rax, 40(%rsp)		/* store calling SS */
387	movq	32(%rsp), %rax		/* reload calling RSP */
388	movq	%rbp, (%rax)		/* store %rbp there */
389	popq	%rax			/* pop off temp */
390	jmp	tr_iret_kernel		/* return from interrupt */
391	/*NOTREACHED*/
392
393ud_leave:
394	/*
395	 * We must emulate a "leave", which is the same as a "movq %rbp,
396	 * %rsp" followed by a "popq %rbp".  We can exploit the fact
397	 * that the %rsp is explicitly saved to effect the pop without
398	 * having to reshuffle the other data pushed for the trap.
399	 */
400
401	INTR_POP
402	pushq	%rax			/* push temp */
403	movq	8(%rsp), %rax		/* load calling RIP */
404	addq	$1, %rax		/* increment over trapping instr */
405	movq	%rax, 8(%rsp)		/* store calling RIP */
406	movq	(%rbp), %rax		/* get new %rbp */
407	addq	$8, %rbp		/* adjust new %rsp */
408	movq	%rbp, 32(%rsp)		/* store new %rsp */
409	movq	%rax, %rbp		/* set new %rbp */
410	popq	%rax			/* pop off temp */
411	jmp	tr_iret_kernel		/* return from interrupt */
412	/*NOTREACHED*/
413
414ud_nop:
415	/*
416	 * We must emulate a "nop".  This is obviously not hard:  we need only
417	 * advance the %rip by one.
418	 */
419	INTR_POP
420	incq	(%rsp)
421	jmp	tr_iret_kernel
422	/*NOTREACHED*/
423
424ud_ret:
425	INTR_POP
426	pushq	%rax			/* push temp */
427	movq	32(%rsp), %rax		/* load %rsp */
428	movq	(%rax), %rax		/* load calling RIP */
429	movq	%rax, 8(%rsp)		/* store calling RIP */
430	addq	$8, 32(%rsp)		/* adjust new %rsp */
431	popq	%rax			/* pop off temp */
432	jmp	tr_iret_kernel		/* return from interrupt */
433	/*NOTREACHED*/
434
435ud_trap:
436	/*
437	 * We're going to let the kernel handle this as a normal #UD.  If,
438	 * however, we came through #BP and are spoofing #UD (in this case,
439	 * the stored error value will be non-zero), we need to de-spoof
440	 * the trap by incrementing %rip and pushing T_BPTFLT.
441	 */
442	cmpq	$0, REGOFF_ERR(%rsp)
443	je	ud_ud
444	incq	REGOFF_RIP(%rsp)
445	addq	$REGOFF_RIP, %rsp
446	NPTRAP_NOERR(T_BPTFLT)	/* $3 */
447	jmp	cmntrap
448
449ud_ud:
450	addq	$REGOFF_RIP, %rsp
451ud_user:
452	NPTRAP_NOERR(T_ILLINST)
453	jmp	cmntrap
454	SET_SIZE(invoptrap)
455
456	/*
457	 * #NM
458	 */
459
460	ENTRY_NP(ndptrap)
461	TRAP_NOERR(T_NOEXTFLT)	/* $0 */
462	SET_CPU_GSBASE
463	jmp	cmntrap
464	SET_SIZE(ndptrap)
465
466#if !defined(__xpv)
467
468	/*
469	 * #DF
470	 */
471	ENTRY_NP(syserrtrap)
472	pushq	$T_DBLFLT
473	SET_CPU_GSBASE
474
475	/*
476	 * We share this handler with kmdb (if kmdb is loaded).  As such, we
477	 * may have reached this point after encountering a #df in kmdb.  If
478	 * that happens, we'll still be on kmdb's IDT.  We need to switch back
479	 * to this CPU's IDT before proceeding.  Furthermore, if we did arrive
480	 * here from kmdb, kmdb is probably in a very sickly state, and
481	 * shouldn't be entered from the panic flow.  We'll suppress that
482	 * entry by setting nopanicdebug.
483	 */
484	pushq	%rax
485	subq	$DESCTBR_SIZE, %rsp
486	sidt	(%rsp)
487	movq	%gs:CPU_IDT, %rax
488	cmpq	%rax, DTR_BASE(%rsp)
489	je	1f
490
491	movq	%rax, DTR_BASE(%rsp)
492	movw	$_MUL(NIDT, GATE_DESC_SIZE), DTR_LIMIT(%rsp)
493	lidt	(%rsp)
494
495	movl	$1, nopanicdebug
496
4971:	addq	$DESCTBR_SIZE, %rsp
498	popq	%rax
499
500	DFTRAP_PUSH
501
502	/*
503	 * freeze trap trace.
504	 */
505#ifdef TRAPTRACE
506	leaq	trap_trace_freeze(%rip), %r11
507	incl	(%r11)
508#endif
509
510	ENABLE_INTR_FLAGS
511
512	movq	%rsp, %rdi	/* &regs */
513	xorl	%esi, %esi	/* clear address */
514	xorl	%edx, %edx	/* cpuid = 0 */
515	call	trap
516
517	SET_SIZE(syserrtrap)
518
519#endif	/* !__xpv */
520
521	/*
522	 * #TS
523	 */
524	ENTRY_NP(invtsstrap)
525	TRAP_ERR(T_TSSFLT)	/* $10 already have error code on stack */
526	jmp	cmntrap
527	SET_SIZE(invtsstrap)
528
529	/*
530	 * #NP
531	 */
532	ENTRY_NP(segnptrap)
533	TRAP_ERR(T_SEGFLT)	/* $11 already have error code on stack */
534	SET_CPU_GSBASE
535	jmp	cmntrap
536	SET_SIZE(segnptrap)
537
538	/*
539	 * #SS
540	 */
541	ENTRY_NP(stktrap)
542	TRAP_ERR(T_STKFLT)	/* $12 already have error code on stack */
543	SET_CPU_GSBASE
544	jmp	cmntrap
545	SET_SIZE(stktrap)
546
547	/*
548	 * #GP
549	 */
550	ENTRY_NP(gptrap)
551	TRAP_ERR(T_GPFLT)	/* $13 already have error code on stack */
552	SET_CPU_GSBASE
553	jmp	cmntrap
554	SET_SIZE(gptrap)
555
556	/*
557	 * #PF
558	 */
559	ENTRY_NP(pftrap)
560	TRAP_ERR(T_PGFLT)	/* $14 already have error code on stack */
561	INTR_PUSH
562#if defined(__xpv)
563
564	movq	%gs:CPU_VCPU_INFO, %r15
565	movq	VCPU_INFO_ARCH_CR2(%r15), %r15	/* vcpu[].arch.cr2 */
566
567#else	/* __xpv */
568
569	movq	%cr2, %r15
570
571#endif	/* __xpv */
572	jmp	cmntrap_pushed
573	SET_SIZE(pftrap)
574
575	ENTRY_NP(resvtrap)
576	TRAP_NOERR(T_RESVTRAP)	/* (reserved)  */
577	jmp	cmntrap
578	SET_SIZE(resvtrap)
579
580	/*
581	 * #MF
582	 */
583	ENTRY_NP(ndperr)
584	TRAP_NOERR(T_EXTERRFLT)	/* $16 */
585	jmp	cmninttrap
586	SET_SIZE(ndperr)
587
588	/*
589	 * #AC
590	 */
591	ENTRY_NP(achktrap)
592	TRAP_ERR(T_ALIGNMENT)	/* $17 */
593	jmp	cmntrap
594	SET_SIZE(achktrap)
595
596	/*
597	 * #MC
598	 */
599	.globl	cmi_mca_trap	/* see uts/i86pc/os/cmi.c */
600
601	ENTRY_NP(mcetrap)
602	TRAP_NOERR(T_MCE)	/* $18 */
603
604	SET_CPU_GSBASE
605
606	INTR_PUSH
607	INTGATE_INIT_KERNEL_FLAGS
608
609	TRACE_PTR(%rdi, %rbx, %ebx, %rcx, $TT_TRAP)
610	TRACE_REGS(%rdi, %rsp, %rbx, %rcx)
611	TRACE_STAMP(%rdi)
612
613	movq	%rsp, %rbp
614
615	movq	%rsp, %rdi	/* arg0 = struct regs *rp */
616	call	cmi_mca_trap	/* cmi_mca_trap(rp); */
617
618	jmp	_sys_rtt
619	SET_SIZE(mcetrap)
620
621	/*
622	 * #XF
623	 */
624	ENTRY_NP(xmtrap)
625	TRAP_NOERR(T_SIMDFPE)	/* $19 */
626	jmp	cmninttrap
627	SET_SIZE(xmtrap)
628
629	ENTRY_NP(invaltrap)
630	TRAP_NOERR(T_INVALTRAP)	/* very invalid */
631	jmp	cmntrap
632	SET_SIZE(invaltrap)
633
634	.globl	fasttable
635
636	ENTRY_NP(fasttrap)
637	cmpl	$T_LASTFAST, %eax
638	ja	1f
639	orl	%eax, %eax	/* (zero extend top 32-bits) */
640	leaq	fasttable(%rip), %r11
641	leaq	(%r11, %rax, CLONGSIZE), %r11
642	movq	(%r11), %r11
643	INDIRECT_JMP_REG(r11)
6441:
645	/*
646	 * Fast syscall number was illegal.  Make it look
647	 * as if the INT failed.  Modify %rip to point before the
648	 * INT, push the expected error code and fake a GP fault.
649	 *
650	 * XXX Why make the error code be offset into idt + 1?
651	 * Instead we should push a real (soft?) error code
652	 * on the stack and #gp handler could know about fasttraps?
653	 */
654	XPV_TRAP_POP
655
656	subq	$2, (%rsp)	/* XXX int insn 2-bytes */
657	pushq	$_CONST(_MUL(T_FASTTRAP, GATE_DESC_SIZE) + 2)
658
659#if defined(__xpv)
660	pushq	%r11
661	pushq	%rcx
662#endif
663	jmp	gptrap
664	SET_SIZE(fasttrap)
665
666	ENTRY_NP(dtrace_ret)
667	TRAP_NOERR(T_DTRACE_RET)
668	jmp	dtrace_trap
669	SET_SIZE(dtrace_ret)
670
671	/*
672	 * RFLAGS 24 bytes up the stack from %rsp.
673	 * XXX a constant would be nicer.
674	 */
675	ENTRY_NP(fast_null)
676	XPV_TRAP_POP
677	orq	$PS_C, 24(%rsp)	/* set carry bit in user flags */
678	call	x86_md_clear
679	jmp	tr_iret_auto
680	/*NOTREACHED*/
681	SET_SIZE(fast_null)
682
683	/*
684	 * Interrupts start at 32
685	 */
686#define MKIVCT(n)			\
687	ENTRY_NP(ivct##n)		\
688	push	$0;			\
689	push	$n - 0x20;		\
690	jmp	cmnint;			\
691	SET_SIZE(ivct##n)
692
693	MKIVCT(32)
694	MKIVCT(33)
695	MKIVCT(34)
696	MKIVCT(35)
697	MKIVCT(36)
698	MKIVCT(37)
699	MKIVCT(38)
700	MKIVCT(39)
701	MKIVCT(40)
702	MKIVCT(41)
703	MKIVCT(42)
704	MKIVCT(43)
705	MKIVCT(44)
706	MKIVCT(45)
707	MKIVCT(46)
708	MKIVCT(47)
709	MKIVCT(48)
710	MKIVCT(49)
711	MKIVCT(50)
712	MKIVCT(51)
713	MKIVCT(52)
714	MKIVCT(53)
715	MKIVCT(54)
716	MKIVCT(55)
717	MKIVCT(56)
718	MKIVCT(57)
719	MKIVCT(58)
720	MKIVCT(59)
721	MKIVCT(60)
722	MKIVCT(61)
723	MKIVCT(62)
724	MKIVCT(63)
725	MKIVCT(64)
726	MKIVCT(65)
727	MKIVCT(66)
728	MKIVCT(67)
729	MKIVCT(68)
730	MKIVCT(69)
731	MKIVCT(70)
732	MKIVCT(71)
733	MKIVCT(72)
734	MKIVCT(73)
735	MKIVCT(74)
736	MKIVCT(75)
737	MKIVCT(76)
738	MKIVCT(77)
739	MKIVCT(78)
740	MKIVCT(79)
741	MKIVCT(80)
742	MKIVCT(81)
743	MKIVCT(82)
744	MKIVCT(83)
745	MKIVCT(84)
746	MKIVCT(85)
747	MKIVCT(86)
748	MKIVCT(87)
749	MKIVCT(88)
750	MKIVCT(89)
751	MKIVCT(90)
752	MKIVCT(91)
753	MKIVCT(92)
754	MKIVCT(93)
755	MKIVCT(94)
756	MKIVCT(95)
757	MKIVCT(96)
758	MKIVCT(97)
759	MKIVCT(98)
760	MKIVCT(99)
761	MKIVCT(100)
762	MKIVCT(101)
763	MKIVCT(102)
764	MKIVCT(103)
765	MKIVCT(104)
766	MKIVCT(105)
767	MKIVCT(106)
768	MKIVCT(107)
769	MKIVCT(108)
770	MKIVCT(109)
771	MKIVCT(110)
772	MKIVCT(111)
773	MKIVCT(112)
774	MKIVCT(113)
775	MKIVCT(114)
776	MKIVCT(115)
777	MKIVCT(116)
778	MKIVCT(117)
779	MKIVCT(118)
780	MKIVCT(119)
781	MKIVCT(120)
782	MKIVCT(121)
783	MKIVCT(122)
784	MKIVCT(123)
785	MKIVCT(124)
786	MKIVCT(125)
787	MKIVCT(126)
788	MKIVCT(127)
789	MKIVCT(128)
790	MKIVCT(129)
791	MKIVCT(130)
792	MKIVCT(131)
793	MKIVCT(132)
794	MKIVCT(133)
795	MKIVCT(134)
796	MKIVCT(135)
797	MKIVCT(136)
798	MKIVCT(137)
799	MKIVCT(138)
800	MKIVCT(139)
801	MKIVCT(140)
802	MKIVCT(141)
803	MKIVCT(142)
804	MKIVCT(143)
805	MKIVCT(144)
806	MKIVCT(145)
807	MKIVCT(146)
808	MKIVCT(147)
809	MKIVCT(148)
810	MKIVCT(149)
811	MKIVCT(150)
812	MKIVCT(151)
813	MKIVCT(152)
814	MKIVCT(153)
815	MKIVCT(154)
816	MKIVCT(155)
817	MKIVCT(156)
818	MKIVCT(157)
819	MKIVCT(158)
820	MKIVCT(159)
821	MKIVCT(160)
822	MKIVCT(161)
823	MKIVCT(162)
824	MKIVCT(163)
825	MKIVCT(164)
826	MKIVCT(165)
827	MKIVCT(166)
828	MKIVCT(167)
829	MKIVCT(168)
830	MKIVCT(169)
831	MKIVCT(170)
832	MKIVCT(171)
833	MKIVCT(172)
834	MKIVCT(173)
835	MKIVCT(174)
836	MKIVCT(175)
837	MKIVCT(176)
838	MKIVCT(177)
839	MKIVCT(178)
840	MKIVCT(179)
841	MKIVCT(180)
842	MKIVCT(181)
843	MKIVCT(182)
844	MKIVCT(183)
845	MKIVCT(184)
846	MKIVCT(185)
847	MKIVCT(186)
848	MKIVCT(187)
849	MKIVCT(188)
850	MKIVCT(189)
851	MKIVCT(190)
852	MKIVCT(191)
853	MKIVCT(192)
854	MKIVCT(193)
855	MKIVCT(194)
856	MKIVCT(195)
857	MKIVCT(196)
858	MKIVCT(197)
859	MKIVCT(198)
860	MKIVCT(199)
861	MKIVCT(200)
862	MKIVCT(201)
863	MKIVCT(202)
864	MKIVCT(203)
865	MKIVCT(204)
866	MKIVCT(205)
867	MKIVCT(206)
868	MKIVCT(207)
869	MKIVCT(208)
870	MKIVCT(209)
871	MKIVCT(210)
872	MKIVCT(211)
873	MKIVCT(212)
874	MKIVCT(213)
875	MKIVCT(214)
876	MKIVCT(215)
877	MKIVCT(216)
878	MKIVCT(217)
879	MKIVCT(218)
880	MKIVCT(219)
881	MKIVCT(220)
882	MKIVCT(221)
883	MKIVCT(222)
884	MKIVCT(223)
885	MKIVCT(224)
886	MKIVCT(225)
887	MKIVCT(226)
888	MKIVCT(227)
889	MKIVCT(228)
890	MKIVCT(229)
891	MKIVCT(230)
892	MKIVCT(231)
893	MKIVCT(232)
894	MKIVCT(233)
895	MKIVCT(234)
896	MKIVCT(235)
897	MKIVCT(236)
898	MKIVCT(237)
899	MKIVCT(238)
900	MKIVCT(239)
901	MKIVCT(240)
902	MKIVCT(241)
903	MKIVCT(242)
904	MKIVCT(243)
905	MKIVCT(244)
906	MKIVCT(245)
907	MKIVCT(246)
908	MKIVCT(247)
909	MKIVCT(248)
910	MKIVCT(249)
911	MKIVCT(250)
912	MKIVCT(251)
913	MKIVCT(252)
914	MKIVCT(253)
915	MKIVCT(254)
916	MKIVCT(255)
917