17c478bdstevel@tonic-gate/*
27af88acKuriakose Kuruvilla * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
38e50396Adam H. Leventhal * Copyright (c) 2013, 2014 by Delphix. All rights reserved.
4a9cc46cRobert Mustacchi * Copyright 2019 Joyent, Inc.
57c478bdstevel@tonic-gate */
67c478bdstevel@tonic-gate
77c478bdstevel@tonic-gate/*
87c478bdstevel@tonic-gate * Copyright (c) 1989, 1990 William F. Jolitz.
97c478bdstevel@tonic-gate * Copyright (c) 1990 The Regents of the University of California.
107c478bdstevel@tonic-gate * All rights reserved.
117c478bdstevel@tonic-gate *
127c478bdstevel@tonic-gate * Redistribution and use in source and binary forms, with or without
137c478bdstevel@tonic-gate * modification, are permitted provided that the following conditions
147c478bdstevel@tonic-gate * are met:
157c478bdstevel@tonic-gate * 1. Redistributions of source code must retain the above copyright
167c478bdstevel@tonic-gate *    notice, this list of conditions and the following disclaimer.
177c478bdstevel@tonic-gate * 2. Redistributions in binary form must reproduce the above copyright
187c478bdstevel@tonic-gate *    notice, this list of conditions and the following disclaimer in the
197c478bdstevel@tonic-gate *    documentation and/or other materials provided with the distribution.
207c478bdstevel@tonic-gate * 3. All advertising materials mentioning features or use of this software
217c478bdstevel@tonic-gate *    must display the following acknowledgement:
227c478bdstevel@tonic-gate *	This product includes software developed by the University of
237c478bdstevel@tonic-gate *	California, Berkeley and its contributors.
247c478bdstevel@tonic-gate * 4. Neither the name of the University nor the names of its contributors
257c478bdstevel@tonic-gate *    may be used to endorse or promote products derived from this software
267c478bdstevel@tonic-gate *    without specific prior written permission.
277c478bdstevel@tonic-gate *
287c478bdstevel@tonic-gate * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
297c478bdstevel@tonic-gate * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
307c478bdstevel@tonic-gate * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
317c478bdstevel@tonic-gate * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
327c478bdstevel@tonic-gate * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
337c478bdstevel@tonic-gate * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
347c478bdstevel@tonic-gate * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
357c478bdstevel@tonic-gate * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
367c478bdstevel@tonic-gate * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
377c478bdstevel@tonic-gate * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
387c478bdstevel@tonic-gate * SUCH DAMAGE.
397c478bdstevel@tonic-gate *
407c478bdstevel@tonic-gate * $FreeBSD: src/sys/amd64/amd64/exception.S,v 1.113 2003/10/15 02:04:52 peter Exp $
417c478bdstevel@tonic-gate */
427c478bdstevel@tonic-gate
437c478bdstevel@tonic-gate#include <sys/asm_linkage.h>
447c478bdstevel@tonic-gate#include <sys/asm_misc.h>
457c478bdstevel@tonic-gate#include <sys/trap.h>
467c478bdstevel@tonic-gate#include <sys/psw.h>
477c478bdstevel@tonic-gate#include <sys/regset.h>
487c478bdstevel@tonic-gate#include <sys/privregs.h>
497c478bdstevel@tonic-gate#include <sys/dtrace.h>
50ae115bcmrj#include <sys/x86_archext.h>
517c478bdstevel@tonic-gate#include <sys/traptrace.h>
524179143andrei#include <sys/machparam.h>
537c478bdstevel@tonic-gate
547c478bdstevel@tonic-gate#include "assym.h"
557c478bdstevel@tonic-gate
567c478bdstevel@tonic-gate/*
577c478bdstevel@tonic-gate * push $0 on stack for traps that do not
587c478bdstevel@tonic-gate * generate an error code. This is so the rest
597c478bdstevel@tonic-gate * of the kernel can expect a consistent stack
607c478bdstevel@tonic-gate * from from any exception.
61843e198johnlev *
62843e198johnlev * Note that for all exceptions for amd64
63843e198johnlev * %r11 and %rcx are on the stack. Just pop
64843e198johnlev * them back into their appropriate registers and let
65843e198johnlev * it get saved as is running native.
66843e198johnlev */
67843e198johnlev
689b0bb79John Levon#if defined(__xpv)
69843e198johnlev
70843e198johnlev#define	NPTRAP_NOERR(trapno)	\
71843e198johnlev	pushq	$0;		\
7274ecdb5John Levon	pushq	$trapno
73843e198johnlev
74843e198johnlev#define	TRAP_NOERR(trapno)	\
75843e198johnlev	XPV_TRAP_POP;		\
76843e198johnlev	NPTRAP_NOERR(trapno)
77843e198johnlev
78843e198johnlev/*
79843e198johnlev * error code already pushed by hw
80843e198johnlev * onto stack.
817c478bdstevel@tonic-gate */
82843e198johnlev#define	TRAP_ERR(trapno)	\
83843e198johnlev	XPV_TRAP_POP;		\
8474ecdb5John Levon	pushq	$trapno
85843e198johnlev
869b0bb79John Levon#else /* __xpv */
87ae115bcmrj
887c478bdstevel@tonic-gate#define	TRAP_NOERR(trapno)	\
897c478bdstevel@tonic-gate	push	$0;		\
9074ecdb5John Levon	push	$trapno
917c478bdstevel@tonic-gate
92ae115bcmrj#define	NPTRAP_NOERR(trapno) TRAP_NOERR(trapno)
93ae115bcmrj
947c478bdstevel@tonic-gate/*
957c478bdstevel@tonic-gate * error code already pushed by hw
967c478bdstevel@tonic-gate * onto stack.
977c478bdstevel@tonic-gate */
987c478bdstevel@tonic-gate#define	TRAP_ERR(trapno)	\
9974ecdb5John Levon	push	$trapno
1007c478bdstevel@tonic-gate
1019b0bb79John Levon#endif	/* __xpv */
102843e198johnlev
10374ecdb5John Levon	/*
10474ecdb5John Levon	 * These are the stacks used on cpu0 for taking double faults,
1059b0bb79John Levon	 * NMIs and MCEs.
10674ecdb5John Levon	 *
10774ecdb5John Levon	 * We define them here instead of in a C file so that we can page-align
10874ecdb5John Levon	 * them (gcc won't do that in a .c file).
10974ecdb5John Levon	 */
11074ecdb5John Levon	.data
11174ecdb5John Levon	DGDEF3(dblfault_stack0, DEFAULTSTKSZ, MMU_PAGESIZE)
11274ecdb5John Levon	.fill	DEFAULTSTKSZ, 1, 0
11374ecdb5John Levon	DGDEF3(nmi_stack0, DEFAULTSTKSZ, MMU_PAGESIZE)
11474ecdb5John Levon	.fill	DEFAULTSTKSZ, 1, 0
11574ecdb5John Levon	DGDEF3(mce_stack0, DEFAULTSTKSZ, MMU_PAGESIZE)
11674ecdb5John Levon	.fill	DEFAULTSTKSZ, 1, 0
117ae115bcmrj
1187c478bdstevel@tonic-gate	/*
1197c478bdstevel@tonic-gate	 * #DE
1207c478bdstevel@tonic-gate	 */
1217c478bdstevel@tonic-gate	ENTRY_NP(div0trap)
1227c478bdstevel@tonic-gate	TRAP_NOERR(T_ZERODIV)	/* $0 */
1237c478bdstevel@tonic-gate	jmp	cmntrap
1247c478bdstevel@tonic-gate	SET_SIZE(div0trap)
1257c478bdstevel@tonic-gate
1267c478bdstevel@tonic-gate	/*
1277c478bdstevel@tonic-gate	 * #DB
1287c478bdstevel@tonic-gate	 *
129ae115bcmrj	 * Fetch %dr6 and clear it, handing off the value to the
130ae115bcmrj	 * cmntrap code in %r15/%esi
131ae115bcmrj	 */
132ae115bcmrj	ENTRY_NP(dbgtrap)
133ae115bcmrj	TRAP_NOERR(T_SGLSTP)	/* $1 */
134ae115bcmrj
135843e198johnlev#if !defined(__xpv)		/* no sysenter support yet */
136ae115bcmrj	/*
1377c478bdstevel@tonic-gate	 * If we get here as a result of single-stepping a sysenter
1387c478bdstevel@tonic-gate	 * instruction, we suddenly find ourselves taking a #db
1397c478bdstevel@tonic-gate	 * in kernel mode -before- we've swapgs'ed.  So before we can
1407c478bdstevel@tonic-gate	 * take the trap, we do the swapgs here, and fix the return
1417c478bdstevel@tonic-gate	 * %rip in trap() so that we return immediately after the
1427c478bdstevel@tonic-gate	 * swapgs in the sysenter handler to avoid doing the swapgs again.
1437c478bdstevel@tonic-gate	 *
1447c478bdstevel@tonic-gate	 * Nobody said that the design of sysenter was particularly
1457c478bdstevel@tonic-gate	 * elegant, did they?
1467c478bdstevel@tonic-gate	 */
14705456efsherrym
1487c478bdstevel@tonic-gate	pushq	%r11
14905456efsherrym
15005456efsherrym	/*
15105456efsherrym	 * At this point the stack looks like this:
15205456efsherrym	 *
15365f2042Robert Mustacchi	 * (high address)	r_ss
15405456efsherrym	 *			r_rsp
15505456efsherrym	 *			r_rfl
15605456efsherrym	 *			r_cs
15705456efsherrym	 *			r_rip		<-- %rsp + 24
15805456efsherrym	 *			r_err		<-- %rsp + 16
15905456efsherrym	 *			r_trapno	<-- %rsp + 8
16005456efsherrym	 * (low address)	%r11		<-- %rsp
16105456efsherrym	 */
1627c478bdstevel@tonic-gate	leaq	sys_sysenter(%rip), %r11
16305456efsherrym	cmpq	%r11, 24(%rsp)	/* Compare to saved r_rip on the stack */
1640022570jv	je	1f
1650022570jv	leaq	brand_sys_sysenter(%rip), %r11
1660022570jv	cmpq	%r11, 24(%rsp)	/* Compare to saved r_rip on the stack */
16774ecdb5John Levon	je	1f
16874ecdb5John Levon	leaq	tr_sys_sysenter(%rip), %r11
16974ecdb5John Levon	cmpq	%r11, 24(%rsp)
17074ecdb5John Levon	je	1f
17174ecdb5John Levon	leaq	tr_brand_sys_sysenter(%rip), %r11
17274ecdb5John Levon	cmpq	%r11, 24(%rsp)
1730022570jv	jne	2f
17442cd193John Levon1:	swapgs
17542cd193John Levon2:	lfence /* swapgs mitigation */
17642cd193John Levon	popq	%r11
177843e198johnlev#endif	/* !__xpv */
178ae115bcmrj
179ae115bcmrj	INTR_PUSH
180843e198johnlev#if defined(__xpv)
181843e198johnlev	movl	$6, %edi
182843e198johnlev	call	kdi_dreg_get
183843e198johnlev	movq	%rax, %r15		/* %db6 -> %r15 */
184843e198johnlev	movl	$6, %edi
185843e198johnlev	movl	$0, %esi
186843e198johnlev	call	kdi_dreg_set		/* 0 -> %db6 */
187843e198johnlev#else
188