xref: /illumos-gate/usr/src/uts/intel/ml/i86_subr.s (revision f0089e39)
17c478bd9Sstevel@tonic-gate/*
27c478bd9Sstevel@tonic-gate * CDDL HEADER START
37c478bd9Sstevel@tonic-gate *
47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the
5ee88d2b9Skchow * Common Development and Distribution License (the "License").
6ee88d2b9Skchow * You may not use this file except in compliance with the License.
77c478bd9Sstevel@tonic-gate *
87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing.
107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions
117c478bd9Sstevel@tonic-gate * and limitations under the License.
127c478bd9Sstevel@tonic-gate *
137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each
147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the
167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying
177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bd9Sstevel@tonic-gate *
197c478bd9Sstevel@tonic-gate * CDDL HEADER END
207c478bd9Sstevel@tonic-gate */
21ae115bc7Smrj
227c478bd9Sstevel@tonic-gate/*
237417cfdeSKuriakose Kuruvilla * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
24549e8c97SMarcel Telka * Copyright 2014 Nexenta Systems, Inc.  All rights reserved.
25bf16b11eSMatthew Ahrens * Copyright (c) 2014 by Delphix. All rights reserved.
26a9cc46cfSRobert Mustacchi * Copyright 2019 Joyent, Inc.
277c478bd9Sstevel@tonic-gate */
287c478bd9Sstevel@tonic-gate
297c478bd9Sstevel@tonic-gate/*
307c478bd9Sstevel@tonic-gate *  Copyright (c) 1990, 1991 UNIX System Laboratories, Inc.
317c478bd9Sstevel@tonic-gate *  Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T
327c478bd9Sstevel@tonic-gate *    All Rights Reserved
337c478bd9Sstevel@tonic-gate */
347c478bd9Sstevel@tonic-gate
357af88ac7SKuriakose Kuruvilla/*
367af88ac7SKuriakose Kuruvilla * Copyright (c) 2009, Intel Corporation.
377af88ac7SKuriakose Kuruvilla * All rights reserved.
387af88ac7SKuriakose Kuruvilla */
397af88ac7SKuriakose Kuruvilla
407c478bd9Sstevel@tonic-gate/*
417c478bd9Sstevel@tonic-gate * General assembly language routines.
427c478bd9Sstevel@tonic-gate * It is the intent of this file to contain routines that are
437c478bd9Sstevel@tonic-gate * independent of the specific kernel architecture, and those that are
447c478bd9Sstevel@tonic-gate * common across kernel architectures.
457c478bd9Sstevel@tonic-gate * As architectures diverge, and implementations of specific
467c478bd9Sstevel@tonic-gate * architecture-dependent routines change, the routines should be moved
477c478bd9Sstevel@tonic-gate * from this file into the respective ../`arch -k`/subr.s file.
487c478bd9Sstevel@tonic-gate */
497c478bd9Sstevel@tonic-gate
507c478bd9Sstevel@tonic-gate#include <sys/asm_linkage.h>
517c478bd9Sstevel@tonic-gate#include <sys/asm_misc.h>
527c478bd9Sstevel@tonic-gate#include <sys/panic.h>
537c478bd9Sstevel@tonic-gate#include <sys/ontrap.h>
547c478bd9Sstevel@tonic-gate#include <sys/regset.h>
557c478bd9Sstevel@tonic-gate#include <sys/privregs.h>
567c478bd9Sstevel@tonic-gate#include <sys/reboot.h>
577c478bd9Sstevel@tonic-gate#include <sys/psw.h>
587c478bd9Sstevel@tonic-gate#include <sys/x86_archext.h>
597c478bd9Sstevel@tonic-gate
607c478bd9Sstevel@tonic-gate#include "assym.h"
617c478bd9Sstevel@tonic-gate#include <sys/dditypes.h>
627c478bd9Sstevel@tonic-gate
637c478bd9Sstevel@tonic-gate/*
647c478bd9Sstevel@tonic-gate * on_fault()
653ce2fcdcSRobert Mustacchi *
667c478bd9Sstevel@tonic-gate * Catch lofault faults. Like setjmp except it returns one
677c478bd9Sstevel@tonic-gate * if code following causes uncorrectable fault. Turned off
683ce2fcdcSRobert Mustacchi * by calling no_fault(). Note that while under on_fault(),
693ce2fcdcSRobert Mustacchi * SMAP is disabled. For more information see
70*f0089e39SRichard Lowe * uts/intel/ml/copy.s.
717c478bd9Sstevel@tonic-gate */
727c478bd9Sstevel@tonic-gate
737c478bd9Sstevel@tonic-gate	ENTRY(on_fault)
747c478bd9Sstevel@tonic-gate	movq	%gs:CPU_THREAD, %rsi
757c478bd9Sstevel@tonic-gate	leaq	catch_fault(%rip), %rdx
767c478bd9Sstevel@tonic-gate	movq	%rdi, T_ONFAULT(%rsi)		/* jumpbuf in t_onfault */
777c478bd9Sstevel@tonic-gate	movq	%rdx, T_LOFAULT(%rsi)		/* catch_fault in t_lofault */
783ce2fcdcSRobert Mustacchi	call	smap_disable			/* allow user accesses */
797c478bd9Sstevel@tonic-gate	jmp	setjmp				/* let setjmp do the rest */
807c478bd9Sstevel@tonic-gate
817c478bd9Sstevel@tonic-gatecatch_fault:
827c478bd9Sstevel@tonic-gate	movq	%gs:CPU_THREAD, %rsi
837c478bd9Sstevel@tonic-gate	movq	T_ONFAULT(%rsi), %rdi		/* address of save area */
847c478bd9Sstevel@tonic-gate	xorl	%eax, %eax
857c478bd9Sstevel@tonic-gate	movq	%rax, T_ONFAULT(%rsi)		/* turn off onfault */
867c478bd9Sstevel@tonic-gate	movq	%rax, T_LOFAULT(%rsi)		/* turn off lofault */
873ce2fcdcSRobert Mustacchi	call	smap_enable			/* disallow user accesses */
887c478bd9Sstevel@tonic-gate	jmp	longjmp				/* let longjmp do the rest */
897c478bd9Sstevel@tonic-gate	SET_SIZE(on_fault)
907c478bd9Sstevel@tonic-gate
917c478bd9Sstevel@tonic-gate	ENTRY(no_fault)
927c478bd9Sstevel@tonic-gate	movq	%gs:CPU_THREAD, %rsi
937c478bd9Sstevel@tonic-gate	xorl	%eax, %eax
947c478bd9Sstevel@tonic-gate	movq	%rax, T_ONFAULT(%rsi)		/* turn off onfault */
957c478bd9Sstevel@tonic-gate	movq	%rax, T_LOFAULT(%rsi)		/* turn off lofault */
963ce2fcdcSRobert Mustacchi	call	smap_enable			/* disallow user accesses */
977c478bd9Sstevel@tonic-gate	ret
987c478bd9Sstevel@tonic-gate	SET_SIZE(no_fault)
997c478bd9Sstevel@tonic-gate
1007c478bd9Sstevel@tonic-gate/*
1017c478bd9Sstevel@tonic-gate * Default trampoline code for on_trap() (see <sys/ontrap.h>).  We just
1027c478bd9Sstevel@tonic-gate * do a longjmp(&curthread->t_ontrap->ot_jmpbuf) if this is ever called.
1037c478bd9Sstevel@tonic-gate */
1047c478bd9Sstevel@tonic-gate
1057c478bd9Sstevel@tonic-gate	ENTRY(on_trap_trampoline)
1067c478bd9Sstevel@tonic-gate	movq	%gs:CPU_THREAD, %rsi
1077c478bd9Sstevel@tonic-gate	movq	T_ONTRAP(%rsi), %rdi
1087c478bd9Sstevel@tonic-gate	addq	$OT_JMPBUF, %rdi
1097c478bd9Sstevel@tonic-gate	jmp	longjmp
1107c478bd9Sstevel@tonic-gate	SET_SIZE(on_trap_trampoline)
1117c478bd9Sstevel@tonic-gate
1127c478bd9Sstevel@tonic-gate/*
1137c478bd9Sstevel@tonic-gate * Push a new element on to the t_ontrap stack.  Refer to <sys/ontrap.h> for
1147c478bd9Sstevel@tonic-gate * more information about the on_trap() mechanism.  If the on_trap_data is the
1157c478bd9Sstevel@tonic-gate * same as the topmost stack element, we just modify that element.
1167c478bd9Sstevel@tonic-gate */
1177c478bd9Sstevel@tonic-gate
1187c478bd9Sstevel@tonic-gate	ENTRY(on_trap)
1197c478bd9Sstevel@tonic-gate	movw	%si, OT_PROT(%rdi)		/* ot_prot = prot */
1207c478bd9Sstevel@tonic-gate	movw	$0, OT_TRAP(%rdi)		/* ot_trap = 0 */
1217c478bd9Sstevel@tonic-gate	leaq	on_trap_trampoline(%rip), %rdx	/* rdx = &on_trap_trampoline */
1227c478bd9Sstevel@tonic-gate	movq	%rdx, OT_TRAMPOLINE(%rdi)	/* ot_trampoline = rdx */
1237c478bd9Sstevel@tonic-gate	xorl	%ecx, %ecx
1247c478bd9Sstevel@tonic-gate	movq	%rcx, OT_HANDLE(%rdi)		/* ot_handle = NULL */
1257c478bd9Sstevel@tonic-gate	movq	%rcx, OT_PAD1(%rdi)		/* ot_pad1 = NULL */
1267c478bd9Sstevel@tonic-gate	movq	%gs:CPU_THREAD, %rdx		/* rdx = curthread */
1277c478bd9Sstevel@tonic-gate	movq	T_ONTRAP(%rdx), %rcx		/* rcx = curthread->t_ontrap */
1287c478bd9Sstevel@tonic-gate	cmpq	%rdi, %rcx			/* if (otp == %rcx)	*/
1297c478bd9Sstevel@tonic-gate	je	0f				/*	don't modify t_ontrap */
1307c478bd9Sstevel@tonic-gate
1317c478bd9Sstevel@tonic-gate	movq	%rcx, OT_PREV(%rdi)		/* ot_prev = t_ontrap */
1327c478bd9Sstevel@tonic-gate	movq	%rdi, T_ONTRAP(%rdx)		/* curthread->t_ontrap = otp */
1337c478bd9Sstevel@tonic-gate
1347c478bd9Sstevel@tonic-gate0:	addq	$OT_JMPBUF, %rdi		/* &ot_jmpbuf */
1357c478bd9Sstevel@tonic-gate	jmp	setjmp
1367c478bd9Sstevel@tonic-gate	SET_SIZE(on_trap)
1377c478bd9Sstevel@tonic-gate
1387c478bd9Sstevel@tonic-gate/*
1397c478bd9Sstevel@tonic-gate * Setjmp and longjmp implement non-local gotos using state vectors
1407c478bd9Sstevel@tonic-gate * type label_t.
1417c478bd9Sstevel@tonic-gate */
1427c478bd9Sstevel@tonic-gate
1437c478bd9Sstevel@tonic-gate#if LABEL_PC != 0
1447c478bd9Sstevel@tonic-gate#error LABEL_PC MUST be defined as 0 for setjmp/longjmp to work as coded
1457c478bd9Sstevel@tonic-gate#endif	/* LABEL_PC != 0 */
1467c478bd9Sstevel@tonic-gate
1477c478bd9Sstevel@tonic-gate	ENTRY(setjmp)
1487c478bd9Sstevel@tonic-gate	movq	%rsp, LABEL_SP(%rdi)
1497c478bd9Sstevel@tonic-gate	movq	%rbp, LABEL_RBP(%rdi)
1507c478bd9Sstevel@tonic-gate	movq	%rbx, LABEL_RBX(%rdi)
1517c478bd9Sstevel@tonic-gate	movq	%r12, LABEL_R12(%rdi)
1527c478bd9Sstevel@tonic-gate	movq	%r13, LABEL_R13(%rdi)
1537c478bd9Sstevel@tonic-gate	movq	%r14, LABEL_R14(%rdi)
1547c478bd9Sstevel@tonic-gate	movq	%r15, LABEL_R15(%rdi)
1557c478bd9Sstevel@tonic-gate	movq	(%rsp), %rdx		/* return address */
1567c478bd9Sstevel@tonic-gate	movq	%rdx, (%rdi)		/* LABEL_PC is 0 */
1577c478bd9Sstevel@tonic-gate	xorl	%eax, %eax		/* return 0 */
1587c478bd9Sstevel@tonic-gate	ret
1597c478bd9Sstevel@tonic-gate	SET_SIZE(setjmp)
1607c478bd9Sstevel@tonic-gate
1617c478bd9Sstevel@tonic-gate	ENTRY(longjmp)
1627c478bd9Sstevel@tonic-gate	movq	LABEL_SP(%rdi), %rsp
1637c478bd9Sstevel@tonic-gate	movq	LABEL_RBP(%rdi), %rbp
1647c478bd9Sstevel@tonic-gate	movq	LABEL_RBX(%rdi), %rbx
1657c478bd9Sstevel@tonic-gate	movq	LABEL_R12(%rdi), %r12
1667c478bd9Sstevel@tonic-gate	movq	LABEL_R13(%rdi), %r13
1677c478bd9Sstevel@tonic-gate	movq	LABEL_R14(%rdi), %r14
1687c478bd9Sstevel@tonic-gate	movq	LABEL_R15(%rdi), %r15
1697c478bd9Sstevel@tonic-gate	movq	(%rdi), %rdx		/* return address; LABEL_PC is 0 */
1707c478bd9Sstevel@tonic-gate	movq	%rdx, (%rsp)
1717c478bd9Sstevel@tonic-gate	xorl	%eax, %eax
1727c478bd9Sstevel@tonic-gate	incl	%eax			/* return 1 */
1737c478bd9Sstevel@tonic-gate	ret
1747c478bd9Sstevel@tonic-gate	SET_SIZE(longjmp)
1757c478bd9Sstevel@tonic-gate
1767c478bd9Sstevel@tonic-gate/*
1777c478bd9Sstevel@tonic-gate * if a() calls b() calls caller(),
1787c478bd9Sstevel@tonic-gate * caller() returns return address in a().
1797c478bd9Sstevel@tonic-gate * (Note: We assume a() and b() are C routines which do the normal entry/exit
1807c478bd9Sstevel@tonic-gate *  sequence.)
1817c478bd9Sstevel@tonic-gate */
1827c478bd9Sstevel@tonic-gate
1837c478bd9Sstevel@tonic-gate	ENTRY(caller)
1847c478bd9Sstevel@tonic-gate	movq	8(%rbp), %rax		/* b()'s return pc, in a() */
1857c478bd9Sstevel@tonic-gate	ret
1867c478bd9Sstevel@tonic-gate	SET_SIZE(caller)
1877c478bd9Sstevel@tonic-gate
1887c478bd9Sstevel@tonic-gate/*
1897c478bd9Sstevel@tonic-gate * if a() calls callee(), callee() returns the
1907c478bd9Sstevel@tonic-gate * return address in a();
1917c478bd9Sstevel@tonic-gate */
1927c478bd9Sstevel@tonic-gate
1937c478bd9Sstevel@tonic-gate	ENTRY(callee)
1947c478bd9Sstevel@tonic-gate	movq	(%rsp), %rax		/* callee()'s return pc, in a() */
1957c478bd9Sstevel@tonic-gate	ret
1967c478bd9Sstevel@tonic-gate	SET_SIZE(callee)
1977c478bd9Sstevel@tonic-gate
1987c478bd9Sstevel@tonic-gate/*
1997c478bd9Sstevel@tonic-gate * return the current frame pointer
2007c478bd9Sstevel@tonic-gate */
2017c478bd9Sstevel@tonic-gate
2027c478bd9Sstevel@tonic-gate	ENTRY(getfp)
2037c478bd9Sstevel@tonic-gate	movq	%rbp, %rax
2047c478bd9Sstevel@tonic-gate	ret
2057c478bd9Sstevel@tonic-gate	SET_SIZE(getfp)
2067c478bd9Sstevel@tonic-gate
2077c478bd9Sstevel@tonic-gate/*
2087c478bd9Sstevel@tonic-gate * Invalidate a single page table entry in the TLB
2097c478bd9Sstevel@tonic-gate */
2107c478bd9Sstevel@tonic-gate
21174ecdb51SJohn Levon	ENTRY(mmu_invlpg)
2127c478bd9Sstevel@tonic-gate	invlpg	(%rdi)
2137c478bd9Sstevel@tonic-gate	ret
21474ecdb51SJohn Levon	SET_SIZE(mmu_invlpg)
2157c478bd9Sstevel@tonic-gate
2167c478bd9Sstevel@tonic-gate
2177c478bd9Sstevel@tonic-gate/*
2187c478bd9Sstevel@tonic-gate * Get/Set the value of various control registers
2197c478bd9Sstevel@tonic-gate */
2207c478bd9Sstevel@tonic-gate
2217c478bd9Sstevel@tonic-gate	ENTRY(getcr0)
2227c478bd9Sstevel@tonic-gate	movq	%cr0, %rax
2237c478bd9Sstevel@tonic-gate	ret
2247c478bd9Sstevel@tonic-gate	SET_SIZE(getcr0)
2257c478bd9Sstevel@tonic-gate
2267c478bd9Sstevel@tonic-gate	ENTRY(setcr0)
2277c478bd9Sstevel@tonic-gate	movq	%rdi, %cr0
2287c478bd9Sstevel@tonic-gate	ret
2297c478bd9Sstevel@tonic-gate	SET_SIZE(setcr0)
2307c478bd9Sstevel@tonic-gate
231ae115bc7Smrj        ENTRY(getcr2)
232843e1988Sjohnlev#if defined(__xpv)
233843e1988Sjohnlev	movq	%gs:CPU_VCPU_INFO, %rax
234843e1988Sjohnlev	movq	VCPU_INFO_ARCH_CR2(%rax), %rax
235843e1988Sjohnlev#else
236ae115bc7Smrj        movq    %cr2, %rax
237843e1988Sjohnlev#endif
238ae115bc7Smrj        ret
2397c478bd9Sstevel@tonic-gate	SET_SIZE(getcr2)
2407c478bd9Sstevel@tonic-gate
2417c478bd9Sstevel@tonic-gate	ENTRY(getcr3)
242ae115bc7Smrj	movq    %cr3, %rax
2437c478bd9Sstevel@tonic-gate	ret
2447c478bd9Sstevel@tonic-gate	SET_SIZE(getcr3)
2457c478bd9Sstevel@tonic-gate
246843e1988Sjohnlev#if !defined(__xpv)
247843e1988Sjohnlev
248ae115bc7Smrj        ENTRY(setcr3)
249ae115bc7Smrj        movq    %rdi, %cr3
250ae115bc7Smrj        ret
2517c478bd9Sstevel@tonic-gate	SET_SIZE(setcr3)
2527c478bd9Sstevel@tonic-gate
2537c478bd9Sstevel@tonic-gate	ENTRY(reload_cr3)
2547c478bd9Sstevel@tonic-gate	movq	%cr3, %rdi
2557c478bd9Sstevel@tonic-gate	movq	%rdi, %cr3
2567c478bd9Sstevel@tonic-gate	ret
2577c478bd9Sstevel@tonic-gate	SET_SIZE(reload_cr3)
2587c478bd9Sstevel@tonic-gate
259843e1988Sjohnlev#endif	/* __xpv */
260843e1988Sjohnlev
2617c478bd9Sstevel@tonic-gate	ENTRY(getcr4)
2627c478bd9Sstevel@tonic-gate	movq	%cr4, %rax
2637c478bd9Sstevel@tonic-gate	ret
2647c478bd9Sstevel@tonic-gate	SET_SIZE(getcr4)
2657c478bd9Sstevel@tonic-gate
2667c478bd9Sstevel@tonic-gate	ENTRY(setcr4)
2677c478bd9Sstevel@tonic-gate	movq	%rdi, %cr4
2687c478bd9Sstevel@tonic-gate	ret
2697c478bd9Sstevel@tonic-gate	SET_SIZE(setcr4)
2707c478bd9Sstevel@tonic-gate
2717c478bd9Sstevel@tonic-gate	ENTRY(getcr8)
2727c478bd9Sstevel@tonic-gate	movq	%cr8, %rax
2737c478bd9Sstevel@tonic-gate	ret
2747c478bd9Sstevel@tonic-gate	SET_SIZE(getcr8)
2757c478bd9Sstevel@tonic-gate
2767c478bd9Sstevel@tonic-gate	ENTRY(setcr8)
2777c478bd9Sstevel@tonic-gate	movq	%rdi, %cr8
2787c478bd9Sstevel@tonic-gate	ret
2797c478bd9Sstevel@tonic-gate	SET_SIZE(setcr8)
2807c478bd9Sstevel@tonic-gate
2817c478bd9Sstevel@tonic-gate	ENTRY(__cpuid_insn)
2828949bcd6Sandrei	movq	%rbx, %r8
2838949bcd6Sandrei	movq	%rcx, %r9
2848949bcd6Sandrei	movq	%rdx, %r11
2858949bcd6Sandrei	movl	(%rdi), %eax		/* %eax = regs->cp_eax */
2868949bcd6Sandrei	movl	0x4(%rdi), %ebx		/* %ebx = regs->cp_ebx */
2878949bcd6Sandrei	movl	0x8(%rdi), %ecx		/* %ecx = regs->cp_ecx */
2888949bcd6Sandrei	movl	0xc(%rdi), %edx		/* %edx = regs->cp_edx */
2897c478bd9Sstevel@tonic-gate	cpuid
2908949bcd6Sandrei	movl	%eax, (%rdi)		/* regs->cp_eax = %eax */
2918949bcd6Sandrei	movl	%ebx, 0x4(%rdi)		/* regs->cp_ebx = %ebx */
2928949bcd6Sandrei	movl	%ecx, 0x8(%rdi)		/* regs->cp_ecx = %ecx */
2938949bcd6Sandrei	movl	%edx, 0xc(%rdi)		/* regs->cp_edx = %edx */
2948949bcd6Sandrei	movq	%r8, %rbx
2958949bcd6Sandrei	movq	%r9, %rcx
2968949bcd6Sandrei	movq	%r11, %rdx
2977c478bd9Sstevel@tonic-gate	ret
2987c478bd9Sstevel@tonic-gate	SET_SIZE(__cpuid_insn)
2997c478bd9Sstevel@tonic-gate
300f98fbcecSbholler	ENTRY_NP(i86_monitor)
301f98fbcecSbholler	pushq	%rbp
302f98fbcecSbholler	movq	%rsp, %rbp
303f98fbcecSbholler	movq	%rdi, %rax		/* addr */
304f98fbcecSbholler	movq	%rsi, %rcx		/* extensions */
305f98fbcecSbholler	/* rdx contains input arg3: hints */
3061d1a3942SBill Holler	clflush	(%rax)
307f98fbcecSbholler	.byte	0x0f, 0x01, 0xc8	/* monitor */
308f98fbcecSbholler	leave
309f98fbcecSbholler	ret
310f98fbcecSbholler	SET_SIZE(i86_monitor)
311f98fbcecSbholler
312f98fbcecSbholler	ENTRY_NP(i86_mwait)
313f98fbcecSbholler	pushq	%rbp
31465f20420SRobert Mustacchi	call	x86_md_clear
315f98fbcecSbholler	movq	%rsp, %rbp
316f98fbcecSbholler	movq	%rdi, %rax		/* data */
317f98fbcecSbholler	movq	%rsi, %rcx		/* extensions */
318f98fbcecSbholler	.byte	0x0f, 0x01, 0xc9	/* mwait */
319f98fbcecSbholler	leave
320f98fbcecSbholler	ret
321f98fbcecSbholler	SET_SIZE(i86_mwait)
322f98fbcecSbholler
323f34a7178SJoe Bonasera#if defined(__xpv)
324f34a7178SJoe Bonasera	/*
325f34a7178SJoe Bonasera	 * Defined in C
326f34a7178SJoe Bonasera	 */
327f34a7178SJoe Bonasera#else
328f34a7178SJoe Bonasera
329ae115bc7Smrj	ENTRY_NP(tsc_read)
330247dbb3dSsudheer	movq	%rbx, %r11
331247dbb3dSsudheer	movl	$0, %eax
332247dbb3dSsudheer	cpuid
333247dbb3dSsudheer	rdtsc
334247dbb3dSsudheer	movq	%r11, %rbx
335247dbb3dSsudheer	shlq	$32, %rdx
336247dbb3dSsudheer	orq	%rdx, %rax
337247dbb3dSsudheer	ret
338247dbb3dSsudheer	.globl _tsc_mfence_start
339247dbb3dSsudheer_tsc_mfence_start:
340247dbb3dSsudheer	mfence
341ae115bc7Smrj	rdtsc
342ae115bc7Smrj	shlq	$32, %rdx
343ae115bc7Smrj	orq	%rdx, %rax
344ae115bc7Smrj	ret
345247dbb3dSsudheer	.globl _tsc_mfence_end
346247dbb3dSsudheer_tsc_mfence_end:
347247dbb3dSsudheer	.globl _tscp_start
348247dbb3dSsudheer_tscp_start:
349247dbb3dSsudheer	.byte	0x0f, 0x01, 0xf9	/* rdtscp instruction */
350247dbb3dSsudheer	shlq	$32, %rdx
351247dbb3dSsudheer	orq	%rdx, %rax
352247dbb3dSsudheer	ret
353247dbb3dSsudheer	.globl _tscp_end
354247dbb3dSsudheer_tscp_end:
355247dbb3dSsudheer	.globl _no_rdtsc_start
356247dbb3dSsudheer_no_rdtsc_start:
357247dbb3dSsudheer	xorl	%edx, %edx
358247dbb3dSsudheer	xorl	%eax, %eax
359247dbb3dSsudheer	ret
360247dbb3dSsudheer	.globl _no_rdtsc_end
361247dbb3dSsudheer_no_rdtsc_end:
36215363b27Ssudheer	.globl _tsc_lfence_start
36315363b27Ssudheer_tsc_lfence_start:
36415363b27Ssudheer	lfence
36515363b27Ssudheer	rdtsc
36615363b27Ssudheer	shlq	$32, %rdx
36715363b27Ssudheer	orq	%rdx, %rax
36815363b27Ssudheer	ret
36915363b27Ssudheer	.globl _tsc_lfence_end
37015363b27Ssudheer_tsc_lfence_end:
371ae115bc7Smrj	SET_SIZE(tsc_read)
372ae115bc7Smrj
37306fb6a36Sdv
374843e1988Sjohnlev#endif	/* __xpv */
375843e1988Sjohnlev
37606fb6a36Sdv	ENTRY_NP(randtick)
37706fb6a36Sdv	rdtsc
37806fb6a36Sdv	shlq    $32, %rdx
37906fb6a36Sdv	orq     %rdx, %rax
38006fb6a36Sdv	ret
38106fb6a36Sdv	SET_SIZE(randtick)
3827c478bd9Sstevel@tonic-gate/*
3837c478bd9Sstevel@tonic-gate * Insert entryp after predp in a doubly linked list.
3847c478bd9Sstevel@tonic-gate */
3857c478bd9Sstevel@tonic-gate
3867c478bd9Sstevel@tonic-gate	ENTRY(_insque)
3874b56a003SDaniel Anderson	movq	(%rsi), %rax		/* predp->forw			*/
3887c478bd9Sstevel@tonic-gate	movq	%rsi, CPTRSIZE(%rdi)	/* entryp->back = predp		*/
3897c478bd9Sstevel@tonic-gate	movq	%rax, (%rdi)		/* entryp->forw = predp->forw	*/
3907c478bd9Sstevel@tonic-gate	movq	%rdi, (%rsi)		/* predp->forw = entryp		*/
3917c478bd9Sstevel@tonic-gate	movq	%rdi, CPTRSIZE(%rax)	/* predp->forw->back = entryp	*/
3927c478bd9Sstevel@tonic-gate	ret
3937c478bd9Sstevel@tonic-gate	SET_SIZE(_insque)
3947c478bd9Sstevel@tonic-gate
3957c478bd9Sstevel@tonic-gate/*
3967c478bd9Sstevel@tonic-gate * Remove entryp from a doubly linked list
3977c478bd9Sstevel@tonic-gate */
3987c478bd9Sstevel@tonic-gate
3997c478bd9Sstevel@tonic-gate	ENTRY(_remque)
4007c478bd9Sstevel@tonic-gate	movq	(%rdi), %rax		/* entry->forw */
4017c478bd9Sstevel@tonic-gate	movq	CPTRSIZE(%rdi), %rdx	/* entry->back */
4027c478bd9Sstevel@tonic-gate	movq	%rax, (%rdx)		/* entry->back->forw = entry->forw */
4037c478bd9Sstevel@tonic-gate	movq	%rdx, CPTRSIZE(%rax)	/* entry->forw->back = entry->back */
4047c478bd9Sstevel@tonic-gate	ret
4057c478bd9Sstevel@tonic-gate	SET_SIZE(_remque)
4067c478bd9Sstevel@tonic-gate
4077c478bd9Sstevel@tonic-gate/*
4087c478bd9Sstevel@tonic-gate * Returns the number of
4097c478bd9Sstevel@tonic-gate * non-NULL bytes in string argument.
4107c478bd9Sstevel@tonic-gate */
4117c478bd9Sstevel@tonic-gate
4127c478bd9Sstevel@tonic-gate/*
4137c478bd9Sstevel@tonic-gate * This is close to a simple transliteration of a C version of this
4147c478bd9Sstevel@tonic-gate * routine.  We should either just -make- this be a C version, or
4157c478bd9Sstevel@tonic-gate * justify having it in assembler by making it significantly faster.
4167c478bd9Sstevel@tonic-gate *
4177c478bd9Sstevel@tonic-gate * size_t
4187c478bd9Sstevel@tonic-gate * strlen(const char *s)
4197c478bd9Sstevel@tonic-gate * {
4207c478bd9Sstevel@tonic-gate *	const char *s0;
4217c478bd9Sstevel@tonic-gate * #if defined(DEBUG)
4227c478bd9Sstevel@tonic-gate *	if ((uintptr_t)s < KERNELBASE)
4237c478bd9Sstevel@tonic-gate *		panic(.str_panic_msg);
4247c478bd9Sstevel@tonic-gate * #endif
4257c478bd9Sstevel@tonic-gate *	for (s0 = s; *s; s++)
4267c478bd9Sstevel@tonic-gate *		;
4277c478bd9Sstevel@tonic-gate *	return (s - s0);
4287c478bd9Sstevel@tonic-gate * }
4297c478bd9Sstevel@tonic-gate */
4307c478bd9Sstevel@tonic-gate
4317c478bd9Sstevel@tonic-gate	ENTRY(strlen)
4327c478bd9Sstevel@tonic-gate#ifdef DEBUG
433ae115bc7Smrj	movq	postbootkernelbase(%rip), %rax
4347c478bd9Sstevel@tonic-gate	cmpq	%rax, %rdi
4357c478bd9Sstevel@tonic-gate	jae	str_valid
4367c478bd9Sstevel@tonic-gate	pushq	%rbp
4377c478bd9Sstevel@tonic-gate	movq	%rsp, %rbp
4387c478bd9Sstevel@tonic-gate	leaq	.str_panic_msg(%rip), %rdi
4397c478bd9Sstevel@tonic-gate	xorl	%eax, %eax
4407c478bd9Sstevel@tonic-gate	call	panic
4417c478bd9Sstevel@tonic-gate#endif	/* DEBUG */
4427c478bd9Sstevel@tonic-gatestr_valid:
4437c478bd9Sstevel@tonic-gate	cmpb	$0, (%rdi)
4447c478bd9Sstevel@tonic-gate	movq	%rdi, %rax
4457c478bd9Sstevel@tonic-gate	je	.null_found
4467c478bd9Sstevel@tonic-gate	.align	4
4477c478bd9Sstevel@tonic-gate.strlen_loop:
4487c478bd9Sstevel@tonic-gate	incq	%rdi
4497c478bd9Sstevel@tonic-gate	cmpb	$0, (%rdi)
4507c478bd9Sstevel@tonic-gate	jne	.strlen_loop
4517c478bd9Sstevel@tonic-gate.null_found:
4527c478bd9Sstevel@tonic-gate	subq	%rax, %rdi
4537c478bd9Sstevel@tonic-gate	movq	%rdi, %rax
4547c478bd9Sstevel@tonic-gate	ret
4557c478bd9Sstevel@tonic-gate	SET_SIZE(strlen)
4567c478bd9Sstevel@tonic-gate
4577c478bd9Sstevel@tonic-gate#ifdef DEBUG
4587c478bd9Sstevel@tonic-gate	.text
4597c478bd9Sstevel@tonic-gate.str_panic_msg:
4607c478bd9Sstevel@tonic-gate	.string "strlen: argument below kernelbase"
4617c478bd9Sstevel@tonic-gate#endif /* DEBUG */
4627c478bd9Sstevel@tonic-gate
4637c478bd9Sstevel@tonic-gate	/*
4644b56a003SDaniel Anderson	 * Berkeley 4.3 introduced symbolically named interrupt levels
4657c478bd9Sstevel@tonic-gate	 * as a way deal with priority in a machine independent fashion.
4667c478bd9Sstevel@tonic-gate	 * Numbered priorities are machine specific, and should be
4677c478bd9Sstevel@tonic-gate	 * discouraged where possible.
4687c478bd9Sstevel@tonic-gate	 *
4697c478bd9Sstevel@tonic-gate	 * Note, for the machine specific priorities there are
4707c478bd9Sstevel@tonic-gate	 * examples listed for devices that use a particular priority.
4717c478bd9Sstevel@tonic-gate	 * It should not be construed that all devices of that
4727c478bd9Sstevel@tonic-gate	 * type should be at that priority.  It is currently were
4737c478bd9Sstevel@tonic-gate	 * the current devices fit into the priority scheme based
4747c478bd9Sstevel@tonic-gate	 * upon time criticalness.
4757c478bd9Sstevel@tonic-gate	 *
4767c478bd9Sstevel@tonic-gate	 * The underlying assumption of these assignments is that
4777c478bd9Sstevel@tonic-gate	 * IPL 10 is the highest level from which a device
4787c478bd9Sstevel@tonic-gate	 * routine can call wakeup.  Devices that interrupt from higher
4797c478bd9Sstevel@tonic-gate	 * levels are restricted in what they can do.  If they need
4807c478bd9Sstevel@tonic-gate	 * kernels services they should schedule a routine at a lower
4817c478bd9Sstevel@tonic-gate	 * level (via software interrupt) to do the required
4827c478bd9Sstevel@tonic-gate	 * processing.
4837c478bd9Sstevel@tonic-gate	 *
4847c478bd9Sstevel@tonic-gate	 * Examples of this higher usage:
4857c478bd9Sstevel@tonic-gate	 *	Level	Usage
4867c478bd9Sstevel@tonic-gate	 *	14	Profiling clock (and PROM uart polling clock)
4877c478bd9Sstevel@tonic-gate	 *	12	Serial ports
4887c478bd9Sstevel@tonic-gate	 *
4897c478bd9Sstevel@tonic-gate	 * The serial ports request lower level processing on level 6.
4907c478bd9Sstevel@tonic-gate	 *
4917c478bd9Sstevel@tonic-gate	 * Also, almost all splN routines (where N is a number or a
4927c478bd9Sstevel@tonic-gate	 * mnemonic) will do a RAISE(), on the assumption that they are
4937c478bd9Sstevel@tonic-gate	 * never used to lower our priority.
4947c478bd9Sstevel@tonic-gate	 * The exceptions are:
4957c478bd9Sstevel@tonic-gate	 *	spl8()		Because you can't be above 15 to begin with!
4967c478bd9Sstevel@tonic-gate	 *	splzs()		Because this is used at boot time to lower our
4977c478bd9Sstevel@tonic-gate	 *			priority, to allow the PROM to poll the uart.
4987c478bd9Sstevel@tonic-gate	 *	spl0()		Used to lower priority to 0.
4997c478bd9Sstevel@tonic-gate	 */
5007c478bd9Sstevel@tonic-gate
501ae115bc7Smrj#define	SETPRI(level) \
502ae115bc7Smrj	movl	$/**/level, %edi;	/* new priority */		\
503ae115bc7Smrj	jmp	do_splx			/* redirect to do_splx */
5047c478bd9Sstevel@tonic-gate
5057c478bd9Sstevel@tonic-gate#define	RAISE(level) \
506ae115bc7Smrj	movl	$/**/level, %edi;	/* new priority */		\
507ae115bc7Smrj	jmp	splr			/* redirect to splr */
5087c478bd9Sstevel@tonic-gate
5097c478bd9Sstevel@tonic-gate	/* locks out all interrupts, including memory errors */
5107c478bd9Sstevel@tonic-gate	ENTRY(spl8)
5117c478bd9Sstevel@tonic-gate	SETPRI(15)
5127c478bd9Sstevel@tonic-gate	SET_SIZE(spl8)
5137c478bd9Sstevel@tonic-gate
5147c478bd9Sstevel@tonic-gate	/* just below the level that profiling runs */
5157c478bd9Sstevel@tonic-gate	ENTRY(spl7)
5167c478bd9Sstevel@tonic-gate	RAISE(13)
5177c478bd9Sstevel@tonic-gate	SET_SIZE(spl7)
5187c478bd9Sstevel@tonic-gate
5197c478bd9Sstevel@tonic-gate	/* sun specific - highest priority onboard serial i/o asy ports */
5207c478bd9Sstevel@tonic-gate	ENTRY(splzs)
5217c478bd9Sstevel@tonic-gate	SETPRI(12)	/* Can't be a RAISE, as it's used to lower us */
5227c478bd9Sstevel@tonic-gate	SET_SIZE(splzs)
5237c478bd9Sstevel@tonic-gate
5247c478bd9Sstevel@tonic-gate	ENTRY(splhi)
5257c478bd9Sstevel@tonic-gate	ALTENTRY(splhigh)
5267c478bd9Sstevel@tonic-gate	ALTENTRY(spl6)
5277c478bd9Sstevel@tonic-gate	ALTENTRY(i_ddi_splhigh)
5287c478bd9Sstevel@tonic-gate
529ae115bc7Smrj	RAISE(DISP_LEVEL)
5307c478bd9Sstevel@tonic-gate
5317c478bd9Sstevel@tonic-gate	SET_SIZE(i_ddi_splhigh)
5327c478bd9Sstevel@tonic-gate	SET_SIZE(spl6)
5337c478bd9Sstevel@tonic-gate	SET_SIZE(splhigh)
5347c478bd9Sstevel@tonic-gate	SET_SIZE(splhi)
5357c478bd9Sstevel@tonic-gate
5367c478bd9Sstevel@tonic-gate	/* allow all interrupts */
5377c478bd9Sstevel@tonic-gate	ENTRY(spl0)
5387c478bd9Sstevel@tonic-gate	SETPRI(0)
5397c478bd9Sstevel@tonic-gate	SET_SIZE(spl0)
5407c478bd9Sstevel@tonic-gate
5417c478bd9Sstevel@tonic-gate
5424b56a003SDaniel Anderson	/* splx implementation */
5437c478bd9Sstevel@tonic-gate	ENTRY(splx)
544ae115bc7Smrj	jmp	do_splx		/* redirect to common splx code */
5457c478bd9Sstevel@tonic-gate	SET_SIZE(splx)
5467c478bd9Sstevel@tonic-gate
547085f98c4Ssethg	ENTRY(wait_500ms)
5481b30f017Sgvasick	pushq	%rbx
54932206069Ssethg	movl	$50000, %ebx
550085f98c4Ssethg1:
551085f98c4Ssethg	call	tenmicrosec
55232206069Ssethg	decl	%ebx
55332206069Ssethg	jnz	1b
5541b30f017Sgvasick	popq	%rbx
55565f20420SRobert Mustacchi	ret
556085f98c4Ssethg	SET_SIZE(wait_500ms)
557085f98c4Ssethg
558085f98c4Ssethg#define	RESET_METHOD_KBC	1
559085f98c4Ssethg#define	RESET_METHOD_PORT92	2
560085f98c4Ssethg#define RESET_METHOD_PCI	4
561085f98c4Ssethg
562085f98c4Ssethg	DGDEF3(pc_reset_methods, 4, 8)
563085f98c4Ssethg	.long RESET_METHOD_KBC|RESET_METHOD_PORT92|RESET_METHOD_PCI;
564085f98c4Ssethg
5657c478bd9Sstevel@tonic-gate	ENTRY(pc_reset)
566085f98c4Ssethg
56732206069Ssethg	testl	$RESET_METHOD_KBC, pc_reset_methods(%rip)
568085f98c4Ssethg	jz	1f
569085f98c4Ssethg
5707303b3c3Ssethg	/
5717303b3c3Ssethg	/ Try the classic keyboard controller-triggered reset.
5727303b3c3Ssethg	/
5737c478bd9Sstevel@tonic-gate	movw	$0x64, %dx
5747c478bd9Sstevel@tonic-gate	movb	$0xfe, %al
5757c478bd9Sstevel@tonic-gate	outb	(%dx)
5767303b3c3Ssethg
577085f98c4Ssethg	/ Wait up to 500 milliseconds here for the keyboard controller
578085f98c4Ssethg	/ to pull the reset line.  On some systems where the keyboard
579085f98c4Ssethg	/ controller is slow to pull the reset line, the next reset method
580085f98c4Ssethg	/ may be executed (which may be bad if those systems hang when the
581085f98c4Ssethg	/ next reset method is used, e.g. Ferrari 3400 (doesn't like port 92),
582085f98c4Ssethg	/ and Ferrari 4000 (doesn't like the cf9 reset method))
583085f98c4Ssethg
584085f98c4Ssethg	call	wait_500ms
585085f98c4Ssethg
586085f98c4Ssethg1:
58732206069Ssethg	testl	$RESET_METHOD_PORT92, pc_reset_methods(%rip)
588085f98c4Ssethg	jz	3f
589085f98c4Ssethg
5907303b3c3Ssethg	/
5917303b3c3Ssethg	/ Try port 0x92 fast reset
5927303b3c3Ssethg	/
5937303b3c3Ssethg	movw	$0x92, %dx
5947303b3c3Ssethg	inb	(%dx)
5957303b3c3Ssethg	cmpb	$0xff, %al	/ If port's not there, we should get back 0xFF
5967303b3c3Ssethg	je	1f
5977303b3c3Ssethg	testb	$1, %al		/ If bit 0
5987303b3c3Ssethg	jz	2f		/ is clear, jump to perform the reset
5997303b3c3Ssethg	andb	$0xfe, %al	/ otherwise,
6007303b3c3Ssethg	outb	(%dx)		/ clear bit 0 first, then
6017303b3c3Ssethg2:
6027303b3c3Ssethg	orb	$1, %al		/ Set bit 0
6037303b3c3Ssethg	outb	(%dx)		/ and reset the system
6047303b3c3Ssethg1:
6051a48003fSsethg
606085f98c4Ssethg	call	wait_500ms
607085f98c4Ssethg
608085f98c4Ssethg3:
60932206069Ssethg	testl	$RESET_METHOD_PCI, pc_reset_methods(%rip)
610085f98c4Ssethg	jz	4f
611085f98c4Ssethg
6121a48003fSsethg	/ Try the PCI (soft) reset vector (should work on all modern systems,
6131a48003fSsethg	/ but has been shown to cause problems on 450NX systems, and some newer
6141a48003fSsethg	/ systems (e.g. ATI IXP400-equipped systems))
6151a48003fSsethg	/ When resetting via this method, 2 writes are required.  The first
6161a48003fSsethg	/ targets bit 1 (0=hard reset without power cycle, 1=hard reset with
6171a48003fSsethg	/ power cycle).
6181a48003fSsethg	/ The reset occurs on the second write, during bit 2's transition from
6191a48003fSsethg	/ 0->1.
6201a48003fSsethg	movw	$0xcf9, %dx
6211a48003fSsethg	movb	$0x2, %al	/ Reset mode = hard, no power cycle
6221a48003fSsethg	outb	(%dx)
6231a48003fSsethg	movb	$0x6, %al
6241a48003fSsethg	outb	(%dx)
6251a48003fSsethg
626085f98c4Ssethg	call	wait_500ms
627085f98c4Ssethg
628085f98c4Ssethg4:
6297303b3c3Ssethg	/
6301a48003fSsethg	/ port 0xcf9 failed also.  Last-ditch effort is to
6317303b3c3Ssethg	/ triple-fault the CPU.
632f2be5148Sszhou	/ Also, use triple fault for EFI firmware
6337303b3c3Ssethg	/
634f2be5148Sszhou	ENTRY(efi_reset)
6357303b3c3Ssethg	pushq	$0x0
6367303b3c3Ssethg	pushq	$0x0		/ IDT base of 0, limit of 0 + 2 unused bytes
6377303b3c3Ssethg	lidt	(%rsp)
6387303b3c3Ssethg	int	$0x0		/ Trigger interrupt, generate triple-fault
639085f98c4Ssethg
640085f98c4Ssethg	cli
641085f98c4Ssethg	hlt			/ Wait forever
6427c478bd9Sstevel@tonic-gate	/*NOTREACHED*/
643f2be5148Sszhou	SET_SIZE(efi_reset)
6447c478bd9Sstevel@tonic-gate	SET_SIZE(pc_reset)
6457c478bd9Sstevel@tonic-gate
6467c478bd9Sstevel@tonic-gate/*
6477c478bd9Sstevel@tonic-gate * C callable in and out routines
6487c478bd9Sstevel@tonic-gate */
6497c478bd9Sstevel@tonic-gate
6507c478bd9Sstevel@tonic-gate	ENTRY(outl)
6517c478bd9Sstevel@tonic-gate	movw	%di, %dx
6527c478bd9Sstevel@tonic-gate	movl	%esi, %eax
6537c478bd9Sstevel@tonic-gate	outl	(%dx)
6547c478bd9Sstevel@tonic-gate	ret
6557c478bd9Sstevel@tonic-gate	SET_SIZE(outl)
6567c478bd9Sstevel@tonic-gate
6577c478bd9Sstevel@tonic-gate	ENTRY(outw)
6587c478bd9Sstevel@tonic-gate	movw	%di, %dx
6597c478bd9Sstevel@tonic-gate	movw	%si, %ax
6607c478bd9Sstevel@tonic-gate	D16 outl (%dx)		/* XX64 why not outw? */
6617c478bd9Sstevel@tonic-gate	ret
6627c478bd9Sstevel@tonic-gate	SET_SIZE(outw)
6637c478bd9Sstevel@tonic-gate
6647c478bd9Sstevel@tonic-gate	ENTRY(outb)
6657c478bd9Sstevel@tonic-gate	movw	%di, %dx
6667c478bd9Sstevel@tonic-gate	movb	%sil, %al
6677c478bd9Sstevel@tonic-gate	outb	(%dx)
6687c478bd9Sstevel@tonic-gate	ret
6697c478bd9Sstevel@tonic-gate	SET_SIZE(outb)
6707c478bd9Sstevel@tonic-gate
6717c478bd9Sstevel@tonic-gate	ENTRY(inl)
6727c478bd9Sstevel@tonic-gate	xorl	%eax, %eax
6737c478bd9Sstevel@tonic-gate	movw	%di, %dx
6747c478bd9Sstevel@tonic-gate	inl	(%dx)
6757c478bd9Sstevel@tonic-gate	ret
6767c478bd9Sstevel@tonic-gate	SET_SIZE(inl)
6777c478bd9Sstevel@tonic-gate
6787c478bd9Sstevel@tonic-gate	ENTRY(inw)
6797c478bd9Sstevel@tonic-gate	xorl	%eax, %eax
6807c478bd9Sstevel@tonic-gate	movw	%di, %dx
6817c478bd9Sstevel@tonic-gate	D16 inl	(%dx)
6827c478bd9Sstevel@tonic-gate	ret
6837c478bd9Sstevel@tonic-gate	SET_SIZE(inw)
6847c478bd9Sstevel@tonic-gate
6857c478bd9Sstevel@tonic-gate
6867c478bd9Sstevel@tonic-gate	ENTRY(inb)
6877c478bd9Sstevel@tonic-gate	xorl	%eax, %eax
6887c478bd9Sstevel@tonic-gate	movw	%di, %dx
6897c478bd9Sstevel@tonic-gate	inb	(%dx)
6907c478bd9Sstevel@tonic-gate	ret
6917c478bd9Sstevel@tonic-gate	SET_SIZE(inb)
6927c478bd9Sstevel@tonic-gate
6937c478bd9Sstevel@tonic-gate/*
6947aec1d6eScindi * void int3(void)
6957aec1d6eScindi * void int18(void)
6967c478bd9Sstevel@tonic-gate * void int20(void)
697e3d60c9bSAdrian Frost * void int_cmci(void)
6987c478bd9Sstevel@tonic-gate */
6997c478bd9Sstevel@tonic-gate
7007aec1d6eScindi	ENTRY(int3)
7017aec1d6eScindi	int	$T_BPTFLT
7027aec1d6eScindi	ret
7037aec1d6eScindi	SET_SIZE(int3)
7047aec1d6eScindi
7057aec1d6eScindi	ENTRY(int18)
7067aec1d6eScindi	int	$T_MCE
7077aec1d6eScindi	ret
7087aec1d6eScindi	SET_SIZE(int18)
7097aec1d6eScindi
7107c478bd9Sstevel@tonic-gate	ENTRY(int20)
7117c478bd9Sstevel@tonic-gate	movl	boothowto, %eax
7127c478bd9Sstevel@tonic-gate	andl	$RB_DEBUG, %eax
7137c478bd9Sstevel@tonic-gate	jz	1f
7147c478bd9Sstevel@tonic-gate
7157aec1d6eScindi	int	$T_DBGENTR
7167c478bd9Sstevel@tonic-gate1:
71785641879Skalai	rep;	ret	/* use 2 byte return instruction when branch target */
71885641879Skalai			/* AMD Software Optimization Guide - Section 6.2 */
7197c478bd9Sstevel@tonic-gate	SET_SIZE(int20)
7207c478bd9Sstevel@tonic-gate
721e3d60c9bSAdrian Frost	ENTRY(int_cmci)
722e3d60c9bSAdrian Frost	int	$T_ENOEXTFLT
723e3d60c9bSAdrian Frost	ret
724e3d60c9bSAdrian Frost	SET_SIZE(int_cmci)
725e3d60c9bSAdrian Frost
7267c478bd9Sstevel@tonic-gate	ENTRY(scanc)
7277c478bd9Sstevel@tonic-gate					/* rdi == size */
7287c478bd9Sstevel@tonic-gate					/* rsi == cp */
7297c478bd9Sstevel@tonic-gate					/* rdx == table */
7307c478bd9Sstevel@tonic-gate					/* rcx == mask */
7317c478bd9Sstevel@tonic-gate	addq	%rsi, %rdi		/* end = &cp[size] */
7324b56a003SDaniel Anderson.scanloop:
7337c478bd9Sstevel@tonic-gate	cmpq	%rdi, %rsi		/* while (cp < end */
7347c478bd9Sstevel@tonic-gate	jnb	.scandone
7357c478bd9Sstevel@tonic-gate	movzbq	(%rsi), %r8		/* %r8 = *cp */
7367c478bd9Sstevel@tonic-gate	incq	%rsi			/* cp++ */
7377c478bd9Sstevel@tonic-gate	testb	%cl, (%r8, %rdx)
7387c478bd9Sstevel@tonic-gate	jz	.scanloop		/*  && (table[*cp] & mask) == 0) */
7397c478bd9Sstevel@tonic-gate	decq	%rsi			/* (fix post-increment) */
7407c478bd9Sstevel@tonic-gate.scandone:
7417c478bd9Sstevel@tonic-gate	movl	%edi, %eax
7427c478bd9Sstevel@tonic-gate	subl	%esi, %eax		/* return (end - cp) */
7437c478bd9Sstevel@tonic-gate	ret
7447c478bd9Sstevel@tonic-gate	SET_SIZE(scanc)
7457c478bd9Sstevel@tonic-gate
7467c478bd9Sstevel@tonic-gate/*
7477c478bd9Sstevel@tonic-gate * Replacement functions for ones that are normally inlined.
7487c478bd9Sstevel@tonic-gate * In addition to the copy in i86.il, they are defined here just in case.
7497c478bd9Sstevel@tonic-gate */
7507c478bd9Sstevel@tonic-gate
7517c478bd9Sstevel@tonic-gate	ENTRY(intr_clear)
7527c478bd9Sstevel@tonic-gate	ENTRY(clear_int_flag)
7537c478bd9Sstevel@tonic-gate	pushfq
7547c478bd9Sstevel@tonic-gate	popq	%rax
755843e1988Sjohnlev#if defined(__xpv)
756843e1988Sjohnlev	leaq	xpv_panicking, %rdi
757843e1988Sjohnlev	movl	(%rdi), %edi
758843e1988Sjohnlev	cmpl	$0, %edi
759843e1988Sjohnlev	jne	2f
760843e1988Sjohnlev	CLIRET(%rdi, %dl)	/* returns event mask in %dl */
761843e1988Sjohnlev	/*
762843e1988Sjohnlev	 * Synthesize the PS_IE bit from the event mask bit
763843e1988Sjohnlev	 */
764843e1988Sjohnlev	andq    $_BITNOT(PS_IE), %rax
765843e1988Sjohnlev	testb	$1, %dl
766843e1988Sjohnlev	jnz	1f
767843e1988Sjohnlev	orq	$PS_IE, %rax
768843e1988Sjohnlev1:
769843e1988Sjohnlev	ret
770843e1988Sjohnlev2:
771843e1988Sjohnlev#endif
772ae115bc7Smrj	CLI(%rdi)
7737c478bd9Sstevel@tonic-gate	ret
7747c478bd9Sstevel@tonic-gate	SET_SIZE(clear_int_flag)
7757c478bd9Sstevel@tonic-gate	SET_SIZE(intr_clear)
7767c478bd9Sstevel@tonic-gate
7777c478bd9Sstevel@tonic-gate	ENTRY(curcpup)
7787c478bd9Sstevel@tonic-gate	movq	%gs:CPU_SELF, %rax
7797c478bd9Sstevel@tonic-gate	ret
7807c478bd9Sstevel@tonic-gate	SET_SIZE(curcpup)
7817c478bd9Sstevel@tonic-gate
7824b56a003SDaniel Anderson/* htonll(), ntohll(), htonl(), ntohl(), htons(), ntohs()
7834b56a003SDaniel Anderson * These functions reverse the byte order of the input parameter and returns
7844b56a003SDaniel Anderson * the result.  This is to convert the byte order from host byte order
7854b56a003SDaniel Anderson * (little endian) to network byte order (big endian), or vice versa.
7864b56a003SDaniel Anderson */
7874b56a003SDaniel Anderson
7884b56a003SDaniel Anderson	ENTRY(htonll)
7894b56a003SDaniel Anderson	ALTENTRY(ntohll)
7904b56a003SDaniel Anderson	movq	%rdi, %rax
7914b56a003SDaniel Anderson	bswapq	%rax
7927c478bd9Sstevel@tonic-gate	ret
7934b56a003SDaniel Anderson	SET_SIZE(ntohll)
7944b56a003SDaniel Anderson	SET_SIZE(htonll)
7957c478bd9Sstevel@tonic-gate
7964b56a003SDaniel Anderson	/* XX64 there must be shorter sequences for this */
7977c478bd9Sstevel@tonic-gate	ENTRY(htonl)
7987c478bd9Sstevel@tonic-gate	ALTENTRY(ntohl)
7994b56a003SDaniel Anderson	movl	%edi, %eax
8007c478bd9Sstevel@tonic-gate	bswap	%eax
8017c478bd9Sstevel@tonic-gate	ret
8027c478bd9Sstevel@tonic-gate	SET_SIZE(ntohl)
8037c478bd9Sstevel@tonic-gate	SET_SIZE(htonl)
8047c478bd9Sstevel@tonic-gate
8057c478bd9Sstevel@tonic-gate	/* XX64 there must be better sequences for this */
8067c478bd9Sstevel@tonic-gate	ENTRY(htons)
8077c478bd9Sstevel@tonic-gate	ALTENTRY(ntohs)
8087c478bd9Sstevel@tonic-gate	movl	%edi, %eax
8097c478bd9Sstevel@tonic-gate	bswap	%eax
8107c478bd9Sstevel@tonic-gate	shrl	$16, %eax
8117c478bd9Sstevel@tonic-gate	ret
8124b56a003SDaniel Anderson	SET_SIZE(ntohs)
8137c478bd9Sstevel@tonic-gate	SET_SIZE(htons)
8147c478bd9Sstevel@tonic-gate
8157c478bd9Sstevel@tonic-gate
8167c478bd9Sstevel@tonic-gate	ENTRY(intr_restore)
8177c478bd9Sstevel@tonic-gate	ENTRY(restore_int_flag)
818a563a037Sbholler	testq	$PS_IE, %rdi
819a563a037Sbholler	jz	1f
820843e1988Sjohnlev#if defined(__xpv)
821843e1988Sjohnlev	leaq	xpv_panicking, %rsi
822843e1988Sjohnlev	movl	(%rsi), %esi
823843e1988Sjohnlev	cmpl	$0, %esi
824843e1988Sjohnlev	jne	1f
825843e1988Sjohnlev	/*
826843e1988Sjohnlev	 * Since we're -really- running unprivileged, our attempt
827843e1988Sjohnlev	 * to change the state of the IF bit will be ignored.
828843e1988Sjohnlev	 * The virtual IF bit is tweaked by CLI and STI.
829843e1988Sjohnlev	 */
830843e1988Sjohnlev	IE_TO_EVENT_MASK(%rsi, %rdi)
831a563a037Sbholler#else
832a563a037Sbholler	sti
833843e1988Sjohnlev#endif
834a563a037Sbholler1:
8357c478bd9Sstevel@tonic-gate	ret
8367c478bd9Sstevel@tonic-gate	SET_SIZE(restore_int_flag)
8377c478bd9Sstevel@tonic-gate	SET_SIZE(intr_restore)
8387c478bd9Sstevel@tonic-gate
8397c478bd9Sstevel@tonic-gate	ENTRY(sti)
840ae115bc7Smrj	STI
8417c478bd9Sstevel@tonic-gate	ret
8427c478bd9Sstevel@tonic-gate	SET_SIZE(sti)
8437c478bd9Sstevel@tonic-gate
844ae115bc7Smrj	ENTRY(cli)
845ae115bc7Smrj	CLI(%rax)
846ae115bc7Smrj	ret
847ae115bc7Smrj	SET_SIZE(cli)
848ae115bc7Smrj
8497c478bd9Sstevel@tonic-gate	ENTRY(dtrace_interrupt_disable)
8507c478bd9Sstevel@tonic-gate	pushfq
8517c478bd9Sstevel@tonic-gate	popq	%rax
852843e1988Sjohnlev#if defined(__xpv)
853843e1988Sjohnlev	leaq	xpv_panicking, %rdi
854843e1988Sjohnlev	movl	(%rdi), %edi
855843e1988Sjohnlev	cmpl	$0, %edi
856e4b86885SCheng Sean Ye	jne	.dtrace_interrupt_disable_done
857843e1988Sjohnlev	CLIRET(%rdi, %dl)	/* returns event mask in %dl */
858843e1988Sjohnlev	/*
859843e1988Sjohnlev	 * Synthesize the PS_IE bit from the event mask bit
860843e1988Sjohnlev	 */
861843e1988Sjohnlev	andq    $_BITNOT(PS_IE), %rax
862843e1988Sjohnlev	testb	$1, %dl
863e4b86885SCheng Sean Ye	jnz	.dtrace_interrupt_disable_done
864843e1988Sjohnlev	orq	$PS_IE, %rax
865843e1988Sjohnlev#else
866ae115bc7Smrj	CLI(%rdx)
867843e1988Sjohnlev#endif
868e4b86885SCheng Sean Ye.dtrace_interrupt_disable_done:
8697c478bd9Sstevel@tonic-gate	ret
8707c478bd9Sstevel@tonic-gate	SET_SIZE(dtrace_interrupt_disable)
8717c478bd9Sstevel@tonic-gate
8727c478bd9Sstevel@tonic-gate	ENTRY(dtrace_interrupt_enable)
8737c478bd9Sstevel@tonic-gate	pushq	%rdi
8747c478bd9Sstevel@tonic-gate	popfq
875843e1988Sjohnlev#if defined(__xpv)
876843e1988Sjohnlev	leaq	xpv_panicking, %rdx
877843e1988Sjohnlev	movl	(%rdx), %edx
878843e1988Sjohnlev	cmpl	$0, %edx
879e4b86885SCheng Sean Ye	jne	.dtrace_interrupt_enable_done
880843e1988Sjohnlev	/*
881843e1988Sjohnlev	 * Since we're -really- running unprivileged, our attempt
882843e1988Sjohnlev	 * to change the state of the IF bit will be ignored. The
883843e1988Sjohnlev	 * virtual IF bit is tweaked by CLI and STI.
884843e1988Sjohnlev	 */
885843e1988Sjohnlev	IE_TO_EVENT_MASK(%rdx, %rdi)
886843e1988Sjohnlev#endif
887e4b86885SCheng Sean Ye.dtrace_interrupt_enable_done:
8887c478bd9Sstevel@tonic-gate	ret
8897c478bd9Sstevel@tonic-gate	SET_SIZE(dtrace_interrupt_enable)
8907c478bd9Sstevel@tonic-gate
8917c478bd9Sstevel@tonic-gate
8927c478bd9Sstevel@tonic-gate	ENTRY(dtrace_membar_producer)
89385641879Skalai	rep;	ret	/* use 2 byte return instruction when branch target */
89485641879Skalai			/* AMD Software Optimization Guide - Section 6.2 */
8957c478bd9Sstevel@tonic-gate	SET_SIZE(dtrace_membar_producer)
8967c478bd9Sstevel@tonic-gate
8977c478bd9Sstevel@tonic-gate	ENTRY(dtrace_membar_consumer)
89885641879Skalai	rep;	ret	/* use 2 byte return instruction when branch target */
89985641879Skalai			/* AMD Software Optimization Guide - Section 6.2 */
9007c478bd9Sstevel@tonic-gate	SET_SIZE(dtrace_membar_consumer)
9017c478bd9Sstevel@tonic-gate
9027c478bd9Sstevel@tonic-gate	ENTRY(threadp)
9037c478bd9Sstevel@tonic-gate	movq	%gs:CPU_THREAD, %rax
9047c478bd9Sstevel@tonic-gate	ret
9057c478bd9Sstevel@tonic-gate	SET_SIZE(threadp)
9067c478bd9Sstevel@tonic-gate
9077c478bd9Sstevel@tonic-gate/*
9087c478bd9Sstevel@tonic-gate *   Checksum routine for Internet Protocol Headers
9097c478bd9Sstevel@tonic-gate */
9107c478bd9Sstevel@tonic-gate
9117c478bd9Sstevel@tonic-gate	ENTRY(ip_ocsum)
9127c478bd9Sstevel@tonic-gate	pushq	%rbp
9137c478bd9Sstevel@tonic-gate	movq	%rsp, %rbp
9147c478bd9Sstevel@tonic-gate#ifdef DEBUG
915ae115bc7Smrj	movq	postbootkernelbase(%rip), %rax
9167c478bd9Sstevel@tonic-gate	cmpq	%rax, %rdi
9177c478bd9Sstevel@tonic-gate	jnb	1f
9187c478bd9Sstevel@tonic-gate	xorl	%eax, %eax
9197c478bd9Sstevel@tonic-gate	movq	%rdi, %rsi
9207c478bd9Sstevel@tonic-gate	leaq	.ip_ocsum_panic_msg(%rip), %rdi
9217c478bd9Sstevel@tonic-gate	call	panic
9227c478bd9Sstevel@tonic-gate	/*NOTREACHED*/
9237c478bd9Sstevel@tonic-gate.ip_ocsum_panic_msg:
9247c478bd9Sstevel@tonic-gate	.string	"ip_ocsum: address 0x%p below kernelbase\n"
9257c478bd9Sstevel@tonic-gate1:
9267c478bd9Sstevel@tonic-gate#endif
9277c478bd9Sstevel@tonic-gate	movl	%esi, %ecx	/* halfword_count */
9287c478bd9Sstevel@tonic-gate	movq	%rdi, %rsi	/* address */
9297c478bd9Sstevel@tonic-gate				/* partial sum in %edx */
9307c478bd9Sstevel@tonic-gate	xorl	%eax, %eax
9317c478bd9Sstevel@tonic-gate	testl	%ecx, %ecx
9327c478bd9Sstevel@tonic-gate	jz	.ip_ocsum_done
9337c478bd9Sstevel@tonic-gate	testq	$3, %rsi
9347c478bd9Sstevel@tonic-gate	jnz	.ip_csum_notaligned
9357c478bd9Sstevel@tonic-gate.ip_csum_aligned:	/* XX64 opportunities for 8-byte operations? */
9367c478bd9Sstevel@tonic-gate.next_iter:
9377c478bd9Sstevel@tonic-gate	/* XX64 opportunities for prefetch? */
9387c478bd9Sstevel@tonic-gate	/* XX64 compute csum with 64 bit quantities? */
9397c478bd9Sstevel@tonic-gate	subl	$32, %ecx
9407c478bd9Sstevel@tonic-gate	jl	.less_than_32
9417c478bd9Sstevel@tonic-gate
9427c478bd9Sstevel@tonic-gate	addl	0(%rsi), %edx
9437c478bd9Sstevel@tonic-gate.only60:
9447c478bd9Sstevel@tonic-gate	adcl	4(%rsi), %eax
9457c478bd9Sstevel@tonic-gate.only56:
9467c478bd9Sstevel@tonic-gate	adcl	8(%rsi), %edx
9477c478bd9Sstevel@tonic-gate.only52:
9487c478bd9Sstevel@tonic-gate	adcl	12(%rsi), %eax
9497c478bd9Sstevel@tonic-gate.only48:
9507c478bd9Sstevel@tonic-gate	adcl	16(%rsi), %edx
9517c478bd9Sstevel@tonic-gate.only44:
9527c478bd9Sstevel@tonic-gate	adcl	20(%rsi), %eax
9537c478bd9Sstevel@tonic-gate.only40:
9547c478bd9Sstevel@tonic-gate	adcl	24(%rsi), %edx
9557c478bd9Sstevel@tonic-gate.only36:
9567c478bd9Sstevel@tonic-gate	adcl	28(%rsi), %eax
9577c478bd9Sstevel@tonic-gate.only32:
9587c478bd9Sstevel@tonic-gate	adcl	32(%rsi), %edx
9597c478bd9Sstevel@tonic-gate.only28:
9607c478bd9Sstevel@tonic-gate	adcl	36(%rsi), %eax
9617c478bd9Sstevel@tonic-gate.only24:
9627c478bd9Sstevel@tonic-gate	adcl	40(%rsi), %edx
9637c478bd9Sstevel@tonic-gate.only20:
9647c478bd9Sstevel@tonic-gate	adcl	44(%rsi), %eax
9657c478bd9Sstevel@tonic-gate.only16:
9667c478bd9Sstevel@tonic-gate	adcl	48(%rsi), %edx
9677c478bd9Sstevel@tonic-gate.only12:
9687c478bd9Sstevel@tonic-gate	adcl	52(%rsi), %eax
9697c478bd9Sstevel@tonic-gate.only8:
9707c478bd9Sstevel@tonic-gate	adcl	56(%rsi), %edx
9717c478bd9Sstevel@tonic-gate.only4:
9727c478bd9Sstevel@tonic-gate	adcl	60(%rsi), %eax	/* could be adding -1 and -1 with a carry */
9737c478bd9Sstevel@tonic-gate.only0:
9747c478bd9Sstevel@tonic-gate	adcl	$0, %eax	/* could be adding -1 in eax with a carry */
9757c478bd9Sstevel@tonic-gate	adcl	$0, %eax
9767c478bd9Sstevel@tonic-gate
9777c478bd9Sstevel@tonic-gate	addq	$64, %rsi
9787c478bd9Sstevel@tonic-gate	testl	%ecx, %ecx
9797c478bd9Sstevel@tonic-gate	jnz	.next_iter
9807c478bd9Sstevel@tonic-gate
9817c478bd9Sstevel@tonic-gate.ip_ocsum_done:
9827c478bd9Sstevel@tonic-gate	addl	%eax, %edx
9837c478bd9Sstevel@tonic-gate	adcl	$0, %edx
9847c478bd9Sstevel@tonic-gate	movl	%edx, %eax	/* form a 16 bit checksum by */
9857c478bd9Sstevel@tonic-gate	shrl	$16, %eax	/* adding two halves of 32 bit checksum */
9867c478bd9Sstevel@tonic-gate	addw	%dx, %ax
9877c478bd9Sstevel@tonic-gate	adcw	$0, %ax
9887c478bd9Sstevel@tonic-gate	andl	$0xffff, %eax
9897c478bd9Sstevel@tonic-gate	leave
9907c478bd9Sstevel@tonic-gate	ret
9917c478bd9Sstevel@tonic-gate
9927c478bd9Sstevel@tonic-gate.ip_csum_notaligned:
9937c478bd9Sstevel@tonic-gate	xorl	%edi, %edi
9947c478bd9Sstevel@tonic-gate	movw	(%rsi), %di
9957c478bd9Sstevel@tonic-gate	addl	%edi, %edx
9967c478bd9Sstevel@tonic-gate	adcl	$0, %edx
9977c478bd9Sstevel@tonic-gate	addq	$2, %rsi
9987c478bd9Sstevel@tonic-gate	decl	%ecx
9997c478bd9Sstevel@tonic-gate	jmp	.ip_csum_aligned
10007c478bd9Sstevel@tonic-gate
10017c478bd9Sstevel@tonic-gate.less_than_32:
10027c478bd9Sstevel@tonic-gate	addl	$32, %ecx
10037c478bd9Sstevel@tonic-gate	testl	$1, %ecx
10047c478bd9Sstevel@tonic-gate	jz	.size_aligned
10057c478bd9Sstevel@tonic-gate	andl	$0xfe, %ecx
10067c478bd9Sstevel@tonic-gate	movzwl	(%rsi, %rcx, 2), %edi
10077c478bd9Sstevel@tonic-gate	addl	%edi, %edx
10087c478bd9Sstevel@tonic-gate	adcl	$0, %edx
10097c478bd9Sstevel@tonic-gate.size_aligned:
10107c478bd9Sstevel@tonic-gate	movl	%ecx, %edi
10117c478bd9Sstevel@tonic-gate	shrl	$1, %ecx
10127c478bd9Sstevel@tonic-gate	shl	$1, %edi
10137c478bd9Sstevel@tonic-gate	subq	$64, %rdi
10147c478bd9Sstevel@tonic-gate	addq	%rdi, %rsi
10157c478bd9Sstevel@tonic-gate	leaq    .ip_ocsum_jmptbl(%rip), %rdi
10167c478bd9Sstevel@tonic-gate	leaq	(%rdi, %rcx, 8), %rdi
10177c478bd9Sstevel@tonic-gate	xorl	%ecx, %ecx
10187c478bd9Sstevel@tonic-gate	clc
101965f20420SRobert Mustacchi	movq	(%rdi), %rdi
102065f20420SRobert Mustacchi	INDIRECT_JMP_REG(rdi)
10217c478bd9Sstevel@tonic-gate
10227c478bd9Sstevel@tonic-gate	.align	8
10237c478bd9Sstevel@tonic-gate.ip_ocsum_jmptbl:
10247c478bd9Sstevel@tonic-gate	.quad	.only0, .only4, .only8, .only12, .only16, .only20
10257c478bd9Sstevel@tonic-gate	.quad	.only24, .only28, .only32, .only36, .only40, .only44
10267c478bd9Sstevel@tonic-gate	.quad	.only48, .only52, .only56, .only60
10277c478bd9Sstevel@tonic-gate	SET_SIZE(ip_ocsum)
10287c478bd9Sstevel@tonic-gate
10297c478bd9Sstevel@tonic-gate/*
10307c478bd9Sstevel@tonic-gate * multiply two long numbers and yield a u_longlong_t result, callable from C.
10317c478bd9Sstevel@tonic-gate * Provided to manipulate hrtime_t values.
10327c478bd9Sstevel@tonic-gate */
10337c478bd9Sstevel@tonic-gate
10347c478bd9Sstevel@tonic-gate	ENTRY(mul32)
10357c478bd9Sstevel@tonic-gate	xorl	%edx, %edx	/* XX64 joe, paranoia? */
10367c478bd9Sstevel@tonic-gate	movl	%edi, %eax
10377c478bd9Sstevel@tonic-gate	mull	%esi
103865f20420SRobert Mustacchi	shlq	$32, %rdx
10397c478bd9Sstevel@tonic-gate	orq	%rdx, %rax
10407c478bd9Sstevel@tonic-gate	ret
10417c478bd9Sstevel@tonic-gate	SET_SIZE(mul32)
10427c478bd9Sstevel@tonic-gate
10437c478bd9Sstevel@tonic-gate	ENTRY(scan_memory)
10447c478bd9Sstevel@tonic-gate	shrq	$3, %rsi	/* convert %rsi from byte to quadword count */
10457c478bd9Sstevel@tonic-gate	jz	.scanm_done
10467c478bd9Sstevel@tonic-gate	movq	%rsi, %rcx	/* move count into rep control register */
10477c478bd9Sstevel@tonic-gate	movq	%rdi, %rsi	/* move addr into lodsq control reg. */
10487c478bd9Sstevel@tonic-gate	rep lodsq		/* scan the memory range */
10497c478bd9Sstevel@tonic-gate.scanm_done:
105085641879Skalai	rep;	ret	/* use 2 byte return instruction when branch target */
105185641879Skalai			/* AMD Software Optimization Guide - Section 6.2 */
10527c478bd9Sstevel@tonic-gate	SET_SIZE(scan_memory)
10537c478bd9Sstevel@tonic-gate
10547c478bd9Sstevel@tonic-gate
10557c478bd9Sstevel@tonic-gate	ENTRY(lowbit)
10567c478bd9Sstevel@tonic-gate	movl	$-1, %eax
1057fdb8cf8cSJosef 'Jeff' Sipek	bsfq	%rdi, %rdi
1058fdb8cf8cSJosef 'Jeff' Sipek	cmovnz	%edi, %eax
10597c478bd9Sstevel@tonic-gate	incl	%eax
10607c478bd9Sstevel@tonic-gate	ret
10617c478bd9Sstevel@tonic-gate	SET_SIZE(lowbit)
10627c478bd9Sstevel@tonic-gate
10637c478bd9Sstevel@tonic-gate	ENTRY(highbit)
1064fdb8cf8cSJosef 'Jeff' Sipek	ALTENTRY(highbit64)
10657c478bd9Sstevel@tonic-gate	movl	$-1, %eax
1066fdb8cf8cSJosef 'Jeff' Sipek	bsrq	%rdi, %rdi
1067fdb8cf8cSJosef 'Jeff' Sipek	cmovnz	%edi, %eax
10687c478bd9Sstevel@tonic-gate	incl	%eax
10697c478bd9Sstevel@tonic-gate	ret
1070fdb8cf8cSJosef 'Jeff' Sipek	SET_SIZE(highbit64)
10717c478bd9Sstevel@tonic-gate	SET_SIZE(highbit)
10727c478bd9Sstevel@tonic-gate
1073ee88d2b9Skchow#define	XMSR_ACCESS_VAL		$0x9c5a203a
1074ee88d2b9Skchow
10757c478bd9Sstevel@tonic-gate	ENTRY(rdmsr)
10767c478bd9Sstevel@tonic-gate	movl	%edi, %ecx
10777c478bd9Sstevel@tonic-gate	rdmsr
10787c478bd9Sstevel@tonic-gate	shlq	$32, %rdx
10797c478bd9Sstevel@tonic-gate	orq	%rdx, %rax
10807c478bd9Sstevel@tonic-gate	ret
10817c478bd9Sstevel@tonic-gate	SET_SIZE(rdmsr)
10827c478bd9Sstevel@tonic-gate
10837c478bd9Sstevel@tonic-gate	ENTRY(wrmsr)
10840ac7d7d8Skucharsk	movq	%rsi, %rdx
10850ac7d7d8Skucharsk	shrq	$32, %rdx
10860ac7d7d8Skucharsk	movl	%esi, %eax
10877c478bd9Sstevel@tonic-gate	movl	%edi, %ecx
10887c478bd9Sstevel@tonic-gate	wrmsr
10897c478bd9Sstevel@tonic-gate	ret
10907c478bd9Sstevel@tonic-gate	SET_SIZE(wrmsr)
10917c478bd9Sstevel@tonic-gate
1092ee88d2b9Skchow	ENTRY(xrdmsr)
1093ae115bc7Smrj	pushq	%rbp
1094ae115bc7Smrj	movq	%rsp, %rbp
1095ee88d2b9Skchow	movl	%edi, %ecx
1096ee88d2b9Skchow	movl	XMSR_ACCESS_VAL, %edi	/* this value is needed to access MSR */
1097ee88d2b9Skchow	rdmsr
1098ee88d2b9Skchow	shlq	$32, %rdx
1099ee88d2b9Skchow	orq	%rdx, %rax
1100ae115bc7Smrj	leave
1101ee88d2b9Skchow	ret
1102ee88d2b9Skchow	SET_SIZE(xrdmsr)
1103ee88d2b9Skchow
1104ee88d2b9Skchow	ENTRY(xwrmsr)
1105ae115bc7Smrj	pushq	%rbp
1106ae115bc7Smrj	movq	%rsp, %rbp
1107ee88d2b9Skchow	movl	%edi, %ecx
1108ee88d2b9Skchow	movl	XMSR_ACCESS_VAL, %edi	/* this value is needed to access MSR */
1109ee88d2b9Skchow	movq	%rsi, %rdx
1110ee88d2b9Skchow	shrq	$32, %rdx
1111ee88d2b9Skchow	movl	%esi, %eax
1112ee88d2b9Skchow	wrmsr
1113ae115bc7Smrj	leave
1114ee88d2b9Skchow	ret
1115ee88d2b9Skchow	SET_SIZE(xwrmsr)
11167af88ac7SKuriakose Kuruvilla
11177af88ac7SKuriakose Kuruvilla	ENTRY(get_xcr)
11187af88ac7SKuriakose Kuruvilla	movl	%edi, %ecx
11197af88ac7SKuriakose Kuruvilla	#xgetbv
11207af88ac7SKuriakose Kuruvilla	.byte	0x0f,0x01,0xd0
11217af88ac7SKuriakose Kuruvilla	shlq	$32, %rdx
11227af88ac7SKuriakose Kuruvilla	orq	%rdx, %rax
11237af88ac7SKuriakose Kuruvilla	ret
11247af88ac7SKuriakose Kuruvilla	SET_SIZE(get_xcr)
11257af88ac7SKuriakose Kuruvilla
11267af88ac7SKuriakose Kuruvilla	ENTRY(set_xcr)
11277af88ac7SKuriakose Kuruvilla	movq	%rsi, %rdx
11287af88ac7SKuriakose Kuruvilla	shrq	$32, %rdx
11297af88ac7SKuriakose Kuruvilla	movl	%esi, %eax
11307af88ac7SKuriakose Kuruvilla	movl	%edi, %ecx
11317af88ac7SKuriakose Kuruvilla	#xsetbv
11327af88ac7SKuriakose Kuruvilla	.byte	0x0f,0x01,0xd1
11337af88ac7SKuriakose Kuruvilla	ret
11347af88ac7SKuriakose Kuruvilla	SET_SIZE(set_xcr)
11357af88ac7SKuriakose Kuruvilla
11367c478bd9Sstevel@tonic-gate	ENTRY(invalidate_cache)
11377c478bd9Sstevel@tonic-gate	wbinvd
11387c478bd9Sstevel@tonic-gate	ret
11397c478bd9Sstevel@tonic-gate	SET_SIZE(invalidate_cache)
11407c478bd9Sstevel@tonic-gate
1141ae115bc7Smrj	ENTRY_NP(getcregs)
1142843e1988Sjohnlev#if defined(__xpv)
1143843e1988Sjohnlev	/*
1144843e1988Sjohnlev	 * Only a few of the hardware control registers or descriptor tables
1145843e1988Sjohnlev	 * are directly accessible to us, so just zero the structure.
1146843e1988Sjohnlev	 *
1147843e1988Sjohnlev	 * XXPV	Perhaps it would be helpful for the hypervisor to return
1148843e1988Sjohnlev	 *	virtualized versions of these for post-mortem use.
1149843e1988Sjohnlev	 *	(Need to reevaluate - perhaps it already does!)
1150843e1988Sjohnlev	 */
1151843e1988Sjohnlev	pushq	%rdi		/* save *crp */
1152843e1988Sjohnlev	movq	$CREGSZ, %rsi
1153843e1988Sjohnlev	call	bzero
1154843e1988Sjohnlev	popq	%rdi
1155843e1988Sjohnlev
1156843e1988Sjohnlev	/*
1157843e1988Sjohnlev	 * Dump what limited information we can
1158843e1988Sjohnlev	 */
1159843e1988Sjohnlev	movq	%cr0, %rax
1160843e1988Sjohnlev	movq	%rax, CREG_CR0(%rdi)	/* cr0 */
1161843e1988Sjohnlev	movq	%cr2, %rax
1162843e1988Sjohnlev	movq	%rax, CREG_CR2(%rdi)	/* cr2 */
1163843e1988Sjohnlev	movq	%cr3, %rax
1164843e1988Sjohnlev	movq	%rax, CREG_CR3(%rdi)	/* cr3 */
1165843e1988Sjohnlev	movq	%cr4, %rax
1166843e1988Sjohnlev	movq	%rax, CREG_CR4(%rdi)	/* cr4 */
1167843e1988Sjohnlev
1168843e1988Sjohnlev#else	/* __xpv */
1169ae115bc7Smrj
11707c478bd9Sstevel@tonic-gate#define	GETMSR(r, off, d)	\
11717c478bd9Sstevel@tonic-gate	movl	$r, %ecx;	\
11727c478bd9Sstevel@tonic-gate	rdmsr;			\
11737c478bd9Sstevel@tonic-gate	movl	%eax, off(d);	\
11747c478bd9Sstevel@tonic-gate	movl	%edx, off+4(d)
11757c478bd9Sstevel@tonic-gate
11767c478bd9Sstevel@tonic-gate	xorl	%eax, %eax
11777c478bd9Sstevel@tonic-gate	movq	%rax, CREG_GDT+8(%rdi)
11787c478bd9Sstevel@tonic-gate	sgdt	CREG_GDT(%rdi)		/* 10 bytes */
11797c478bd9Sstevel@tonic-gate	movq	%rax, CREG_IDT+8(%rdi)
11807c478bd9Sstevel@tonic-gate	sidt	CREG_IDT(%rdi)		/* 10 bytes */
11817c478bd9Sstevel@tonic-gate	movq	%rax, CREG_LDT(%rdi)
11827c478bd9Sstevel@tonic-gate	sldt	CREG_LDT(%rdi)		/* 2 bytes */
11837c478bd9Sstevel@tonic-gate	movq	%rax, CREG_TASKR(%rdi)
11847c478bd9Sstevel@tonic-gate	str	CREG_TASKR(%rdi)	/* 2 bytes */
11857c478bd9Sstevel@tonic-gate	movq	%cr0, %rax
11867c478bd9Sstevel@tonic-gate	movq	%rax, CREG_CR0(%rdi)	/* cr0 */
11877c478bd9Sstevel@tonic-gate	movq	%cr2, %rax
11887c478bd9Sstevel@tonic-gate	movq	%rax, CREG_CR2(%rdi)	/* cr2 */
11897c478bd9Sstevel@tonic-gate	movq	%cr3, %rax
11907c478bd9Sstevel@tonic-gate	movq	%rax, CREG_CR3(%rdi)	/* cr3 */
11917c478bd9Sstevel@tonic-gate	movq	%cr4, %rax
1192ae115bc7Smrj	movq	%rax, CREG_CR4(%rdi)	/* cr4 */
11937c478bd9Sstevel@tonic-gate	movq	%cr8, %rax
11947c478bd9Sstevel@tonic-gate	movq	%rax, CREG_CR8(%rdi)	/* cr8 */
11957c478bd9Sstevel@tonic-gate	GETMSR(MSR_AMD_KGSBASE, CREG_KGSBASE, %rdi)
11967c478bd9Sstevel@tonic-gate	GETMSR(MSR_AMD_EFER, CREG_EFER, %rdi)
1197843e1988Sjohnlev#endif	/* __xpv */
1198ae115bc7Smrj	ret
11997c478bd9Sstevel@tonic-gate	SET_SIZE(getcregs)
12007c478bd9Sstevel@tonic-gate
12017c478bd9Sstevel@tonic-gate#undef GETMSR
12027c478bd9Sstevel@tonic-gate
12037c478bd9Sstevel@tonic-gate
12047c478bd9Sstevel@tonic-gate/*
12057c478bd9Sstevel@tonic-gate * A panic trigger is a word which is updated atomically and can only be set
12067c478bd9Sstevel@tonic-gate * once.  We atomically store 0xDEFACEDD and load the old value.  If the
12077c478bd9Sstevel@tonic-gate * previous value was 0, we succeed and return 1; otherwise return 0.
12087c478bd9Sstevel@tonic-gate * This allows a partially corrupt trigger to still trigger correctly.  DTrace
12097c478bd9Sstevel@tonic-gate * has its own version of this function to allow it to panic correctly from
12107c478bd9Sstevel@tonic-gate * probe context.
12117c478bd9Sstevel@tonic-gate */
12127c478bd9Sstevel@tonic-gate
12137c478bd9Sstevel@tonic-gate	ENTRY_NP(panic_trigger)
12147c478bd9Sstevel@tonic-gate	xorl	%eax, %eax
12157c478bd9Sstevel@tonic-gate	movl	$0xdefacedd, %edx
12167c478bd9Sstevel@tonic-gate	lock
12177c478bd9Sstevel@tonic-gate	  xchgl	%edx, (%rdi)
12187c478bd9Sstevel@tonic-gate	cmpl	$0, %edx
121965f20420SRobert Mustacchi	je	0f
12207c478bd9Sstevel@tonic-gate	movl	$0, %eax
12217c478bd9Sstevel@tonic-gate	ret
12227c478bd9Sstevel@tonic-gate0:	movl	$1, %eax
12237c478bd9Sstevel@tonic-gate	ret
12247c478bd9Sstevel@tonic-gate	SET_SIZE(panic_trigger)
122565f20420SRobert Mustacchi
12267c478bd9Sstevel@tonic-gate	ENTRY_NP(dtrace_panic_trigger)
12277c478bd9Sstevel@tonic-gate	xorl	%eax, %eax
12287c478bd9Sstevel@tonic-gate	movl	$0xdefacedd, %edx
12297c478bd9Sstevel@tonic-gate	lock
12307c478bd9Sstevel@tonic-gate	  xchgl	%edx, (%rdi)
12317c478bd9Sstevel@tonic-gate	cmpl	$0, %edx
12327c478bd9Sstevel@tonic-gate	je	0f
12337c478bd9Sstevel@tonic-gate	movl	$0, %eax
12347c478bd9Sstevel@tonic-gate	ret
12357c478bd9Sstevel@tonic-gate0:	movl	$1, %eax
12367c478bd9Sstevel@tonic-gate	ret
12377c478bd9Sstevel@tonic-gate	SET_SIZE(dtrace_panic_trigger)
12387c478bd9Sstevel@tonic-gate
12397c478bd9Sstevel@tonic-gate/*
12407c478bd9Sstevel@tonic-gate * The panic() and cmn_err() functions invoke vpanic() as a common entry point
12417c478bd9Sstevel@tonic-gate * into the panic code implemented in panicsys().  vpanic() is responsible
12427c478bd9Sstevel@tonic-gate * for passing through the format string and arguments, and constructing a
12437c478bd9Sstevel@tonic-gate * regs structure on the stack into which it saves the current register
12447c478bd9Sstevel@tonic-gate * values.  If we are not dying due to a fatal trap, these registers will
12457c478bd9Sstevel@tonic-gate * then be preserved in panicbuf as the current processor state.  Before
12467c478bd9Sstevel@tonic-gate * invoking panicsys(), vpanic() activates the first panic trigger (see
12477c478bd9Sstevel@tonic-gate * common/os/panic.c) and switches to the panic_stack if successful.  Note that
12487c478bd9Sstevel@tonic-gate * DTrace takes a slightly different panic path if it must panic from probe
12497c478bd9Sstevel@tonic-gate * context.  Instead of calling panic, it calls into dtrace_vpanic(), which
12507c478bd9Sstevel@tonic-gate * sets up the initial stack as vpanic does, calls dtrace_panic_trigger(), and
12517c478bd9Sstevel@tonic-gate * branches back into vpanic().
12527c478bd9Sstevel@tonic-gate */
12537c478bd9Sstevel@tonic-gate
12547c478bd9Sstevel@tonic-gate	ENTRY_NP(vpanic)			/* Initial stack layout: */
125565f20420SRobert Mustacchi
125665f20420SRobert Mustacchi	pushq	%rbp				/* | %rip |	0x60	*/
12577c478bd9Sstevel@tonic-gate	movq	%rsp, %rbp			/* | %rbp |	0x58	*/
12587c478bd9Sstevel@tonic-gate	pushfq					/* | rfl  |	0x50	*/
12597c478bd9Sstevel@tonic-gate	pushq	%r11				/* | %r11 |	0x48	*/
12607c478bd9Sstevel@tonic-gate	pushq	%r10				/* | %r10 |	0x40	*/
12617c478bd9Sstevel@tonic-gate	pushq	%rbx				/* | %rbx |	0x38	*/
12627c478bd9Sstevel@tonic-gate	pushq	%rax				/* | %rax |	0x30	*/
12637c478bd9Sstevel@tonic-gate	pushq	%r9				/* | %r9  |	0x28	*/
12647c478bd9Sstevel@tonic-gate	pushq	%r8				/* | %r8  |	0x20	*/
12657c478bd9Sstevel@tonic-gate	pushq	%rcx				/* | %rcx |	0x18	*/
12667c478bd9Sstevel@tonic-gate	pushq	%rdx				/* | %rdx |	0x10	*/
12677c478bd9Sstevel@tonic-gate	pushq	%rsi				/* | %rsi |	0x8 alist */
12687c478bd9Sstevel@tonic-gate	pushq	%rdi				/* | %rdi |	0x0 format */
12697c478bd9Sstevel@tonic-gate
12707c478bd9Sstevel@tonic-gate	movq	%rsp, %rbx			/* %rbx = current %rsp */
12717c478bd9Sstevel@tonic-gate
12727c478bd9Sstevel@tonic-gate	leaq	panic_quiesce(%rip), %rdi	/* %rdi = &panic_quiesce */
12737c478bd9Sstevel@tonic-gate	call	panic_trigger			/* %eax = panic_trigger() */
12747c478bd9Sstevel@tonic-gate
12757c478bd9Sstevel@tonic-gatevpanic_common:
127652d60c84Sgavinm	/*
127752d60c84Sgavinm	 * The panic_trigger result is in %eax from the call above, and
127852d60c84Sgavinm	 * dtrace_panic places it in %eax before branching here.
127952d60c84Sgavinm	 * The rdmsr instructions that follow below will clobber %eax so
128052d60c84Sgavinm	 * we stash the panic_trigger result in %r11d.
128152d60c84Sgavinm	 */
128252d60c84Sgavinm	movl	%eax, %r11d
128352d60c84Sgavinm	cmpl	$0, %r11d
12847c478bd9Sstevel@tonic-gate	je	0f
12857c478bd9Sstevel@tonic-gate
12867c478bd9Sstevel@tonic-gate	/*
12877c478bd9Sstevel@tonic-gate	 * If panic_trigger() was successful, we are the first to initiate a
12887c478bd9Sstevel@tonic-gate	 * panic: we now switch to the reserved panic_stack before continuing.
12897c478bd9Sstevel@tonic-gate	 */
12907c478bd9Sstevel@tonic-gate	leaq	panic_stack(%rip), %rsp
12917c478bd9Sstevel@tonic-gate	addq	$PANICSTKSIZE, %rsp
12927c478bd9Sstevel@tonic-gate0:	subq	$REGSIZE, %rsp
12937c478bd9Sstevel@tonic-gate	/*
12947c478bd9Sstevel@tonic-gate	 * Now that we've got everything set up, store the register values as
12957c478bd9Sstevel@tonic-gate	 * they were when we entered vpanic() to the designated location in
12967c478bd9Sstevel@tonic-gate	 * the regs structure we allocated on the stack.
12977c478bd9Sstevel@tonic-gate	 */
12987c478bd9Sstevel@tonic-gate	movq	0x0(%rbx), %rcx
12997c478bd9Sstevel@tonic-gate	movq	%rcx, REGOFF_RDI(%rsp)
13007c478bd9Sstevel@tonic-gate	movq	0x8(%rbx), %rcx
13017c478bd9Sstevel@tonic-gate	movq	%rcx, REGOFF_RSI(%rsp)
13027c478bd9Sstevel@tonic-gate	movq	0x10(%rbx), %rcx
13037c478bd9Sstevel@tonic-gate	movq	%rcx, REGOFF_RDX(%rsp)
13047c478bd9Sstevel@tonic-gate	movq	0x18(%rbx), %rcx
13057c478bd9Sstevel@tonic-gate	movq	%rcx, REGOFF_RCX(%rsp)
13067c478bd9Sstevel@tonic-gate	movq	0x20(%rbx), %rcx
13077c478bd9Sstevel@tonic-gate
13087c478bd9Sstevel@tonic-gate	movq	%rcx, REGOFF_R8(%rsp)
13097c478bd9Sstevel@tonic-gate	movq	0x28(%rbx), %rcx
13107c478bd9Sstevel@tonic-gate	movq	%rcx, REGOFF_R9(%rsp)
13117c478bd9Sstevel@tonic-gate	movq	0x30(%rbx), %rcx
13127c478bd9Sstevel@tonic-gate	movq	%rcx, REGOFF_RAX(%rsp)
13137c478bd9Sstevel@tonic-gate	movq	0x38(%rbx), %rcx
13142d596601Ssethg	movq	%rcx, REGOFF_RBX(%rsp)
13157c478bd9Sstevel@tonic-gate	movq	0x58(%rbx), %rcx
13167c478bd9Sstevel@tonic-gate
13177c478bd9Sstevel@tonic-gate	movq	%rcx, REGOFF_RBP(%rsp)
13187c478bd9Sstevel@tonic-gate	movq	0x40(%rbx), %rcx
13197c478bd9Sstevel@tonic-gate	movq	%rcx, REGOFF_R10(%rsp)
13207c478bd9Sstevel@tonic-gate	movq	0x48(%rbx), %rcx
13217c478bd9Sstevel@tonic-gate	movq	%rcx, REGOFF_R11(%rsp)
13227c478bd9Sstevel@tonic-gate	movq	%r12, REGOFF_R12(%rsp)
13237c478bd9Sstevel@tonic-gate
13247c478bd9Sstevel@tonic-gate	movq	%r13, REGOFF_R13(%rsp)
13257c478bd9Sstevel@tonic-gate	movq	%r14, REGOFF_R14(%rsp)
13267c478bd9Sstevel@tonic-gate	movq	%r15, REGOFF_R15(%rsp)
13277c478bd9Sstevel@tonic-gate
13287c478bd9Sstevel@tonic-gate	xorl	%ecx, %ecx
13297c478bd9Sstevel@tonic-gate	movw	%ds, %cx
13307c478bd9Sstevel@tonic-gate	movq	%rcx, REGOFF_DS(%rsp)
13317c478bd9Sstevel@tonic-gate	movw	%es, %cx
13327c478bd9Sstevel@tonic-gate	movq	%rcx, REGOFF_ES(%rsp)
13337c478bd9Sstevel@tonic-gate	movw	%fs, %cx
13347c478bd9Sstevel@tonic-gate	movq	%rcx, REGOFF_FS(%rsp)
13357c478bd9Sstevel@tonic-gate	movw	%gs, %cx
13367c478bd9Sstevel@tonic-gate	movq	%rcx, REGOFF_GS(%rsp)
13377c478bd9Sstevel@tonic-gate
13387c478bd9Sstevel@tonic-gate	movq	$0, REGOFF_TRAPNO(%rsp)
13397c478bd9Sstevel@tonic-gate
13407c478bd9Sstevel@tonic-gate	movq	$0, REGOFF_ERR(%rsp)
13417c478bd9Sstevel@tonic-gate	leaq	vpanic(%rip), %rcx
13427c478bd9Sstevel@tonic-gate	movq	%rcx, REGOFF_RIP(%rsp)
13437c478bd9Sstevel@tonic-gate	movw	%cs, %cx
13447c478bd9Sstevel@tonic-gate	movzwq	%cx, %rcx
13457c478bd9Sstevel@tonic-gate	movq	%rcx, REGOFF_CS(%rsp)
13467c478bd9Sstevel@tonic-gate	movq	0x50(%rbx), %rcx
13477c478bd9Sstevel@tonic-gate	movq	%rcx, REGOFF_RFL(%rsp)
13487c478bd9Sstevel@tonic-gate	movq	%rbx, %rcx
13497c478bd9Sstevel@tonic-gate	addq	$0x60, %rcx
13507c478bd9Sstevel@tonic-gate	movq	%rcx, REGOFF_RSP(%rsp)
13517c478bd9Sstevel@tonic-gate	movw	%ss, %cx
13527c478bd9Sstevel@tonic-gate	movzwq	%cx, %rcx
13537c478bd9Sstevel@tonic-gate	movq	%rcx, REGOFF_SS(%rsp)
13547c478bd9Sstevel@tonic-gate
13557c478bd9Sstevel@tonic-gate	/*
135665f20420SRobert Mustacchi	 * panicsys(format, alist, rp, on_panic_stack)
135765f20420SRobert Mustacchi	 */
13587c478bd9Sstevel@tonic-gate	movq	REGOFF_RDI(%rsp), %rdi		/* format */
13597c478bd9Sstevel@tonic-gate	movq	REGOFF_RSI(%rsp), %rsi		/* alist */
13607c478bd9Sstevel@tonic-gate	movq	%rsp, %rdx			/* struct regs */
136152d60c84Sgavinm	movl	%r11d, %ecx			/* on_panic_stack */
13627c478bd9Sstevel@tonic-gate	call	panicsys
13637c478bd9Sstevel@tonic-gate	addq	$REGSIZE, %rsp
13647c478bd9Sstevel@tonic-gate	popq	%rdi
13657c478bd9Sstevel@tonic-gate	popq	%rsi
13667c478bd9Sstevel@tonic-gate	popq	%rdx
13677c478bd9Sstevel@tonic-gate	popq	%rcx
13687c478bd9Sstevel@tonic-gate	popq	%r8
13697c478bd9Sstevel@tonic-gate	popq	%r9
13707c478bd9Sstevel@tonic-gate	popq	%rax
13717c478bd9Sstevel@tonic-gate	popq	%rbx
13727c478bd9Sstevel@tonic-gate	popq	%r10
13737c478bd9Sstevel@tonic-gate	popq	%r11
13747c478bd9Sstevel@tonic-gate	popfq
13757c478bd9Sstevel@tonic-gate	leave
13767c478bd9Sstevel@tonic-gate	ret
13777c478bd9Sstevel@tonic-gate	SET_SIZE(vpanic)
13787c478bd9Sstevel@tonic-gate
13797c478bd9Sstevel@tonic-gate	ENTRY_NP(dtrace_vpanic)			/* Initial stack layout: */
13807c478bd9Sstevel@tonic-gate
138165f20420SRobert Mustacchi	pushq	%rbp				/* | %rip |	0x60	*/
13827c478bd9Sstevel@tonic-gate	movq	%rsp, %rbp			/* | %rbp |	0x58	*/
13837c478bd9Sstevel@tonic-gate	pushfq					/* | rfl  |	0x50	*/
13847c478bd9Sstevel@tonic-gate	pushq	%r11				/* | %r11 |	0x48	*/
13857c478bd9Sstevel@tonic-gate	pushq	%r10				/* | %r10 |	0x40	*/
13867c478bd9Sstevel@tonic-gate	pushq	%rbx				/* | %rbx |	0x38	*/
13877c478bd9Sstevel@tonic-gate	pushq	%rax				/* | %rax |	0x30	*/
13887c478bd9Sstevel@tonic-gate	pushq	%r9				/* | %r9  |	0x28	*/
13897c478bd9Sstevel@tonic-gate	pushq	%r8				/* | %r8  |	0x20	*/
13907c478bd9Sstevel@tonic-gate	pushq	%rcx				/* | %rcx |	0x18	*/
13917c478bd9Sstevel@tonic-gate	pushq	%rdx				/* | %rdx |	0x10	*/
13927c478bd9Sstevel@tonic-gate	pushq	%rsi				/* | %rsi |	0x8 alist */
13937c478bd9Sstevel@tonic-gate	pushq	%rdi				/* | %rdi |	0x0 format */
13947c478bd9Sstevel@tonic-gate
13957c478bd9Sstevel@tonic-gate	movq	%rsp, %rbx			/* %rbx = current %rsp */
13967c478bd9Sstevel@tonic-gate
13977c478bd9Sstevel@tonic-gate	leaq	panic_quiesce(%rip), %rdi	/* %rdi = &panic_quiesce */
13987c478bd9Sstevel@tonic-gate	call	dtrace_panic_trigger	/* %eax = dtrace_panic_trigger() */
13997c478bd9Sstevel@tonic-gate	jmp	vpanic_common
14007c478bd9Sstevel@tonic-gate
14017c478bd9Sstevel@tonic-gate	SET_SIZE(dtrace_vpanic)
14027c478bd9Sstevel@tonic-gate
14037c478bd9Sstevel@tonic-gate	DGDEF3(timedelta, 8, 8)
14047c478bd9Sstevel@tonic-gate	.long	0, 0
14057c478bd9Sstevel@tonic-gate
14067c478bd9Sstevel@tonic-gate	/*
14077c478bd9Sstevel@tonic-gate	 * initialized to a non zero value to make pc_gethrtime()
14087c478bd9Sstevel@tonic-gate	 * work correctly even before clock is initialized
14097c478bd9Sstevel@tonic-gate	 */
14107c478bd9Sstevel@tonic-gate	DGDEF3(hrtime_base, 8, 8)
14117c478bd9Sstevel@tonic-gate	.long	_MUL(NSEC_PER_CLOCK_TICK, 6), 0
14127c478bd9Sstevel@tonic-gate
14137c478bd9Sstevel@tonic-gate	DGDEF3(adj_shift, 4, 4)
14147c478bd9Sstevel@tonic-gate	.long	ADJ_SHIFT
14157c478bd9Sstevel@tonic-gate
14167c478bd9Sstevel@tonic-gate	ENTRY_NP(hres_tick)
14177c478bd9Sstevel@tonic-gate	pushq	%rbp
14187c478bd9Sstevel@tonic-gate	movq	%rsp, %rbp
14197c478bd9Sstevel@tonic-gate
14207c478bd9Sstevel@tonic-gate	/*
14217c478bd9Sstevel@tonic-gate	 * We need to call *gethrtimef before picking up CLOCK_LOCK (obviously,
14227c478bd9Sstevel@tonic-gate	 * hres_last_tick can only be modified while holding CLOCK_LOCK).
14237c478bd9Sstevel@tonic-gate	 * At worst, performing this now instead of under CLOCK_LOCK may
14247c478bd9Sstevel@tonic-gate	 * introduce some jitter in pc_gethrestime().
14257c478bd9Sstevel@tonic-gate	 */
142665f20420SRobert Mustacchi	movq	gethrtimef(%rip), %rsi
142765f20420SRobert Mustacchi	INDIRECT_CALL_REG(rsi)
14287c478bd9Sstevel@tonic-gate	movq	%rax, %r8
14297c478bd9Sstevel@tonic-gate
14307c478bd9Sstevel@tonic-gate	leaq	hres_lock(%rip), %rax
14317c478bd9Sstevel@tonic-gate	movb	$-1, %dl
14327c478bd9Sstevel@tonic-gate.CL1:
14337c478bd9Sstevel@tonic-gate	xchgb	%dl, (%rax)
14347c478bd9Sstevel@tonic-gate	testb	%dl, %dl
14357c478bd9Sstevel@tonic-gate	jz	.CL3			/* got it */
14367c478bd9Sstevel@tonic-gate.CL2:
14377c478bd9Sstevel@tonic-gate	cmpb	$0, (%rax)		/* possible to get lock? */
14387c478bd9Sstevel@tonic-gate	pause
14397c478bd9Sstevel@tonic-gate	jne	.CL2
14407c478bd9Sstevel@tonic-gate	jmp	.CL1			/* yes, try again */
14417c478bd9Sstevel@tonic-gate.CL3:
14427c478bd9Sstevel@tonic-gate	/*
14437c478bd9Sstevel@tonic-gate	 * compute the interval since last time hres_tick was called
14447c478bd9Sstevel@tonic-gate	 * and adjust hrtime_base and hrestime accordingly
14457c478bd9Sstevel@tonic-gate	 * hrtime_base is an 8 byte value (in nsec), hrestime is
14467c478bd9Sstevel@tonic-gate	 * a timestruc_t (sec, nsec)
14477c478bd9Sstevel@tonic-gate	 */
14487c478bd9Sstevel@tonic-gate	leaq	hres_last_tick(%rip), %rax
14497c478bd9Sstevel@tonic-gate	movq	%r8, %r11
14507c478bd9Sstevel@tonic-gate	subq	(%rax), %r8
14517c478bd9Sstevel@tonic-gate	addq	%r8, hrtime_base(%rip)	/* add interval to hrtime_base */
14527c478bd9Sstevel@tonic-gate	addq	%r8, hrestime+8(%rip)	/* add interval to hrestime.tv_nsec */
14537c478bd9Sstevel@tonic-gate	/*
14547c478bd9Sstevel@tonic-gate	 * Now that we have CLOCK_LOCK, we can update hres_last_tick
145565f20420SRobert Mustacchi	 */
145665f20420SRobert Mustacchi	movq	%r11, (%rax)
14577c478bd9Sstevel@tonic-gate
14587c478bd9Sstevel@tonic-gate	call	__adj_hrestime
14597c478bd9Sstevel@tonic-gate
14607c478bd9Sstevel@tonic-gate	/*
14617c478bd9Sstevel@tonic-gate	 * release the hres_lock
14627c478bd9Sstevel@tonic-gate	 */
14637c478bd9Sstevel@tonic-gate	incl	hres_lock(%rip)
14647c478bd9Sstevel@tonic-gate	leave
14657c478bd9Sstevel@tonic-gate	ret
14667c478bd9Sstevel@tonic-gate	SET_SIZE(hres_tick)
146765f20420SRobert Mustacchi
14687c478bd9Sstevel@tonic-gate/*
14697c478bd9Sstevel@tonic-gate * void prefetch_smap_w(void *)
14707c478bd9Sstevel@tonic-gate *
14717c478bd9Sstevel@tonic-gate * Prefetch ahead within a linear list of smap structures.
14727c478bd9Sstevel@tonic-gate * Not implemented for ia32.  Stub for compatibility.
14737c478bd9Sstevel@tonic-gate */
14747c478bd9Sstevel@tonic-gate
14757c478bd9Sstevel@tonic-gate	ENTRY(prefetch_smap_w)
147685641879Skalai	rep;	ret	/* use 2 byte return instruction when branch target */
147785641879Skalai			/* AMD Software Optimization Guide - Section 6.2 */
14787c478bd9Sstevel@tonic-gate	SET_SIZE(prefetch_smap_w)
14797c478bd9Sstevel@tonic-gate
14807c478bd9Sstevel@tonic-gate/*
14817c478bd9Sstevel@tonic-gate * prefetch_page_r(page_t *)
14827c478bd9Sstevel@tonic-gate * issue prefetch instructions for a page_t
14837c478bd9Sstevel@tonic-gate */
14847c478bd9Sstevel@tonic-gate
14857c478bd9Sstevel@tonic-gate	ENTRY(prefetch_page_r)
148685641879Skalai	rep;	ret	/* use 2 byte return instruction when branch target */
148785641879Skalai			/* AMD Software Optimization Guide - Section 6.2 */
14887c478bd9Sstevel@tonic-gate	SET_SIZE(prefetch_page_r)
14897c478bd9Sstevel@tonic-gate
14907c478bd9Sstevel@tonic-gate	ENTRY(bcmp)
14917c478bd9Sstevel@tonic-gate	pushq	%rbp
14927c478bd9Sstevel@tonic-gate	movq	%rsp, %rbp
14937c478bd9Sstevel@tonic-gate#ifdef DEBUG
1494549e8c97SMarcel Telka	testq	%rdx,%rdx
1495549e8c97SMarcel Telka	je	1f
1496ae115bc7Smrj	movq	postbootkernelbase(%rip), %r11
14977c478bd9Sstevel@tonic-gate	cmpq	%r11, %rdi
14987c478bd9Sstevel@tonic-gate	jb	0f
14997c478bd9Sstevel@tonic-gate	cmpq	%r11, %rsi
15007c478bd9Sstevel@tonic-gate	jnb	1f
15017c478bd9Sstevel@tonic-gate0:	leaq	.bcmp_panic_msg(%rip), %rdi
15027c478bd9Sstevel@tonic-gate	xorl	%eax, %eax
15037c478bd9Sstevel@tonic-gate	call	panic
15047c478bd9Sstevel@tonic-gate1:
15057c478bd9Sstevel@tonic-gate#endif	/* DEBUG */
15067c478bd9Sstevel@tonic-gate	call	memcmp
15077c478bd9Sstevel@tonic-gate	testl	%eax, %eax
15087c478bd9Sstevel@tonic-gate	setne	%dl
15097c478bd9Sstevel@tonic-gate	leave
15107c478bd9Sstevel@tonic-gate	movzbl	%dl, %eax
15117c478bd9Sstevel@tonic-gate	ret
15127c478bd9Sstevel@tonic-gate	SET_SIZE(bcmp)
151365f20420SRobert Mustacchi
15147c478bd9Sstevel@tonic-gate#ifdef DEBUG
15157c478bd9Sstevel@tonic-gate	.text
15167c478bd9Sstevel@tonic-gate.bcmp_panic_msg:
15177c478bd9Sstevel@tonic-gate	.string "bcmp: arguments below kernelbase"
15187c478bd9Sstevel@tonic-gate#endif	/* DEBUG */
15197c478bd9Sstevel@tonic-gate
1520ae115bc7Smrj	ENTRY_NP(bsrw_insn)
1521ae115bc7Smrj	xorl	%eax, %eax
1522ae115bc7Smrj	bsrw	%di, %ax
1523ae115bc7Smrj	ret
1524ae115bc7Smrj	SET_SIZE(bsrw_insn)
1525ae115bc7Smrj
1526ae115bc7Smrj	ENTRY_NP(switch_sp_and_call)
1527ae115bc7Smrj	pushq	%rbp
1528ae115bc7Smrj	movq	%rsp, %rbp		/* set up stack frame */
1529ae115bc7Smrj	movq	%rdi, %rsp		/* switch stack pointer */
1530ae115bc7Smrj	movq	%rdx, %rdi		/* pass func arg 1 */
1531ae115bc7Smrj	movq	%rsi, %r11		/* save function to call */
1532ae115bc7Smrj	movq	%rcx, %rsi		/* pass func arg 2 */
153365f20420SRobert Mustacchi	INDIRECT_CALL_REG(r11)		/* call function */
1534ae115bc7Smrj	leave				/* restore stack */
1535ae115bc7Smrj	ret
1536ae115bc7Smrj	SET_SIZE(switch_sp_and_call)
1537ae115bc7Smrj
1538ae115bc7Smrj	ENTRY_NP(kmdb_enter)
1539ae115bc7Smrj	pushq	%rbp
1540ae115bc7Smrj	movq	%rsp, %rbp
1541ae115bc7Smrj
1542ae115bc7Smrj	/*
1543ae115bc7Smrj	 * Save flags, do a 'cli' then return the saved flags
1544ae115bc7Smrj	 */
1545ae115bc7Smrj	call	intr_clear
1546ae115bc7Smrj
1547ae115bc7Smrj	int	$T_DBGENTR
1548ae115bc7Smrj
1549ae115bc7Smrj	/*
1550ae115bc7Smrj	 * Restore the saved flags
1551ae115bc7Smrj	 */
1552ae115bc7Smrj	movq	%rax, %rdi
1553ae115bc7Smrj	call	intr_restore
1554ae115bc7Smrj
1555ae115bc7Smrj	leave
155665f20420SRobert Mustacchi	ret
1557ae115bc7Smrj	SET_SIZE(kmdb_enter)
1558ae115bc7Smrj
1559ae115bc7Smrj	ENTRY_NP(return_instr)
1560ae115bc7Smrj	rep;	ret	/* use 2 byte instruction when branch target */
1561ae115bc7Smrj			/* AMD Software Optimization Guide - Section 6.2 */
1562ae115bc7Smrj	SET_SIZE(return_instr)
1563ae115bc7Smrj
1564ae115bc7Smrj	ENTRY(getflags)
1565ae115bc7Smrj	pushfq
1566ae115bc7Smrj	popq	%rax
1567843e1988Sjohnlev#if defined(__xpv)
1568843e1988Sjohnlev	CURTHREAD(%rdi)
1569843e1988Sjohnlev	KPREEMPT_DISABLE(%rdi)
1570843e1988Sjohnlev	/*
1571843e1988Sjohnlev	 * Synthesize the PS_IE bit from the event mask bit
1572843e1988Sjohnlev	 */
1573843e1988Sjohnlev	CURVCPU(%r11)
1574843e1988Sjohnlev	andq    $_BITNOT(PS_IE), %rax
1575843e1988Sjohnlev	XEN_TEST_UPCALL_MASK(%r11)
1576843e1988Sjohnlev	jnz	1f
1577843e1988Sjohnlev	orq	$PS_IE, %rax
1578843e1988Sjohnlev1:
1579843e1988Sjohnlev	KPREEMPT_ENABLE_NOKP(%rdi)
1580843e1988Sjohnlev#endif
1581ae115bc7Smrj	ret
1582ae115bc7Smrj	SET_SIZE(getflags)
1583ae115bc7Smrj
15844df4bd60Sbs	ENTRY(ftrace_interrupt_disable)
15854df4bd60Sbs	pushfq
15864df4bd60Sbs	popq	%rax
15874df4bd60Sbs	CLI(%rdx)
15884df4bd60Sbs	ret
15894df4bd60Sbs	SET_SIZE(ftrace_interrupt_disable)
15904df4bd60Sbs
15914df4bd60Sbs	ENTRY(ftrace_interrupt_enable)
15924df4bd60Sbs	pushq	%rdi
15934df4bd60Sbs	popfq
15944df4bd60Sbs	ret
15954df4bd60Sbs	SET_SIZE(ftrace_interrupt_enable)
15964df4bd60Sbs
159786c1f4dcSVikram Hegde	ENTRY(clflush_insn)
159886c1f4dcSVikram Hegde	clflush (%rdi)
159986c1f4dcSVikram Hegde	ret
160086c1f4dcSVikram Hegde	SET_SIZE(clflush_insn)
160186c1f4dcSVikram Hegde
160286c1f4dcSVikram Hegde	ENTRY(mfence_insn)
160386c1f4dcSVikram Hegde	mfence
160486c1f4dcSVikram Hegde	ret
160586c1f4dcSVikram Hegde	SET_SIZE(mfence_insn)
160686c1f4dcSVikram Hegde
16077997e108SSurya Prakki/*
160879ec9da8SYuri Pankov * VMware implements an I/O port that programs can query to detect if software
160979ec9da8SYuri Pankov * is running in a VMware hypervisor. This hypervisor port behaves differently
161079ec9da8SYuri Pankov * depending on magic values in certain registers and modifies some registers
161179ec9da8SYuri Pankov * as a side effect.
161279ec9da8SYuri Pankov *
161365f20420SRobert Mustacchi * References: http://kb.vmware.com/kb/1009458
16147997e108SSurya Prakki */
161579ec9da8SYuri Pankov
161679ec9da8SYuri Pankov	ENTRY(vmware_port)
16177997e108SSurya Prakki	pushq	%rbx
161879ec9da8SYuri Pankov	movl	$VMWARE_HVMAGIC, %eax
161979ec9da8SYuri Pankov	movl	$0xffffffff, %ebx
162079ec9da8SYuri Pankov	movl	%edi, %ecx
162179ec9da8SYuri Pankov	movl	$VMWARE_HVPORT, %edx
16227997e108SSurya Prakki	inl	(%dx)
162379ec9da8SYuri Pankov	movl	%eax, (%rsi)
162479ec9da8SYuri Pankov	movl	%ebx, 4(%rsi)
162579ec9da8SYuri Pankov	movl	%ecx, 8(%rsi)
162679ec9da8SYuri Pankov	movl	%edx, 12(%rsi)
16277997e108SSurya Prakki	popq	%rbx
16287997e108SSurya Prakki	ret
162979ec9da8SYuri Pankov	SET_SIZE(vmware_port)
1630