17c478bd9Sstevel@tonic-gate/* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 5ee88d2b9Skchow * Common Development and Distribution License (the "License"). 6ee88d2b9Skchow * You may not use this file except in compliance with the License. 77c478bd9Sstevel@tonic-gate * 87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 117c478bd9Sstevel@tonic-gate * and limitations under the License. 127c478bd9Sstevel@tonic-gate * 137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 187c478bd9Sstevel@tonic-gate * 197c478bd9Sstevel@tonic-gate * CDDL HEADER END 207c478bd9Sstevel@tonic-gate */ 21ae115bc7Smrj 227c478bd9Sstevel@tonic-gate/* 237417cfdeSKuriakose Kuruvilla * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved. 24549e8c97SMarcel Telka * Copyright 2014 Nexenta Systems, Inc. All rights reserved. 25bf16b11eSMatthew Ahrens * Copyright (c) 2014 by Delphix. All rights reserved. 26a9cc46cfSRobert Mustacchi * Copyright 2019 Joyent, Inc. 277c478bd9Sstevel@tonic-gate */ 287c478bd9Sstevel@tonic-gate 297c478bd9Sstevel@tonic-gate/* 307c478bd9Sstevel@tonic-gate * Copyright (c) 1990, 1991 UNIX System Laboratories, Inc. 317c478bd9Sstevel@tonic-gate * Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T 327c478bd9Sstevel@tonic-gate * All Rights Reserved 337c478bd9Sstevel@tonic-gate */ 347c478bd9Sstevel@tonic-gate 357af88ac7SKuriakose Kuruvilla/* 367af88ac7SKuriakose Kuruvilla * Copyright (c) 2009, Intel Corporation. 377af88ac7SKuriakose Kuruvilla * All rights reserved. 387af88ac7SKuriakose Kuruvilla */ 397af88ac7SKuriakose Kuruvilla 407c478bd9Sstevel@tonic-gate/* 417c478bd9Sstevel@tonic-gate * General assembly language routines. 427c478bd9Sstevel@tonic-gate * It is the intent of this file to contain routines that are 437c478bd9Sstevel@tonic-gate * independent of the specific kernel architecture, and those that are 447c478bd9Sstevel@tonic-gate * common across kernel architectures. 457c478bd9Sstevel@tonic-gate * As architectures diverge, and implementations of specific 467c478bd9Sstevel@tonic-gate * architecture-dependent routines change, the routines should be moved 477c478bd9Sstevel@tonic-gate * from this file into the respective ../`arch -k`/subr.s file. 487c478bd9Sstevel@tonic-gate */ 497c478bd9Sstevel@tonic-gate 507c478bd9Sstevel@tonic-gate#include <sys/asm_linkage.h> 517c478bd9Sstevel@tonic-gate#include <sys/asm_misc.h> 527c478bd9Sstevel@tonic-gate#include <sys/panic.h> 537c478bd9Sstevel@tonic-gate#include <sys/ontrap.h> 547c478bd9Sstevel@tonic-gate#include <sys/regset.h> 557c478bd9Sstevel@tonic-gate#include <sys/privregs.h> 567c478bd9Sstevel@tonic-gate#include <sys/reboot.h> 577c478bd9Sstevel@tonic-gate#include <sys/psw.h> 587c478bd9Sstevel@tonic-gate#include <sys/x86_archext.h> 597c478bd9Sstevel@tonic-gate 607c478bd9Sstevel@tonic-gate#include "assym.h" 617c478bd9Sstevel@tonic-gate#include <sys/dditypes.h> 627c478bd9Sstevel@tonic-gate 637c478bd9Sstevel@tonic-gate/* 647c478bd9Sstevel@tonic-gate * on_fault() 653ce2fcdcSRobert Mustacchi * 667c478bd9Sstevel@tonic-gate * Catch lofault faults. Like setjmp except it returns one 677c478bd9Sstevel@tonic-gate * if code following causes uncorrectable fault. Turned off 683ce2fcdcSRobert Mustacchi * by calling no_fault(). Note that while under on_fault(), 693ce2fcdcSRobert Mustacchi * SMAP is disabled. For more information see 70f0089e39SRichard Lowe * uts/intel/ml/copy.s. 717c478bd9Sstevel@tonic-gate */ 727c478bd9Sstevel@tonic-gate 737c478bd9Sstevel@tonic-gate ENTRY(on_fault) 747c478bd9Sstevel@tonic-gate movq %gs:CPU_THREAD, %rsi 757c478bd9Sstevel@tonic-gate leaq catch_fault(%rip), %rdx 767c478bd9Sstevel@tonic-gate movq %rdi, T_ONFAULT(%rsi) /* jumpbuf in t_onfault */ 777c478bd9Sstevel@tonic-gate movq %rdx, T_LOFAULT(%rsi) /* catch_fault in t_lofault */ 783ce2fcdcSRobert Mustacchi call smap_disable /* allow user accesses */ 797c478bd9Sstevel@tonic-gate jmp setjmp /* let setjmp do the rest */ 807c478bd9Sstevel@tonic-gate 817c478bd9Sstevel@tonic-gatecatch_fault: 827c478bd9Sstevel@tonic-gate movq %gs:CPU_THREAD, %rsi 837c478bd9Sstevel@tonic-gate movq T_ONFAULT(%rsi), %rdi /* address of save area */ 847c478bd9Sstevel@tonic-gate xorl %eax, %eax 857c478bd9Sstevel@tonic-gate movq %rax, T_ONFAULT(%rsi) /* turn off onfault */ 867c478bd9Sstevel@tonic-gate movq %rax, T_LOFAULT(%rsi) /* turn off lofault */ 873ce2fcdcSRobert Mustacchi call smap_enable /* disallow user accesses */ 887c478bd9Sstevel@tonic-gate jmp longjmp /* let longjmp do the rest */ 897c478bd9Sstevel@tonic-gate SET_SIZE(on_fault) 907c478bd9Sstevel@tonic-gate 917c478bd9Sstevel@tonic-gate ENTRY(no_fault) 927c478bd9Sstevel@tonic-gate movq %gs:CPU_THREAD, %rsi 937c478bd9Sstevel@tonic-gate xorl %eax, %eax 947c478bd9Sstevel@tonic-gate movq %rax, T_ONFAULT(%rsi) /* turn off onfault */ 957c478bd9Sstevel@tonic-gate movq %rax, T_LOFAULT(%rsi) /* turn off lofault */ 963ce2fcdcSRobert Mustacchi call smap_enable /* disallow user accesses */ 977c478bd9Sstevel@tonic-gate ret 987c478bd9Sstevel@tonic-gate SET_SIZE(no_fault) 997c478bd9Sstevel@tonic-gate 1007c478bd9Sstevel@tonic-gate/* 1017c478bd9Sstevel@tonic-gate * Default trampoline code for on_trap() (see <sys/ontrap.h>). We just 1027c478bd9Sstevel@tonic-gate * do a longjmp(&curthread->t_ontrap->ot_jmpbuf) if this is ever called. 1037c478bd9Sstevel@tonic-gate */ 1047c478bd9Sstevel@tonic-gate 1057c478bd9Sstevel@tonic-gate ENTRY(on_trap_trampoline) 1067c478bd9Sstevel@tonic-gate movq %gs:CPU_THREAD, %rsi 1077c478bd9Sstevel@tonic-gate movq T_ONTRAP(%rsi), %rdi 1087c478bd9Sstevel@tonic-gate addq $OT_JMPBUF, %rdi 1097c478bd9Sstevel@tonic-gate jmp longjmp 1107c478bd9Sstevel@tonic-gate SET_SIZE(on_trap_trampoline) 1117c478bd9Sstevel@tonic-gate 1127c478bd9Sstevel@tonic-gate/* 1137c478bd9Sstevel@tonic-gate * Push a new element on to the t_ontrap stack. Refer to <sys/ontrap.h> for 1147c478bd9Sstevel@tonic-gate * more information about the on_trap() mechanism. If the on_trap_data is the 1157c478bd9Sstevel@tonic-gate * same as the topmost stack element, we just modify that element. 1167c478bd9Sstevel@tonic-gate */ 1177c478bd9Sstevel@tonic-gate 1187c478bd9Sstevel@tonic-gate ENTRY(on_trap) 1197c478bd9Sstevel@tonic-gate movw %si, OT_PROT(%rdi) /* ot_prot = prot */ 1207c478bd9Sstevel@tonic-gate movw $0, OT_TRAP(%rdi) /* ot_trap = 0 */ 1217c478bd9Sstevel@tonic-gate leaq on_trap_trampoline(%rip), %rdx /* rdx = &on_trap_trampoline */ 1227c478bd9Sstevel@tonic-gate movq %rdx, OT_TRAMPOLINE(%rdi) /* ot_trampoline = rdx */ 1237c478bd9Sstevel@tonic-gate xorl %ecx, %ecx 1247c478bd9Sstevel@tonic-gate movq %rcx, OT_HANDLE(%rdi) /* ot_handle = NULL */ 1257c478bd9Sstevel@tonic-gate movq %rcx, OT_PAD1(%rdi) /* ot_pad1 = NULL */ 1267c478bd9Sstevel@tonic-gate movq %gs:CPU_THREAD, %rdx /* rdx = curthread */ 1277c478bd9Sstevel@tonic-gate movq T_ONTRAP(%rdx), %rcx /* rcx = curthread->t_ontrap */ 1287c478bd9Sstevel@tonic-gate cmpq %rdi, %rcx /* if (otp == %rcx) */ 1297c478bd9Sstevel@tonic-gate je 0f /* don't modify t_ontrap */ 1307c478bd9Sstevel@tonic-gate 1317c478bd9Sstevel@tonic-gate movq %rcx, OT_PREV(%rdi) /* ot_prev = t_ontrap */ 1327c478bd9Sstevel@tonic-gate movq %rdi, T_ONTRAP(%rdx) /* curthread->t_ontrap = otp */ 1337c478bd9Sstevel@tonic-gate 1347c478bd9Sstevel@tonic-gate0: addq $OT_JMPBUF, %rdi /* &ot_jmpbuf */ 1357c478bd9Sstevel@tonic-gate jmp setjmp 1367c478bd9Sstevel@tonic-gate SET_SIZE(on_trap) 1377c478bd9Sstevel@tonic-gate 1387c478bd9Sstevel@tonic-gate/* 1397c478bd9Sstevel@tonic-gate * Setjmp and longjmp implement non-local gotos using state vectors 1407c478bd9Sstevel@tonic-gate * type label_t. 1417c478bd9Sstevel@tonic-gate */ 1427c478bd9Sstevel@tonic-gate 1437c478bd9Sstevel@tonic-gate#if LABEL_PC != 0 1447c478bd9Sstevel@tonic-gate#error LABEL_PC MUST be defined as 0 for setjmp/longjmp to work as coded 1457c478bd9Sstevel@tonic-gate#endif /* LABEL_PC != 0 */ 1467c478bd9Sstevel@tonic-gate 1477c478bd9Sstevel@tonic-gate ENTRY(setjmp) 1487c478bd9Sstevel@tonic-gate movq %rsp, LABEL_SP(%rdi) 1497c478bd9Sstevel@tonic-gate movq %rbp, LABEL_RBP(%rdi) 1507c478bd9Sstevel@tonic-gate movq %rbx, LABEL_RBX(%rdi) 1517c478bd9Sstevel@tonic-gate movq %r12, LABEL_R12(%rdi) 1527c478bd9Sstevel@tonic-gate movq %r13, LABEL_R13(%rdi) 1537c478bd9Sstevel@tonic-gate movq %r14, LABEL_R14(%rdi) 1547c478bd9Sstevel@tonic-gate movq %r15, LABEL_R15(%rdi) 1557c478bd9Sstevel@tonic-gate movq (%rsp), %rdx /* return address */ 1567c478bd9Sstevel@tonic-gate movq %rdx, (%rdi) /* LABEL_PC is 0 */ 1577c478bd9Sstevel@tonic-gate xorl %eax, %eax /* return 0 */ 1587c478bd9Sstevel@tonic-gate ret 1597c478bd9Sstevel@tonic-gate SET_SIZE(setjmp) 1607c478bd9Sstevel@tonic-gate 1617c478bd9Sstevel@tonic-gate ENTRY(longjmp) 1627c478bd9Sstevel@tonic-gate movq LABEL_SP(%rdi), %rsp 1637c478bd9Sstevel@tonic-gate movq LABEL_RBP(%rdi), %rbp 1647c478bd9Sstevel@tonic-gate movq LABEL_RBX(%rdi), %rbx 1657c478bd9Sstevel@tonic-gate movq LABEL_R12(%rdi), %r12 1667c478bd9Sstevel@tonic-gate movq LABEL_R13(%rdi), %r13 1677c478bd9Sstevel@tonic-gate movq LABEL_R14(%rdi), %r14 1687c478bd9Sstevel@tonic-gate movq LABEL_R15(%rdi), %r15 1697c478bd9Sstevel@tonic-gate movq (%rdi), %rdx /* return address; LABEL_PC is 0 */ 1707c478bd9Sstevel@tonic-gate movq %rdx, (%rsp) 1717c478bd9Sstevel@tonic-gate xorl %eax, %eax 1727c478bd9Sstevel@tonic-gate incl %eax /* return 1 */ 1737c478bd9Sstevel@tonic-gate ret 1747c478bd9Sstevel@tonic-gate SET_SIZE(longjmp) 1757c478bd9Sstevel@tonic-gate 1767c478bd9Sstevel@tonic-gate/* 1777c478bd9Sstevel@tonic-gate * if a() calls b() calls caller(), 1787c478bd9Sstevel@tonic-gate * caller() returns return address in a(). 1797c478bd9Sstevel@tonic-gate * (Note: We assume a() and b() are C routines which do the normal entry/exit 1807c478bd9Sstevel@tonic-gate * sequence.) 1817c478bd9Sstevel@tonic-gate */ 1827c478bd9Sstevel@tonic-gate 1837c478bd9Sstevel@tonic-gate ENTRY(caller) 1847c478bd9Sstevel@tonic-gate movq 8(%rbp), %rax /* b()'s return pc, in a() */ 1857c478bd9Sstevel@tonic-gate ret 1867c478bd9Sstevel@tonic-gate SET_SIZE(caller) 1877c478bd9Sstevel@tonic-gate 1887c478bd9Sstevel@tonic-gate/* 1897c478bd9Sstevel@tonic-gate * if a() calls callee(), callee() returns the 1907c478bd9Sstevel@tonic-gate * return address in a(); 1917c478bd9Sstevel@tonic-gate */ 1927c478bd9Sstevel@tonic-gate 1937c478bd9Sstevel@tonic-gate ENTRY(callee) 1947c478bd9Sstevel@tonic-gate movq (%rsp), %rax /* callee()'s return pc, in a() */ 1957c478bd9Sstevel@tonic-gate ret 1967c478bd9Sstevel@tonic-gate SET_SIZE(callee) 1977c478bd9Sstevel@tonic-gate 1987c478bd9Sstevel@tonic-gate/* 1997c478bd9Sstevel@tonic-gate * return the current frame pointer 2007c478bd9Sstevel@tonic-gate */ 2017c478bd9Sstevel@tonic-gate 2027c478bd9Sstevel@tonic-gate ENTRY(getfp) 2037c478bd9Sstevel@tonic-gate movq %rbp, %rax 2047c478bd9Sstevel@tonic-gate ret 2057c478bd9Sstevel@tonic-gate SET_SIZE(getfp) 2067c478bd9Sstevel@tonic-gate 2077c478bd9Sstevel@tonic-gate/* 2087c478bd9Sstevel@tonic-gate * Invalidate a single page table entry in the TLB 2097c478bd9Sstevel@tonic-gate */ 2107c478bd9Sstevel@tonic-gate 21174ecdb51SJohn Levon ENTRY(mmu_invlpg) 2127c478bd9Sstevel@tonic-gate invlpg (%rdi) 2137c478bd9Sstevel@tonic-gate ret 21474ecdb51SJohn Levon SET_SIZE(mmu_invlpg) 2157c478bd9Sstevel@tonic-gate 2167c478bd9Sstevel@tonic-gate 2177c478bd9Sstevel@tonic-gate/* 2187c478bd9Sstevel@tonic-gate * Get/Set the value of various control registers 2197c478bd9Sstevel@tonic-gate */ 2207c478bd9Sstevel@tonic-gate 2217c478bd9Sstevel@tonic-gate ENTRY(getcr0) 2227c478bd9Sstevel@tonic-gate movq %cr0, %rax 2237c478bd9Sstevel@tonic-gate ret 2247c478bd9Sstevel@tonic-gate SET_SIZE(getcr0) 2257c478bd9Sstevel@tonic-gate 2267c478bd9Sstevel@tonic-gate ENTRY(setcr0) 2277c478bd9Sstevel@tonic-gate movq %rdi, %cr0 2287c478bd9Sstevel@tonic-gate ret 2297c478bd9Sstevel@tonic-gate SET_SIZE(setcr0) 2307c478bd9Sstevel@tonic-gate 231ae115bc7Smrj ENTRY(getcr2) 232843e1988Sjohnlev#if defined(__xpv) 233843e1988Sjohnlev movq %gs:CPU_VCPU_INFO, %rax 234843e1988Sjohnlev movq VCPU_INFO_ARCH_CR2(%rax), %rax 235843e1988Sjohnlev#else 236ae115bc7Smrj movq %cr2, %rax 237843e1988Sjohnlev#endif 238ae115bc7Smrj ret 2397c478bd9Sstevel@tonic-gate SET_SIZE(getcr2) 2407c478bd9Sstevel@tonic-gate 2417c478bd9Sstevel@tonic-gate ENTRY(getcr3) 242ae115bc7Smrj movq %cr3, %rax 2437c478bd9Sstevel@tonic-gate ret 2447c478bd9Sstevel@tonic-gate SET_SIZE(getcr3) 2457c478bd9Sstevel@tonic-gate 246843e1988Sjohnlev#if !defined(__xpv) 247843e1988Sjohnlev 248ae115bc7Smrj ENTRY(setcr3) 249ae115bc7Smrj movq %rdi, %cr3 250ae115bc7Smrj ret 2517c478bd9Sstevel@tonic-gate SET_SIZE(setcr3) 2527c478bd9Sstevel@tonic-gate 2537c478bd9Sstevel@tonic-gate ENTRY(reload_cr3) 2547c478bd9Sstevel@tonic-gate movq %cr3, %rdi 2557c478bd9Sstevel@tonic-gate movq %rdi, %cr3 2567c478bd9Sstevel@tonic-gate ret 2577c478bd9Sstevel@tonic-gate SET_SIZE(reload_cr3) 2587c478bd9Sstevel@tonic-gate 259843e1988Sjohnlev#endif /* __xpv */ 260843e1988Sjohnlev 2617c478bd9Sstevel@tonic-gate ENTRY(getcr4) 2627c478bd9Sstevel@tonic-gate movq %cr4, %rax 2637c478bd9Sstevel@tonic-gate ret 2647c478bd9Sstevel@tonic-gate SET_SIZE(getcr4) 2657c478bd9Sstevel@tonic-gate 2667c478bd9Sstevel@tonic-gate ENTRY(setcr4) 2677c478bd9Sstevel@tonic-gate movq %rdi, %cr4 2687c478bd9Sstevel@tonic-gate ret 2697c478bd9Sstevel@tonic-gate SET_SIZE(setcr4) 2707c478bd9Sstevel@tonic-gate 2717c478bd9Sstevel@tonic-gate ENTRY(getcr8) 2727c478bd9Sstevel@tonic-gate movq %cr8, %rax 2737c478bd9Sstevel@tonic-gate ret 2747c478bd9Sstevel@tonic-gate SET_SIZE(getcr8) 2757c478bd9Sstevel@tonic-gate 2767c478bd9Sstevel@tonic-gate ENTRY(setcr8) 2777c478bd9Sstevel@tonic-gate movq %rdi, %cr8 2787c478bd9Sstevel@tonic-gate ret 2797c478bd9Sstevel@tonic-gate SET_SIZE(setcr8) 2807c478bd9Sstevel@tonic-gate 2817c478bd9Sstevel@tonic-gate ENTRY(__cpuid_insn) 2828949bcd6Sandrei movq %rbx, %r8 2838949bcd6Sandrei movq %rcx, %r9 2848949bcd6Sandrei movq %rdx, %r11 2858949bcd6Sandrei movl (%rdi), %eax /* %eax = regs->cp_eax */ 2868949bcd6Sandrei movl 0x4(%rdi), %ebx /* %ebx = regs->cp_ebx */ 2878949bcd6Sandrei movl 0x8(%rdi), %ecx /* %ecx = regs->cp_ecx */ 2888949bcd6Sandrei movl 0xc(%rdi), %edx /* %edx = regs->cp_edx */ 2897c478bd9Sstevel@tonic-gate cpuid 2908949bcd6Sandrei movl %eax, (%rdi) /* regs->cp_eax = %eax */ 2918949bcd6Sandrei movl %ebx, 0x4(%rdi) /* regs->cp_ebx = %ebx */ 2928949bcd6Sandrei movl %ecx, 0x8(%rdi) /* regs->cp_ecx = %ecx */ 2938949bcd6Sandrei movl %edx, 0xc(%rdi) /* regs->cp_edx = %edx */ 2948949bcd6Sandrei movq %r8, %rbx 2958949bcd6Sandrei movq %r9, %rcx 2968949bcd6Sandrei movq %r11, %rdx 2977c478bd9Sstevel@tonic-gate ret 2987c478bd9Sstevel@tonic-gate SET_SIZE(__cpuid_insn) 2997c478bd9Sstevel@tonic-gate 300f98fbcecSbholler ENTRY_NP(i86_monitor) 301f98fbcecSbholler pushq %rbp 302f98fbcecSbholler movq %rsp, %rbp 303f98fbcecSbholler movq %rdi, %rax /* addr */ 304f98fbcecSbholler movq %rsi, %rcx /* extensions */ 305f98fbcecSbholler /* rdx contains input arg3: hints */ 3061d1a3942SBill Holler clflush (%rax) 307f98fbcecSbholler .byte 0x0f, 0x01, 0xc8 /* monitor */ 308f98fbcecSbholler leave 309f98fbcecSbholler ret 310f98fbcecSbholler SET_SIZE(i86_monitor) 311f98fbcecSbholler 312f98fbcecSbholler ENTRY_NP(i86_mwait) 313f98fbcecSbholler pushq %rbp 31465f20420SRobert Mustacchi call x86_md_clear 315f98fbcecSbholler movq %rsp, %rbp 316f98fbcecSbholler movq %rdi, %rax /* data */ 317f98fbcecSbholler movq %rsi, %rcx /* extensions */ 318f98fbcecSbholler .byte 0x0f, 0x01, 0xc9 /* mwait */ 319f98fbcecSbholler leave 320f98fbcecSbholler ret 321f98fbcecSbholler SET_SIZE(i86_mwait) 322f98fbcecSbholler 323f34a7178SJoe Bonasera#if defined(__xpv) 324f34a7178SJoe Bonasera /* 325f34a7178SJoe Bonasera * Defined in C 326f34a7178SJoe Bonasera */ 327f34a7178SJoe Bonasera#else 328f34a7178SJoe Bonasera 329ae115bc7Smrj ENTRY_NP(tsc_read) 330247dbb3dSsudheer movq %rbx, %r11 331247dbb3dSsudheer movl $0, %eax 332247dbb3dSsudheer cpuid 333247dbb3dSsudheer rdtsc 334247dbb3dSsudheer movq %r11, %rbx 335247dbb3dSsudheer shlq $32, %rdx 336247dbb3dSsudheer orq %rdx, %rax 337247dbb3dSsudheer ret 338247dbb3dSsudheer .globl _tsc_mfence_start 339247dbb3dSsudheer_tsc_mfence_start: 340247dbb3dSsudheer mfence 341ae115bc7Smrj rdtsc 342ae115bc7Smrj shlq $32, %rdx 343ae115bc7Smrj orq %rdx, %rax 344ae115bc7Smrj ret 345247dbb3dSsudheer .globl _tsc_mfence_end 346247dbb3dSsudheer_tsc_mfence_end: 347247dbb3dSsudheer .globl _tscp_start 348247dbb3dSsudheer_tscp_start: 349247dbb3dSsudheer .byte 0x0f, 0x01, 0xf9 /* rdtscp instruction */ 350247dbb3dSsudheer shlq $32, %rdx 351247dbb3dSsudheer orq %rdx, %rax 352247dbb3dSsudheer ret 353247dbb3dSsudheer .globl _tscp_end 354247dbb3dSsudheer_tscp_end: 355247dbb3dSsudheer .globl _no_rdtsc_start 356247dbb3dSsudheer_no_rdtsc_start: 357247dbb3dSsudheer xorl %edx, %edx 358247dbb3dSsudheer xorl %eax, %eax 359247dbb3dSsudheer ret 360247dbb3dSsudheer .globl _no_rdtsc_end 361247dbb3dSsudheer_no_rdtsc_end: 36215363b27Ssudheer .globl _tsc_lfence_start 36315363b27Ssudheer_tsc_lfence_start: 36415363b27Ssudheer lfence 36515363b27Ssudheer rdtsc 36615363b27Ssudheer shlq $32, %rdx 36715363b27Ssudheer orq %rdx, %rax 36815363b27Ssudheer ret 36915363b27Ssudheer .globl _tsc_lfence_end 37015363b27Ssudheer_tsc_lfence_end: 371ae115bc7Smrj SET_SIZE(tsc_read) 372ae115bc7Smrj 37306fb6a36Sdv 374843e1988Sjohnlev#endif /* __xpv */ 375843e1988Sjohnlev 37606fb6a36Sdv ENTRY_NP(randtick) 37706fb6a36Sdv rdtsc 37806fb6a36Sdv shlq $32, %rdx 37906fb6a36Sdv orq %rdx, %rax 38006fb6a36Sdv ret 38106fb6a36Sdv SET_SIZE(randtick) 3827c478bd9Sstevel@tonic-gate/* 3837c478bd9Sstevel@tonic-gate * Insert entryp after predp in a doubly linked list. 3847c478bd9Sstevel@tonic-gate */ 3857c478bd9Sstevel@tonic-gate 3867c478bd9Sstevel@tonic-gate ENTRY(_insque) 3874b56a003SDaniel Anderson movq (%rsi), %rax /* predp->forw */ 3887c478bd9Sstevel@tonic-gate movq %rsi, CPTRSIZE(%rdi) /* entryp->back = predp */ 3897c478bd9Sstevel@tonic-gate movq %rax, (%rdi) /* entryp->forw = predp->forw */ 3907c478bd9Sstevel@tonic-gate movq %rdi, (%rsi) /* predp->forw = entryp */ 3917c478bd9Sstevel@tonic-gate movq %rdi, CPTRSIZE(%rax) /* predp->forw->back = entryp */ 3927c478bd9Sstevel@tonic-gate ret 3937c478bd9Sstevel@tonic-gate SET_SIZE(_insque) 3947c478bd9Sstevel@tonic-gate 3957c478bd9Sstevel@tonic-gate/* 3967c478bd9Sstevel@tonic-gate * Remove entryp from a doubly linked list 3977c478bd9Sstevel@tonic-gate */ 3987c478bd9Sstevel@tonic-gate 3997c478bd9Sstevel@tonic-gate ENTRY(_remque) 4007c478bd9Sstevel@tonic-gate movq (%rdi), %rax /* entry->forw */ 4017c478bd9Sstevel@tonic-gate movq CPTRSIZE(%rdi), %rdx /* entry->back */ 4027c478bd9Sstevel@tonic-gate movq %rax, (%rdx) /* entry->back->forw = entry->forw */ 4037c478bd9Sstevel@tonic-gate movq %rdx, CPTRSIZE(%rax) /* entry->forw->back = entry->back */ 4047c478bd9Sstevel@tonic-gate ret 4057c478bd9Sstevel@tonic-gate SET_SIZE(_remque) 4067c478bd9Sstevel@tonic-gate 4077c478bd9Sstevel@tonic-gate/* 4087c478bd9Sstevel@tonic-gate * Returns the number of 4097c478bd9Sstevel@tonic-gate * non-NULL bytes in string argument. 4107c478bd9Sstevel@tonic-gate */ 4117c478bd9Sstevel@tonic-gate 4127c478bd9Sstevel@tonic-gate/* 4137c478bd9Sstevel@tonic-gate * This is close to a simple transliteration of a C version of this 4147c478bd9Sstevel@tonic-gate * routine. We should either just -make- this be a C version, or 4157c478bd9Sstevel@tonic-gate * justify having it in assembler by making it significantly faster. 4167c478bd9Sstevel@tonic-gate * 4177c478bd9Sstevel@tonic-gate * size_t 4187c478bd9Sstevel@tonic-gate * strlen(const char *s) 4197c478bd9Sstevel@tonic-gate * { 4207c478bd9Sstevel@tonic-gate * const char *s0; 4217c478bd9Sstevel@tonic-gate * #if defined(DEBUG) 4227c478bd9Sstevel@tonic-gate * if ((uintptr_t)s < KERNELBASE) 4237c478bd9Sstevel@tonic-gate * panic(.str_panic_msg); 4247c478bd9Sstevel@tonic-gate * #endif 4257c478bd9Sstevel@tonic-gate * for (s0 = s; *s; s++) 4267c478bd9Sstevel@tonic-gate * ; 4277c478bd9Sstevel@tonic-gate * return (s - s0); 4287c478bd9Sstevel@tonic-gate * } 4297c478bd9Sstevel@tonic-gate */ 4307c478bd9Sstevel@tonic-gate 4317c478bd9Sstevel@tonic-gate ENTRY(strlen) 4327c478bd9Sstevel@tonic-gate#ifdef DEBUG 433ae115bc7Smrj movq postbootkernelbase(%rip), %rax 4347c478bd9Sstevel@tonic-gate cmpq %rax, %rdi 4357c478bd9Sstevel@tonic-gate jae str_valid 4367c478bd9Sstevel@tonic-gate pushq %rbp 4377c478bd9Sstevel@tonic-gate movq %rsp, %rbp 4387c478bd9Sstevel@tonic-gate leaq .str_panic_msg(%rip), %rdi 4397c478bd9Sstevel@tonic-gate xorl %eax, %eax 4407c478bd9Sstevel@tonic-gate call panic 4417c478bd9Sstevel@tonic-gate#endif /* DEBUG */ 4427c478bd9Sstevel@tonic-gatestr_valid: 4437c478bd9Sstevel@tonic-gate cmpb $0, (%rdi) 4447c478bd9Sstevel@tonic-gate movq %rdi, %rax 4457c478bd9Sstevel@tonic-gate je .null_found 4467c478bd9Sstevel@tonic-gate .align 4 4477c478bd9Sstevel@tonic-gate.strlen_loop: 4487c478bd9Sstevel@tonic-gate incq %rdi 4497c478bd9Sstevel@tonic-gate cmpb $0, (%rdi) 4507c478bd9Sstevel@tonic-gate jne .strlen_loop 4517c478bd9Sstevel@tonic-gate.null_found: 4527c478bd9Sstevel@tonic-gate subq %rax, %rdi 4537c478bd9Sstevel@tonic-gate movq %rdi, %rax 4547c478bd9Sstevel@tonic-gate ret 4557c478bd9Sstevel@tonic-gate SET_SIZE(strlen) 4567c478bd9Sstevel@tonic-gate 4577c478bd9Sstevel@tonic-gate#ifdef DEBUG 4587c478bd9Sstevel@tonic-gate .text 4597c478bd9Sstevel@tonic-gate.str_panic_msg: 4607c478bd9Sstevel@tonic-gate .string "strlen: argument below kernelbase" 4617c478bd9Sstevel@tonic-gate#endif /* DEBUG */ 4627c478bd9Sstevel@tonic-gate 4637c478bd9Sstevel@tonic-gate /* 4644b56a003SDaniel Anderson * Berkeley 4.3 introduced symbolically named interrupt levels 4657c478bd9Sstevel@tonic-gate * as a way deal with priority in a machine independent fashion. 4667c478bd9Sstevel@tonic-gate * Numbered priorities are machine specific, and should be 4677c478bd9Sstevel@tonic-gate * discouraged where possible. 4687c478bd9Sstevel@tonic-gate * 4697c478bd9Sstevel@tonic-gate * Note, for the machine specific priorities there are 4707c478bd9Sstevel@tonic-gate * examples listed for devices that use a particular priority. 4717c478bd9Sstevel@tonic-gate * It should not be construed that all devices of that 4727c478bd9Sstevel@tonic-gate * type should be at that priority. It is currently were 4737c478bd9Sstevel@tonic-gate * the current devices fit into the priority scheme based 4747c478bd9Sstevel@tonic-gate * upon time criticalness. 4757c478bd9Sstevel@tonic-gate * 4767c478bd9Sstevel@tonic-gate * The underlying assumption of these assignments is that 4777c478bd9Sstevel@tonic-gate * IPL 10 is the highest level from which a device 4787c478bd9Sstevel@tonic-gate * routine can call wakeup. Devices that interrupt from higher 4797c478bd9Sstevel@tonic-gate * levels are restricted in what they can do. If they need 4807c478bd9Sstevel@tonic-gate * kernels services they should schedule a routine at a lower 4817c478bd9Sstevel@tonic-gate * level (via software interrupt) to do the required 4827c478bd9Sstevel@tonic-gate * processing. 4837c478bd9Sstevel@tonic-gate * 4847c478bd9Sstevel@tonic-gate * Examples of this higher usage: 4857c478bd9Sstevel@tonic-gate * Level Usage 4867c478bd9Sstevel@tonic-gate * 14 Profiling clock (and PROM uart polling clock) 4877c478bd9Sstevel@tonic-gate * 12 Serial ports 4887c478bd9Sstevel@tonic-gate * 4897c478bd9Sstevel@tonic-gate * The serial ports request lower level processing on level 6. 4907c478bd9Sstevel@tonic-gate * 4917c478bd9Sstevel@tonic-gate * Also, almost all splN routines (where N is a number or a 4927c478bd9Sstevel@tonic-gate * mnemonic) will do a RAISE(), on the assumption that they are 4937c478bd9Sstevel@tonic-gate * never used to lower our priority. 4947c478bd9Sstevel@tonic-gate * The exceptions are: 4957c478bd9Sstevel@tonic-gate * spl8() Because you can't be above 15 to begin with! 4967c478bd9Sstevel@tonic-gate * splzs() Because this is used at boot time to lower our 4977c478bd9Sstevel@tonic-gate * priority, to allow the PROM to poll the uart. 4987c478bd9Sstevel@tonic-gate * spl0() Used to lower priority to 0. 4997c478bd9Sstevel@tonic-gate */ 5007c478bd9Sstevel@tonic-gate 501ae115bc7Smrj#define SETPRI(level) \ 502*5d9d9091SRichard Lowe movl $##level, %edi; /* new priority */ \ 503ae115bc7Smrj jmp do_splx /* redirect to do_splx */ 5047c478bd9Sstevel@tonic-gate 5057c478bd9Sstevel@tonic-gate#define RAISE(level) \ 506*5d9d9091SRichard Lowe movl $##level, %edi; /* new priority */ \ 507ae115bc7Smrj jmp splr /* redirect to splr */ 5087c478bd9Sstevel@tonic-gate 5097c478bd9Sstevel@tonic-gate /* locks out all interrupts, including memory errors */ 5107c478bd9Sstevel@tonic-gate ENTRY(spl8) 5117c478bd9Sstevel@tonic-gate SETPRI(15) 5127c478bd9Sstevel@tonic-gate SET_SIZE(spl8) 5137c478bd9Sstevel@tonic-gate 5147c478bd9Sstevel@tonic-gate /* just below the level that profiling runs */ 5157c478bd9Sstevel@tonic-gate ENTRY(spl7) 5167c478bd9Sstevel@tonic-gate RAISE(13) 5177c478bd9Sstevel@tonic-gate SET_SIZE(spl7) 5187c478bd9Sstevel@tonic-gate 5197c478bd9Sstevel@tonic-gate /* sun specific - highest priority onboard serial i/o asy ports */ 5207c478bd9Sstevel@tonic-gate ENTRY(splzs) 5217c478bd9Sstevel@tonic-gate SETPRI(12) /* Can't be a RAISE, as it's used to lower us */ 5227c478bd9Sstevel@tonic-gate SET_SIZE(splzs) 5237c478bd9Sstevel@tonic-gate 5247c478bd9Sstevel@tonic-gate ENTRY(splhi) 5257c478bd9Sstevel@tonic-gate ALTENTRY(splhigh) 5267c478bd9Sstevel@tonic-gate ALTENTRY(spl6) 5277c478bd9Sstevel@tonic-gate ALTENTRY(i_ddi_splhigh) 5287c478bd9Sstevel@tonic-gate 529ae115bc7Smrj RAISE(DISP_LEVEL) 5307c478bd9Sstevel@tonic-gate 5317c478bd9Sstevel@tonic-gate SET_SIZE(i_ddi_splhigh) 5327c478bd9Sstevel@tonic-gate SET_SIZE(spl6) 5337c478bd9Sstevel@tonic-gate SET_SIZE(splhigh) 5347c478bd9Sstevel@tonic-gate SET_SIZE(splhi) 5357c478bd9Sstevel@tonic-gate 5367c478bd9Sstevel@tonic-gate /* allow all interrupts */ 5377c478bd9Sstevel@tonic-gate ENTRY(spl0) 5387c478bd9Sstevel@tonic-gate SETPRI(0) 5397c478bd9Sstevel@tonic-gate SET_SIZE(spl0) 5407c478bd9Sstevel@tonic-gate 5417c478bd9Sstevel@tonic-gate 5424b56a003SDaniel Anderson /* splx implementation */ 5437c478bd9Sstevel@tonic-gate ENTRY(splx) 544ae115bc7Smrj jmp do_splx /* redirect to common splx code */ 5457c478bd9Sstevel@tonic-gate SET_SIZE(splx) 5467c478bd9Sstevel@tonic-gate 547085f98c4Ssethg ENTRY(wait_500ms) 5481b30f017Sgvasick pushq %rbx 54932206069Ssethg movl $50000, %ebx 550085f98c4Ssethg1: 551085f98c4Ssethg call tenmicrosec 55232206069Ssethg decl %ebx 55332206069Ssethg jnz 1b 5541b30f017Sgvasick popq %rbx 55565f20420SRobert Mustacchi ret 556085f98c4Ssethg SET_SIZE(wait_500ms) 557085f98c4Ssethg 558085f98c4Ssethg#define RESET_METHOD_KBC 1 559085f98c4Ssethg#define RESET_METHOD_PORT92 2 560085f98c4Ssethg#define RESET_METHOD_PCI 4 561085f98c4Ssethg 562085f98c4Ssethg DGDEF3(pc_reset_methods, 4, 8) 563085f98c4Ssethg .long RESET_METHOD_KBC|RESET_METHOD_PORT92|RESET_METHOD_PCI; 564085f98c4Ssethg 5657c478bd9Sstevel@tonic-gate ENTRY(pc_reset) 566085f98c4Ssethg 56732206069Ssethg testl $RESET_METHOD_KBC, pc_reset_methods(%rip) 568085f98c4Ssethg jz 1f 569085f98c4Ssethg 5707303b3c3Ssethg / 5717303b3c3Ssethg / Try the classic keyboard controller-triggered reset. 5727303b3c3Ssethg / 5737c478bd9Sstevel@tonic-gate movw $0x64, %dx 5747c478bd9Sstevel@tonic-gate movb $0xfe, %al 5757c478bd9Sstevel@tonic-gate outb (%dx) 5767303b3c3Ssethg 577085f98c4Ssethg / Wait up to 500 milliseconds here for the keyboard controller 578085f98c4Ssethg / to pull the reset line. On some systems where the keyboard 579085f98c4Ssethg / controller is slow to pull the reset line, the next reset method 580085f98c4Ssethg / may be executed (which may be bad if those systems hang when the 581085f98c4Ssethg / next reset method is used, e.g. Ferrari 3400 (doesn't like port 92), 582085f98c4Ssethg / and Ferrari 4000 (doesn't like the cf9 reset method)) 583085f98c4Ssethg 584085f98c4Ssethg call wait_500ms 585085f98c4Ssethg 586085f98c4Ssethg1: 58732206069Ssethg testl $RESET_METHOD_PORT92, pc_reset_methods(%rip) 588085f98c4Ssethg jz 3f 589085f98c4Ssethg 5907303b3c3Ssethg / 5917303b3c3Ssethg / Try port 0x92 fast reset 5927303b3c3Ssethg / 5937303b3c3Ssethg movw $0x92, %dx 5947303b3c3Ssethg inb (%dx) 5957303b3c3Ssethg cmpb $0xff, %al / If port's not there, we should get back 0xFF 5967303b3c3Ssethg je 1f 5977303b3c3Ssethg testb $1, %al / If bit 0 5987303b3c3Ssethg jz 2f / is clear, jump to perform the reset 5997303b3c3Ssethg andb $0xfe, %al / otherwise, 6007303b3c3Ssethg outb (%dx) / clear bit 0 first, then 6017303b3c3Ssethg2: 6027303b3c3Ssethg orb $1, %al / Set bit 0 6037303b3c3Ssethg outb (%dx) / and reset the system 6047303b3c3Ssethg1: 6051a48003fSsethg 606085f98c4Ssethg call wait_500ms 607085f98c4Ssethg 608085f98c4Ssethg3: 60932206069Ssethg testl $RESET_METHOD_PCI, pc_reset_methods(%rip) 610085f98c4Ssethg jz 4f 611085f98c4Ssethg 6121a48003fSsethg / Try the PCI (soft) reset vector (should work on all modern systems, 6131a48003fSsethg / but has been shown to cause problems on 450NX systems, and some newer 6141a48003fSsethg / systems (e.g. ATI IXP400-equipped systems)) 6151a48003fSsethg / When resetting via this method, 2 writes are required. The first 6161a48003fSsethg / targets bit 1 (0=hard reset without power cycle, 1=hard reset with 6171a48003fSsethg / power cycle). 6181a48003fSsethg / The reset occurs on the second write, during bit 2's transition from 6191a48003fSsethg / 0->1. 6201a48003fSsethg movw $0xcf9, %dx 6211a48003fSsethg movb $0x2, %al / Reset mode = hard, no power cycle 6221a48003fSsethg outb (%dx) 6231a48003fSsethg movb $0x6, %al 6241a48003fSsethg outb (%dx) 6251a48003fSsethg 626085f98c4Ssethg call wait_500ms 627085f98c4Ssethg 628085f98c4Ssethg4: 6297303b3c3Ssethg / 6301a48003fSsethg / port 0xcf9 failed also. Last-ditch effort is to 6317303b3c3Ssethg / triple-fault the CPU. 632f2be5148Sszhou / Also, use triple fault for EFI firmware 6337303b3c3Ssethg / 634f2be5148Sszhou ENTRY(efi_reset) 6357303b3c3Ssethg pushq $0x0 6367303b3c3Ssethg pushq $0x0 / IDT base of 0, limit of 0 + 2 unused bytes 6377303b3c3Ssethg lidt (%rsp) 6387303b3c3Ssethg int $0x0 / Trigger interrupt, generate triple-fault 639085f98c4Ssethg 640085f98c4Ssethg cli 641085f98c4Ssethg hlt / Wait forever 6427c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 643f2be5148Sszhou SET_SIZE(efi_reset) 6447c478bd9Sstevel@tonic-gate SET_SIZE(pc_reset) 6457c478bd9Sstevel@tonic-gate 6467c478bd9Sstevel@tonic-gate/* 6477c478bd9Sstevel@tonic-gate * C callable in and out routines 6487c478bd9Sstevel@tonic-gate */ 6497c478bd9Sstevel@tonic-gate 6507c478bd9Sstevel@tonic-gate ENTRY(outl) 6517c478bd9Sstevel@tonic-gate movw %di, %dx 6527c478bd9Sstevel@tonic-gate movl %esi, %eax 6537c478bd9Sstevel@tonic-gate outl (%dx) 6547c478bd9Sstevel@tonic-gate ret 6557c478bd9Sstevel@tonic-gate SET_SIZE(outl) 6567c478bd9Sstevel@tonic-gate 6577c478bd9Sstevel@tonic-gate ENTRY(outw) 6587c478bd9Sstevel@tonic-gate movw %di, %dx 6597c478bd9Sstevel@tonic-gate movw %si, %ax 6607c478bd9Sstevel@tonic-gate D16 outl (%dx) /* XX64 why not outw? */ 6617c478bd9Sstevel@tonic-gate ret 6627c478bd9Sstevel@tonic-gate SET_SIZE(outw) 6637c478bd9Sstevel@tonic-gate 6647c478bd9Sstevel@tonic-gate ENTRY(outb) 6657c478bd9Sstevel@tonic-gate movw %di, %dx 6667c478bd9Sstevel@tonic-gate movb %sil, %al 6677c478bd9Sstevel@tonic-gate outb (%dx) 6687c478bd9Sstevel@tonic-gate ret 6697c478bd9Sstevel@tonic-gate SET_SIZE(outb) 6707c478bd9Sstevel@tonic-gate 6717c478bd9Sstevel@tonic-gate ENTRY(inl) 6727c478bd9Sstevel@tonic-gate xorl %eax, %eax 6737c478bd9Sstevel@tonic-gate movw %di, %dx 6747c478bd9Sstevel@tonic-gate inl (%dx) 6757c478bd9Sstevel@tonic-gate ret 6767c478bd9Sstevel@tonic-gate SET_SIZE(inl) 6777c478bd9Sstevel@tonic-gate 6787c478bd9Sstevel@tonic-gate ENTRY(inw) 6797c478bd9Sstevel@tonic-gate xorl %eax, %eax 6807c478bd9Sstevel@tonic-gate movw %di, %dx 6817c478bd9Sstevel@tonic-gate D16 inl (%dx) 6827c478bd9Sstevel@tonic-gate ret 6837c478bd9Sstevel@tonic-gate SET_SIZE(inw) 6847c478bd9Sstevel@tonic-gate 6857c478bd9Sstevel@tonic-gate 6867c478bd9Sstevel@tonic-gate ENTRY(inb) 6877c478bd9Sstevel@tonic-gate xorl %eax, %eax 6887c478bd9Sstevel@tonic-gate movw %di, %dx 6897c478bd9Sstevel@tonic-gate inb (%dx) 6907c478bd9Sstevel@tonic-gate ret 6917c478bd9Sstevel@tonic-gate SET_SIZE(inb) 6927c478bd9Sstevel@tonic-gate 6937c478bd9Sstevel@tonic-gate/* 6947aec1d6eScindi * void int3(void) 6957aec1d6eScindi * void int18(void) 6967c478bd9Sstevel@tonic-gate * void int20(void) 697e3d60c9bSAdrian Frost * void int_cmci(void) 6987c478bd9Sstevel@tonic-gate */ 6997c478bd9Sstevel@tonic-gate 7007aec1d6eScindi ENTRY(int3) 7017aec1d6eScindi int $T_BPTFLT 7027aec1d6eScindi ret 7037aec1d6eScindi SET_SIZE(int3) 7047aec1d6eScindi 7057aec1d6eScindi ENTRY(int18) 7067aec1d6eScindi int $T_MCE 7077aec1d6eScindi ret 7087aec1d6eScindi SET_SIZE(int18) 7097aec1d6eScindi 7107c478bd9Sstevel@tonic-gate ENTRY(int20) 7117c478bd9Sstevel@tonic-gate movl boothowto, %eax 7127c478bd9Sstevel@tonic-gate andl $RB_DEBUG, %eax 7137c478bd9Sstevel@tonic-gate jz 1f 7147c478bd9Sstevel@tonic-gate 7157aec1d6eScindi int $T_DBGENTR 7167c478bd9Sstevel@tonic-gate1: 71785641879Skalai rep; ret /* use 2 byte return instruction when branch target */ 71885641879Skalai /* AMD Software Optimization Guide - Section 6.2 */ 7197c478bd9Sstevel@tonic-gate SET_SIZE(int20) 7207c478bd9Sstevel@tonic-gate 721e3d60c9bSAdrian Frost ENTRY(int_cmci) 722e3d60c9bSAdrian Frost int $T_ENOEXTFLT 723e3d60c9bSAdrian Frost ret 724e3d60c9bSAdrian Frost SET_SIZE(int_cmci) 725e3d60c9bSAdrian Frost 7267c478bd9Sstevel@tonic-gate ENTRY(scanc) 7277c478bd9Sstevel@tonic-gate /* rdi == size */ 7287c478bd9Sstevel@tonic-gate /* rsi == cp */ 7297c478bd9Sstevel@tonic-gate /* rdx == table */ 7307c478bd9Sstevel@tonic-gate /* rcx == mask */ 7317c478bd9Sstevel@tonic-gate addq %rsi, %rdi /* end = &cp[size] */ 7324b56a003SDaniel Anderson.scanloop: 7337c478bd9Sstevel@tonic-gate cmpq %rdi, %rsi /* while (cp < end */ 7347c478bd9Sstevel@tonic-gate jnb .scandone 7357c478bd9Sstevel@tonic-gate movzbq (%rsi), %r8 /* %r8 = *cp */ 7367c478bd9Sstevel@tonic-gate incq %rsi /* cp++ */ 7377c478bd9Sstevel@tonic-gate testb %cl, (%r8, %rdx) 7387c478bd9Sstevel@tonic-gate jz .scanloop /* && (table[*cp] & mask) == 0) */ 7397c478bd9Sstevel@tonic-gate decq %rsi /* (fix post-increment) */ 7407c478bd9Sstevel@tonic-gate.scandone: 7417c478bd9Sstevel@tonic-gate movl %edi, %eax 7427c478bd9Sstevel@tonic-gate subl %esi, %eax /* return (end - cp) */ 7437c478bd9Sstevel@tonic-gate ret 7447c478bd9Sstevel@tonic-gate SET_SIZE(scanc) 7457c478bd9Sstevel@tonic-gate 7467c478bd9Sstevel@tonic-gate/* 7477c478bd9Sstevel@tonic-gate * Replacement functions for ones that are normally inlined. 748564d5236SRichard Lowe * In addition to the inline copies, they are defined here just in case. 7497c478bd9Sstevel@tonic-gate */ 7507c478bd9Sstevel@tonic-gate 7517c478bd9Sstevel@tonic-gate ENTRY(intr_clear) 7527c478bd9Sstevel@tonic-gate ENTRY(clear_int_flag) 7537c478bd9Sstevel@tonic-gate pushfq 7547c478bd9Sstevel@tonic-gate popq %rax 755843e1988Sjohnlev#if defined(__xpv) 756843e1988Sjohnlev leaq xpv_panicking, %rdi 757843e1988Sjohnlev movl (%rdi), %edi 758843e1988Sjohnlev cmpl $0, %edi 759843e1988Sjohnlev jne 2f 760843e1988Sjohnlev CLIRET(%rdi, %dl) /* returns event mask in %dl */ 761843e1988Sjohnlev /* 762843e1988Sjohnlev * Synthesize the PS_IE bit from the event mask bit 763843e1988Sjohnlev */ 764843e1988Sjohnlev andq $_BITNOT(PS_IE), %rax 765843e1988Sjohnlev testb $1, %dl 766843e1988Sjohnlev jnz 1f 767843e1988Sjohnlev orq $PS_IE, %rax 768843e1988Sjohnlev1: 769843e1988Sjohnlev ret 770843e1988Sjohnlev2: 771843e1988Sjohnlev#endif 772ae115bc7Smrj CLI(%rdi) 7737c478bd9Sstevel@tonic-gate ret 7747c478bd9Sstevel@tonic-gate SET_SIZE(clear_int_flag) 7757c478bd9Sstevel@tonic-gate SET_SIZE(intr_clear) 7767c478bd9Sstevel@tonic-gate 7777c478bd9Sstevel@tonic-gate ENTRY(curcpup) 7787c478bd9Sstevel@tonic-gate movq %gs:CPU_SELF, %rax 7797c478bd9Sstevel@tonic-gate ret 7807c478bd9Sstevel@tonic-gate SET_SIZE(curcpup) 7817c478bd9Sstevel@tonic-gate 7824b56a003SDaniel Anderson/* htonll(), ntohll(), htonl(), ntohl(), htons(), ntohs() 7834b56a003SDaniel Anderson * These functions reverse the byte order of the input parameter and returns 7844b56a003SDaniel Anderson * the result. This is to convert the byte order from host byte order 7854b56a003SDaniel Anderson * (little endian) to network byte order (big endian), or vice versa. 7864b56a003SDaniel Anderson */ 7874b56a003SDaniel Anderson 7884b56a003SDaniel Anderson ENTRY(htonll) 7894b56a003SDaniel Anderson ALTENTRY(ntohll) 7904b56a003SDaniel Anderson movq %rdi, %rax 7914b56a003SDaniel Anderson bswapq %rax 7927c478bd9Sstevel@tonic-gate ret 7934b56a003SDaniel Anderson SET_SIZE(ntohll) 7944b56a003SDaniel Anderson SET_SIZE(htonll) 7957c478bd9Sstevel@tonic-gate 7964b56a003SDaniel Anderson /* XX64 there must be shorter sequences for this */ 7977c478bd9Sstevel@tonic-gate ENTRY(htonl) 7987c478bd9Sstevel@tonic-gate ALTENTRY(ntohl) 7994b56a003SDaniel Anderson movl %edi, %eax 8007c478bd9Sstevel@tonic-gate bswap %eax 8017c478bd9Sstevel@tonic-gate ret 8027c478bd9Sstevel@tonic-gate SET_SIZE(ntohl) 8037c478bd9Sstevel@tonic-gate SET_SIZE(htonl) 8047c478bd9Sstevel@tonic-gate 8057c478bd9Sstevel@tonic-gate /* XX64 there must be better sequences for this */ 8067c478bd9Sstevel@tonic-gate ENTRY(htons) 8077c478bd9Sstevel@tonic-gate ALTENTRY(ntohs) 8087c478bd9Sstevel@tonic-gate movl %edi, %eax 8097c478bd9Sstevel@tonic-gate bswap %eax 8107c478bd9Sstevel@tonic-gate shrl $16, %eax 8117c478bd9Sstevel@tonic-gate ret 8124b56a003SDaniel Anderson SET_SIZE(ntohs) 8137c478bd9Sstevel@tonic-gate SET_SIZE(htons) 8147c478bd9Sstevel@tonic-gate 8157c478bd9Sstevel@tonic-gate 8167c478bd9Sstevel@tonic-gate ENTRY(intr_restore) 8177c478bd9Sstevel@tonic-gate ENTRY(restore_int_flag) 818a563a037Sbholler testq $PS_IE, %rdi 819a563a037Sbholler jz 1f 820843e1988Sjohnlev#if defined(__xpv) 821843e1988Sjohnlev leaq xpv_panicking, %rsi 822843e1988Sjohnlev movl (%rsi), %esi 823843e1988Sjohnlev cmpl $0, %esi 824843e1988Sjohnlev jne 1f 825843e1988Sjohnlev /* 826843e1988Sjohnlev * Since we're -really- running unprivileged, our attempt 827843e1988Sjohnlev * to change the state of the IF bit will be ignored. 828843e1988Sjohnlev * The virtual IF bit is tweaked by CLI and STI. 829843e1988Sjohnlev */ 830843e1988Sjohnlev IE_TO_EVENT_MASK(%rsi, %rdi) 831a563a037Sbholler#else 832a563a037Sbholler sti 833843e1988Sjohnlev#endif 834a563a037Sbholler1: 8357c478bd9Sstevel@tonic-gate ret 8367c478bd9Sstevel@tonic-gate SET_SIZE(restore_int_flag) 8377c478bd9Sstevel@tonic-gate SET_SIZE(intr_restore) 8387c478bd9Sstevel@tonic-gate 8397c478bd9Sstevel@tonic-gate ENTRY(sti) 840ae115bc7Smrj STI 8417c478bd9Sstevel@tonic-gate ret 8427c478bd9Sstevel@tonic-gate SET_SIZE(sti) 8437c478bd9Sstevel@tonic-gate 844ae115bc7Smrj ENTRY(cli) 845ae115bc7Smrj CLI(%rax) 846ae115bc7Smrj ret 847ae115bc7Smrj SET_SIZE(cli) 848ae115bc7Smrj 8497c478bd9Sstevel@tonic-gate ENTRY(dtrace_interrupt_disable) 8507c478bd9Sstevel@tonic-gate pushfq 8517c478bd9Sstevel@tonic-gate popq %rax 852843e1988Sjohnlev#if defined(__xpv) 853843e1988Sjohnlev leaq xpv_panicking, %rdi 854843e1988Sjohnlev movl (%rdi), %edi 855843e1988Sjohnlev cmpl $0, %edi 856e4b86885SCheng Sean Ye jne .dtrace_interrupt_disable_done 857843e1988Sjohnlev CLIRET(%rdi, %dl) /* returns event mask in %dl */ 858843e1988Sjohnlev /* 859843e1988Sjohnlev * Synthesize the PS_IE bit from the event mask bit 860843e1988Sjohnlev */ 861843e1988Sjohnlev andq $_BITNOT(PS_IE), %rax 862843e1988Sjohnlev testb $1, %dl 863e4b86885SCheng Sean Ye jnz .dtrace_interrupt_disable_done 864843e1988Sjohnlev orq $PS_IE, %rax 865843e1988Sjohnlev#else 866ae115bc7Smrj CLI(%rdx) 867843e1988Sjohnlev#endif 868e4b86885SCheng Sean Ye.dtrace_interrupt_disable_done: 8697c478bd9Sstevel@tonic-gate ret 8707c478bd9Sstevel@tonic-gate SET_SIZE(dtrace_interrupt_disable) 8717c478bd9Sstevel@tonic-gate 8727c478bd9Sstevel@tonic-gate ENTRY(dtrace_interrupt_enable) 8737c478bd9Sstevel@tonic-gate pushq %rdi 8747c478bd9Sstevel@tonic-gate popfq 875843e1988Sjohnlev#if defined(__xpv) 876843e1988Sjohnlev leaq xpv_panicking, %rdx 877843e1988Sjohnlev movl (%rdx), %edx 878843e1988Sjohnlev cmpl $0, %edx 879e4b86885SCheng Sean Ye jne .dtrace_interrupt_enable_done 880843e1988Sjohnlev /* 881843e1988Sjohnlev * Since we're -really- running unprivileged, our attempt 882843e1988Sjohnlev * to change the state of the IF bit will be ignored. The 883843e1988Sjohnlev * virtual IF bit is tweaked by CLI and STI. 884843e1988Sjohnlev */ 885843e1988Sjohnlev IE_TO_EVENT_MASK(%rdx, %rdi) 886843e1988Sjohnlev#endif 887e4b86885SCheng Sean Ye.dtrace_interrupt_enable_done: 8887c478bd9Sstevel@tonic-gate ret 8897c478bd9Sstevel@tonic-gate SET_SIZE(dtrace_interrupt_enable) 8907c478bd9Sstevel@tonic-gate 8917c478bd9Sstevel@tonic-gate 8927c478bd9Sstevel@tonic-gate ENTRY(dtrace_membar_producer) 89385641879Skalai rep; ret /* use 2 byte return instruction when branch target */ 89485641879Skalai /* AMD Software Optimization Guide - Section 6.2 */ 8957c478bd9Sstevel@tonic-gate SET_SIZE(dtrace_membar_producer) 8967c478bd9Sstevel@tonic-gate 8977c478bd9Sstevel@tonic-gate ENTRY(dtrace_membar_consumer) 89885641879Skalai rep; ret /* use 2 byte return instruction when branch target */ 89985641879Skalai /* AMD Software Optimization Guide - Section 6.2 */ 9007c478bd9Sstevel@tonic-gate SET_SIZE(dtrace_membar_consumer) 9017c478bd9Sstevel@tonic-gate 9027c478bd9Sstevel@tonic-gate ENTRY(threadp) 9037c478bd9Sstevel@tonic-gate movq %gs:CPU_THREAD, %rax 9047c478bd9Sstevel@tonic-gate ret 9057c478bd9Sstevel@tonic-gate SET_SIZE(threadp) 9067c478bd9Sstevel@tonic-gate 9077c478bd9Sstevel@tonic-gate/* 9087c478bd9Sstevel@tonic-gate * Checksum routine for Internet Protocol Headers 9097c478bd9Sstevel@tonic-gate */ 9107c478bd9Sstevel@tonic-gate 9117c478bd9Sstevel@tonic-gate ENTRY(ip_ocsum) 9127c478bd9Sstevel@tonic-gate pushq %rbp 9137c478bd9Sstevel@tonic-gate movq %rsp, %rbp 9147c478bd9Sstevel@tonic-gate#ifdef DEBUG 915ae115bc7Smrj movq postbootkernelbase(%rip), %rax 9167c478bd9Sstevel@tonic-gate cmpq %rax, %rdi 9177c478bd9Sstevel@tonic-gate jnb 1f 9187c478bd9Sstevel@tonic-gate xorl %eax, %eax 9197c478bd9Sstevel@tonic-gate movq %rdi, %rsi 9207c478bd9Sstevel@tonic-gate leaq .ip_ocsum_panic_msg(%rip), %rdi 9217c478bd9Sstevel@tonic-gate call panic 9227c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 9237c478bd9Sstevel@tonic-gate.ip_ocsum_panic_msg: 9247c478bd9Sstevel@tonic-gate .string "ip_ocsum: address 0x%p below kernelbase\n" 9257c478bd9Sstevel@tonic-gate1: 9267c478bd9Sstevel@tonic-gate#endif 9277c478bd9Sstevel@tonic-gate movl %esi, %ecx /* halfword_count */ 9287c478bd9Sstevel@tonic-gate movq %rdi, %rsi /* address */ 9297c478bd9Sstevel@tonic-gate /* partial sum in %edx */ 9307c478bd9Sstevel@tonic-gate xorl %eax, %eax 9317c478bd9Sstevel@tonic-gate testl %ecx, %ecx 9327c478bd9Sstevel@tonic-gate jz .ip_ocsum_done 9337c478bd9Sstevel@tonic-gate testq $3, %rsi 9347c478bd9Sstevel@tonic-gate jnz .ip_csum_notaligned 9357c478bd9Sstevel@tonic-gate.ip_csum_aligned: /* XX64 opportunities for 8-byte operations? */ 9367c478bd9Sstevel@tonic-gate.next_iter: 9377c478bd9Sstevel@tonic-gate /* XX64 opportunities for prefetch? */ 9387c478bd9Sstevel@tonic-gate /* XX64 compute csum with 64 bit quantities? */ 9397c478bd9Sstevel@tonic-gate subl $32, %ecx 9407c478bd9Sstevel@tonic-gate jl .less_than_32 9417c478bd9Sstevel@tonic-gate 9427c478bd9Sstevel@tonic-gate addl 0(%rsi), %edx 9437c478bd9Sstevel@tonic-gate.only60: 9447c478bd9Sstevel@tonic-gate adcl 4(%rsi), %eax 9457c478bd9Sstevel@tonic-gate.only56: 9467c478bd9Sstevel@tonic-gate adcl 8(%rsi), %edx 9477c478bd9Sstevel@tonic-gate.only52: 9487c478bd9Sstevel@tonic-gate adcl 12(%rsi), %eax 9497c478bd9Sstevel@tonic-gate.only48: 9507c478bd9Sstevel@tonic-gate adcl 16(%rsi), %edx 9517c478bd9Sstevel@tonic-gate.only44: 9527c478bd9Sstevel@tonic-gate adcl 20(%rsi), %eax 9537c478bd9Sstevel@tonic-gate.only40: 9547c478bd9Sstevel@tonic-gate adcl 24(%rsi), %edx 9557c478bd9Sstevel@tonic-gate.only36: 9567c478bd9Sstevel@tonic-gate adcl 28(%rsi), %eax 9577c478bd9Sstevel@tonic-gate.only32: 9587c478bd9Sstevel@tonic-gate adcl 32(%rsi), %edx 9597c478bd9Sstevel@tonic-gate.only28: 9607c478bd9Sstevel@tonic-gate adcl 36(%rsi), %eax 9617c478bd9Sstevel@tonic-gate.only24: 9627c478bd9Sstevel@tonic-gate adcl 40(%rsi), %edx 9637c478bd9Sstevel@tonic-gate.only20: 9647c478bd9Sstevel@tonic-gate adcl 44(%rsi), %eax 9657c478bd9Sstevel@tonic-gate.only16: 9667c478bd9Sstevel@tonic-gate adcl 48(%rsi), %edx 9677c478bd9Sstevel@tonic-gate.only12: 9687c478bd9Sstevel@tonic-gate adcl 52(%rsi), %eax 9697c478bd9Sstevel@tonic-gate.only8: 9707c478bd9Sstevel@tonic-gate adcl 56(%rsi), %edx 9717c478bd9Sstevel@tonic-gate.only4: 9727c478bd9Sstevel@tonic-gate adcl 60(%rsi), %eax /* could be adding -1 and -1 with a carry */ 9737c478bd9Sstevel@tonic-gate.only0: 9747c478bd9Sstevel@tonic-gate adcl $0, %eax /* could be adding -1 in eax with a carry */ 9757c478bd9Sstevel@tonic-gate adcl $0, %eax 9767c478bd9Sstevel@tonic-gate 9777c478bd9Sstevel@tonic-gate addq $64, %rsi 9787c478bd9Sstevel@tonic-gate testl %ecx, %ecx 9797c478bd9Sstevel@tonic-gate jnz .next_iter 9807c478bd9Sstevel@tonic-gate 9817c478bd9Sstevel@tonic-gate.ip_ocsum_done: 9827c478bd9Sstevel@tonic-gate addl %eax, %edx 9837c478bd9Sstevel@tonic-gate adcl $0, %edx 9847c478bd9Sstevel@tonic-gate movl %edx, %eax /* form a 16 bit checksum by */ 9857c478bd9Sstevel@tonic-gate shrl $16, %eax /* adding two halves of 32 bit checksum */ 9867c478bd9Sstevel@tonic-gate addw %dx, %ax 9877c478bd9Sstevel@tonic-gate adcw $0, %ax 9887c478bd9Sstevel@tonic-gate andl $0xffff, %eax 9897c478bd9Sstevel@tonic-gate leave 9907c478bd9Sstevel@tonic-gate ret 9917c478bd9Sstevel@tonic-gate 9927c478bd9Sstevel@tonic-gate.ip_csum_notaligned: 9937c478bd9Sstevel@tonic-gate xorl %edi, %edi 9947c478bd9Sstevel@tonic-gate movw (%rsi), %di 9957c478bd9Sstevel@tonic-gate addl %edi, %edx 9967c478bd9Sstevel@tonic-gate adcl $0, %edx 9977c478bd9Sstevel@tonic-gate addq $2, %rsi 9987c478bd9Sstevel@tonic-gate decl %ecx 9997c478bd9Sstevel@tonic-gate jmp .ip_csum_aligned 10007c478bd9Sstevel@tonic-gate 10017c478bd9Sstevel@tonic-gate.less_than_32: 10027c478bd9Sstevel@tonic-gate addl $32, %ecx 10037c478bd9Sstevel@tonic-gate testl $1, %ecx 10047c478bd9Sstevel@tonic-gate jz .size_aligned 10057c478bd9Sstevel@tonic-gate andl $0xfe, %ecx 10067c478bd9Sstevel@tonic-gate movzwl (%rsi, %rcx, 2), %edi 10077c478bd9Sstevel@tonic-gate addl %edi, %edx 10087c478bd9Sstevel@tonic-gate adcl $0, %edx 10097c478bd9Sstevel@tonic-gate.size_aligned: 10107c478bd9Sstevel@tonic-gate movl %ecx, %edi 10117c478bd9Sstevel@tonic-gate shrl $1, %ecx 10127c478bd9Sstevel@tonic-gate shl $1, %edi 10137c478bd9Sstevel@tonic-gate subq $64, %rdi 10147c478bd9Sstevel@tonic-gate addq %rdi, %rsi 10157c478bd9Sstevel@tonic-gate leaq .ip_ocsum_jmptbl(%rip), %rdi 10167c478bd9Sstevel@tonic-gate leaq (%rdi, %rcx, 8), %rdi 10177c478bd9Sstevel@tonic-gate xorl %ecx, %ecx 10187c478bd9Sstevel@tonic-gate clc 101965f20420SRobert Mustacchi movq (%rdi), %rdi 102065f20420SRobert Mustacchi INDIRECT_JMP_REG(rdi) 10217c478bd9Sstevel@tonic-gate 10227c478bd9Sstevel@tonic-gate .align 8 10237c478bd9Sstevel@tonic-gate.ip_ocsum_jmptbl: 10247c478bd9Sstevel@tonic-gate .quad .only0, .only4, .only8, .only12, .only16, .only20 10257c478bd9Sstevel@tonic-gate .quad .only24, .only28, .only32, .only36, .only40, .only44 10267c478bd9Sstevel@tonic-gate .quad .only48, .only52, .only56, .only60 10277c478bd9Sstevel@tonic-gate SET_SIZE(ip_ocsum) 10287c478bd9Sstevel@tonic-gate 10297c478bd9Sstevel@tonic-gate/* 10307c478bd9Sstevel@tonic-gate * multiply two long numbers and yield a u_longlong_t result, callable from C. 10317c478bd9Sstevel@tonic-gate * Provided to manipulate hrtime_t values. 10327c478bd9Sstevel@tonic-gate */ 10337c478bd9Sstevel@tonic-gate 10347c478bd9Sstevel@tonic-gate ENTRY(mul32) 10357c478bd9Sstevel@tonic-gate xorl %edx, %edx /* XX64 joe, paranoia? */ 10367c478bd9Sstevel@tonic-gate movl %edi, %eax 10377c478bd9Sstevel@tonic-gate mull %esi 103865f20420SRobert Mustacchi shlq $32, %rdx 10397c478bd9Sstevel@tonic-gate orq %rdx, %rax 10407c478bd9Sstevel@tonic-gate ret 10417c478bd9Sstevel@tonic-gate SET_SIZE(mul32) 10427c478bd9Sstevel@tonic-gate 10437c478bd9Sstevel@tonic-gate ENTRY(scan_memory) 10447c478bd9Sstevel@tonic-gate shrq $3, %rsi /* convert %rsi from byte to quadword count */ 10457c478bd9Sstevel@tonic-gate jz .scanm_done 10467c478bd9Sstevel@tonic-gate movq %rsi, %rcx /* move count into rep control register */ 10477c478bd9Sstevel@tonic-gate movq %rdi, %rsi /* move addr into lodsq control reg. */ 10487c478bd9Sstevel@tonic-gate rep lodsq /* scan the memory range */ 10497c478bd9Sstevel@tonic-gate.scanm_done: 105085641879Skalai rep; ret /* use 2 byte return instruction when branch target */ 105185641879Skalai /* AMD Software Optimization Guide - Section 6.2 */ 10527c478bd9Sstevel@tonic-gate SET_SIZE(scan_memory) 10537c478bd9Sstevel@tonic-gate 10547c478bd9Sstevel@tonic-gate 10557c478bd9Sstevel@tonic-gate ENTRY(lowbit) 10567c478bd9Sstevel@tonic-gate movl $-1, %eax 1057fdb8cf8cSJosef 'Jeff' Sipek bsfq %rdi, %rdi 1058fdb8cf8cSJosef 'Jeff' Sipek cmovnz %edi, %eax 10597c478bd9Sstevel@tonic-gate incl %eax 10607c478bd9Sstevel@tonic-gate ret 10617c478bd9Sstevel@tonic-gate SET_SIZE(lowbit) 10627c478bd9Sstevel@tonic-gate 10637c478bd9Sstevel@tonic-gate ENTRY(highbit) 1064fdb8cf8cSJosef 'Jeff' Sipek ALTENTRY(highbit64) 10657c478bd9Sstevel@tonic-gate movl $-1, %eax 1066fdb8cf8cSJosef 'Jeff' Sipek bsrq %rdi, %rdi 1067fdb8cf8cSJosef 'Jeff' Sipek cmovnz %edi, %eax 10687c478bd9Sstevel@tonic-gate incl %eax 10697c478bd9Sstevel@tonic-gate ret 1070fdb8cf8cSJosef 'Jeff' Sipek SET_SIZE(highbit64) 10717c478bd9Sstevel@tonic-gate SET_SIZE(highbit) 10727c478bd9Sstevel@tonic-gate 1073ee88d2b9Skchow#define XMSR_ACCESS_VAL $0x9c5a203a 1074ee88d2b9Skchow 10757c478bd9Sstevel@tonic-gate ENTRY(rdmsr) 10767c478bd9Sstevel@tonic-gate movl %edi, %ecx 10777c478bd9Sstevel@tonic-gate rdmsr 10787c478bd9Sstevel@tonic-gate shlq $32, %rdx 10797c478bd9Sstevel@tonic-gate orq %rdx, %rax 10807c478bd9Sstevel@tonic-gate ret 10817c478bd9Sstevel@tonic-gate SET_SIZE(rdmsr) 10827c478bd9Sstevel@tonic-gate 10837c478bd9Sstevel@tonic-gate ENTRY(wrmsr) 10840ac7d7d8Skucharsk movq %rsi, %rdx 10850ac7d7d8Skucharsk shrq $32, %rdx 10860ac7d7d8Skucharsk movl %esi, %eax 10877c478bd9Sstevel@tonic-gate movl %edi, %ecx 10887c478bd9Sstevel@tonic-gate wrmsr 10897c478bd9Sstevel@tonic-gate ret 10907c478bd9Sstevel@tonic-gate SET_SIZE(wrmsr) 10917c478bd9Sstevel@tonic-gate 1092ee88d2b9Skchow ENTRY(xrdmsr) 1093ae115bc7Smrj pushq %rbp 1094ae115bc7Smrj movq %rsp, %rbp 1095ee88d2b9Skchow movl %edi, %ecx 1096ee88d2b9Skchow movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */ 1097ee88d2b9Skchow rdmsr 1098ee88d2b9Skchow shlq $32, %rdx 1099ee88d2b9Skchow orq %rdx, %rax 1100ae115bc7Smrj leave 1101ee88d2b9Skchow ret 1102ee88d2b9Skchow SET_SIZE(xrdmsr) 1103ee88d2b9Skchow 1104ee88d2b9Skchow ENTRY(xwrmsr) 1105ae115bc7Smrj pushq %rbp 1106ae115bc7Smrj movq %rsp, %rbp 1107ee88d2b9Skchow movl %edi, %ecx 1108ee88d2b9Skchow movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */ 1109ee88d2b9Skchow movq %rsi, %rdx 1110ee88d2b9Skchow shrq $32, %rdx 1111ee88d2b9Skchow movl %esi, %eax 1112ee88d2b9Skchow wrmsr 1113ae115bc7Smrj leave 1114ee88d2b9Skchow ret 1115ee88d2b9Skchow SET_SIZE(xwrmsr) 11167af88ac7SKuriakose Kuruvilla 11177af88ac7SKuriakose Kuruvilla ENTRY(get_xcr) 11187af88ac7SKuriakose Kuruvilla movl %edi, %ecx 11197af88ac7SKuriakose Kuruvilla #xgetbv 11207af88ac7SKuriakose Kuruvilla .byte 0x0f,0x01,0xd0 11217af88ac7SKuriakose Kuruvilla shlq $32, %rdx 11227af88ac7SKuriakose Kuruvilla orq %rdx, %rax 11237af88ac7SKuriakose Kuruvilla ret 11247af88ac7SKuriakose Kuruvilla SET_SIZE(get_xcr) 11257af88ac7SKuriakose Kuruvilla 11267af88ac7SKuriakose Kuruvilla ENTRY(set_xcr) 11277af88ac7SKuriakose Kuruvilla movq %rsi, %rdx 11287af88ac7SKuriakose Kuruvilla shrq $32, %rdx 11297af88ac7SKuriakose Kuruvilla movl %esi, %eax 11307af88ac7SKuriakose Kuruvilla movl %edi, %ecx 11317af88ac7SKuriakose Kuruvilla #xsetbv 11327af88ac7SKuriakose Kuruvilla .byte 0x0f,0x01,0xd1 11337af88ac7SKuriakose Kuruvilla ret 11347af88ac7SKuriakose Kuruvilla SET_SIZE(set_xcr) 11357af88ac7SKuriakose Kuruvilla 11367c478bd9Sstevel@tonic-gate ENTRY(invalidate_cache) 11377c478bd9Sstevel@tonic-gate wbinvd 11387c478bd9Sstevel@tonic-gate ret 11397c478bd9Sstevel@tonic-gate SET_SIZE(invalidate_cache) 11407c478bd9Sstevel@tonic-gate 1141ae115bc7Smrj ENTRY_NP(getcregs) 1142843e1988Sjohnlev#if defined(__xpv) 1143843e1988Sjohnlev /* 1144843e1988Sjohnlev * Only a few of the hardware control registers or descriptor tables 1145843e1988Sjohnlev * are directly accessible to us, so just zero the structure. 1146843e1988Sjohnlev * 1147843e1988Sjohnlev * XXPV Perhaps it would be helpful for the hypervisor to return 1148843e1988Sjohnlev * virtualized versions of these for post-mortem use. 1149843e1988Sjohnlev * (Need to reevaluate - perhaps it already does!) 1150843e1988Sjohnlev */ 1151843e1988Sjohnlev pushq %rdi /* save *crp */ 1152843e1988Sjohnlev movq $CREGSZ, %rsi 1153843e1988Sjohnlev call bzero 1154843e1988Sjohnlev popq %rdi 1155843e1988Sjohnlev 1156843e1988Sjohnlev /* 1157843e1988Sjohnlev * Dump what limited information we can 1158843e1988Sjohnlev */ 1159843e1988Sjohnlev movq %cr0, %rax 1160843e1988Sjohnlev movq %rax, CREG_CR0(%rdi) /* cr0 */ 1161843e1988Sjohnlev movq %cr2, %rax 1162843e1988Sjohnlev movq %rax, CREG_CR2(%rdi) /* cr2 */ 1163843e1988Sjohnlev movq %cr3, %rax 1164843e1988Sjohnlev movq %rax, CREG_CR3(%rdi) /* cr3 */ 1165843e1988Sjohnlev movq %cr4, %rax 1166843e1988Sjohnlev movq %rax, CREG_CR4(%rdi) /* cr4 */ 1167843e1988Sjohnlev 1168843e1988Sjohnlev#else /* __xpv */ 1169ae115bc7Smrj 11707c478bd9Sstevel@tonic-gate#define GETMSR(r, off, d) \ 11717c478bd9Sstevel@tonic-gate movl $r, %ecx; \ 11727c478bd9Sstevel@tonic-gate rdmsr; \ 11737c478bd9Sstevel@tonic-gate movl %eax, off(d); \ 11747c478bd9Sstevel@tonic-gate movl %edx, off+4(d) 11757c478bd9Sstevel@tonic-gate 11767c478bd9Sstevel@tonic-gate xorl %eax, %eax 11777c478bd9Sstevel@tonic-gate movq %rax, CREG_GDT+8(%rdi) 11787c478bd9Sstevel@tonic-gate sgdt CREG_GDT(%rdi) /* 10 bytes */ 11797c478bd9Sstevel@tonic-gate movq %rax, CREG_IDT+8(%rdi) 11807c478bd9Sstevel@tonic-gate sidt CREG_IDT(%rdi) /* 10 bytes */ 11817c478bd9Sstevel@tonic-gate movq %rax, CREG_LDT(%rdi) 11827c478bd9Sstevel@tonic-gate sldt CREG_LDT(%rdi) /* 2 bytes */ 11837c478bd9Sstevel@tonic-gate movq %rax, CREG_TASKR(%rdi) 11847c478bd9Sstevel@tonic-gate str CREG_TASKR(%rdi) /* 2 bytes */ 11857c478bd9Sstevel@tonic-gate movq %cr0, %rax 11867c478bd9Sstevel@tonic-gate movq %rax, CREG_CR0(%rdi) /* cr0 */ 11877c478bd9Sstevel@tonic-gate movq %cr2, %rax 11887c478bd9Sstevel@tonic-gate movq %rax, CREG_CR2(%rdi) /* cr2 */ 11897c478bd9Sstevel@tonic-gate movq %cr3, %rax 11907c478bd9Sstevel@tonic-gate movq %rax, CREG_CR3(%rdi) /* cr3 */ 11917c478bd9Sstevel@tonic-gate movq %cr4, %rax 1192ae115bc7Smrj movq %rax, CREG_CR4(%rdi) /* cr4 */ 11937c478bd9Sstevel@tonic-gate movq %cr8, %rax 11947c478bd9Sstevel@tonic-gate movq %rax, CREG_CR8(%rdi) /* cr8 */ 11957c478bd9Sstevel@tonic-gate GETMSR(MSR_AMD_KGSBASE, CREG_KGSBASE, %rdi) 11967c478bd9Sstevel@tonic-gate GETMSR(MSR_AMD_EFER, CREG_EFER, %rdi) 1197843e1988Sjohnlev#endif /* __xpv */ 1198ae115bc7Smrj ret 11997c478bd9Sstevel@tonic-gate SET_SIZE(getcregs) 12007c478bd9Sstevel@tonic-gate 12017c478bd9Sstevel@tonic-gate#undef GETMSR 12027c478bd9Sstevel@tonic-gate 12037c478bd9Sstevel@tonic-gate 12047c478bd9Sstevel@tonic-gate/* 12057c478bd9Sstevel@tonic-gate * A panic trigger is a word which is updated atomically and can only be set 12067c478bd9Sstevel@tonic-gate * once. We atomically store 0xDEFACEDD and load the old value. If the 12077c478bd9Sstevel@tonic-gate * previous value was 0, we succeed and return 1; otherwise return 0. 12087c478bd9Sstevel@tonic-gate * This allows a partially corrupt trigger to still trigger correctly. DTrace 12097c478bd9Sstevel@tonic-gate * has its own version of this function to allow it to panic correctly from 12107c478bd9Sstevel@tonic-gate * probe context. 12117c478bd9Sstevel@tonic-gate */ 12127c478bd9Sstevel@tonic-gate 12137c478bd9Sstevel@tonic-gate ENTRY_NP(panic_trigger) 12147c478bd9Sstevel@tonic-gate xorl %eax, %eax 12157c478bd9Sstevel@tonic-gate movl $0xdefacedd, %edx 12167c478bd9Sstevel@tonic-gate lock 12177c478bd9Sstevel@tonic-gate xchgl %edx, (%rdi) 12187c478bd9Sstevel@tonic-gate cmpl $0, %edx 121965f20420SRobert Mustacchi je 0f 12207c478bd9Sstevel@tonic-gate movl $0, %eax 12217c478bd9Sstevel@tonic-gate ret 12227c478bd9Sstevel@tonic-gate0: movl $1, %eax 12237c478bd9Sstevel@tonic-gate ret 12247c478bd9Sstevel@tonic-gate SET_SIZE(panic_trigger) 122565f20420SRobert Mustacchi 12267c478bd9Sstevel@tonic-gate ENTRY_NP(dtrace_panic_trigger) 12277c478bd9Sstevel@tonic-gate xorl %eax, %eax 12287c478bd9Sstevel@tonic-gate movl $0xdefacedd, %edx 12297c478bd9Sstevel@tonic-gate lock 12307c478bd9Sstevel@tonic-gate xchgl %edx, (%rdi) 12317c478bd9Sstevel@tonic-gate cmpl $0, %edx 12327c478bd9Sstevel@tonic-gate je 0f 12337c478bd9Sstevel@tonic-gate movl $0, %eax 12347c478bd9Sstevel@tonic-gate ret 12357c478bd9Sstevel@tonic-gate0: movl $1, %eax 12367c478bd9Sstevel@tonic-gate ret 12377c478bd9Sstevel@tonic-gate SET_SIZE(dtrace_panic_trigger) 12387c478bd9Sstevel@tonic-gate 12397c478bd9Sstevel@tonic-gate/* 12407c478bd9Sstevel@tonic-gate * The panic() and cmn_err() functions invoke vpanic() as a common entry point 12417c478bd9Sstevel@tonic-gate * into the panic code implemented in panicsys(). vpanic() is responsible 12427c478bd9Sstevel@tonic-gate * for passing through the format string and arguments, and constructing a 12437c478bd9Sstevel@tonic-gate * regs structure on the stack into which it saves the current register 12447c478bd9Sstevel@tonic-gate * values. If we are not dying due to a fatal trap, these registers will 12457c478bd9Sstevel@tonic-gate * then be preserved in panicbuf as the current processor state. Before 12467c478bd9Sstevel@tonic-gate * invoking panicsys(), vpanic() activates the first panic trigger (see 12477c478bd9Sstevel@tonic-gate * common/os/panic.c) and switches to the panic_stack if successful. Note that 12487c478bd9Sstevel@tonic-gate * DTrace takes a slightly different panic path if it must panic from probe 12497c478bd9Sstevel@tonic-gate * context. Instead of calling panic, it calls into dtrace_vpanic(), which 12507c478bd9Sstevel@tonic-gate * sets up the initial stack as vpanic does, calls dtrace_panic_trigger(), and 12517c478bd9Sstevel@tonic-gate * branches back into vpanic(). 12527c478bd9Sstevel@tonic-gate */ 12537c478bd9Sstevel@tonic-gate 12547c478bd9Sstevel@tonic-gate ENTRY_NP(vpanic) /* Initial stack layout: */ 125565f20420SRobert Mustacchi 125665f20420SRobert Mustacchi pushq %rbp /* | %rip | 0x60 */ 12577c478bd9Sstevel@tonic-gate movq %rsp, %rbp /* | %rbp | 0x58 */ 12587c478bd9Sstevel@tonic-gate pushfq /* | rfl | 0x50 */ 12597c478bd9Sstevel@tonic-gate pushq %r11 /* | %r11 | 0x48 */ 12607c478bd9Sstevel@tonic-gate pushq %r10 /* | %r10 | 0x40 */ 12617c478bd9Sstevel@tonic-gate pushq %rbx /* | %rbx | 0x38 */ 12627c478bd9Sstevel@tonic-gate pushq %rax /* | %rax | 0x30 */ 12637c478bd9Sstevel@tonic-gate pushq %r9 /* | %r9 | 0x28 */ 12647c478bd9Sstevel@tonic-gate pushq %r8 /* | %r8 | 0x20 */ 12657c478bd9Sstevel@tonic-gate pushq %rcx /* | %rcx | 0x18 */ 12667c478bd9Sstevel@tonic-gate pushq %rdx /* | %rdx | 0x10 */ 12677c478bd9Sstevel@tonic-gate pushq %rsi /* | %rsi | 0x8 alist */ 12687c478bd9Sstevel@tonic-gate pushq %rdi /* | %rdi | 0x0 format */ 12697c478bd9Sstevel@tonic-gate 12707c478bd9Sstevel@tonic-gate movq %rsp, %rbx /* %rbx = current %rsp */ 12717c478bd9Sstevel@tonic-gate 12727c478bd9Sstevel@tonic-gate leaq panic_quiesce(%rip), %rdi /* %rdi = &panic_quiesce */ 12737c478bd9Sstevel@tonic-gate call panic_trigger /* %eax = panic_trigger() */ 12747c478bd9Sstevel@tonic-gate 12757c478bd9Sstevel@tonic-gatevpanic_common: 127652d60c84Sgavinm /* 127752d60c84Sgavinm * The panic_trigger result is in %eax from the call above, and 127852d60c84Sgavinm * dtrace_panic places it in %eax before branching here. 127952d60c84Sgavinm * The rdmsr instructions that follow below will clobber %eax so 128052d60c84Sgavinm * we stash the panic_trigger result in %r11d. 128152d60c84Sgavinm */ 128252d60c84Sgavinm movl %eax, %r11d 128352d60c84Sgavinm cmpl $0, %r11d 12847c478bd9Sstevel@tonic-gate je 0f 12857c478bd9Sstevel@tonic-gate 12867c478bd9Sstevel@tonic-gate /* 12877c478bd9Sstevel@tonic-gate * If panic_trigger() was successful, we are the first to initiate a 12887c478bd9Sstevel@tonic-gate * panic: we now switch to the reserved panic_stack before continuing. 12897c478bd9Sstevel@tonic-gate */ 12907c478bd9Sstevel@tonic-gate leaq panic_stack(%rip), %rsp 12917c478bd9Sstevel@tonic-gate addq $PANICSTKSIZE, %rsp 12927c478bd9Sstevel@tonic-gate0: subq $REGSIZE, %rsp 12937c478bd9Sstevel@tonic-gate /* 12947c478bd9Sstevel@tonic-gate * Now that we've got everything set up, store the register values as 12957c478bd9Sstevel@tonic-gate * they were when we entered vpanic() to the designated location in 12967c478bd9Sstevel@tonic-gate * the regs structure we allocated on the stack. 12977c478bd9Sstevel@tonic-gate */ 12987c478bd9Sstevel@tonic-gate movq 0x0(%rbx), %rcx 12997c478bd9Sstevel@tonic-gate movq %rcx, REGOFF_RDI(%rsp) 13007c478bd9Sstevel@tonic-gate movq 0x8(%rbx), %rcx 13017c478bd9Sstevel@tonic-gate movq %rcx, REGOFF_RSI(%rsp) 13027c478bd9Sstevel@tonic-gate movq 0x10(%rbx), %rcx 13037c478bd9Sstevel@tonic-gate movq %rcx, REGOFF_RDX(%rsp) 13047c478bd9Sstevel@tonic-gate movq 0x18(%rbx), %rcx 13057c478bd9Sstevel@tonic-gate movq %rcx, REGOFF_RCX(%rsp) 13067c478bd9Sstevel@tonic-gate movq 0x20(%rbx), %rcx 13077c478bd9Sstevel@tonic-gate 13087c478bd9Sstevel@tonic-gate movq %rcx, REGOFF_R8(%rsp) 13097c478bd9Sstevel@tonic-gate movq 0x28(%rbx), %rcx 13107c478bd9Sstevel@tonic-gate movq %rcx, REGOFF_R9(%rsp) 13117c478bd9Sstevel@tonic-gate movq 0x30(%rbx), %rcx 13127c478bd9Sstevel@tonic-gate movq %rcx, REGOFF_RAX(%rsp) 13137c478bd9Sstevel@tonic-gate movq 0x38(%rbx), %rcx 13142d596601Ssethg movq %rcx, REGOFF_RBX(%rsp) 13157c478bd9Sstevel@tonic-gate movq 0x58(%rbx), %rcx 13167c478bd9Sstevel@tonic-gate 13177c478bd9Sstevel@tonic-gate movq %rcx, REGOFF_RBP(%rsp) 13187c478bd9Sstevel@tonic-gate movq 0x40(%rbx), %rcx 13197c478bd9Sstevel@tonic-gate movq %rcx, REGOFF_R10(%rsp) 13207c478bd9Sstevel@tonic-gate movq 0x48(%rbx), %rcx 13217c478bd9Sstevel@tonic-gate movq %rcx, REGOFF_R11(%rsp) 13227c478bd9Sstevel@tonic-gate movq %r12, REGOFF_R12(%rsp) 13237c478bd9Sstevel@tonic-gate 13247c478bd9Sstevel@tonic-gate movq %r13, REGOFF_R13(%rsp) 13257c478bd9Sstevel@tonic-gate movq %r14, REGOFF_R14(%rsp) 13267c478bd9Sstevel@tonic-gate movq %r15, REGOFF_R15(%rsp) 13277c478bd9Sstevel@tonic-gate 13287c478bd9Sstevel@tonic-gate xorl %ecx, %ecx 13297c478bd9Sstevel@tonic-gate movw %ds, %cx 13307c478bd9Sstevel@tonic-gate movq %rcx, REGOFF_DS(%rsp) 13317c478bd9Sstevel@tonic-gate movw %es, %cx 13327c478bd9Sstevel@tonic-gate movq %rcx, REGOFF_ES(%rsp) 13337c478bd9Sstevel@tonic-gate movw %fs, %cx 13347c478bd9Sstevel@tonic-gate movq %rcx, REGOFF_FS(%rsp) 13357c478bd9Sstevel@tonic-gate movw %gs, %cx 13367c478bd9Sstevel@tonic-gate movq %rcx, REGOFF_GS(%rsp) 13377c478bd9Sstevel@tonic-gate 13387c478bd9Sstevel@tonic-gate movq $0, REGOFF_TRAPNO(%rsp) 13397c478bd9Sstevel@tonic-gate 13407c478bd9Sstevel@tonic-gate movq $0, REGOFF_ERR(%rsp) 13417c478bd9Sstevel@tonic-gate leaq vpanic(%rip), %rcx 13427c478bd9Sstevel@tonic-gate movq %rcx, REGOFF_RIP(%rsp) 13437c478bd9Sstevel@tonic-gate movw %cs, %cx 13447c478bd9Sstevel@tonic-gate movzwq %cx, %rcx 13457c478bd9Sstevel@tonic-gate movq %rcx, REGOFF_CS(%rsp) 13467c478bd9Sstevel@tonic-gate movq 0x50(%rbx), %rcx 13477c478bd9Sstevel@tonic-gate movq %rcx, REGOFF_RFL(%rsp) 13487c478bd9Sstevel@tonic-gate movq %rbx, %rcx 13497c478bd9Sstevel@tonic-gate addq $0x60, %rcx 13507c478bd9Sstevel@tonic-gate movq %rcx, REGOFF_RSP(%rsp) 13517c478bd9Sstevel@tonic-gate movw %ss, %cx 13527c478bd9Sstevel@tonic-gate movzwq %cx, %rcx 13537c478bd9Sstevel@tonic-gate movq %rcx, REGOFF_SS(%rsp) 13547c478bd9Sstevel@tonic-gate 13557c478bd9Sstevel@tonic-gate /* 135665f20420SRobert Mustacchi * panicsys(format, alist, rp, on_panic_stack) 135765f20420SRobert Mustacchi */ 13587c478bd9Sstevel@tonic-gate movq REGOFF_RDI(%rsp), %rdi /* format */ 13597c478bd9Sstevel@tonic-gate movq REGOFF_RSI(%rsp), %rsi /* alist */ 13607c478bd9Sstevel@tonic-gate movq %rsp, %rdx /* struct regs */ 136152d60c84Sgavinm movl %r11d, %ecx /* on_panic_stack */ 13627c478bd9Sstevel@tonic-gate call panicsys 13637c478bd9Sstevel@tonic-gate addq $REGSIZE, %rsp 13647c478bd9Sstevel@tonic-gate popq %rdi 13657c478bd9Sstevel@tonic-gate popq %rsi 13667c478bd9Sstevel@tonic-gate popq %rdx 13677c478bd9Sstevel@tonic-gate popq %rcx 13687c478bd9Sstevel@tonic-gate popq %r8 13697c478bd9Sstevel@tonic-gate popq %r9 13707c478bd9Sstevel@tonic-gate popq %rax 13717c478bd9Sstevel@tonic-gate popq %rbx 13727c478bd9Sstevel@tonic-gate popq %r10 13737c478bd9Sstevel@tonic-gate popq %r11 13747c478bd9Sstevel@tonic-gate popfq 13757c478bd9Sstevel@tonic-gate leave 13767c478bd9Sstevel@tonic-gate ret 13777c478bd9Sstevel@tonic-gate SET_SIZE(vpanic) 13787c478bd9Sstevel@tonic-gate 13797c478bd9Sstevel@tonic-gate ENTRY_NP(dtrace_vpanic) /* Initial stack layout: */ 13807c478bd9Sstevel@tonic-gate 138165f20420SRobert Mustacchi pushq %rbp /* | %rip | 0x60 */ 13827c478bd9Sstevel@tonic-gate movq %rsp, %rbp /* | %rbp | 0x58 */ 13837c478bd9Sstevel@tonic-gate pushfq /* | rfl | 0x50 */ 13847c478bd9Sstevel@tonic-gate pushq %r11 /* | %r11 | 0x48 */ 13857c478bd9Sstevel@tonic-gate pushq %r10 /* | %r10 | 0x40 */ 13867c478bd9Sstevel@tonic-gate pushq %rbx /* | %rbx | 0x38 */ 13877c478bd9Sstevel@tonic-gate pushq %rax /* | %rax | 0x30 */ 13887c478bd9Sstevel@tonic-gate pushq %r9 /* | %r9 | 0x28 */ 13897c478bd9Sstevel@tonic-gate pushq %r8 /* | %r8 | 0x20 */ 13907c478bd9Sstevel@tonic-gate pushq %rcx /* | %rcx | 0x18 */ 13917c478bd9Sstevel@tonic-gate pushq %rdx /* | %rdx | 0x10 */ 13927c478bd9Sstevel@tonic-gate pushq %rsi /* | %rsi | 0x8 alist */ 13937c478bd9Sstevel@tonic-gate pushq %rdi /* | %rdi | 0x0 format */ 13947c478bd9Sstevel@tonic-gate 13957c478bd9Sstevel@tonic-gate movq %rsp, %rbx /* %rbx = current %rsp */ 13967c478bd9Sstevel@tonic-gate 13977c478bd9Sstevel@tonic-gate leaq panic_quiesce(%rip), %rdi /* %rdi = &panic_quiesce */ 13987c478bd9Sstevel@tonic-gate call dtrace_panic_trigger /* %eax = dtrace_panic_trigger() */ 13997c478bd9Sstevel@tonic-gate jmp vpanic_common 14007c478bd9Sstevel@tonic-gate 14017c478bd9Sstevel@tonic-gate SET_SIZE(dtrace_vpanic) 14027c478bd9Sstevel@tonic-gate 14037c478bd9Sstevel@tonic-gate DGDEF3(timedelta, 8, 8) 14047c478bd9Sstevel@tonic-gate .long 0, 0 14057c478bd9Sstevel@tonic-gate 14067c478bd9Sstevel@tonic-gate /* 14077c478bd9Sstevel@tonic-gate * initialized to a non zero value to make pc_gethrtime() 14087c478bd9Sstevel@tonic-gate * work correctly even before clock is initialized 14097c478bd9Sstevel@tonic-gate */ 14107c478bd9Sstevel@tonic-gate DGDEF3(hrtime_base, 8, 8) 14117c478bd9Sstevel@tonic-gate .long _MUL(NSEC_PER_CLOCK_TICK, 6), 0 14127c478bd9Sstevel@tonic-gate 14137c478bd9Sstevel@tonic-gate DGDEF3(adj_shift, 4, 4) 14147c478bd9Sstevel@tonic-gate .long ADJ_SHIFT 14157c478bd9Sstevel@tonic-gate 14167c478bd9Sstevel@tonic-gate ENTRY_NP(hres_tick) 14177c478bd9Sstevel@tonic-gate pushq %rbp 14187c478bd9Sstevel@tonic-gate movq %rsp, %rbp 14197c478bd9Sstevel@tonic-gate 14207c478bd9Sstevel@tonic-gate /* 14217c478bd9Sstevel@tonic-gate * We need to call *gethrtimef before picking up CLOCK_LOCK (obviously, 14227c478bd9Sstevel@tonic-gate * hres_last_tick can only be modified while holding CLOCK_LOCK). 14237c478bd9Sstevel@tonic-gate * At worst, performing this now instead of under CLOCK_LOCK may 14247c478bd9Sstevel@tonic-gate * introduce some jitter in pc_gethrestime(). 14257c478bd9Sstevel@tonic-gate */ 142665f20420SRobert Mustacchi movq gethrtimef(%rip), %rsi 142765f20420SRobert Mustacchi INDIRECT_CALL_REG(rsi) 14287c478bd9Sstevel@tonic-gate movq %rax, %r8 14297c478bd9Sstevel@tonic-gate 14307c478bd9Sstevel@tonic-gate leaq hres_lock(%rip), %rax 14317c478bd9Sstevel@tonic-gate movb $-1, %dl 14327c478bd9Sstevel@tonic-gate.CL1: 14337c478bd9Sstevel@tonic-gate xchgb %dl, (%rax) 14347c478bd9Sstevel@tonic-gate testb %dl, %dl 14357c478bd9Sstevel@tonic-gate jz .CL3 /* got it */ 14367c478bd9Sstevel@tonic-gate.CL2: 14377c478bd9Sstevel@tonic-gate cmpb $0, (%rax) /* possible to get lock? */ 14387c478bd9Sstevel@tonic-gate pause 14397c478bd9Sstevel@tonic-gate jne .CL2 14407c478bd9Sstevel@tonic-gate jmp .CL1 /* yes, try again */ 14417c478bd9Sstevel@tonic-gate.CL3: 14427c478bd9Sstevel@tonic-gate /* 14437c478bd9Sstevel@tonic-gate * compute the interval since last time hres_tick was called 14447c478bd9Sstevel@tonic-gate * and adjust hrtime_base and hrestime accordingly 14457c478bd9Sstevel@tonic-gate * hrtime_base is an 8 byte value (in nsec), hrestime is 14467c478bd9Sstevel@tonic-gate * a timestruc_t (sec, nsec) 14477c478bd9Sstevel@tonic-gate */ 14487c478bd9Sstevel@tonic-gate leaq hres_last_tick(%rip), %rax 14497c478bd9Sstevel@tonic-gate movq %r8, %r11 14507c478bd9Sstevel@tonic-gate subq (%rax), %r8 14517c478bd9Sstevel@tonic-gate addq %r8, hrtime_base(%rip) /* add interval to hrtime_base */ 14527c478bd9Sstevel@tonic-gate addq %r8, hrestime+8(%rip) /* add interval to hrestime.tv_nsec */ 14537c478bd9Sstevel@tonic-gate /* 14547c478bd9Sstevel@tonic-gate * Now that we have CLOCK_LOCK, we can update hres_last_tick 145565f20420SRobert Mustacchi */ 145665f20420SRobert Mustacchi movq %r11, (%rax) 14577c478bd9Sstevel@tonic-gate 14587c478bd9Sstevel@tonic-gate call __adj_hrestime 14597c478bd9Sstevel@tonic-gate 14607c478bd9Sstevel@tonic-gate /* 14617c478bd9Sstevel@tonic-gate * release the hres_lock 14627c478bd9Sstevel@tonic-gate */ 14637c478bd9Sstevel@tonic-gate incl hres_lock(%rip) 14647c478bd9Sstevel@tonic-gate leave 14657c478bd9Sstevel@tonic-gate ret 14667c478bd9Sstevel@tonic-gate SET_SIZE(hres_tick) 146765f20420SRobert Mustacchi 14687c478bd9Sstevel@tonic-gate/* 14697c478bd9Sstevel@tonic-gate * void prefetch_smap_w(void *) 14707c478bd9Sstevel@tonic-gate * 14717c478bd9Sstevel@tonic-gate * Prefetch ahead within a linear list of smap structures. 14727c478bd9Sstevel@tonic-gate * Not implemented for ia32. Stub for compatibility. 14737c478bd9Sstevel@tonic-gate */ 14747c478bd9Sstevel@tonic-gate 14757c478bd9Sstevel@tonic-gate ENTRY(prefetch_smap_w) 147685641879Skalai rep; ret /* use 2 byte return instruction when branch target */ 147785641879Skalai /* AMD Software Optimization Guide - Section 6.2 */ 14787c478bd9Sstevel@tonic-gate SET_SIZE(prefetch_smap_w) 14797c478bd9Sstevel@tonic-gate 14807c478bd9Sstevel@tonic-gate/* 14817c478bd9Sstevel@tonic-gate * prefetch_page_r(page_t *) 14827c478bd9Sstevel@tonic-gate * issue prefetch instructions for a page_t 14837c478bd9Sstevel@tonic-gate */ 14847c478bd9Sstevel@tonic-gate 14857c478bd9Sstevel@tonic-gate ENTRY(prefetch_page_r) 148685641879Skalai rep; ret /* use 2 byte return instruction when branch target */ 148785641879Skalai /* AMD Software Optimization Guide - Section 6.2 */ 14887c478bd9Sstevel@tonic-gate SET_SIZE(prefetch_page_r) 14897c478bd9Sstevel@tonic-gate 14907c478bd9Sstevel@tonic-gate ENTRY(bcmp) 14917c478bd9Sstevel@tonic-gate pushq %rbp 14927c478bd9Sstevel@tonic-gate movq %rsp, %rbp 14937c478bd9Sstevel@tonic-gate#ifdef DEBUG 1494549e8c97SMarcel Telka testq %rdx,%rdx 1495549e8c97SMarcel Telka je 1f 1496ae115bc7Smrj movq postbootkernelbase(%rip), %r11 14977c478bd9Sstevel@tonic-gate cmpq %r11, %rdi 14987c478bd9Sstevel@tonic-gate jb 0f 14997c478bd9Sstevel@tonic-gate cmpq %r11, %rsi 15007c478bd9Sstevel@tonic-gate jnb 1f 15017c478bd9Sstevel@tonic-gate0: leaq .bcmp_panic_msg(%rip), %rdi 15027c478bd9Sstevel@tonic-gate xorl %eax, %eax 15037c478bd9Sstevel@tonic-gate call panic 15047c478bd9Sstevel@tonic-gate1: 15057c478bd9Sstevel@tonic-gate#endif /* DEBUG */ 15067c478bd9Sstevel@tonic-gate call memcmp 15077c478bd9Sstevel@tonic-gate testl %eax, %eax 15087c478bd9Sstevel@tonic-gate setne %dl 15097c478bd9Sstevel@tonic-gate leave 15107c478bd9Sstevel@tonic-gate movzbl %dl, %eax 15117c478bd9Sstevel@tonic-gate ret 15127c478bd9Sstevel@tonic-gate SET_SIZE(bcmp) 151365f20420SRobert Mustacchi 15147c478bd9Sstevel@tonic-gate#ifdef DEBUG 15157c478bd9Sstevel@tonic-gate .text 15167c478bd9Sstevel@tonic-gate.bcmp_panic_msg: 15177c478bd9Sstevel@tonic-gate .string "bcmp: arguments below kernelbase" 15187c478bd9Sstevel@tonic-gate#endif /* DEBUG */ 15197c478bd9Sstevel@tonic-gate 1520ae115bc7Smrj ENTRY_NP(bsrw_insn) 1521ae115bc7Smrj xorl %eax, %eax 1522ae115bc7Smrj bsrw %di, %ax 1523ae115bc7Smrj ret 1524ae115bc7Smrj SET_SIZE(bsrw_insn) 1525ae115bc7Smrj 1526ae115bc7Smrj ENTRY_NP(switch_sp_and_call) 1527ae115bc7Smrj pushq %rbp 1528ae115bc7Smrj movq %rsp, %rbp /* set up stack frame */ 1529ae115bc7Smrj movq %rdi, %rsp /* switch stack pointer */ 1530ae115bc7Smrj movq %rdx, %rdi /* pass func arg 1 */ 1531ae115bc7Smrj movq %rsi, %r11 /* save function to call */ 1532ae115bc7Smrj movq %rcx, %rsi /* pass func arg 2 */ 153365f20420SRobert Mustacchi INDIRECT_CALL_REG(r11) /* call function */ 1534ae115bc7Smrj leave /* restore stack */ 1535ae115bc7Smrj ret 1536ae115bc7Smrj SET_SIZE(switch_sp_and_call) 1537ae115bc7Smrj 1538ae115bc7Smrj ENTRY_NP(kmdb_enter) 1539ae115bc7Smrj pushq %rbp 1540ae115bc7Smrj movq %rsp, %rbp 1541ae115bc7Smrj 1542ae115bc7Smrj /* 1543ae115bc7Smrj * Save flags, do a 'cli' then return the saved flags 1544ae115bc7Smrj */ 1545ae115bc7Smrj call intr_clear 1546ae115bc7Smrj 1547ae115bc7Smrj int $T_DBGENTR 1548ae115bc7Smrj 1549ae115bc7Smrj /* 1550ae115bc7Smrj * Restore the saved flags 1551ae115bc7Smrj */ 1552ae115bc7Smrj movq %rax, %rdi 1553ae115bc7Smrj call intr_restore 1554ae115bc7Smrj 1555ae115bc7Smrj leave 155665f20420SRobert Mustacchi ret 1557ae115bc7Smrj SET_SIZE(kmdb_enter) 1558ae115bc7Smrj 1559ae115bc7Smrj ENTRY_NP(return_instr) 1560ae115bc7Smrj rep; ret /* use 2 byte instruction when branch target */ 1561ae115bc7Smrj /* AMD Software Optimization Guide - Section 6.2 */ 1562ae115bc7Smrj SET_SIZE(return_instr) 1563ae115bc7Smrj 1564ae115bc7Smrj ENTRY(getflags) 1565ae115bc7Smrj pushfq 1566ae115bc7Smrj popq %rax 1567843e1988Sjohnlev#if defined(__xpv) 1568843e1988Sjohnlev CURTHREAD(%rdi) 1569843e1988Sjohnlev KPREEMPT_DISABLE(%rdi) 1570843e1988Sjohnlev /* 1571843e1988Sjohnlev * Synthesize the PS_IE bit from the event mask bit 1572843e1988Sjohnlev */ 1573843e1988Sjohnlev CURVCPU(%r11) 1574843e1988Sjohnlev andq $_BITNOT(PS_IE), %rax 1575843e1988Sjohnlev XEN_TEST_UPCALL_MASK(%r11) 1576843e1988Sjohnlev jnz 1f 1577843e1988Sjohnlev orq $PS_IE, %rax 1578843e1988Sjohnlev1: 1579843e1988Sjohnlev KPREEMPT_ENABLE_NOKP(%rdi) 1580843e1988Sjohnlev#endif 1581ae115bc7Smrj ret 1582ae115bc7Smrj SET_SIZE(getflags) 1583ae115bc7Smrj 15844df4bd60Sbs ENTRY(ftrace_interrupt_disable) 15854df4bd60Sbs pushfq 15864df4bd60Sbs popq %rax 15874df4bd60Sbs CLI(%rdx) 15884df4bd60Sbs ret 15894df4bd60Sbs SET_SIZE(ftrace_interrupt_disable) 15904df4bd60Sbs 15914df4bd60Sbs ENTRY(ftrace_interrupt_enable) 15924df4bd60Sbs pushq %rdi 15934df4bd60Sbs popfq 15944df4bd60Sbs ret 15954df4bd60Sbs SET_SIZE(ftrace_interrupt_enable) 15964df4bd60Sbs 159786c1f4dcSVikram Hegde ENTRY(clflush_insn) 159886c1f4dcSVikram Hegde clflush (%rdi) 159986c1f4dcSVikram Hegde ret 160086c1f4dcSVikram Hegde SET_SIZE(clflush_insn) 160186c1f4dcSVikram Hegde 160286c1f4dcSVikram Hegde ENTRY(mfence_insn) 160386c1f4dcSVikram Hegde mfence 160486c1f4dcSVikram Hegde ret 160586c1f4dcSVikram Hegde SET_SIZE(mfence_insn) 160686c1f4dcSVikram Hegde 16077997e108SSurya Prakki/* 160879ec9da8SYuri Pankov * VMware implements an I/O port that programs can query to detect if software 160979ec9da8SYuri Pankov * is running in a VMware hypervisor. This hypervisor port behaves differently 161079ec9da8SYuri Pankov * depending on magic values in certain registers and modifies some registers 161179ec9da8SYuri Pankov * as a side effect. 161279ec9da8SYuri Pankov * 161365f20420SRobert Mustacchi * References: http://kb.vmware.com/kb/1009458 16147997e108SSurya Prakki */ 161579ec9da8SYuri Pankov 161679ec9da8SYuri Pankov ENTRY(vmware_port) 16177997e108SSurya Prakki pushq %rbx 161879ec9da8SYuri Pankov movl $VMWARE_HVMAGIC, %eax 161979ec9da8SYuri Pankov movl $0xffffffff, %ebx 162079ec9da8SYuri Pankov movl %edi, %ecx 162179ec9da8SYuri Pankov movl $VMWARE_HVPORT, %edx 16227997e108SSurya Prakki inl (%dx) 162379ec9da8SYuri Pankov movl %eax, (%rsi) 162479ec9da8SYuri Pankov movl %ebx, 4(%rsi) 162579ec9da8SYuri Pankov movl %ecx, 8(%rsi) 162679ec9da8SYuri Pankov movl %edx, 12(%rsi) 16277997e108SSurya Prakki popq %rbx 16287997e108SSurya Prakki ret 162979ec9da8SYuri Pankov SET_SIZE(vmware_port) 1630