17c478bd9Sstevel@tonic-gate/* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 5ee88d2b9Skchow * Common Development and Distribution License (the "License"). 6ee88d2b9Skchow * You may not use this file except in compliance with the License. 77c478bd9Sstevel@tonic-gate * 87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 117c478bd9Sstevel@tonic-gate * and limitations under the License. 127c478bd9Sstevel@tonic-gate * 137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 187c478bd9Sstevel@tonic-gate * 197c478bd9Sstevel@tonic-gate * CDDL HEADER END 207c478bd9Sstevel@tonic-gate */ 217c478bd9Sstevel@tonic-gate/* 222850d85bSmv * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 237c478bd9Sstevel@tonic-gate * Use is subject to license terms. 247c478bd9Sstevel@tonic-gate */ 257c478bd9Sstevel@tonic-gate 269b0bb795SJohn Levon/* 279b0bb795SJohn Levon * Copyright 2019 Joyent, Inc. 28ee6ee36aSPatrick Mooney * Copyright 2022 Oxide Computer Company 299b0bb795SJohn Levon */ 309b0bb795SJohn Levon 317c478bd9Sstevel@tonic-gate#include "assym.h" 327c478bd9Sstevel@tonic-gate 33575a7426Spt#include <sys/mutex_impl.h> 347c478bd9Sstevel@tonic-gate#include <sys/asm_linkage.h> 357c478bd9Sstevel@tonic-gate#include <sys/asm_misc.h> 367c478bd9Sstevel@tonic-gate#include <sys/regset.h> 377c478bd9Sstevel@tonic-gate#include <sys/rwlock_impl.h> 387c478bd9Sstevel@tonic-gate#include <sys/lockstat.h> 397c478bd9Sstevel@tonic-gate 40ee6ee36aSPatrick Mooney 41ee6ee36aSPatrick Mooney#if defined(OPTERON_ERRATUM_147) 42ee6ee36aSPatrick Mooney 43ee6ee36aSPatrick Mooney/* 44ee6ee36aSPatrick Mooney * Leave space for an lfence to be inserted if required by a CPU which suffers 45ee6ee36aSPatrick Mooney * from this erratum. Pad (with nops) the location for the lfence so that it 46ee6ee36aSPatrick Mooney * is adequately aligned for atomic hotpatching. 47ee6ee36aSPatrick Mooney */ 48ee6ee36aSPatrick Mooney#define ERRATUM147_PATCH_POINT(name) \ 49ee6ee36aSPatrick Mooney .align 4, NOP_INSTR; \ 50*5d9d9091SRichard Lowe.##name##_147_patch_point: \ 51ee6ee36aSPatrick Mooney nop; \ 52ee6ee36aSPatrick Mooney nop; \ 53ee6ee36aSPatrick Mooney nop; \ 54ee6ee36aSPatrick Mooney nop; 55ee6ee36aSPatrick Mooney 56ee6ee36aSPatrick Mooney#else /* defined(OPTERON_ERRATUM_147) */ 57ee6ee36aSPatrick Mooney 58ee6ee36aSPatrick Mooney/* Empty macro so ifdefs are not required for all of the erratum sites. */ 59ee6ee36aSPatrick Mooney#define ERRATUM147_PATCH_POINT(name) 60ee6ee36aSPatrick Mooney 61ee6ee36aSPatrick Mooney#endif /* defined(OPTERON_ERRATUM_147) */ 62ee6ee36aSPatrick Mooney 63ee6ee36aSPatrick Mooney/* 64ee6ee36aSPatrick Mooney * Patch point for lockstat probes. When the associated probe is disabled, it 65ee6ee36aSPatrick Mooney * will 'ret' from the function. It is hotpatched to allow execution to fall 66ee6ee36aSPatrick Mooney * through when the probe is enabled. 67ee6ee36aSPatrick Mooney */ 68ee6ee36aSPatrick Mooney#define LOCKSTAT_RET(name) \ 69*5d9d9091SRichard Lowe.##name##_lockstat_patch_point: \ 70ee6ee36aSPatrick Mooney ret; 71ee6ee36aSPatrick Mooney 727c478bd9Sstevel@tonic-gate/* 737c478bd9Sstevel@tonic-gate * lock_try(lp), ulock_try(lp) 747c478bd9Sstevel@tonic-gate * - returns non-zero on success. 757c478bd9Sstevel@tonic-gate * - doesn't block interrupts so don't use this to spin on a lock. 767c478bd9Sstevel@tonic-gate * 777c478bd9Sstevel@tonic-gate * ulock_try() is for a lock in the user address space. 787c478bd9Sstevel@tonic-gate */ 797c478bd9Sstevel@tonic-gate 807c478bd9Sstevel@tonic-gate .globl kernelbase 817c478bd9Sstevel@tonic-gate 827c478bd9Sstevel@tonic-gate ENTRY(lock_try) 837c478bd9Sstevel@tonic-gate movb $-1, %dl 847c478bd9Sstevel@tonic-gate movzbq %dl, %rax 857c478bd9Sstevel@tonic-gate xchgb %dl, (%rdi) 867c478bd9Sstevel@tonic-gate xorb %dl, %al 87ee6ee36aSPatrick Mooney LOCKSTAT_RET(lock_try) 88ee6ee36aSPatrick Mooney 897c478bd9Sstevel@tonic-gate testb %al, %al 907c478bd9Sstevel@tonic-gate jnz 0f 917c478bd9Sstevel@tonic-gate ret 927c478bd9Sstevel@tonic-gate0: 937c478bd9Sstevel@tonic-gate movq %gs:CPU_THREAD, %rdx /* rdx = thread addr */ 947c478bd9Sstevel@tonic-gate movq %rdi, %rsi /* rsi = lock addr */ 957c478bd9Sstevel@tonic-gate movl $LS_LOCK_TRY_ACQUIRE, %edi /* edi = event */ 967c478bd9Sstevel@tonic-gate jmp lockstat_wrapper 977c478bd9Sstevel@tonic-gate SET_SIZE(lock_try) 987c478bd9Sstevel@tonic-gate 997c478bd9Sstevel@tonic-gate ENTRY(lock_spin_try) 1007c478bd9Sstevel@tonic-gate movb $-1, %dl 1017c478bd9Sstevel@tonic-gate movzbq %dl, %rax 1027c478bd9Sstevel@tonic-gate xchgb %dl, (%rdi) 1037c478bd9Sstevel@tonic-gate xorb %dl, %al 1047c478bd9Sstevel@tonic-gate ret 1057c478bd9Sstevel@tonic-gate SET_SIZE(lock_spin_try) 1067c478bd9Sstevel@tonic-gate 1077c478bd9Sstevel@tonic-gate ENTRY(ulock_try) 1087c478bd9Sstevel@tonic-gate#ifdef DEBUG 1097c478bd9Sstevel@tonic-gate movq kernelbase(%rip), %rax 1107c478bd9Sstevel@tonic-gate cmpq %rax, %rdi /* test uaddr < kernelbase */ 1117c478bd9Sstevel@tonic-gate jb ulock_pass /* uaddr < kernelbase, proceed */ 1127c478bd9Sstevel@tonic-gate 1137c478bd9Sstevel@tonic-gate movq %rdi, %r12 /* preserve lock ptr for debugging */ 1147c478bd9Sstevel@tonic-gate leaq .ulock_panic_msg(%rip), %rdi 115ee6ee36aSPatrick Mooney pushq %rbp 1167c478bd9Sstevel@tonic-gate movq %rsp, %rbp 1177c478bd9Sstevel@tonic-gate xorl %eax, %eax /* clear for varargs */ 1187c478bd9Sstevel@tonic-gate call panic 1197c478bd9Sstevel@tonic-gate 1207c478bd9Sstevel@tonic-gate#endif /* DEBUG */ 1217c478bd9Sstevel@tonic-gate 1227c478bd9Sstevel@tonic-gateulock_pass: 1237c478bd9Sstevel@tonic-gate movl $1, %eax 1247c478bd9Sstevel@tonic-gate xchgb %al, (%rdi) 1257c478bd9Sstevel@tonic-gate xorb $1, %al 1267c478bd9Sstevel@tonic-gate ret 1277c478bd9Sstevel@tonic-gate SET_SIZE(ulock_try) 1287c478bd9Sstevel@tonic-gate 1297c478bd9Sstevel@tonic-gate#ifdef DEBUG 1307c478bd9Sstevel@tonic-gate .data 1317c478bd9Sstevel@tonic-gate.ulock_panic_msg: 1327c478bd9Sstevel@tonic-gate .string "ulock_try: Argument is above kernelbase" 1337c478bd9Sstevel@tonic-gate .text 1347c478bd9Sstevel@tonic-gate#endif /* DEBUG */ 1357c478bd9Sstevel@tonic-gate 1367c478bd9Sstevel@tonic-gate/* 1377c478bd9Sstevel@tonic-gate * lock_clear(lp) 1387c478bd9Sstevel@tonic-gate * - unlock lock without changing interrupt priority level. 1397c478bd9Sstevel@tonic-gate */ 1407c478bd9Sstevel@tonic-gate 1417c478bd9Sstevel@tonic-gate ENTRY(lock_clear) 1427c478bd9Sstevel@tonic-gate movb $0, (%rdi) 143ee6ee36aSPatrick Mooney LOCKSTAT_RET(lock_clear) 144ee6ee36aSPatrick Mooney 1457c478bd9Sstevel@tonic-gate movq %rdi, %rsi /* rsi = lock addr */ 1467c478bd9Sstevel@tonic-gate movq %gs:CPU_THREAD, %rdx /* rdx = thread addr */ 1477c478bd9Sstevel@tonic-gate movl $LS_LOCK_CLEAR_RELEASE, %edi /* edi = event */ 1487c478bd9Sstevel@tonic-gate jmp lockstat_wrapper 1497c478bd9Sstevel@tonic-gate SET_SIZE(lock_clear) 1507c478bd9Sstevel@tonic-gate 1517c478bd9Sstevel@tonic-gate ENTRY(ulock_clear) 1527c478bd9Sstevel@tonic-gate#ifdef DEBUG 1537c478bd9Sstevel@tonic-gate movq kernelbase(%rip), %rcx 1547c478bd9Sstevel@tonic-gate cmpq %rcx, %rdi /* test uaddr < kernelbase */ 1557c478bd9Sstevel@tonic-gate jb ulock_clr /* uaddr < kernelbase, proceed */ 1567c478bd9Sstevel@tonic-gate 1577c478bd9Sstevel@tonic-gate leaq .ulock_clear_msg(%rip), %rdi 158ee6ee36aSPatrick Mooney pushq %rbp 1597c478bd9Sstevel@tonic-gate movq %rsp, %rbp 1607c478bd9Sstevel@tonic-gate xorl %eax, %eax /* clear for varargs */ 1617c478bd9Sstevel@tonic-gate call panic 1627c478bd9Sstevel@tonic-gate#endif 1637c478bd9Sstevel@tonic-gate 1647c478bd9Sstevel@tonic-gateulock_clr: 1657c478bd9Sstevel@tonic-gate movb $0, (%rdi) 1667c478bd9Sstevel@tonic-gate ret 1677c478bd9Sstevel@tonic-gate SET_SIZE(ulock_clear) 1687c478bd9Sstevel@tonic-gate 1697c478bd9Sstevel@tonic-gate#ifdef DEBUG 1707c478bd9Sstevel@tonic-gate .data 1717c478bd9Sstevel@tonic-gate.ulock_clear_msg: 1727c478bd9Sstevel@tonic-gate .string "ulock_clear: Argument is above kernelbase" 1737c478bd9Sstevel@tonic-gate .text 1747c478bd9Sstevel@tonic-gate#endif /* DEBUG */ 1757c478bd9Sstevel@tonic-gate 1767c478bd9Sstevel@tonic-gate 1777c478bd9Sstevel@tonic-gate/* 1787c478bd9Sstevel@tonic-gate * lock_set_spl(lock_t *lp, int new_pil, u_short *old_pil) 1797c478bd9Sstevel@tonic-gate * Drops lp, sets pil to new_pil, stores old pil in *old_pil. 1807c478bd9Sstevel@tonic-gate */ 1817c478bd9Sstevel@tonic-gate 1827c478bd9Sstevel@tonic-gate ENTRY(lock_set_spl) 1837c478bd9Sstevel@tonic-gate pushq %rbp 1847c478bd9Sstevel@tonic-gate movq %rsp, %rbp 1857c478bd9Sstevel@tonic-gate subq $32, %rsp 1867c478bd9Sstevel@tonic-gate movl %esi, 8(%rsp) /* save priority level */ 1877c478bd9Sstevel@tonic-gate movq %rdx, 16(%rsp) /* save old pil ptr */ 1887c478bd9Sstevel@tonic-gate movq %rdi, 24(%rsp) /* save lock pointer */ 1897c478bd9Sstevel@tonic-gate movl %esi, %edi /* pass priority level */ 1907c478bd9Sstevel@tonic-gate call splr /* raise priority level */ 1917c478bd9Sstevel@tonic-gate movq 24(%rsp), %rdi /* rdi = lock addr */ 1927c478bd9Sstevel@tonic-gate movb $-1, %dl 1937c478bd9Sstevel@tonic-gate xchgb %dl, (%rdi) /* try to set lock */ 1947c478bd9Sstevel@tonic-gate testb %dl, %dl /* did we get the lock? ... */ 1957c478bd9Sstevel@tonic-gate jnz .lss_miss /* ... no, go to C for the hard case */ 1967c478bd9Sstevel@tonic-gate movq 16(%rsp), %rdx /* rdx = old pil addr */ 1977c478bd9Sstevel@tonic-gate movw %ax, (%rdx) /* store old pil */ 1987c478bd9Sstevel@tonic-gate leave 199ee6ee36aSPatrick Mooney LOCKSTAT_RET(lock_set_spl) 200ee6ee36aSPatrick Mooney 2017c478bd9Sstevel@tonic-gate movq %rdi, %rsi /* rsi = lock addr */ 2027c478bd9Sstevel@tonic-gate movq %gs:CPU_THREAD, %rdx /* rdx = thread addr */ 2037c478bd9Sstevel@tonic-gate movl $LS_LOCK_SET_SPL_ACQUIRE, %edi 2047c478bd9Sstevel@tonic-gate jmp lockstat_wrapper 205ee6ee36aSPatrick Mooney 2067c478bd9Sstevel@tonic-gate.lss_miss: 2077c478bd9Sstevel@tonic-gate movl 8(%rsp), %esi /* new_pil */ 2087c478bd9Sstevel@tonic-gate movq 16(%rsp), %rdx /* old_pil_addr */ 2097c478bd9Sstevel@tonic-gate movl %eax, %ecx /* original pil */ 2107c478bd9Sstevel@tonic-gate leave /* unwind stack */ 2117c478bd9Sstevel@tonic-gate jmp lock_set_spl_spin 2127c478bd9Sstevel@tonic-gate SET_SIZE(lock_set_spl) 2137c478bd9Sstevel@tonic-gate 2147c478bd9Sstevel@tonic-gate/* 2157c478bd9Sstevel@tonic-gate * void 2167c478bd9Sstevel@tonic-gate * lock_init(lp) 2177c478bd9Sstevel@tonic-gate */ 2187c478bd9Sstevel@tonic-gate 2197c478bd9Sstevel@tonic-gate ENTRY(lock_init) 2207c478bd9Sstevel@tonic-gate movb $0, (%rdi) 2217c478bd9Sstevel@tonic-gate ret 2227c478bd9Sstevel@tonic-gate SET_SIZE(lock_init) 2237c478bd9Sstevel@tonic-gate 2247c478bd9Sstevel@tonic-gate/* 2257c478bd9Sstevel@tonic-gate * void 2267c478bd9Sstevel@tonic-gate * lock_set(lp) 2277c478bd9Sstevel@tonic-gate */ 2287c478bd9Sstevel@tonic-gate 2297c478bd9Sstevel@tonic-gate ENTRY(lock_set) 2307c478bd9Sstevel@tonic-gate movb $-1, %dl 2317c478bd9Sstevel@tonic-gate xchgb %dl, (%rdi) /* try to set lock */ 2327c478bd9Sstevel@tonic-gate testb %dl, %dl /* did we get it? */ 2337c478bd9Sstevel@tonic-gate jnz lock_set_spin /* no, go to C for the hard case */ 234ee6ee36aSPatrick Mooney LOCKSTAT_RET(lock_set) 235ee6ee36aSPatrick Mooney 2367c478bd9Sstevel@tonic-gate movq %rdi, %rsi /* rsi = lock addr */ 2377c478bd9Sstevel@tonic-gate movq %gs:CPU_THREAD, %rdx /* rdx = thread addr */ 2387c478bd9Sstevel@tonic-gate movl $LS_LOCK_SET_ACQUIRE, %edi 2397c478bd9Sstevel@tonic-gate jmp lockstat_wrapper 2407c478bd9Sstevel@tonic-gate SET_SIZE(lock_set) 2417c478bd9Sstevel@tonic-gate 2427c478bd9Sstevel@tonic-gate/* 2437c478bd9Sstevel@tonic-gate * lock_clear_splx(lp, s) 2447c478bd9Sstevel@tonic-gate */ 2457c478bd9Sstevel@tonic-gate 2467c478bd9Sstevel@tonic-gate ENTRY(lock_clear_splx) 247ee6ee36aSPatrick Mooney pushq %rbp 2487c478bd9Sstevel@tonic-gate movq %rsp, %rbp 249ee6ee36aSPatrick Mooney pushq %rdi /* save lp across call for lockstat */ 250ee6ee36aSPatrick Mooney movb $0, (%rdi) /* clear lock */ 251ee6ee36aSPatrick Mooney movl %esi, %edi /* arg for splx */ 252ee6ee36aSPatrick Mooney call splx /* let splx do its thing */ 253ee6ee36aSPatrick Mooney popq %rsi /* retreive lp for lockstat */ 254ee6ee36aSPatrick Mooney leave 255ee6ee36aSPatrick Mooney LOCKSTAT_RET(lock_clear_splx) 256ee6ee36aSPatrick Mooney 2577c478bd9Sstevel@tonic-gate movq %gs:CPU_THREAD, %rdx /* rdx = thread addr */ 2587c478bd9Sstevel@tonic-gate movl $LS_LOCK_CLEAR_SPLX_RELEASE, %edi 2597c478bd9Sstevel@tonic-gate jmp lockstat_wrapper 2607c478bd9Sstevel@tonic-gate SET_SIZE(lock_clear_splx) 2617c478bd9Sstevel@tonic-gate 2627c478bd9Sstevel@tonic-gate/* 2637c478bd9Sstevel@tonic-gate * mutex_enter() and mutex_exit(). 2647c478bd9Sstevel@tonic-gate * 2657c478bd9Sstevel@tonic-gate * These routines handle the simple cases of mutex_enter() (adaptive 2667c478bd9Sstevel@tonic-gate * lock, not held) and mutex_exit() (adaptive lock, held, no waiters). 2677c478bd9Sstevel@tonic-gate * If anything complicated is going on we punt to mutex_vector_enter(). 2687c478bd9Sstevel@tonic-gate * 2697c478bd9Sstevel@tonic-gate * mutex_tryenter() is similar to mutex_enter() but returns zero if 2707c478bd9Sstevel@tonic-gate * the lock cannot be acquired, nonzero on success. 2717c478bd9Sstevel@tonic-gate * 2727c478bd9Sstevel@tonic-gate * If mutex_exit() gets preempted in the window between checking waiters 2737c478bd9Sstevel@tonic-gate * and clearing the lock, we can miss wakeups. Disabling preemption 2747c478bd9Sstevel@tonic-gate * in the mutex code is prohibitively expensive, so instead we detect 2757c478bd9Sstevel@tonic-gate * mutex preemption by examining the trapped PC in the interrupt path. 2767c478bd9Sstevel@tonic-gate * If we interrupt a thread in mutex_exit() that has not yet cleared 2777c478bd9Sstevel@tonic-gate * the lock, cmnint() resets its PC back to the beginning of 2787c478bd9Sstevel@tonic-gate * mutex_exit() so it will check again for waiters when it resumes. 2797c478bd9Sstevel@tonic-gate */ 2807c478bd9Sstevel@tonic-gate 2817c478bd9Sstevel@tonic-gate ENTRY_NP(mutex_enter) 2827c478bd9Sstevel@tonic-gate movq %gs:CPU_THREAD, %rdx /* rdx = thread ptr */ 2837c478bd9Sstevel@tonic-gate xorl %eax, %eax /* rax = 0 (unheld adaptive) */ 2847c478bd9Sstevel@tonic-gate lock 2857c478bd9Sstevel@tonic-gate cmpxchgq %rdx, (%rdi) 2867c478bd9Sstevel@tonic-gate jnz mutex_vector_enter 287ee6ee36aSPatrick Mooney 288ee6ee36aSPatrick Mooney ERRATUM147_PATCH_POINT(mutex_enter) 289ee6ee36aSPatrick Mooney 290ee6ee36aSPatrick Mooney LOCKSTAT_RET(mutex_enter) 291ee6ee36aSPatrick Mooney 2927c478bd9Sstevel@tonic-gate movq %rdi, %rsi 2937c478bd9Sstevel@tonic-gate movl $LS_MUTEX_ENTER_ACQUIRE, %edi 294ee6ee36aSPatrick Mooney jmp lockstat_wrapper 295ee6ee36aSPatrick Mooney SET_SIZE(mutex_enter) 296ee6ee36aSPatrick Mooney 297ee6ee36aSPatrick Mooney 2987c478bd9Sstevel@tonic-gate/* 2997c478bd9Sstevel@tonic-gate * expects %rdx=thread, %rsi=lock, %edi=lockstat event 3007c478bd9Sstevel@tonic-gate */ 301ee6ee36aSPatrick Mooney ENTRY_NP(lockstat_wrapper) 3027c478bd9Sstevel@tonic-gate incb T_LOCKSTAT(%rdx) /* curthread->t_lockstat++ */ 3037c478bd9Sstevel@tonic-gate leaq lockstat_probemap(%rip), %rax 3047c478bd9Sstevel@tonic-gate movl (%rax, %rdi, DTRACE_IDSIZE), %eax 3057c478bd9Sstevel@tonic-gate testl %eax, %eax /* check for non-zero probe */ 3067c478bd9Sstevel@tonic-gate jz 1f 307ee6ee36aSPatrick Mooney pushq %rbp 3087c478bd9Sstevel@tonic-gate movq %rsp, %rbp 3097c478bd9Sstevel@tonic-gate movl %eax, %edi 31065f20420SRobert Mustacchi movq lockstat_probe, %rax 31165f20420SRobert Mustacchi INDIRECT_CALL_REG(rax) 3127c478bd9Sstevel@tonic-gate leave /* unwind stack */ 3137c478bd9Sstevel@tonic-gate1: 3147c478bd9Sstevel@tonic-gate movq %gs:CPU_THREAD, %rdx /* reload thread ptr */ 3157c478bd9Sstevel@tonic-gate decb T_LOCKSTAT(%rdx) /* curthread->t_lockstat-- */ 3167c478bd9Sstevel@tonic-gate movl $1, %eax /* return success if tryenter */ 3177c478bd9Sstevel@tonic-gate ret 3187c478bd9Sstevel@tonic-gate SET_SIZE(lockstat_wrapper) 3197c478bd9Sstevel@tonic-gate 3207c478bd9Sstevel@tonic-gate/* 3217c478bd9Sstevel@tonic-gate * expects %rcx=thread, %rdx=arg, %rsi=lock, %edi=lockstat event 3227c478bd9Sstevel@tonic-gate */ 3237c478bd9Sstevel@tonic-gate ENTRY(lockstat_wrapper_arg) 3247c478bd9Sstevel@tonic-gate incb T_LOCKSTAT(%rcx) /* curthread->t_lockstat++ */ 3257c478bd9Sstevel@tonic-gate leaq lockstat_probemap(%rip), %rax 3267c478bd9Sstevel@tonic-gate movl (%rax, %rdi, DTRACE_IDSIZE), %eax 3277c478bd9Sstevel@tonic-gate testl %eax, %eax /* check for non-zero probe */ 3287c478bd9Sstevel@tonic-gate jz 1f 329ee6ee36aSPatrick Mooney pushq %rbp 3307c478bd9Sstevel@tonic-gate movq %rsp, %rbp 3317c478bd9Sstevel@tonic-gate movl %eax, %edi 33265f20420SRobert Mustacchi movq lockstat_probe, %rax 33365f20420SRobert Mustacchi INDIRECT_CALL_REG(rax) 3347c478bd9Sstevel@tonic-gate leave /* unwind stack */ 3357c478bd9Sstevel@tonic-gate1: 3367c478bd9Sstevel@tonic-gate movq %gs:CPU_THREAD, %rdx /* reload thread ptr */ 3377c478bd9Sstevel@tonic-gate decb T_LOCKSTAT(%rdx) /* curthread->t_lockstat-- */ 3387c478bd9Sstevel@tonic-gate movl $1, %eax /* return success if tryenter */ 3397c478bd9Sstevel@tonic-gate ret 3407c478bd9Sstevel@tonic-gate SET_SIZE(lockstat_wrapper_arg) 3417c478bd9Sstevel@tonic-gate 3427c478bd9Sstevel@tonic-gate 3437c478bd9Sstevel@tonic-gate ENTRY(mutex_tryenter) 3447c478bd9Sstevel@tonic-gate movq %gs:CPU_THREAD, %rdx /* rdx = thread ptr */ 3457c478bd9Sstevel@tonic-gate xorl %eax, %eax /* rax = 0 (unheld adaptive) */ 3467c478bd9Sstevel@tonic-gate lock 3477c478bd9Sstevel@tonic-gate cmpxchgq %rdx, (%rdi) 3487c478bd9Sstevel@tonic-gate jnz mutex_vector_tryenter 3497c478bd9Sstevel@tonic-gate not %eax /* return success (nonzero) */ 350ee6ee36aSPatrick Mooney 351ee6ee36aSPatrick Mooney ERRATUM147_PATCH_POINT(mutex_tryenter) 352ee6ee36aSPatrick Mooney 353ee6ee36aSPatrick Mooney LOCKSTAT_RET(mutex_tryenter) 354ee6ee36aSPatrick Mooney 3557c478bd9Sstevel@tonic-gate movq %rdi, %rsi 356ee6ee36aSPatrick Mooney movl $LS_MUTEX_TRYENTER_ACQUIRE, %edi 3577c478bd9Sstevel@tonic-gate jmp lockstat_wrapper 3587c478bd9Sstevel@tonic-gate SET_SIZE(mutex_tryenter) 3597c478bd9Sstevel@tonic-gate 3607c478bd9Sstevel@tonic-gate ENTRY(mutex_adaptive_tryenter) 3617c478bd9Sstevel@tonic-gate movq %gs:CPU_THREAD, %rdx /* rdx = thread ptr */ 3627c478bd9Sstevel@tonic-gate xorl %eax, %eax /* rax = 0 (unheld adaptive) */ 3637c478bd9Sstevel@tonic-gate lock 3647c478bd9Sstevel@tonic-gate cmpxchgq %rdx, (%rdi) 3657c478bd9Sstevel@tonic-gate jnz 0f 3667c478bd9Sstevel@tonic-gate not %eax /* return success (nonzero) */ 367ee6ee36aSPatrick Mooney 368ee6ee36aSPatrick Mooney ERRATUM147_PATCH_POINT(mutex_atryenter) 369ee6ee36aSPatrick Mooney 3707c478bd9Sstevel@tonic-gate ret 3717c478bd9Sstevel@tonic-gate0: 3727c478bd9Sstevel@tonic-gate xorl %eax, %eax /* return failure */ 3737c478bd9Sstevel@tonic-gate ret 3747c478bd9Sstevel@tonic-gate SET_SIZE(mutex_adaptive_tryenter) 3757c478bd9Sstevel@tonic-gate 376575a7426Spt .globl mutex_owner_running_critical_start 377575a7426Spt 378575a7426Spt ENTRY(mutex_owner_running) 379575a7426Sptmutex_owner_running_critical_start: 380575a7426Spt movq (%rdi), %r11 /* get owner field */ 381575a7426Spt andq $MUTEX_THREAD, %r11 /* remove waiters bit */ 382575a7426Spt cmpq $0, %r11 /* if free, skip */ 383575a7426Spt je 1f /* go return 0 */ 384575a7426Spt movq T_CPU(%r11), %r8 /* get owner->t_cpu */ 385575a7426Spt movq CPU_THREAD(%r8), %r9 /* get t_cpu->cpu_thread */ 386575a7426Spt.mutex_owner_running_critical_end: 387575a7426Spt cmpq %r11, %r9 /* owner == running thread? */ 388575a7426Spt je 2f /* yes, go return cpu */ 389575a7426Spt1: 390575a7426Spt xorq %rax, %rax /* return 0 */ 391575a7426Spt ret 392575a7426Spt2: 393575a7426Spt movq %r8, %rax /* return cpu */ 394575a7426Spt ret 395575a7426Spt SET_SIZE(mutex_owner_running) 396575a7426Spt 397575a7426Spt .globl mutex_owner_running_critical_size 398575a7426Spt .type mutex_owner_running_critical_size, @object 399575a7426Spt .align CPTRSIZE 400575a7426Sptmutex_owner_running_critical_size: 401575a7426Spt .quad .mutex_owner_running_critical_end - mutex_owner_running_critical_start 402575a7426Spt SET_SIZE(mutex_owner_running_critical_size) 403575a7426Spt 404ee88d2b9Skchow .globl mutex_exit_critical_start 4057c478bd9Sstevel@tonic-gate 4067c478bd9Sstevel@tonic-gate ENTRY(mutex_exit) 4077c478bd9Sstevel@tonic-gatemutex_exit_critical_start: /* If interrupted, restart here */ 4087c478bd9Sstevel@tonic-gate movq %gs:CPU_THREAD, %rdx 4097c478bd9Sstevel@tonic-gate cmpq %rdx, (%rdi) 4107c478bd9Sstevel@tonic-gate jne mutex_vector_exit /* wrong type or wrong owner */ 4117c478bd9Sstevel@tonic-gate movq $0, (%rdi) /* clear owner AND lock */ 4127c478bd9Sstevel@tonic-gate.mutex_exit_critical_end: 413ee6ee36aSPatrick Mooney LOCKSTAT_RET(mutex_exit) 414ee6ee36aSPatrick Mooney 4157c478bd9Sstevel@tonic-gate movq %rdi, %rsi 4167c478bd9Sstevel@tonic-gate movl $LS_MUTEX_EXIT_RELEASE, %edi 4177c478bd9Sstevel@tonic-gate jmp lockstat_wrapper 4187c478bd9Sstevel@tonic-gate SET_SIZE(mutex_exit) 4197c478bd9Sstevel@tonic-gate 4207c478bd9Sstevel@tonic-gate .globl mutex_exit_critical_size 4217c478bd9Sstevel@tonic-gate .type mutex_exit_critical_size, @object 4227c478bd9Sstevel@tonic-gate .align CPTRSIZE 4237c478bd9Sstevel@tonic-gatemutex_exit_critical_size: 4247c478bd9Sstevel@tonic-gate .quad .mutex_exit_critical_end - mutex_exit_critical_start 4257c478bd9Sstevel@tonic-gate SET_SIZE(mutex_exit_critical_size) 4267c478bd9Sstevel@tonic-gate 4277c478bd9Sstevel@tonic-gate/* 4287c478bd9Sstevel@tonic-gate * rw_enter() and rw_exit(). 4297c478bd9Sstevel@tonic-gate * 4307c478bd9Sstevel@tonic-gate * These routines handle the simple cases of rw_enter (write-locking an unheld 4317c478bd9Sstevel@tonic-gate * lock or read-locking a lock that's neither write-locked nor write-wanted) 4327c478bd9Sstevel@tonic-gate * and rw_exit (no waiters or not the last reader). If anything complicated 4337c478bd9Sstevel@tonic-gate * is going on we punt to rw_enter_sleep() and rw_exit_wakeup(), respectively. 4347c478bd9Sstevel@tonic-gate */ 4357c478bd9Sstevel@tonic-gate 4367c478bd9Sstevel@tonic-gate ENTRY(rw_enter) 4377c478bd9Sstevel@tonic-gate cmpl $RW_WRITER, %esi 4387c478bd9Sstevel@tonic-gate je .rw_write_enter 4397c478bd9Sstevel@tonic-gate movq (%rdi), %rax /* rax = old rw_wwwh value */ 4407c478bd9Sstevel@tonic-gate testl $RW_WRITE_LOCKED|RW_WRITE_WANTED, %eax 4417c478bd9Sstevel@tonic-gate jnz rw_enter_sleep 4427c478bd9Sstevel@tonic-gate leaq RW_READ_LOCK(%rax), %rdx /* rdx = new rw_wwwh value */ 4437c478bd9Sstevel@tonic-gate lock 4447c478bd9Sstevel@tonic-gate cmpxchgq %rdx, (%rdi) /* try to grab read lock */ 4457c478bd9Sstevel@tonic-gate jnz rw_enter_sleep 446ee6ee36aSPatrick Mooney LOCKSTAT_RET(rw_read_enter) 447ee6ee36aSPatrick Mooney 4487c478bd9Sstevel@tonic-gate movq %gs:CPU_THREAD, %rcx /* rcx = thread ptr */ 4497c478bd9Sstevel@tonic-gate movq %rdi, %rsi /* rsi = lock ptr */ 4507c478bd9Sstevel@tonic-gate movl $LS_RW_ENTER_ACQUIRE, %edi 4517c478bd9Sstevel@tonic-gate movl $RW_READER, %edx 4527c478bd9Sstevel@tonic-gate jmp lockstat_wrapper_arg 453ee6ee36aSPatrick Mooney 4547c478bd9Sstevel@tonic-gate.rw_write_enter: 4556a0b1217SPatrick Mooney movq %gs:CPU_THREAD, %rdx 4567c478bd9Sstevel@tonic-gate orq $RW_WRITE_LOCKED, %rdx /* rdx = write-locked value */ 4577c478bd9Sstevel@tonic-gate xorl %eax, %eax /* rax = unheld value */ 4587c478bd9Sstevel@tonic-gate lock 4597c478bd9Sstevel@tonic-gate cmpxchgq %rdx, (%rdi) /* try to grab write lock */ 4607c478bd9Sstevel@tonic-gate jnz rw_enter_sleep 461ee88d2b9Skchow 462ee6ee36aSPatrick Mooney ERRATUM147_PATCH_POINT(rw_write_enter) 463ee6ee36aSPatrick Mooney 464ee6ee36aSPatrick Mooney LOCKSTAT_RET(rw_write_enter) 465ee88d2b9Skchow 4667c478bd9Sstevel@tonic-gate movq %gs:CPU_THREAD, %rcx /* rcx = thread ptr */ 4677c478bd9Sstevel@tonic-gate movq %rdi, %rsi /* rsi = lock ptr */ 4687c478bd9Sstevel@tonic-gate movl $LS_RW_ENTER_ACQUIRE, %edi 4697c478bd9Sstevel@tonic-gate movl $RW_WRITER, %edx 4707c478bd9Sstevel@tonic-gate jmp lockstat_wrapper_arg 4717c478bd9Sstevel@tonic-gate SET_SIZE(rw_enter) 4727c478bd9Sstevel@tonic-gate 4737c478bd9Sstevel@tonic-gate ENTRY(rw_exit) 4747c478bd9Sstevel@tonic-gate movq (%rdi), %rax /* rax = old rw_wwwh value */ 4757c478bd9Sstevel@tonic-gate cmpl $RW_READ_LOCK, %eax /* single-reader, no waiters? */ 4767c478bd9Sstevel@tonic-gate jne .rw_not_single_reader 4777c478bd9Sstevel@tonic-gate xorl %edx, %edx /* rdx = new value (unheld) */ 4787c478bd9Sstevel@tonic-gate.rw_read_exit: 4797c478bd9Sstevel@tonic-gate lock 4807c478bd9Sstevel@tonic-gate cmpxchgq %rdx, (%rdi) /* try to drop read lock */ 4817c478bd9Sstevel@tonic-gate jnz rw_exit_wakeup 482ee6ee36aSPatrick Mooney LOCKSTAT_RET(rw_read_exit) 483ee6ee36aSPatrick Mooney 484a24f6116SJohn Levon movq %gs:CPU_THREAD, %rcx /* rcx = thread ptr */ 4857c478bd9Sstevel@tonic-gate movq %rdi, %rsi /* rsi = lock ptr */ 4867c478bd9Sstevel@tonic-gate movl $LS_RW_EXIT_RELEASE, %edi 4877c478bd9Sstevel@tonic-gate movl $RW_READER, %edx 4887c478bd9Sstevel@tonic-gate jmp lockstat_wrapper_arg 489ee6ee36aSPatrick Mooney 4907c478bd9Sstevel@tonic-gate.rw_not_single_reader: 4917c478bd9Sstevel@tonic-gate testl $RW_WRITE_LOCKED, %eax /* write-locked or write-wanted? */ 4927c478bd9Sstevel@tonic-gate jnz .rw_write_exit 4937c478bd9Sstevel@tonic-gate leaq -RW_READ_LOCK(%rax), %rdx /* rdx = new value */ 4947c478bd9Sstevel@tonic-gate cmpl $RW_READ_LOCK, %edx 4957c478bd9Sstevel@tonic-gate jge .rw_read_exit /* not last reader, safe to drop */ 4967c478bd9Sstevel@tonic-gate jmp rw_exit_wakeup /* last reader with waiters */ 4977c478bd9Sstevel@tonic-gate.rw_write_exit: 4987c478bd9Sstevel@tonic-gate movq %gs:CPU_THREAD, %rax /* rax = thread ptr */ 4997c478bd9Sstevel@tonic-gate xorl %edx, %edx /* rdx = new value (unheld) */ 5007c478bd9Sstevel@tonic-gate orq $RW_WRITE_LOCKED, %rax /* eax = write-locked value */ 5017c478bd9Sstevel@tonic-gate lock 5027c478bd9Sstevel@tonic-gate cmpxchgq %rdx, (%rdi) /* try to drop read lock */ 5037c478bd9Sstevel@tonic-gate jnz rw_exit_wakeup 504ee6ee36aSPatrick Mooney LOCKSTAT_RET(rw_write_exit) 505ee6ee36aSPatrick Mooney 5067c478bd9Sstevel@tonic-gate movq %gs:CPU_THREAD, %rcx /* rcx = thread ptr */ 5077c478bd9Sstevel@tonic-gate movq %rdi, %rsi /* rsi - lock ptr */ 5087c478bd9Sstevel@tonic-gate movl $LS_RW_EXIT_RELEASE, %edi 5097c478bd9Sstevel@tonic-gate movl $RW_WRITER, %edx 5107c478bd9Sstevel@tonic-gate jmp lockstat_wrapper_arg 5117c478bd9Sstevel@tonic-gate SET_SIZE(rw_exit) 5127c478bd9Sstevel@tonic-gate 513ee6ee36aSPatrick Mooney#if defined(OPTERON_ERRATUM_147) 514ee88d2b9Skchow 515ee88d2b9Skchow/* 516ee6ee36aSPatrick Mooney * Track if erratum 147 workaround has been hotpatched into place. 517ee88d2b9Skchow */ 518ee6ee36aSPatrick Mooney DGDEF3(erratum_147_patched, 4, 4) 519ee88d2b9Skchow .long 0 520ee88d2b9Skchow 521ee6ee36aSPatrick Mooney#define HOT_MUTEX_PATCH(iaddr, insn_reg) \ 522ee6ee36aSPatrick Mooney movq $iaddr, %rdi; \ 523ee6ee36aSPatrick Mooney movl %insn_reg, %esi; \ 524ee6ee36aSPatrick Mooney movl $4, %edx; \ 525ee6ee36aSPatrick Mooney call hot_patch_kernel_text; 526ee6ee36aSPatrick Mooney 527ee88d2b9Skchow 528ee88d2b9Skchow/* 529ee6ee36aSPatrick Mooney * void 530ee6ee36aSPatrick Mooney * patch_erratum_147(void) 531ee6ee36aSPatrick Mooney * 532ee6ee36aSPatrick Mooney * Patch lock operations to work around erratum 147. 533ee88d2b9Skchow * 534ee88d2b9Skchow * The workaround is to place a fencing instruction (lfence) between the 535ee88d2b9Skchow * mutex operation and the subsequent read-modify-write instruction. 536ee88d2b9Skchow */ 537ee6ee36aSPatrick Mooney 538ee6ee36aSPatrick Mooney ENTRY_NP(patch_erratum_147) 539ee88d2b9Skchow pushq %rbp 540ee88d2b9Skchow movq %rsp, %rbp 541ee88d2b9Skchow pushq %r12 542ee88d2b9Skchow 543ee88d2b9Skchow /* 544ee6ee36aSPatrick Mooney * Patch `nop; nop; nop; nop` sequence to `lfence; nop`. Since those 545ee6ee36aSPatrick Mooney * patch points have been aligned to a 4-byte boundary, we can be 546ee6ee36aSPatrick Mooney * confident that hot_patch_kernel_text() will be able to proceed 547ee6ee36aSPatrick Mooney * safely and successfully. 548ee88d2b9Skchow */ 549ee6ee36aSPatrick Mooney movl $0x90e8ae0f, %r12d 550ee6ee36aSPatrick Mooney HOT_MUTEX_PATCH(.mutex_enter_147_patch_point, r12d) 551ee6ee36aSPatrick Mooney HOT_MUTEX_PATCH(.mutex_tryenter_147_patch_point, r12d) 552ee6ee36aSPatrick Mooney HOT_MUTEX_PATCH(.mutex_atryenter_147_patch_point, r12d) 553ee6ee36aSPatrick Mooney HOT_MUTEX_PATCH(.rw_write_enter_147_patch_point, r12d) 554ee88d2b9Skchow 555ee6ee36aSPatrick Mooney /* Record that erratum 147 points have been hotpatched */ 556ee6ee36aSPatrick Mooney movl $1, erratum_147_patched 557ee88d2b9Skchow 558ee88d2b9Skchow popq %r12 559ee88d2b9Skchow movq %rbp, %rsp 560ee88d2b9Skchow popq %rbp 561ee88d2b9Skchow ret 562ee6ee36aSPatrick Mooney SET_SIZE(patch_erratum_147) 563ee6ee36aSPatrick Mooney 564ee6ee36aSPatrick Mooney#endif /* OPTERON_ERRATUM_147 */ 565ee88d2b9Skchow 566ee6ee36aSPatrick Mooney /* 567ee6ee36aSPatrick Mooney * void 568ee6ee36aSPatrick Mooney * lockstat_hotpatch_site(caddr_t instr_addr, int do_enable) 569ee6ee36aSPatrick Mooney */ 570ee6ee36aSPatrick Mooney ENTRY(lockstat_hotpatch_site) 571ee6ee36aSPatrick Mooney pushq %rbp 572ee6ee36aSPatrick Mooney movq %rsp, %rbp 573ee6ee36aSPatrick Mooney pushq %rdi 574ee6ee36aSPatrick Mooney pushq %rsi 575ee88d2b9Skchow 576ee6ee36aSPatrick Mooney testl %esi, %esi 577ee6ee36aSPatrick Mooney jz .do_disable 578ee88d2b9Skchow 579ee6ee36aSPatrick Mooney /* enable the probe (replace ret with nop) */ 580ee6ee36aSPatrick Mooney movl $NOP_INSTR, %esi 581ee6ee36aSPatrick Mooney movl $1, %edx 582ee6ee36aSPatrick Mooney call hot_patch_kernel_text 583ee6ee36aSPatrick Mooney leave 584ee6ee36aSPatrick Mooney ret 585ee88d2b9Skchow 586ee6ee36aSPatrick Mooney.do_disable: 587ee6ee36aSPatrick Mooney /* disable the probe (replace nop with ret) */ 588ee6ee36aSPatrick Mooney movl $RET_INSTR, %esi 589ee6ee36aSPatrick Mooney movl $1, %edx 5907c478bd9Sstevel@tonic-gate call hot_patch_kernel_text 591ee6ee36aSPatrick Mooney leave 592ee6ee36aSPatrick Mooney ret 593ee6ee36aSPatrick Mooney SET_SIZE(lockstat_hotpatch_site) 594ee6ee36aSPatrick Mooney 595ee6ee36aSPatrick Mooney#define HOT_PATCH_MATCH(name, probe, reg) \ 596ee6ee36aSPatrick Mooney cmpl $probe, %reg; \ 597ee6ee36aSPatrick Mooney jne 1f; \ 598ee6ee36aSPatrick Mooney leaq lockstat_probemap(%rip), %rax; \ 599ee6ee36aSPatrick Mooney movl _MUL(probe, DTRACE_IDSIZE)(%rax), %esi; \ 600*5d9d9091SRichard Lowe movq $.##name##_lockstat_patch_point, %rdi; \ 601ee6ee36aSPatrick Mooney call lockstat_hotpatch_site; \ 602ee6ee36aSPatrick Mooney 1: 603ee6ee36aSPatrick Mooney 604ee6ee36aSPatrick Mooney/* 605ee6ee36aSPatrick Mooney * void 606ee6ee36aSPatrick Mooney * lockstat_hotpatch_probe(int ls_probe) 607ee6ee36aSPatrick Mooney * 608ee6ee36aSPatrick Mooney * Given a lockstat probe identifier, hotpatch any associated lockstat 609ee6ee36aSPatrick Mooney * primitive routine(s) so they fall through into the lockstat_probe() call (if 610ee6ee36aSPatrick Mooney * the probe is enabled) or return normally (when the probe is disabled). 611ee6ee36aSPatrick Mooney */ 6127c478bd9Sstevel@tonic-gate 613ee6ee36aSPatrick Mooney ENTRY(lockstat_hotpatch_probe) 614ee6ee36aSPatrick Mooney pushq %rbp 6157c478bd9Sstevel@tonic-gate movq %rsp, %rbp 616ee6ee36aSPatrick Mooney pushq %r12 617ee6ee36aSPatrick Mooney movl %edi, %r12d 618ee88d2b9Skchow 619ee6ee36aSPatrick Mooney HOT_PATCH_MATCH(mutex_enter, LS_MUTEX_ENTER_ACQUIRE, r12d) 620ee6ee36aSPatrick Mooney HOT_PATCH_MATCH(mutex_tryenter, LS_MUTEX_TRYENTER_ACQUIRE, r12d) 621ee6ee36aSPatrick Mooney HOT_PATCH_MATCH(mutex_exit, LS_MUTEX_EXIT_RELEASE, r12d) 622ee6ee36aSPatrick Mooney 623ee6ee36aSPatrick Mooney HOT_PATCH_MATCH(rw_write_enter, LS_RW_ENTER_ACQUIRE, r12d) 624ee6ee36aSPatrick Mooney HOT_PATCH_MATCH(rw_read_enter, LS_RW_ENTER_ACQUIRE, r12d) 625ee6ee36aSPatrick Mooney HOT_PATCH_MATCH(rw_write_exit, LS_RW_EXIT_RELEASE, r12d) 626ee6ee36aSPatrick Mooney HOT_PATCH_MATCH(rw_read_exit, LS_RW_EXIT_RELEASE, r12d) 627ee6ee36aSPatrick Mooney 628ee6ee36aSPatrick Mooney HOT_PATCH_MATCH(lock_set, LS_LOCK_SET_ACQUIRE, r12d) 629ee6ee36aSPatrick Mooney HOT_PATCH_MATCH(lock_try, LS_LOCK_TRY_ACQUIRE, r12d) 630ee6ee36aSPatrick Mooney HOT_PATCH_MATCH(lock_clear, LS_LOCK_CLEAR_RELEASE, r12d) 631ee6ee36aSPatrick Mooney HOT_PATCH_MATCH(lock_set_spl, LS_LOCK_SET_SPL_ACQUIRE, r12d) 632ee6ee36aSPatrick Mooney HOT_PATCH_MATCH(lock_clear_splx, LS_LOCK_CLEAR_SPLX_RELEASE, r12d) 633ee6ee36aSPatrick Mooney 634ee6ee36aSPatrick Mooney popq %r12 635ee6ee36aSPatrick Mooney leave 6367c478bd9Sstevel@tonic-gate ret 637ee6ee36aSPatrick Mooney SET_SIZE(lockstat_hotpatch_probe) 6387c478bd9Sstevel@tonic-gate 6397c478bd9Sstevel@tonic-gate ENTRY(membar_enter) 6407c478bd9Sstevel@tonic-gate ALTENTRY(membar_exit) 6412850d85bSmv ALTENTRY(membar_sync) 6427c478bd9Sstevel@tonic-gate mfence /* lighter weight than lock; xorq $0,(%rsp) */ 6437c478bd9Sstevel@tonic-gate ret 6442850d85bSmv SET_SIZE(membar_sync) 6457c478bd9Sstevel@tonic-gate SET_SIZE(membar_exit) 6467c478bd9Sstevel@tonic-gate SET_SIZE(membar_enter) 6477c478bd9Sstevel@tonic-gate 6487c478bd9Sstevel@tonic-gate ENTRY(membar_producer) 6497c478bd9Sstevel@tonic-gate sfence 6507c478bd9Sstevel@tonic-gate ret 6517c478bd9Sstevel@tonic-gate SET_SIZE(membar_producer) 6527c478bd9Sstevel@tonic-gate 6537c478bd9Sstevel@tonic-gate ENTRY(membar_consumer) 6547c478bd9Sstevel@tonic-gate lfence 6557c478bd9Sstevel@tonic-gate ret 6567c478bd9Sstevel@tonic-gate SET_SIZE(membar_consumer) 6577c478bd9Sstevel@tonic-gate 6587c478bd9Sstevel@tonic-gate/* 6597c478bd9Sstevel@tonic-gate * thread_onproc() 6607c478bd9Sstevel@tonic-gate * Set thread in onproc state for the specified CPU. 6617c478bd9Sstevel@tonic-gate * Also set the thread lock pointer to the CPU's onproc lock. 6627c478bd9Sstevel@tonic-gate * Since the new lock isn't held, the store ordering is important. 6637c478bd9Sstevel@tonic-gate * If not done in assembler, the compiler could reorder the stores. 6647c478bd9Sstevel@tonic-gate */ 6657c478bd9Sstevel@tonic-gate 6667c478bd9Sstevel@tonic-gate ENTRY(thread_onproc) 6677c478bd9Sstevel@tonic-gate addq $CPU_THREAD_LOCK, %rsi /* pointer to disp_lock while running */ 6687c478bd9Sstevel@tonic-gate movl $ONPROC_THREAD, T_STATE(%rdi) /* set state to TS_ONPROC */ 6697c478bd9Sstevel@tonic-gate movq %rsi, T_LOCKP(%rdi) /* store new lock pointer */ 6707c478bd9Sstevel@tonic-gate ret 6717c478bd9Sstevel@tonic-gate SET_SIZE(thread_onproc) 6727c478bd9Sstevel@tonic-gate 673575a7426Spt/* 674575a7426Spt * mutex_delay_default(void) 675575a7426Spt * Spins for approx a few hundred processor cycles and returns to caller. 676575a7426Spt */ 677575a7426Spt 678575a7426Spt ENTRY(mutex_delay_default) 679575a7426Spt movq $92,%r11 680575a7426Spt0: decq %r11 681575a7426Spt jg 0b 682575a7426Spt ret 683575a7426Spt SET_SIZE(mutex_delay_default) 684