1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22/*
23 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright 2014 Nexenta Systems, Inc.  All rights reserved.
25 * Copyright (c) 2014 by Delphix. All rights reserved.
26 * Copyright 2019 Joyent, Inc.
27 */
28
29/*
30 *  Copyright (c) 1990, 1991 UNIX System Laboratories, Inc.
31 *  Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T
32 *    All Rights Reserved
33 */
34
35/*
36 * Copyright (c) 2009, Intel Corporation.
37 * All rights reserved.
38 */
39
40/*
41 * General assembly language routines.
42 * It is the intent of this file to contain routines that are
43 * independent of the specific kernel architecture, and those that are
44 * common across kernel architectures.
45 * As architectures diverge, and implementations of specific
46 * architecture-dependent routines change, the routines should be moved
47 * from this file into the respective ../`arch -k`/subr.s file.
48 */
49
50#include <sys/asm_linkage.h>
51#include <sys/asm_misc.h>
52#include <sys/panic.h>
53#include <sys/ontrap.h>
54#include <sys/regset.h>
55#include <sys/privregs.h>
56#include <sys/reboot.h>
57#include <sys/psw.h>
58#include <sys/x86_archext.h>
59
60#if defined(__lint)
61#include <sys/types.h>
62#include <sys/systm.h>
63#include <sys/thread.h>
64#include <sys/archsystm.h>
65#include <sys/byteorder.h>
66#include <sys/dtrace.h>
67#include <sys/ftrace.h>
68#else	/* __lint */
69#include "assym.h"
70#endif	/* __lint */
71#include <sys/dditypes.h>
72
73/*
74 * on_fault()
75 *
76 * Catch lofault faults. Like setjmp except it returns one
77 * if code following causes uncorrectable fault. Turned off
78 * by calling no_fault(). Note that while under on_fault(),
79 * SMAP is disabled. For more information see
80 * uts/intel/ia32/ml/copy.s.
81 */
82
83#if defined(__lint)
84
85/* ARGSUSED */
86int
87on_fault(label_t *ljb)
88{ return (0); }
89
90void
91no_fault(void)
92{}
93
94#else	/* __lint */
95
96#if defined(__amd64)
97
98	ENTRY(on_fault)
99	movq	%gs:CPU_THREAD, %rsi
100	leaq	catch_fault(%rip), %rdx
101	movq	%rdi, T_ONFAULT(%rsi)		/* jumpbuf in t_onfault */
102	movq	%rdx, T_LOFAULT(%rsi)		/* catch_fault in t_lofault */
103	call	smap_disable			/* allow user accesses */
104	jmp	setjmp				/* let setjmp do the rest */
105
106catch_fault:
107	movq	%gs:CPU_THREAD, %rsi
108	movq	T_ONFAULT(%rsi), %rdi		/* address of save area */
109	xorl	%eax, %eax
110	movq	%rax, T_ONFAULT(%rsi)		/* turn off onfault */
111	movq	%rax, T_LOFAULT(%rsi)		/* turn off lofault */
112	call	smap_enable			/* disallow user accesses */
113	jmp	longjmp				/* let longjmp do the rest */
114	SET_SIZE(on_fault)
115
116	ENTRY(no_fault)
117	movq	%gs:CPU_THREAD, %rsi
118	xorl	%eax, %eax
119	movq	%rax, T_ONFAULT(%rsi)		/* turn off onfault */
120	movq	%rax, T_LOFAULT(%rsi)		/* turn off lofault */
121	call	smap_enable			/* disallow user accesses */
122	ret
123	SET_SIZE(no_fault)
124
125#elif defined(__i386)
126
127	ENTRY(on_fault)
128	movl	%gs:CPU_THREAD, %edx
129	movl	4(%esp), %eax			/* jumpbuf address */
130	leal	catch_fault, %ecx
131	movl	%eax, T_ONFAULT(%edx)		/* jumpbuf in t_onfault */
132	movl	%ecx, T_LOFAULT(%edx)		/* catch_fault in t_lofault */
133	jmp	setjmp				/* let setjmp do the rest */
134
135catch_fault:
136	movl	%gs:CPU_THREAD, %edx
137	xorl	%eax, %eax
138	movl	T_ONFAULT(%edx), %ecx		/* address of save area */
139	movl	%eax, T_ONFAULT(%edx)		/* turn off onfault */
140	movl	%eax, T_LOFAULT(%edx)		/* turn off lofault */
141	pushl	%ecx
142	call	longjmp				/* let longjmp do the rest */
143	SET_SIZE(on_fault)
144
145	ENTRY(no_fault)
146	movl	%gs:CPU_THREAD, %edx
147	xorl	%eax, %eax
148	movl	%eax, T_ONFAULT(%edx)		/* turn off onfault */
149	movl	%eax, T_LOFAULT(%edx)		/* turn off lofault */
150	ret
151	SET_SIZE(no_fault)
152
153#endif	/* __i386 */
154#endif	/* __lint */
155
156/*
157 * Default trampoline code for on_trap() (see <sys/ontrap.h>).  We just
158 * do a longjmp(&curthread->t_ontrap->ot_jmpbuf) if this is ever called.
159 */
160
161#if defined(lint)
162
163void
164on_trap_trampoline(void)
165{}
166
167#else	/* __lint */
168
169#if defined(__amd64)
170
171	ENTRY(on_trap_trampoline)
172	movq	%gs:CPU_THREAD, %rsi
173	movq	T_ONTRAP(%rsi), %rdi
174	addq	$OT_JMPBUF, %rdi
175	jmp	longjmp
176	SET_SIZE(on_trap_trampoline)
177
178#elif defined(__i386)
179
180	ENTRY(on_trap_trampoline)
181	movl	%gs:CPU_THREAD, %eax
182	movl	T_ONTRAP(%eax), %eax
183	addl	$OT_JMPBUF, %eax
184	pushl	%eax
185	call	longjmp
186	SET_SIZE(on_trap_trampoline)
187
188#endif	/* __i386 */
189#endif	/* __lint */
190
191/*
192 * Push a new element on to the t_ontrap stack.  Refer to <sys/ontrap.h> for
193 * more information about the on_trap() mechanism.  If the on_trap_data is the
194 * same as the topmost stack element, we just modify that element.
195 */
196#if defined(lint)
197
198/*ARGSUSED*/
199int
200on_trap(on_trap_data_t *otp, uint_t prot)
201{ return (0); }
202
203#else	/* __lint */
204
205#if defined(__amd64)
206
207	ENTRY(on_trap)
208	movw	%si, OT_PROT(%rdi)		/* ot_prot = prot */
209	movw	$0, OT_TRAP(%rdi)		/* ot_trap = 0 */
210	leaq	on_trap_trampoline(%rip), %rdx	/* rdx = &on_trap_trampoline */
211	movq	%rdx, OT_TRAMPOLINE(%rdi)	/* ot_trampoline = rdx */
212	xorl	%ecx, %ecx
213	movq	%rcx, OT_HANDLE(%rdi)		/* ot_handle = NULL */
214	movq	%rcx, OT_PAD1(%rdi)		/* ot_pad1 = NULL */
215	movq	%gs:CPU_THREAD, %rdx		/* rdx = curthread */
216	movq	T_ONTRAP(%rdx), %rcx		/* rcx = curthread->t_ontrap */
217	cmpq	%rdi, %rcx			/* if (otp == %rcx)	*/
218	je	0f				/*	don't modify t_ontrap */
219
220	movq	%rcx, OT_PREV(%rdi)		/* ot_prev = t_ontrap */
221	movq	%rdi, T_ONTRAP(%rdx)		/* curthread->t_ontrap = otp */
222
2230:	addq	$OT_JMPBUF, %rdi		/* &ot_jmpbuf */
224	jmp	setjmp
225	SET_SIZE(on_trap)
226
227#elif defined(__i386)
228
229	ENTRY(on_trap)
230	movl	4(%esp), %eax			/* %eax = otp */
231	movl	8(%esp), %edx			/* %edx = prot */
232
233	movw	%dx, OT_PROT(%eax)		/* ot_prot = prot */
234	movw	$0, OT_TRAP(%eax)		/* ot_trap = 0 */
235	leal	on_trap_trampoline, %edx	/* %edx = &on_trap_trampoline */
236	movl	%edx, OT_TRAMPOLINE(%eax)	/* ot_trampoline = %edx */
237	movl	$0, OT_HANDLE(%eax)		/* ot_handle = NULL */
238	movl	$0, OT_PAD1(%eax)		/* ot_pad1 = NULL */
239	movl	%gs:CPU_THREAD, %edx		/* %edx = curthread */
240	movl	T_ONTRAP(%edx), %ecx		/* %ecx = curthread->t_ontrap */
241	cmpl	%eax, %ecx			/* if (otp == %ecx) */
242	je	0f				/*    don't modify t_ontrap */
243
244	movl	%ecx, OT_PREV(%eax)		/* ot_prev = t_ontrap */
245	movl	%eax, T_ONTRAP(%edx)		/* curthread->t_ontrap = otp */
246
2470:	addl	$OT_JMPBUF, %eax		/* %eax = &ot_jmpbuf */
248	movl	%eax, 4(%esp)			/* put %eax back on the stack */
249	jmp	setjmp				/* let setjmp do the rest */
250	SET_SIZE(on_trap)
251
252#endif	/* __i386 */
253#endif	/* __lint */
254
255/*
256 * Setjmp and longjmp implement non-local gotos using state vectors
257 * type label_t.
258 */
259
260#if defined(__lint)
261
262/* ARGSUSED */
263int
264setjmp(label_t *lp)
265{ return (0); }
266
267/* ARGSUSED */
268void
269longjmp(label_t *lp)
270{}
271
272#else	/* __lint */
273
274#if LABEL_PC != 0
275#error LABEL_PC MUST be defined as 0 for setjmp/longjmp to work as coded
276#endif	/* LABEL_PC != 0 */
277
278#if defined(__amd64)
279
280	ENTRY(setjmp)
281	movq	%rsp, LABEL_SP(%rdi)
282	movq	%rbp, LABEL_RBP(%rdi)
283	movq	%rbx, LABEL_RBX(%rdi)
284	movq	%r12, LABEL_R12(%rdi)
285	movq	%r13, LABEL_R13(%rdi)
286	movq	%r14, LABEL_R14(%rdi)
287	movq	%r15, LABEL_R15(%rdi)
288	movq	(%rsp), %rdx		/* return address */
289	movq	%rdx, (%rdi)		/* LABEL_PC is 0 */
290	xorl	%eax, %eax		/* return 0 */
291	ret
292	SET_SIZE(setjmp)
293
294	ENTRY(longjmp)
295	movq	LABEL_SP(%rdi), %rsp
296	movq	LABEL_RBP(%rdi), %rbp
297	movq	LABEL_RBX(%rdi), %rbx
298	movq	LABEL_R12(%rdi), %r12
299	movq	LABEL_R13(%rdi), %r13
300	movq	LABEL_R14(%rdi), %r14
301	movq	LABEL_R15(%rdi), %r15
302	movq	(%rdi), %rdx		/* return address; LABEL_PC is 0 */
303	movq	%rdx, (%rsp)
304	xorl	%eax, %eax
305	incl	%eax			/* return 1 */
306	ret
307	SET_SIZE(longjmp)
308
309#elif defined(__i386)
310
311	ENTRY(setjmp)
312	movl	4(%esp), %edx		/* address of save area */
313	movl	%ebp, LABEL_EBP(%edx)
314	movl	%ebx, LABEL_EBX(%edx)
315	movl	%esi, LABEL_ESI(%edx)
316	movl	%edi, LABEL_EDI(%edx)
317	movl	%esp, 4(%edx)
318	movl	(%esp), %ecx		/* %eip (return address) */
319	movl	%ecx, (%edx)		/* LABEL_PC is 0 */
320	subl	%eax, %eax		/* return 0 */
321	ret
322	SET_SIZE(setjmp)
323
324	ENTRY(longjmp)
325	movl	4(%esp), %edx		/* address of save area */
326	movl	LABEL_EBP(%edx), %ebp
327	movl	LABEL_EBX(%edx), %ebx
328	movl	LABEL_ESI(%edx), %esi
329	movl	LABEL_EDI(%edx), %edi
330	movl	4(%edx), %esp
331	movl	(%edx), %ecx		/* %eip (return addr); LABEL_PC is 0 */
332	movl	$1, %eax
333	addl	$4, %esp		/* pop ret adr */
334	jmp	*%ecx			/* indirect */
335	SET_SIZE(longjmp)
336
337#endif	/* __i386 */
338#endif	/* __lint */
339
340/*
341 * if a() calls b() calls caller(),
342 * caller() returns return address in a().
343 * (Note: We assume a() and b() are C routines which do the normal entry/exit
344 *  sequence.)
345 */
346
347#if defined(__lint)
348
349caddr_t
350caller(void)
351{ return (0); }
352
353#else	/* __lint */
354
355#if defined(__amd64)
356
357	ENTRY(caller)
358	movq	8(%rbp), %rax		/* b()'s return pc, in a() */
359	ret
360	SET_SIZE(caller)
361
362#elif defined(__i386)
363
364	ENTRY(caller)
365	movl	4(%ebp), %eax		/* b()'s return pc, in a() */
366	ret
367	SET_SIZE(caller)
368
369#endif	/* __i386 */
370#endif	/* __lint */
371
372/*
373 * if a() calls callee(), callee() returns the
374 * return address in a();
375 */
376
377#if defined(__lint)
378
379caddr_t
380callee(void)
381{ return (0); }
382
383#else	/* __lint */
384
385#if defined(__amd64)
386
387	ENTRY(callee)
388	movq	(%rsp), %rax		/* callee()'s return pc, in a() */
389	ret
390	SET_SIZE(callee)
391
392#elif defined(__i386)
393
394	ENTRY(callee)
395	movl	(%esp), %eax		/* callee()'s return pc, in a() */
396	ret
397	SET_SIZE(callee)
398
399#endif	/* __i386 */
400#endif	/* __lint */
401
402/*
403 * return the current frame pointer
404 */
405
406#if defined(__lint)
407
408greg_t
409getfp(void)
410{ return (0); }
411
412#else	/* __lint */
413
414#if defined(__amd64)
415
416	ENTRY(getfp)
417	movq	%rbp, %rax
418	ret
419	SET_SIZE(getfp)
420
421#elif defined(__i386)
422
423	ENTRY(getfp)
424	movl	%ebp, %eax
425	ret
426	SET_SIZE(getfp)
427
428#endif	/* __i386 */
429#endif	/* __lint */
430
431/*
432 * Invalidate a single page table entry in the TLB
433 */
434
435#if defined(__lint)
436
437/* ARGSUSED */
438void
439mmu_invlpg(caddr_t m)
440{}
441
442#else	/* __lint */
443
444	ENTRY(mmu_invlpg)
445	invlpg	(%rdi)
446	ret
447	SET_SIZE(mmu_invlpg)
448
449#endif	/* __lint */
450
451
452/*
453 * Get/Set the value of various control registers
454 */
455
456#if defined(__lint)
457
458ulong_t
459getcr0(void)
460{ return (0); }
461
462/* ARGSUSED */
463void
464setcr0(ulong_t value)
465{}
466
467ulong_t
468getcr2(void)
469{ return (0); }
470
471ulong_t
472getcr3(void)
473{ return (0); }
474
475#if !defined(__xpv)
476/* ARGSUSED */
477void
478setcr3(ulong_t val)
479{}
480
481void
482reload_cr3(void)
483{}
484#endif
485
486ulong_t
487getcr4(void)
488{ return (0); }
489
490/* ARGSUSED */
491void
492setcr4(ulong_t val)
493{}
494
495#if defined(__amd64)
496
497ulong_t
498getcr8(void)
499{ return (0); }
500
501/* ARGSUSED */
502void
503setcr8(ulong_t val)
504{}
505
506#endif	/* __amd64 */
507
508#else	/* __lint */
509
510#if defined(__amd64)
511
512	ENTRY(getcr0)
513	movq	%cr0, %rax
514	ret
515	SET_SIZE(getcr0)
516
517	ENTRY(setcr0)
518	movq	%rdi, %cr0
519	ret
520	SET_SIZE(setcr0)
521
522        ENTRY(getcr2)
523#if defined(__xpv)
524	movq	%gs:CPU_VCPU_INFO, %rax
525	movq	VCPU_INFO_ARCH_CR2(%rax), %rax
526#else
527        movq    %cr2, %rax
528#endif
529        ret
530	SET_SIZE(getcr2)
531
532	ENTRY(getcr3)
533	movq    %cr3, %rax
534	ret
535	SET_SIZE(getcr3)
536
537#if !defined(__xpv)
538
539        ENTRY(setcr3)
540        movq    %rdi, %cr3
541        ret
542	SET_SIZE(setcr3)
543
544	ENTRY(reload_cr3)
545	movq	%cr3, %rdi
546	movq	%rdi, %cr3
547	ret
548	SET_SIZE(reload_cr3)
549
550#endif	/* __xpv */
551
552	ENTRY(getcr4)
553	movq	%cr4, %rax
554	ret
555	SET_SIZE(getcr4)
556
557	ENTRY(setcr4)
558	movq	%rdi, %cr4
559	ret
560	SET_SIZE(setcr4)
561
562	ENTRY(getcr8)
563	movq	%cr8, %rax
564	ret
565	SET_SIZE(getcr8)
566
567	ENTRY(setcr8)
568	movq	%rdi, %cr8
569	ret
570	SET_SIZE(setcr8)
571
572#elif defined(__i386)
573
574        ENTRY(getcr0)
575        movl    %cr0, %eax
576        ret
577	SET_SIZE(getcr0)
578
579        ENTRY(setcr0)
580        movl    4(%esp), %eax
581        movl    %eax, %cr0
582        ret
583	SET_SIZE(setcr0)
584
585	/*
586	 * "lock mov %cr0" is used on processors which indicate it is
587	 * supported via CPUID. Normally the 32 bit TPR is accessed via
588	 * the local APIC.
589	 */
590	ENTRY(getcr8)
591	lock
592	movl	%cr0, %eax
593	ret
594	SET_SIZE(getcr8)
595
596	ENTRY(setcr8)
597        movl    4(%esp), %eax
598	lock
599        movl    %eax, %cr0
600	ret
601	SET_SIZE(setcr8)
602
603        ENTRY(getcr2)
604#if defined(__xpv)
605	movl	%gs:CPU_VCPU_INFO, %eax
606	movl	VCPU_INFO_ARCH_CR2(%eax), %eax
607#else
608        movl    %cr2, %eax
609#endif
610        ret
611	SET_SIZE(getcr2)
612
613	ENTRY(getcr3)
614	movl    %cr3, %eax
615	ret
616	SET_SIZE(getcr3)
617
618#if !defined(__xpv)
619
620        ENTRY(setcr3)
621        movl    4(%esp), %eax
622        movl    %eax, %cr3
623        ret
624	SET_SIZE(setcr3)
625
626	ENTRY(reload_cr3)
627	movl    %cr3, %eax
628	movl    %eax, %cr3
629	ret
630	SET_SIZE(reload_cr3)
631
632#endif	/* __xpv */
633
634	ENTRY(getcr4)
635	movl    %cr4, %eax
636	ret
637	SET_SIZE(getcr4)
638
639        ENTRY(setcr4)
640        movl    4(%esp), %eax
641        movl    %eax, %cr4
642        ret
643	SET_SIZE(setcr4)
644
645#endif	/* __i386 */
646#endif	/* __lint */
647
648#if defined(__lint)
649
650/*ARGSUSED*/
651uint32_t
652__cpuid_insn(struct cpuid_regs *regs)
653{ return (0); }
654
655#else	/* __lint */
656
657#if defined(__amd64)
658
659	ENTRY(__cpuid_insn)
660	movq	%rbx, %r8
661	movq	%rcx, %r9
662	movq	%rdx, %r11
663	movl	(%rdi), %eax		/* %eax = regs->cp_eax */
664	movl	0x4(%rdi), %ebx		/* %ebx = regs->cp_ebx */
665	movl	0x8(%rdi), %ecx		/* %ecx = regs->cp_ecx */
666	movl	0xc(%rdi), %edx		/* %edx = regs->cp_edx */
667	cpuid
668	movl	%eax, (%rdi)		/* regs->cp_eax = %eax */
669	movl	%ebx, 0x4(%rdi)		/* regs->cp_ebx = %ebx */
670	movl	%ecx, 0x8(%rdi)		/* regs->cp_ecx = %ecx */
671	movl	%edx, 0xc(%rdi)		/* regs->cp_edx = %edx */
672	movq	%r8, %rbx
673	movq	%r9, %rcx
674	movq	%r11, %rdx
675	ret
676	SET_SIZE(__cpuid_insn)
677
678#elif defined(__i386)
679
680        ENTRY(__cpuid_insn)
681	pushl	%ebp
682	movl	0x8(%esp), %ebp		/* %ebp = regs */
683	pushl	%ebx
684	pushl	%ecx
685	pushl	%edx
686	movl	(%ebp), %eax		/* %eax = regs->cp_eax */
687	movl	0x4(%ebp), %ebx		/* %ebx = regs->cp_ebx */
688	movl	0x8(%ebp), %ecx		/* %ecx = regs->cp_ecx */
689	movl	0xc(%ebp), %edx		/* %edx = regs->cp_edx */
690	cpuid
691	movl	%eax, (%ebp)		/* regs->cp_eax = %eax */
692	movl	%ebx, 0x4(%ebp)		/* regs->cp_ebx = %ebx */
693	movl	%ecx, 0x8(%ebp)		/* regs->cp_ecx = %ecx */
694	movl	%edx, 0xc(%ebp)		/* regs->cp_edx = %edx */
695	popl	%edx
696	popl	%ecx
697	popl	%ebx
698	popl	%ebp
699	ret
700	SET_SIZE(__cpuid_insn)
701
702#endif	/* __i386 */
703#endif	/* __lint */
704
705#if defined(__lint)
706
707/*ARGSUSED*/
708void
709i86_monitor(volatile uint32_t *addr, uint32_t extensions, uint32_t hints)
710{}
711
712#else   /* __lint */
713
714#if defined(__amd64)
715
716	ENTRY_NP(i86_monitor)
717	pushq	%rbp
718	movq	%rsp, %rbp
719	movq	%rdi, %rax		/* addr */
720	movq	%rsi, %rcx		/* extensions */
721	/* rdx contains input arg3: hints */
722	clflush	(%rax)
723	.byte	0x0f, 0x01, 0xc8	/* monitor */
724	leave
725	ret
726	SET_SIZE(i86_monitor)
727
728#elif defined(__i386)
729
730ENTRY_NP(i86_monitor)
731	pushl	%ebp
732	movl	%esp, %ebp
733	movl	0x8(%ebp),%eax		/* addr */
734	movl	0xc(%ebp),%ecx		/* extensions */
735	movl	0x10(%ebp),%edx		/* hints */
736	clflush	(%eax)
737	.byte	0x0f, 0x01, 0xc8	/* monitor */
738	leave
739	ret
740	SET_SIZE(i86_monitor)
741
742#endif	/* __i386 */
743#endif	/* __lint */
744
745#if defined(__lint)
746
747/*ARGSUSED*/
748void
749i86_mwait(uint32_t data, uint32_t extensions)
750{}
751
752#else	/* __lint */
753
754#if defined(__amd64)
755
756	ENTRY_NP(i86_mwait)
757	pushq	%rbp
758	call	x86_md_clear
759	movq	%rsp, %rbp
760	movq	%rdi, %rax		/* data */
761	movq	%rsi, %rcx		/* extensions */
762	.byte	0x0f, 0x01, 0xc9	/* mwait */
763	leave
764	ret
765	SET_SIZE(i86_mwait)
766
767#elif defined(__i386)
768
769	ENTRY_NP(i86_mwait)
770	pushl	%ebp
771	movl	%esp, %ebp
772	movl	0x8(%ebp),%eax		/* data */
773	movl	0xc(%ebp),%ecx		/* extensions */
774	.byte	0x0f, 0x01, 0xc9	/* mwait */
775	leave
776	ret
777	SET_SIZE(i86_mwait)
778
779#endif	/* __i386 */
780#endif	/* __lint */
781
782#if defined(__xpv)
783	/*
784	 * Defined in C
785	 */
786#else
787
788#if defined(__lint)
789
790hrtime_t
791tsc_read(void)
792{
793	return (0);
794}
795
796#else	/* __lint */
797
798#if defined(__amd64)
799
800	ENTRY_NP(tsc_read)
801	movq	%rbx, %r11
802	movl	$0, %eax
803	cpuid
804	rdtsc
805	movq	%r11, %rbx
806	shlq	$32, %rdx
807	orq	%rdx, %rax
808	ret
809	.globl _tsc_mfence_start
810_tsc_mfence_start:
811	mfence
812	rdtsc
813	shlq	$32, %rdx
814	orq	%rdx, %rax
815	ret
816	.globl _tsc_mfence_end
817_tsc_mfence_end:
818	.globl _tscp_start
819_tscp_start:
820	.byte	0x0f, 0x01, 0xf9	/* rdtscp instruction */
821	shlq	$32, %rdx
822	orq	%rdx, %rax
823	ret
824	.globl _tscp_end
825_tscp_end:
826	.globl _no_rdtsc_start
827_no_rdtsc_start:
828	xorl	%edx, %edx
829	xorl	%eax, %eax
830	ret
831	.globl _no_rdtsc_end
832_no_rdtsc_end:
833	.globl _tsc_lfence_start
834_tsc_lfence_start:
835	lfence
836	rdtsc
837	shlq	$32, %rdx
838	orq	%rdx, %rax
839	ret
840	.globl _tsc_lfence_end
841_tsc_lfence_end:
842	SET_SIZE(tsc_read)
843
844#else /* __i386 */
845
846	ENTRY_NP(tsc_read)
847	pushl	%ebx
848	movl	$0, %eax
849	cpuid
850	rdtsc
851	popl	%ebx
852	ret
853	.globl _tsc_mfence_start
854_tsc_mfence_start:
855	mfence
856	rdtsc
857	ret
858	.globl _tsc_mfence_end
859_tsc_mfence_end:
860	.globl	_tscp_start
861_tscp_start:
862	.byte	0x0f, 0x01, 0xf9	/* rdtscp instruction */
863	ret
864	.globl _tscp_end
865_tscp_end:
866	.globl _no_rdtsc_start
867_no_rdtsc_start:
868	xorl	%edx, %edx
869	xorl	%eax, %eax
870	ret
871	.globl _no_rdtsc_end
872_no_rdtsc_end:
873	.globl _tsc_lfence_start
874_tsc_lfence_start:
875	lfence
876	rdtsc
877	ret
878	.globl _tsc_lfence_end
879_tsc_lfence_end:
880	SET_SIZE(tsc_read)
881
882#endif	/* __i386 */
883
884#endif	/* __lint */
885
886
887#endif	/* __xpv */
888
889#ifdef __lint
890/*
891 * Do not use this function for obtaining clock tick.  This
892 * is called by callers who do not need to have a guarenteed
893 * correct tick value.  The proper routine to use is tsc_read().
894 */
895u_longlong_t
896randtick(void)
897{
898	return (0);
899}
900#else
901#if defined(__amd64)
902	ENTRY_NP(randtick)
903	rdtsc
904	shlq    $32, %rdx
905	orq     %rdx, %rax
906	ret
907	SET_SIZE(randtick)
908#else
909	ENTRY_NP(randtick)
910	rdtsc
911	ret
912	SET_SIZE(randtick)
913#endif /* __i386 */
914#endif /* __lint */
915/*
916 * Insert entryp after predp in a doubly linked list.
917 */
918
919#if defined(__lint)
920
921/*ARGSUSED*/
922void
923_insque(caddr_t entryp, caddr_t predp)
924{}
925
926#else	/* __lint */
927
928#if defined(__amd64)
929
930	ENTRY(_insque)
931	movq	(%rsi), %rax		/* predp->forw			*/
932	movq	%rsi, CPTRSIZE(%rdi)	/* entryp->back = predp		*/
933	movq	%rax, (%rdi)		/* entryp->forw = predp->forw	*/
934	movq	%rdi, (%rsi)		/* predp->forw = entryp		*/
935	movq	%rdi, CPTRSIZE(%rax)	/* predp->forw->back = entryp	*/
936	ret
937	SET_SIZE(_insque)
938
939#elif defined(__i386)
940
941	ENTRY(_insque)
942	movl	8(%esp), %edx
943	movl	4(%esp), %ecx
944	movl	(%edx), %eax		/* predp->forw			*/
945	movl	%edx, CPTRSIZE(%ecx)	/* entryp->back = predp		*/
946	movl	%eax, (%ecx)		/* entryp->forw = predp->forw	*/
947	movl	%ecx, (%edx)		/* predp->forw = entryp		*/
948	movl	%ecx, CPTRSIZE(%eax)	/* predp->forw->back = entryp	*/
949	ret
950	SET_SIZE(_insque)
951
952#endif	/* __i386 */
953#endif	/* __lint */
954
955/*
956 * Remove entryp from a doubly linked list
957 */
958
959#if defined(__lint)
960
961/*ARGSUSED*/
962void
963_remque(caddr_t entryp)
964{}
965
966#else	/* __lint */
967
968#if defined(__amd64)
969
970	ENTRY(_remque)
971	movq	(%rdi), %rax		/* entry->forw */
972	movq	CPTRSIZE(%rdi), %rdx	/* entry->back */
973	movq	%rax, (%rdx)		/* entry->back->forw = entry->forw */
974	movq	%rdx, CPTRSIZE(%rax)	/* entry->forw->back = entry->back */
975	ret
976	SET_SIZE(_remque)
977
978#elif defined(__i386)
979
980	ENTRY(_remque)
981	movl	4(%esp), %ecx
982	movl	(%ecx), %eax		/* entry->forw */
983	movl	CPTRSIZE(%ecx), %edx	/* entry->back */
984	movl	%eax, (%edx)		/* entry->back->forw = entry->forw */
985	movl	%edx, CPTRSIZE(%eax)	/* entry->forw->back = entry->back */
986	ret
987	SET_SIZE(_remque)
988
989#endif	/* __i386 */
990#endif	/* __lint */
991
992/*
993 * Returns the number of
994 * non-NULL bytes in string argument.
995 */
996
997#if defined(__lint)
998
999/* ARGSUSED */
1000size_t
1001strlen(const char *str)
1002{ return (0); }
1003
1004#else	/* __lint */
1005
1006#if defined(__amd64)
1007
1008/*
1009 * This is close to a simple transliteration of a C version of this
1010 * routine.  We should either just -make- this be a C version, or
1011 * justify having it in assembler by making it significantly faster.
1012 *
1013 * size_t
1014 * strlen(const char *s)
1015 * {
1016 *	const char *s0;
1017 * #if defined(DEBUG)
1018 *	if ((uintptr_t)s < KERNELBASE)
1019 *		panic(.str_panic_msg);
1020 * #endif
1021 *	for (s0 = s; *s; s++)
1022 *		;
1023 *	return (s - s0);
1024 * }
1025 */
1026
1027	ENTRY(strlen)
1028#ifdef DEBUG
1029	movq	postbootkernelbase(%rip), %rax
1030	cmpq	%rax, %rdi
1031	jae	str_valid
1032	pushq	%rbp
1033	movq	%rsp, %rbp
1034	leaq	.str_panic_msg(%rip), %rdi
1035	xorl	%eax, %eax
1036	call	panic
1037#endif	/* DEBUG */
1038str_valid:
1039	cmpb	$0, (%rdi)
1040	movq	%rdi, %rax
1041	je	.null_found
1042	.align	4
1043.strlen_loop:
1044	incq	%rdi
1045	cmpb	$0, (%rdi)
1046	jne	.strlen_loop
1047.null_found:
1048	subq	%rax, %rdi
1049	movq	%rdi, %rax
1050	ret
1051	SET_SIZE(strlen)
1052
1053#elif defined(__i386)
1054
1055	ENTRY(strlen)
1056#ifdef DEBUG
1057	movl	postbootkernelbase, %eax
1058	cmpl	%eax, 4(%esp)
1059	jae	str_valid
1060	pushl	%ebp
1061	movl	%esp, %ebp
1062	pushl	$.str_panic_msg
1063	call	panic
1064#endif /* DEBUG */
1065
1066str_valid:
1067	movl	4(%esp), %eax		/* %eax = string address */
1068	testl	$3, %eax		/* if %eax not word aligned */
1069	jnz	.not_word_aligned	/* goto .not_word_aligned */
1070	.align	4
1071.word_aligned:
1072	movl	(%eax), %edx		/* move 1 word from (%eax) to %edx */
1073	movl	$0x7f7f7f7f, %ecx
1074	andl	%edx, %ecx		/* %ecx = %edx & 0x7f7f7f7f */
1075	addl	$4, %eax		/* next word */
1076	addl	$0x7f7f7f7f, %ecx	/* %ecx += 0x7f7f7f7f */
1077	orl	%edx, %ecx		/* %ecx |= %edx */
1078	andl	$0x80808080, %ecx	/* %ecx &= 0x80808080 */
1079	cmpl	$0x80808080, %ecx	/* if no null byte in this word */
1080	je	.word_aligned		/* goto .word_aligned */
1081	subl	$4, %eax		/* post-incremented */
1082.not_word_aligned:
1083	cmpb	$0, (%eax)		/* if a byte in (%eax) is null */
1084	je	.null_found		/* goto .null_found */
1085	incl	%eax			/* next byte */
1086	testl	$3, %eax		/* if %eax not word aligned */
1087	jnz	.not_word_aligned	/* goto .not_word_aligned */
1088	jmp	.word_aligned		/* goto .word_aligned */
1089	.align	4
1090.null_found:
1091	subl	4(%esp), %eax		/* %eax -= string address */
1092	ret
1093	SET_SIZE(strlen)
1094
1095#endif	/* __i386 */
1096
1097#ifdef DEBUG
1098	.text
1099.str_panic_msg:
1100	.string "strlen: argument below kernelbase"
1101#endif /* DEBUG */
1102
1103#endif	/* __lint */
1104
1105	/*
1106	 * Berkeley 4.3 introduced symbolically named interrupt levels
1107	 * as a way deal with priority in a machine independent fashion.
1108	 * Numbered priorities are machine specific, and should be
1109	 * discouraged where possible.
1110	 *
1111	 * Note, for the machine specific priorities there are
1112	 * examples listed for devices that use a particular priority.
1113	 * It should not be construed that all devices of that
1114	 * type should be at that priority.  It is currently were
1115	 * the current devices fit into the priority scheme based
1116	 * upon time criticalness.
1117	 *
1118	 * The underlying assumption of these assignments is that
1119	 * IPL 10 is the highest level from which a device
1120	 * routine can call wakeup.  Devices that interrupt from higher
1121	 * levels are restricted in what they can do.  If they need
1122	 * kernels services they should schedule a routine at a lower
1123	 * level (via software interrupt) to do the required
1124	 * processing.
1125	 *
1126	 * Examples of this higher usage:
1127	 *	Level	Usage
1128	 *	14	Profiling clock (and PROM uart polling clock)
1129	 *	12	Serial ports
1130	 *
1131	 * The serial ports request lower level processing on level 6.
1132	 *
1133	 * Also, almost all splN routines (where N is a number or a
1134	 * mnemonic) will do a RAISE(), on the assumption that they are
1135	 * never used to lower our priority.
1136	 * The exceptions are:
1137	 *	spl8()		Because you can't be above 15 to begin with!
1138	 *	splzs()		Because this is used at boot time to lower our
1139	 *			priority, to allow the PROM to poll the uart.
1140	 *	spl0()		Used to lower priority to 0.
1141	 */
1142
1143#if defined(__lint)
1144
1145int spl0(void)		{ return (0); }
1146int spl6(void)		{ return (0); }
1147int spl7(void)		{ return (0); }
1148int spl8(void)		{ return (0); }
1149int splhigh(void)	{ return (0); }
1150int splhi(void)		{ return (0); }
1151int splzs(void)		{ return (0); }
1152
1153/* ARGSUSED */
1154void
1155splx(int level)
1156{}
1157
1158#else	/* __lint */
1159
1160#if defined(__amd64)
1161
1162#define	SETPRI(level) \
1163	movl	$/**/level, %edi;	/* new priority */		\
1164	jmp	do_splx			/* redirect to do_splx */
1165
1166#define	RAISE(level) \
1167	movl	$/**/level, %edi;	/* new priority */		\
1168	jmp	splr			/* redirect to splr */
1169
1170#elif defined(__i386)
1171
1172#define	SETPRI(level) \
1173	pushl	$/**/level;	/* new priority */			\
1174	call	do_splx;	/* invoke common splx code */		\
1175	addl	$4, %esp;	/* unstack arg */			\
1176	ret
1177
1178#define	RAISE(level) \
1179	pushl	$/**/level;	/* new priority */			\
1180	call	splr;		/* invoke common splr code */		\
1181	addl	$4, %esp;	/* unstack args */			\
1182	ret
1183
1184#endif	/* __i386 */
1185
1186	/* locks out all interrupts, including memory errors */
1187	ENTRY(spl8)
1188	SETPRI(15)
1189	SET_SIZE(spl8)
1190
1191	/* just below the level that profiling runs */
1192	ENTRY(spl7)
1193	RAISE(13)
1194	SET_SIZE(spl7)
1195
1196	/* sun specific - highest priority onboard serial i/o asy ports */
1197	ENTRY(splzs)
1198	SETPRI(12)	/* Can't be a RAISE, as it's used to lower us */
1199	SET_SIZE(splzs)
1200
1201	ENTRY(splhi)
1202	ALTENTRY(splhigh)
1203	ALTENTRY(spl6)
1204	ALTENTRY(i_ddi_splhigh)
1205
1206	RAISE(DISP_LEVEL)
1207
1208	SET_SIZE(i_ddi_splhigh)
1209	SET_SIZE(spl6)
1210	SET_SIZE(splhigh)
1211	SET_SIZE(splhi)
1212
1213	/* allow all interrupts */
1214	ENTRY(spl0)
1215	SETPRI(0)
1216	SET_SIZE(spl0)
1217
1218
1219	/* splx implementation */
1220	ENTRY(splx)
1221	jmp	do_splx		/* redirect to common splx code */
1222	SET_SIZE(splx)
1223
1224#endif	/* __lint */
1225
1226#if defined(__i386)
1227
1228/*
1229 * Read and write the %gs register
1230 */
1231
1232#if defined(__lint)
1233
1234/*ARGSUSED*/
1235uint16_t
1236getgs(void)
1237{ return (0); }
1238
1239/*ARGSUSED*/
1240void
1241setgs(uint16_t sel)
1242{}
1243
1244#else	/* __lint */
1245
1246	ENTRY(getgs)
1247	clr	%eax
1248	movw	%gs, %ax
1249	ret
1250	SET_SIZE(getgs)
1251
1252	ENTRY(setgs)
1253	movw	4(%esp), %gs
1254	ret
1255	SET_SIZE(setgs)
1256
1257#endif	/* __lint */
1258#endif	/* __i386 */
1259
1260#if defined(__lint)
1261
1262void
1263pc_reset(void)
1264{}
1265
1266void
1267efi_reset(void)
1268{}
1269
1270#else	/* __lint */
1271
1272	ENTRY(wait_500ms)
1273#if defined(__amd64)
1274	pushq	%rbx
1275#elif defined(__i386)
1276	push	%ebx
1277#endif
1278	movl	$50000, %ebx
12791:
1280	call	tenmicrosec
1281	decl	%ebx
1282	jnz	1b
1283#if defined(__amd64)
1284	popq	%rbx
1285#elif defined(__i386)
1286	pop	%ebx
1287#endif
1288	ret
1289	SET_SIZE(wait_500ms)
1290
1291#define	RESET_METHOD_KBC	1
1292#define	RESET_METHOD_PORT92	2
1293#define RESET_METHOD_PCI	4
1294
1295	DGDEF3(pc_reset_methods, 4, 8)
1296	.long RESET_METHOD_KBC|RESET_METHOD_PORT92|RESET_METHOD_PCI;
1297
1298	ENTRY(pc_reset)
1299
1300#if defined(__i386)
1301	testl	$RESET_METHOD_KBC, pc_reset_methods
1302#elif defined(__amd64)
1303	testl	$RESET_METHOD_KBC, pc_reset_methods(%rip)
1304#endif
1305	jz	1f
1306
1307	/
1308	/ Try the classic keyboard controller-triggered reset.
1309	/
1310	movw	$0x64, %dx
1311	movb	$0xfe, %al
1312	outb	(%dx)
1313
1314	/ Wait up to 500 milliseconds here for the keyboard controller
1315	/ to pull the reset line.  On some systems where the keyboard
1316	/ controller is slow to pull the reset line, the next reset method
1317	/ may be executed (which may be bad if those systems hang when the
1318	/ next reset method is used, e.g. Ferrari 3400 (doesn't like port 92),
1319	/ and Ferrari 4000 (doesn't like the cf9 reset method))
1320
1321	call	wait_500ms
1322
13231:
1324#if defined(__i386)
1325	testl	$RESET_METHOD_PORT92, pc_reset_methods
1326#elif defined(__amd64)
1327	testl	$RESET_METHOD_PORT92, pc_reset_methods(%rip)
1328#endif
1329	jz	3f
1330
1331	/
1332	/ Try port 0x92 fast reset
1333	/
1334	movw	$0x92, %dx
1335	inb	(%dx)
1336	cmpb	$0xff, %al	/ If port's not there, we should get back 0xFF
1337	je	1f
1338	testb	$1, %al		/ If bit 0
1339	jz	2f		/ is clear, jump to perform the reset
1340	andb	$0xfe, %al	/ otherwise,
1341	outb	(%dx)		/ clear bit 0 first, then
13422:
1343	orb	$1, %al		/ Set bit 0
1344	outb	(%dx)		/ and reset the system
13451:
1346
1347	call	wait_500ms
1348
13493:
1350#if defined(__i386)
1351	testl	$RESET_METHOD_PCI, pc_reset_methods
1352#elif defined(__amd64)
1353	testl	$RESET_METHOD_PCI, pc_reset_methods(%rip)
1354#endif
1355	jz	4f
1356
1357	/ Try the PCI (soft) reset vector (should work on all modern systems,
1358	/ but has been shown to cause problems on 450NX systems, and some newer
1359	/ systems (e.g. ATI IXP400-equipped systems))
1360	/ When resetting via this method, 2 writes are required.  The first
1361	/ targets bit 1 (0=hard reset without power cycle, 1=hard reset with
1362	/ power cycle).
1363	/ The reset occurs on the second write, during bit 2's transition from
1364	/ 0->1.
1365	movw	$0xcf9, %dx
1366	movb	$0x2, %al	/ Reset mode = hard, no power cycle
1367	outb	(%dx)
1368	movb	$0x6, %al
1369	outb	(%dx)
1370
1371	call	wait_500ms
1372
13734:
1374	/
1375	/ port 0xcf9 failed also.  Last-ditch effort is to
1376	/ triple-fault the CPU.
1377	/ Also, use triple fault for EFI firmware
1378	/
1379	ENTRY(efi_reset)
1380#if defined(__amd64)
1381	pushq	$0x0
1382	pushq	$0x0		/ IDT base of 0, limit of 0 + 2 unused bytes
1383	lidt	(%rsp)
1384#elif defined(__i386)
1385	pushl	$0x0
1386	pushl	$0x0		/ IDT base of 0, limit of 0 + 2 unused bytes
1387	lidt	(%esp)
1388#endif
1389	int	$0x0		/ Trigger interrupt, generate triple-fault
1390
1391	cli
1392	hlt			/ Wait forever
1393	/*NOTREACHED*/
1394	SET_SIZE(efi_reset)
1395	SET_SIZE(pc_reset)
1396
1397#endif	/* __lint */
1398
1399/*
1400 * C callable in and out routines
1401 */
1402
1403#if defined(__lint)
1404
1405/* ARGSUSED */
1406void
1407outl(int port_address, uint32_t val)
1408{}
1409
1410#else	/* __lint */
1411
1412#if defined(__amd64)
1413
1414	ENTRY(outl)
1415	movw	%di, %dx
1416	movl	%esi, %eax
1417	outl	(%dx)
1418	ret
1419	SET_SIZE(outl)
1420
1421#elif defined(__i386)
1422
1423	.set	PORT, 4
1424	.set	VAL, 8
1425
1426	ENTRY(outl)
1427	movw	PORT(%esp), %dx
1428	movl	VAL(%esp), %eax
1429	outl	(%dx)
1430	ret
1431	SET_SIZE(outl)
1432
1433#endif	/* __i386 */
1434#endif	/* __lint */
1435
1436#if defined(__lint)
1437
1438/* ARGSUSED */
1439void
1440outw(int port_address, uint16_t val)
1441{}
1442
1443#else	/* __lint */
1444
1445#if defined(__amd64)
1446
1447	ENTRY(outw)
1448	movw	%di, %dx
1449	movw	%si, %ax
1450	D16 outl (%dx)		/* XX64 why not outw? */
1451	ret
1452	SET_SIZE(outw)
1453
1454#elif defined(__i386)
1455
1456	ENTRY(outw)
1457	movw	PORT(%esp), %dx
1458	movw	VAL(%esp), %ax
1459	D16 outl (%dx)
1460	ret
1461	SET_SIZE(outw)
1462
1463#endif	/* __i386 */
1464#endif	/* __lint */
1465
1466#if defined(__lint)
1467
1468/* ARGSUSED */
1469void
1470outb(int port_address, uint8_t val)
1471{}
1472
1473#else	/* __lint */
1474
1475#if defined(__amd64)
1476
1477	ENTRY(outb)
1478	movw	%di, %dx
1479	movb	%sil, %al
1480	outb	(%dx)
1481	ret
1482	SET_SIZE(outb)
1483
1484#elif defined(__i386)
1485
1486	ENTRY(outb)
1487	movw	PORT(%esp), %dx
1488	movb	VAL(%esp), %al
1489	outb	(%dx)
1490	ret
1491	SET_SIZE(outb)
1492
1493#endif	/* __i386 */
1494#endif	/* __lint */
1495
1496#if defined(__lint)
1497
1498/* ARGSUSED */
1499uint32_t
1500inl(int port_address)
1501{ return (0); }
1502
1503#else	/* __lint */
1504
1505#if defined(__amd64)
1506
1507	ENTRY(inl)
1508	xorl	%eax, %eax
1509	movw	%di, %dx
1510	inl	(%dx)
1511	ret
1512	SET_SIZE(inl)
1513
1514#elif defined(__i386)
1515
1516	ENTRY(inl)
1517	movw	PORT(%esp), %dx
1518	inl	(%dx)
1519	ret
1520	SET_SIZE(inl)
1521
1522#endif	/* __i386 */
1523#endif	/* __lint */
1524
1525#if defined(__lint)
1526
1527/* ARGSUSED */
1528uint16_t
1529inw(int port_address)
1530{ return (0); }
1531
1532#else	/* __lint */
1533
1534#if defined(__amd64)
1535
1536	ENTRY(inw)
1537	xorl	%eax, %eax
1538	movw	%di, %dx
1539	D16 inl	(%dx)
1540	ret
1541	SET_SIZE(inw)
1542
1543#elif defined(__i386)
1544
1545	ENTRY(inw)
1546	subl	%eax, %eax
1547	movw	PORT(%esp), %dx
1548	D16 inl	(%dx)
1549	ret
1550	SET_SIZE(inw)
1551
1552#endif	/* __i386 */
1553#endif	/* __lint */
1554
1555
1556#if defined(__lint)
1557
1558/* ARGSUSED */
1559uint8_t
1560inb(int port_address)
1561{ return (0); }
1562
1563#else	/* __lint */
1564
1565#if defined(__amd64)
1566
1567	ENTRY(inb)
1568	xorl	%eax, %eax
1569	movw	%di, %dx
1570	inb	(%dx)
1571	ret
1572	SET_SIZE(inb)
1573
1574#elif defined(__i386)
1575
1576	ENTRY(inb)
1577	subl    %eax, %eax
1578	movw	PORT(%esp), %dx
1579	inb	(%dx)
1580	ret
1581	SET_SIZE(inb)
1582
1583#endif	/* __i386 */
1584#endif	/* __lint */
1585
1586
1587#if defined(__lint)
1588
1589/* ARGSUSED */
1590void
1591repoutsw(int port, uint16_t *addr, int cnt)
1592{}
1593
1594#else	/* __lint */
1595
1596#if defined(__amd64)
1597
1598	ENTRY(repoutsw)
1599	movl	%edx, %ecx
1600	movw	%di, %dx
1601	rep
1602	  D16 outsl
1603	ret
1604	SET_SIZE(repoutsw)
1605
1606#elif defined(__i386)
1607
1608	/*
1609	 * The arguments and saved registers are on the stack in the
1610	 *  following order:
1611	 *      |  cnt  |  +16
1612	 *      | *addr |  +12
1613	 *      | port  |  +8
1614	 *      |  eip  |  +4
1615	 *      |  esi  |  <-- %esp
1616	 * If additional values are pushed onto the stack, make sure
1617	 * to adjust the following constants accordingly.
1618	 */
1619	.set	PORT, 8
1620	.set	ADDR, 12
1621	.set	COUNT, 16
1622
1623	ENTRY(repoutsw)
1624	pushl	%esi
1625	movl	PORT(%esp), %edx
1626	movl	ADDR(%esp), %esi
1627	movl	COUNT(%esp), %ecx
1628	rep
1629	  D16 outsl
1630	popl	%esi
1631	ret
1632	SET_SIZE(repoutsw)
1633
1634#endif	/* __i386 */
1635#endif	/* __lint */
1636
1637
1638#if defined(__lint)
1639
1640/* ARGSUSED */
1641void
1642repinsw(int port_addr, uint16_t *addr, int cnt)
1643{}
1644
1645#else	/* __lint */
1646
1647#if defined(__amd64)
1648
1649	ENTRY(repinsw)
1650	movl	%edx, %ecx
1651	movw	%di, %dx
1652	rep
1653	  D16 insl
1654	ret
1655	SET_SIZE(repinsw)
1656
1657#elif defined(__i386)
1658
1659	ENTRY(repinsw)
1660	pushl	%edi
1661	movl	PORT(%esp), %edx
1662	movl	ADDR(%esp), %edi
1663	movl	COUNT(%esp), %ecx
1664	rep
1665	  D16 insl
1666	popl	%edi
1667	ret
1668	SET_SIZE(repinsw)
1669
1670#endif	/* __i386 */
1671#endif	/* __lint */
1672
1673
1674#if defined(__lint)
1675
1676/* ARGSUSED */
1677void
1678repinsb(int port, uint8_t *addr, int count)
1679{}
1680
1681#else	/* __lint */
1682
1683#if defined(__amd64)
1684
1685	ENTRY(repinsb)
1686	movl	%edx, %ecx
1687	movw	%di, %dx
1688	movq	%rsi, %rdi
1689	rep
1690	  insb
1691	ret
1692	SET_SIZE(repinsb)
1693
1694#elif defined(__i386)
1695
1696	/*
1697	 * The arguments and saved registers are on the stack in the
1698	 *  following order:
1699	 *      |  cnt  |  +16
1700	 *      | *addr |  +12
1701	 *      | port  |  +8
1702	 *      |  eip  |  +4
1703	 *      |  esi  |  <-- %esp
1704	 * If additional values are pushed onto the stack, make sure
1705	 * to adjust the following constants accordingly.
1706	 */
1707	.set	IO_PORT, 8
1708	.set	IO_ADDR, 12
1709	.set	IO_COUNT, 16
1710
1711	ENTRY(repinsb)
1712	pushl	%edi
1713	movl	IO_ADDR(%esp), %edi
1714	movl	IO_COUNT(%esp), %ecx
1715	movl	IO_PORT(%esp), %edx
1716	rep
1717	  insb
1718	popl	%edi
1719	ret
1720	SET_SIZE(repinsb)
1721
1722#endif	/* __i386 */
1723#endif	/* __lint */
1724
1725
1726/*
1727 * Input a stream of 32-bit words.
1728 * NOTE: count is a DWORD count.
1729 */
1730#if defined(__lint)
1731
1732/* ARGSUSED */
1733void
1734repinsd(int port, uint32_t *addr, int count)
1735{}
1736
1737#else	/* __lint */
1738
1739#if defined(__amd64)
1740
1741	ENTRY(repinsd)
1742	movl	%edx, %ecx
1743	movw	%di, %dx
1744	movq	%rsi, %rdi
1745	rep
1746	  insl
1747	ret
1748	SET_SIZE(repinsd)
1749
1750#elif defined(__i386)
1751
1752	ENTRY(repinsd)
1753	pushl	%edi
1754	movl	IO_ADDR(%esp), %edi
1755	movl	IO_COUNT(%esp), %ecx
1756	movl	IO_PORT(%esp), %edx
1757	rep
1758	  insl
1759	popl	%edi
1760	ret
1761	SET_SIZE(repinsd)
1762
1763#endif	/* __i386 */
1764#endif	/* __lint */
1765
1766/*
1767 * Output a stream of bytes
1768 * NOTE: count is a byte count
1769 */
1770#if defined(__lint)
1771
1772/* ARGSUSED */
1773void
1774repoutsb(int port, uint8_t *addr, int count)
1775{}
1776
1777#else	/* __lint */
1778
1779#if defined(__amd64)
1780
1781	ENTRY(repoutsb)
1782	movl	%edx, %ecx
1783	movw	%di, %dx
1784	rep
1785	  outsb
1786	ret
1787	SET_SIZE(repoutsb)
1788
1789#elif defined(__i386)
1790
1791	ENTRY(repoutsb)
1792	pushl	%esi
1793	movl	IO_ADDR(%esp), %esi
1794	movl	IO_COUNT(%esp), %ecx
1795	movl	IO_PORT(%esp), %edx
1796	rep
1797	  outsb
1798	popl	%esi
1799	ret
1800	SET_SIZE(repoutsb)
1801
1802#endif	/* __i386 */
1803#endif	/* __lint */
1804
1805/*
1806 * Output a stream of 32-bit words
1807 * NOTE: count is a DWORD count
1808 */
1809#if defined(__lint)
1810
1811/* ARGSUSED */
1812void
1813repoutsd(int port, uint32_t *addr, int count)
1814{}
1815
1816#else	/* __lint */
1817
1818#if defined(__amd64)
1819
1820	ENTRY(repoutsd)
1821	movl	%edx, %ecx
1822	movw	%di, %dx
1823	rep
1824	  outsl
1825	ret
1826	SET_SIZE(repoutsd)
1827
1828#elif defined(__i386)
1829
1830	ENTRY(repoutsd)
1831	pushl	%esi
1832	movl	IO_ADDR(%esp), %esi
1833	movl	IO_COUNT(%esp), %ecx
1834	movl	IO_PORT(%esp), %edx
1835	rep
1836	  outsl
1837	popl	%esi
1838	ret
1839	SET_SIZE(repoutsd)
1840
1841#endif	/* __i386 */
1842#endif	/* __lint */
1843
1844/*
1845 * void int3(void)
1846 * void int18(void)
1847 * void int20(void)
1848 * void int_cmci(void)
1849 */
1850
1851#if defined(__lint)
1852
1853void
1854int3(void)
1855{}
1856
1857void
1858int18(void)
1859{}
1860
1861void
1862int20(void)
1863{}
1864
1865void
1866int_cmci(void)
1867{}
1868
1869#else	/* __lint */
1870
1871	ENTRY(int3)
1872	int	$T_BPTFLT
1873	ret
1874	SET_SIZE(int3)
1875
1876	ENTRY(int18)
1877	int	$T_MCE
1878	ret
1879	SET_SIZE(int18)
1880
1881	ENTRY(int20)
1882	movl	boothowto, %eax
1883	andl	$RB_DEBUG, %eax
1884	jz	1f
1885
1886	int	$T_DBGENTR
18871:
1888	rep;	ret	/* use 2 byte return instruction when branch target */
1889			/* AMD Software Optimization Guide - Section 6.2 */
1890	SET_SIZE(int20)
1891
1892	ENTRY(int_cmci)
1893	int	$T_ENOEXTFLT
1894	ret
1895	SET_SIZE(int_cmci)
1896
1897#endif	/* __lint */
1898
1899#if defined(__lint)
1900
1901/* ARGSUSED */
1902int
1903scanc(size_t size, uchar_t *cp, uchar_t *table, uchar_t mask)
1904{ return (0); }
1905
1906#else	/* __lint */
1907
1908#if defined(__amd64)
1909
1910	ENTRY(scanc)
1911					/* rdi == size */
1912					/* rsi == cp */
1913					/* rdx == table */
1914					/* rcx == mask */
1915	addq	%rsi, %rdi		/* end = &cp[size] */
1916.scanloop:
1917	cmpq	%rdi, %rsi		/* while (cp < end */
1918	jnb	.scandone
1919	movzbq	(%rsi), %r8		/* %r8 = *cp */
1920	incq	%rsi			/* cp++ */
1921	testb	%cl, (%r8, %rdx)
1922	jz	.scanloop		/*  && (table[*cp] & mask) == 0) */
1923	decq	%rsi			/* (fix post-increment) */
1924.scandone:
1925	movl	%edi, %eax
1926	subl	%esi, %eax		/* return (end - cp) */
1927	ret
1928	SET_SIZE(scanc)
1929
1930#elif defined(__i386)
1931
1932	ENTRY(scanc)
1933	pushl	%edi
1934	pushl	%esi
1935	movb	24(%esp), %cl		/* mask = %cl */
1936	movl	16(%esp), %esi		/* cp = %esi */
1937	movl	20(%esp), %edx		/* table = %edx */
1938	movl	%esi, %edi
1939	addl	12(%esp), %edi		/* end = &cp[size]; */
1940.scanloop:
1941	cmpl	%edi, %esi		/* while (cp < end */
1942	jnb	.scandone
1943	movzbl	(%esi),  %eax		/* %al = *cp */
1944	incl	%esi			/* cp++ */
1945	movb	(%edx,  %eax), %al	/* %al = table[*cp] */
1946	testb	%al, %cl
1947	jz	.scanloop		/*   && (table[*cp] & mask) == 0) */
1948	dec	%esi			/* post-incremented */
1949.scandone:
1950	movl	%edi, %eax
1951	subl	%esi, %eax		/* return (end - cp) */
1952	popl	%esi
1953	popl	%edi
1954	ret
1955	SET_SIZE(scanc)
1956
1957#endif	/* __i386 */
1958#endif	/* __lint */
1959
1960/*
1961 * Replacement functions for ones that are normally inlined.
1962 * In addition to the copy in i86.il, they are defined here just in case.
1963 */
1964
1965#if defined(__lint)
1966
1967ulong_t
1968intr_clear(void)
1969{ return (0); }
1970
1971ulong_t
1972clear_int_flag(void)
1973{ return (0); }
1974
1975#else	/* __lint */
1976
1977#if defined(__amd64)
1978
1979	ENTRY(intr_clear)
1980	ENTRY(clear_int_flag)
1981	pushfq
1982	popq	%rax
1983#if defined(__xpv)
1984	leaq	xpv_panicking, %rdi
1985	movl	(%rdi), %edi
1986	cmpl	$0, %edi
1987	jne	2f
1988	CLIRET(%rdi, %dl)	/* returns event mask in %dl */
1989	/*
1990	 * Synthesize the PS_IE bit from the event mask bit
1991	 */
1992	andq    $_BITNOT(PS_IE), %rax
1993	testb	$1, %dl
1994	jnz	1f
1995	orq	$PS_IE, %rax
19961:
1997	ret
19982:
1999#endif
2000	CLI(%rdi)
2001	ret
2002	SET_SIZE(clear_int_flag)
2003	SET_SIZE(intr_clear)
2004
2005#elif defined(__i386)
2006
2007	ENTRY(intr_clear)
2008	ENTRY(clear_int_flag)
2009	pushfl
2010	popl	%eax
2011#if defined(__xpv)
2012	leal	xpv_panicking, %edx
2013	movl	(%edx), %edx
2014	cmpl	$0, %edx
2015	jne	2f
2016	CLIRET(%edx, %cl)	/* returns event mask in %cl */
2017	/*
2018	 * Synthesize the PS_IE bit from the event mask bit
2019	 */
2020	andl    $_BITNOT(PS_IE), %eax
2021	testb	$1, %cl
2022	jnz	1f
2023	orl	$PS_IE, %eax
20241:
2025	ret
20262:
2027#endif
2028	CLI(%edx)
2029	ret
2030	SET_SIZE(clear_int_flag)
2031	SET_SIZE(intr_clear)
2032
2033#endif	/* __i386 */
2034#endif	/* __lint */
2035
2036#if defined(__lint)
2037
2038struct cpu *
2039curcpup(void)
2040{ return 0; }
2041
2042#else	/* __lint */
2043
2044#if defined(__amd64)
2045
2046	ENTRY(curcpup)
2047	movq	%gs:CPU_SELF, %rax
2048	ret
2049	SET_SIZE(curcpup)
2050
2051#elif defined(__i386)
2052
2053	ENTRY(curcpup)
2054	movl	%gs:CPU_SELF, %eax
2055	ret
2056	SET_SIZE(curcpup)
2057
2058#endif	/* __i386 */
2059#endif	/* __lint */
2060
2061/* htonll(), ntohll(), htonl(), ntohl(), htons(), ntohs()
2062 * These functions reverse the byte order of the input parameter and returns
2063 * the result.  This is to convert the byte order from host byte order
2064 * (little endian) to network byte order (big endian), or vice versa.
2065 */
2066
2067#if defined(__lint)
2068
2069uint64_t
2070htonll(uint64_t i)
2071{ return (i); }
2072
2073uint64_t
2074ntohll(uint64_t i)
2075{ return (i); }
2076
2077uint32_t
2078htonl(uint32_t i)
2079{ return (i); }
2080
2081uint32_t
2082ntohl(uint32_t i)
2083{ return (i); }
2084
2085uint16_t
2086htons(uint16_t i)
2087{ return (i); }
2088
2089uint16_t
2090ntohs(uint16_t i)
2091{ return (i); }
2092
2093#else	/* __lint */
2094
2095#if defined(__amd64)
2096
2097	ENTRY(htonll)
2098	ALTENTRY(ntohll)
2099	movq	%rdi, %rax
2100	bswapq	%rax
2101	ret
2102	SET_SIZE(ntohll)
2103	SET_SIZE(htonll)
2104
2105	/* XX64 there must be shorter sequences for this */
2106	ENTRY(htonl)
2107	ALTENTRY(ntohl)
2108	movl	%edi, %eax
2109	bswap	%eax
2110	ret
2111	SET_SIZE(ntohl)
2112	SET_SIZE(htonl)
2113
2114	/* XX64 there must be better sequences for this */
2115	ENTRY(htons)
2116	ALTENTRY(ntohs)
2117	movl	%edi, %eax
2118	bswap	%eax
2119	shrl	$16, %eax
2120	ret
2121	SET_SIZE(ntohs)
2122	SET_SIZE(htons)
2123
2124#elif defined(__i386)
2125
2126	ENTRY(htonll)
2127	ALTENTRY(ntohll)
2128	movl	4(%esp), %edx
2129	movl	8(%esp), %eax
2130	bswap	%edx
2131	bswap	%eax
2132	ret
2133	SET_SIZE(ntohll)
2134	SET_SIZE(htonll)
2135
2136	ENTRY(htonl)
2137	ALTENTRY(ntohl)
2138	movl	4(%esp), %eax
2139	bswap	%eax
2140	ret
2141	SET_SIZE(ntohl)
2142	SET_SIZE(htonl)
2143
2144	ENTRY(htons)
2145	ALTENTRY(ntohs)
2146	movl	4(%esp), %eax
2147	bswap	%eax
2148	shrl	$16, %eax
2149	ret
2150	SET_SIZE(ntohs)
2151	SET_SIZE(htons)
2152
2153#endif	/* __i386 */
2154#endif	/* __lint */
2155
2156
2157#if defined(__lint)
2158
2159/* ARGSUSED */
2160void
2161intr_restore(ulong_t i)
2162{ return; }
2163
2164/* ARGSUSED */
2165void
2166restore_int_flag(ulong_t i)
2167{ return; }
2168
2169#else	/* __lint */
2170
2171#if defined(__amd64)
2172
2173	ENTRY(intr_restore)
2174	ENTRY(restore_int_flag)
2175	testq	$PS_IE, %rdi
2176	jz	1f
2177#if defined(__xpv)
2178	leaq	xpv_panicking, %rsi
2179	movl	(%rsi), %esi
2180	cmpl	$0, %esi
2181	jne	1f
2182	/*
2183	 * Since we're -really- running unprivileged, our attempt
2184	 * to change the state of the IF bit will be ignored.
2185	 * The virtual IF bit is tweaked by CLI and STI.
2186	 */
2187	IE_TO_EVENT_MASK(%rsi, %rdi)
2188#else
2189	sti
2190#endif
21911:
2192	ret
2193	SET_SIZE(restore_int_flag)
2194	SET_SIZE(intr_restore)
2195
2196#elif defined(__i386)
2197
2198	ENTRY(intr_restore)
2199	ENTRY(restore_int_flag)
2200	testl	$PS_IE, 4(%esp)
2201	jz	1f
2202#if defined(__xpv)
2203	leal	xpv_panicking, %edx
2204	movl	(%edx), %edx
2205	cmpl	$0, %edx
2206	jne	1f
2207	/*
2208	 * Since we're -really- running unprivileged, our attempt
2209	 * to change the state of the IF bit will be ignored.
2210	 * The virtual IF bit is tweaked by CLI and STI.
2211	 */
2212	IE_TO_EVENT_MASK(%edx, 4(%esp))
2213#else
2214	sti
2215#endif
22161:
2217	ret
2218	SET_SIZE(restore_int_flag)
2219	SET_SIZE(intr_restore)
2220
2221#endif	/* __i386 */
2222#endif	/* __lint */
2223
2224#if defined(__lint)
2225
2226void
2227sti(void)
2228{}
2229
2230void
2231cli(void)
2232{}
2233
2234#else	/* __lint */
2235
2236	ENTRY(sti)
2237	STI
2238	ret
2239	SET_SIZE(sti)
2240
2241	ENTRY(cli)
2242#if defined(__amd64)
2243	CLI(%rax)
2244#elif defined(__i386)
2245	CLI(%eax)
2246#endif	/* __i386 */
2247	ret
2248	SET_SIZE(cli)
2249
2250#endif	/* __lint */
2251
2252#if defined(__lint)
2253
2254dtrace_icookie_t
2255dtrace_interrupt_disable(void)
2256{ return (0); }
2257
2258#else   /* __lint */
2259
2260#if defined(__amd64)
2261
2262	ENTRY(dtrace_interrupt_disable)
2263	pushfq
2264	popq	%rax
2265#if defined(__xpv)
2266	leaq	xpv_panicking, %rdi
2267	movl	(%rdi), %edi
2268	cmpl	$0, %edi
2269	jne	.dtrace_interrupt_disable_done
2270	CLIRET(%rdi, %dl)	/* returns event mask in %dl */
2271	/*
2272	 * Synthesize the PS_IE bit from the event mask bit
2273	 */
2274	andq    $_BITNOT(PS_IE), %rax
2275	testb	$1, %dl
2276	jnz	.dtrace_interrupt_disable_done
2277	orq	$PS_IE, %rax
2278#else
2279	CLI(%rdx)
2280#endif
2281.dtrace_interrupt_disable_done:
2282	ret
2283	SET_SIZE(dtrace_interrupt_disable)
2284
2285#elif defined(__i386)
2286
2287	ENTRY(dtrace_interrupt_disable)
2288	pushfl
2289	popl	%eax
2290#if defined(__xpv)
2291	leal	xpv_panicking, %edx
2292	movl	(%edx), %edx
2293	cmpl	$0, %edx
2294	jne	.dtrace_interrupt_disable_done
2295	CLIRET(%edx, %cl)	/* returns event mask in %cl */
2296	/*
2297	 * Synthesize the PS_IE bit from the event mask bit
2298	 */
2299	andl    $_BITNOT(PS_IE), %eax
2300	testb	$1, %cl
2301	jnz	.dtrace_interrupt_disable_done
2302	orl	$PS_IE, %eax
2303#else
2304	CLI(%edx)
2305#endif
2306.dtrace_interrupt_disable_done:
2307	ret
2308	SET_SIZE(dtrace_interrupt_disable)
2309
2310#endif	/* __i386 */
2311#endif	/* __lint */
2312
2313#if defined(__lint)
2314
2315/*ARGSUSED*/
2316void
2317dtrace_interrupt_enable(dtrace_icookie_t cookie)
2318{}
2319
2320#else	/* __lint */
2321
2322#if defined(__amd64)
2323
2324	ENTRY(dtrace_interrupt_enable)
2325	pushq	%rdi
2326	popfq
2327#if defined(__xpv)
2328	leaq	xpv_panicking, %rdx
2329	movl	(%rdx), %edx
2330	cmpl	$0, %edx
2331	jne	.dtrace_interrupt_enable_done
2332	/*
2333	 * Since we're -really- running unprivileged, our attempt
2334	 * to change the state of the IF bit will be ignored. The
2335	 * virtual IF bit is tweaked by CLI and STI.
2336	 */
2337	IE_TO_EVENT_MASK(%rdx, %rdi)
2338#endif
2339.dtrace_interrupt_enable_done:
2340	ret
2341	SET_SIZE(dtrace_interrupt_enable)
2342
2343#elif defined(__i386)
2344
2345	ENTRY(dtrace_interrupt_enable)
2346	movl	4(%esp), %eax
2347	pushl	%eax
2348	popfl
2349#if defined(__xpv)
2350	leal	xpv_panicking, %edx
2351	movl	(%edx), %edx
2352	cmpl	$0, %edx
2353	jne	.dtrace_interrupt_enable_done
2354	/*
2355	 * Since we're -really- running unprivileged, our attempt
2356	 * to change the state of the IF bit will be ignored. The
2357	 * virtual IF bit is tweaked by CLI and STI.
2358	 */
2359	IE_TO_EVENT_MASK(%edx, %eax)
2360#endif
2361.dtrace_interrupt_enable_done:
2362	ret
2363	SET_SIZE(dtrace_interrupt_enable)
2364
2365#endif	/* __i386 */
2366#endif	/* __lint */
2367
2368
2369#if defined(lint)
2370
2371void
2372dtrace_membar_producer(void)
2373{}
2374
2375void
2376dtrace_membar_consumer(void)
2377{}
2378
2379#else	/* __lint */
2380
2381	ENTRY(dtrace_membar_producer)
2382	rep;	ret	/* use 2 byte return instruction when branch target */
2383			/* AMD Software Optimization Guide - Section 6.2 */
2384	SET_SIZE(dtrace_membar_producer)
2385
2386	ENTRY(dtrace_membar_consumer)
2387	rep;	ret	/* use 2 byte return instruction when branch target */
2388			/* AMD Software Optimization Guide - Section 6.2 */
2389	SET_SIZE(dtrace_membar_consumer)
2390
2391#endif	/* __lint */
2392
2393#if defined(__lint)
2394
2395kthread_id_t
2396threadp(void)
2397{ return ((kthread_id_t)0); }
2398
2399#else	/* __lint */
2400
2401#if defined(__amd64)
2402
2403	ENTRY(threadp)
2404	movq	%gs:CPU_THREAD, %rax
2405	ret
2406	SET_SIZE(threadp)
2407
2408#elif defined(__i386)
2409
2410	ENTRY(threadp)
2411	movl	%gs:CPU_THREAD, %eax
2412	ret
2413	SET_SIZE(threadp)
2414
2415#endif	/* __i386 */
2416#endif	/* __lint */
2417
2418/*
2419 *   Checksum routine for Internet Protocol Headers
2420 */
2421
2422#if defined(__lint)
2423
2424/* ARGSUSED */
2425unsigned int
2426ip_ocsum(
2427	ushort_t *address,	/* ptr to 1st message buffer */
2428	int halfword_count,	/* length of data */
2429	unsigned int sum)	/* partial checksum */
2430{
2431	int		i;
2432	unsigned int	psum = 0;	/* partial sum */
2433
2434	for (i = 0; i < halfword_count; i++, address++) {
2435		psum += *address;
2436	}
2437
2438	while ((psum >> 16) != 0) {
2439		psum = (psum & 0xffff) + (psum >> 16);
2440	}
2441
2442	psum += sum;
2443
2444	while ((psum >> 16) != 0) {
2445		psum = (psum & 0xffff) + (psum >> 16);
2446	}
2447
2448	return (psum);
2449}
2450
2451#else	/* __lint */
2452
2453#if defined(__amd64)
2454
2455	ENTRY(ip_ocsum)
2456	pushq	%rbp
2457	movq	%rsp, %rbp
2458#ifdef DEBUG
2459	movq	postbootkernelbase(%rip), %rax
2460	cmpq	%rax, %rdi
2461	jnb	1f
2462	xorl	%eax, %eax
2463	movq	%rdi, %rsi
2464	leaq	.ip_ocsum_panic_msg(%rip), %rdi
2465	call	panic
2466	/*NOTREACHED*/
2467.ip_ocsum_panic_msg:
2468	.string	"ip_ocsum: address 0x%p below kernelbase\n"
24691:
2470#endif
2471	movl	%esi, %ecx	/* halfword_count */
2472	movq	%rdi, %rsi	/* address */
2473				/* partial sum in %edx */
2474	xorl	%eax, %eax
2475	testl	%ecx, %ecx
2476	jz	.ip_ocsum_done
2477	testq	$3, %rsi
2478	jnz	.ip_csum_notaligned
2479.ip_csum_aligned:	/* XX64 opportunities for 8-byte operations? */
2480.next_iter:
2481	/* XX64 opportunities for prefetch? */
2482	/* XX64 compute csum with 64 bit quantities? */
2483	subl	$32, %ecx
2484	jl	.less_than_32
2485
2486	addl	0(%rsi), %edx
2487.only60:
2488	adcl	4(%rsi), %eax
2489.only56:
2490	adcl	8(%rsi), %edx
2491.only52:
2492	adcl	12(%rsi), %eax
2493.only48:
2494	adcl	16(%rsi), %edx
2495.only44:
2496	adcl	20(%rsi), %eax
2497.only40:
2498	adcl	24(%rsi), %edx
2499.only36:
2500	adcl	28(%rsi), %eax
2501.only32:
2502	adcl	32(%rsi), %edx
2503.only28:
2504	adcl	36(%rsi), %eax
2505.only24:
2506	adcl	40(%rsi), %edx
2507.only20:
2508	adcl	44(%rsi), %eax
2509.only16:
2510	adcl	48(%rsi), %edx
2511.only12:
2512	adcl	52(%rsi), %eax
2513.only8:
2514	adcl	56(%rsi), %edx
2515.only4:
2516	adcl	60(%rsi), %eax	/* could be adding -1 and -1 with a carry */
2517.only0:
2518	adcl	$0, %eax	/* could be adding -1 in eax with a carry */
2519	adcl	$0, %eax
2520
2521	addq	$64, %rsi
2522	testl	%ecx, %ecx
2523	jnz	.next_iter
2524
2525.ip_ocsum_done:
2526	addl	%eax, %edx
2527	adcl	$0, %edx
2528	movl	%edx, %eax	/* form a 16 bit checksum by */
2529	shrl	$16, %eax	/* adding two halves of 32 bit checksum */
2530	addw	%dx, %ax
2531	adcw	$0, %ax
2532	andl	$0xffff, %eax
2533	leave
2534	ret
2535
2536.ip_csum_notaligned:
2537	xorl	%edi, %edi
2538	movw	(%rsi), %di
2539	addl	%edi, %edx
2540	adcl	$0, %edx
2541	addq	$2, %rsi
2542	decl	%ecx
2543	jmp	.ip_csum_aligned
2544
2545.less_than_32:
2546	addl	$32, %ecx
2547	testl	$1, %ecx
2548	jz	.size_aligned
2549	andl	$0xfe, %ecx
2550	movzwl	(%rsi, %rcx, 2), %edi
2551	addl	%edi, %edx
2552	adcl	$0, %edx
2553.size_aligned:
2554	movl	%ecx, %edi
2555	shrl	$1, %ecx
2556	shl	$1, %edi
2557	subq	$64, %rdi
2558	addq	%rdi, %rsi
2559	leaq    .ip_ocsum_jmptbl(%rip), %rdi
2560	leaq	(%rdi, %rcx, 8), %rdi
2561	xorl	%ecx, %ecx
2562	clc
2563	movq	(%rdi), %rdi
2564	INDIRECT_JMP_REG(rdi)
2565
2566	.align	8
2567.ip_ocsum_jmptbl:
2568	.quad	.only0, .only4, .only8, .only12, .only16, .only20
2569	.quad	.only24, .only28, .only32, .only36, .only40, .only44
2570	.quad	.only48, .only52, .only56, .only60
2571	SET_SIZE(ip_ocsum)
2572
2573#elif defined(__i386)
2574
2575	ENTRY(ip_ocsum)
2576	pushl	%ebp
2577	movl	%esp, %ebp
2578	pushl	%ebx
2579	pushl	%esi
2580	pushl	%edi
2581	movl	12(%ebp), %ecx	/* count of half words */
2582	movl	16(%ebp), %edx	/* partial checksum */
2583	movl	8(%ebp), %esi
2584	xorl	%eax, %eax
2585	testl	%ecx, %ecx
2586	jz	.ip_ocsum_done
2587
2588	testl	$3, %esi
2589	jnz	.ip_csum_notaligned
2590.ip_csum_aligned:
2591.next_iter:
2592	subl	$32, %ecx
2593	jl	.less_than_32
2594
2595	addl	0(%esi), %edx
2596.only60:
2597	adcl	4(%esi), %eax
2598.only56:
2599	adcl	8(%esi), %edx
2600.only52:
2601	adcl	12(%esi), %eax
2602.only48:
2603	adcl	16(%esi), %edx
2604.only44:
2605	adcl	20(%esi), %eax
2606.only40:
2607	adcl	24(%esi), %edx
2608.only36:
2609	adcl	28(%esi), %eax
2610.only32:
2611	adcl	32(%esi), %edx
2612.only28:
2613	adcl	36(%esi), %eax
2614.only24:
2615	adcl	40(%esi), %edx
2616.only20:
2617	adcl	44(%esi), %eax
2618.only16:
2619	adcl	48(%esi), %edx
2620.only12:
2621	adcl	52(%esi), %eax
2622.only8:
2623	adcl	56(%esi), %edx
2624.only4:
2625	adcl	60(%esi), %eax	/* We could be adding -1 and -1 with a carry */
2626.only0:
2627	adcl	$0, %eax	/* we could be adding -1 in eax with a carry */
2628	adcl	$0, %eax
2629
2630	addl	$64, %esi
2631	andl	%ecx, %ecx
2632	jnz	.next_iter
2633
2634.ip_ocsum_done:
2635	addl	%eax, %edx
2636	adcl	$0, %edx
2637	movl	%edx, %eax	/* form a 16 bit checksum by */
2638	shrl	$16, %eax	/* adding two halves of 32 bit checksum */
2639	addw	%dx, %ax
2640	adcw	$0, %ax
2641	andl	$0xffff, %eax
2642	popl	%edi		/* restore registers */
2643	popl	%esi
2644	popl	%ebx
2645	leave
2646	ret
2647
2648.ip_csum_notaligned:
2649	xorl	%edi, %edi
2650	movw	(%esi), %di
2651	addl	%edi, %edx
2652	adcl	$0, %edx
2653	addl	$2, %esi
2654	decl	%ecx
2655	jmp	.ip_csum_aligned
2656
2657.less_than_32:
2658	addl	$32, %ecx
2659	testl	$1, %ecx
2660	jz	.size_aligned
2661	andl	$0xfe, %ecx
2662	movzwl	(%esi, %ecx, 2), %edi
2663	addl	%edi, %edx
2664	adcl	$0, %edx
2665.size_aligned:
2666	movl	%ecx, %edi
2667	shrl	$1, %ecx
2668	shl	$1, %edi
2669	subl	$64, %edi
2670	addl	%edi, %esi
2671	movl	$.ip_ocsum_jmptbl, %edi
2672	lea	(%edi, %ecx, 4), %edi
2673	xorl	%ecx, %ecx
2674	clc
2675	jmp	*(%edi)
2676	SET_SIZE(ip_ocsum)
2677
2678	.data
2679	.align	4
2680
2681.ip_ocsum_jmptbl:
2682	.long	.only0, .only4, .only8, .only12, .only16, .only20
2683	.long	.only24, .only28, .only32, .only36, .only40, .only44
2684	.long	.only48, .only52, .only56, .only60
2685
2686
2687#endif	/* __i386 */
2688#endif	/* __lint */
2689
2690/*
2691 * multiply two long numbers and yield a u_longlong_t result, callable from C.
2692 * Provided to manipulate hrtime_t values.
2693 */
2694#if defined(__lint)
2695
2696/* result = a * b; */
2697
2698/* ARGSUSED */
2699unsigned long long
2700mul32(uint_t a, uint_t b)
2701{ return (0); }
2702
2703#else	/* __lint */
2704
2705#if defined(__amd64)
2706
2707	ENTRY(mul32)
2708	xorl	%edx, %edx	/* XX64 joe, paranoia? */
2709	movl	%edi, %eax
2710	mull	%esi
2711	shlq	$32, %rdx
2712	orq	%rdx, %rax
2713	ret
2714	SET_SIZE(mul32)
2715
2716#elif defined(__i386)
2717
2718	ENTRY(mul32)
2719	movl	8(%esp), %eax
2720	movl	4(%esp), %ecx
2721	mull	%ecx
2722	ret
2723	SET_SIZE(mul32)
2724
2725#endif	/* __i386 */
2726#endif	/* __lint */
2727
2728#if defined(notused)
2729#if defined(__lint)
2730/* ARGSUSED */
2731void
2732load_pte64(uint64_t *pte, uint64_t pte_value)
2733{}
2734#else	/* __lint */
2735	.globl load_pte64
2736load_pte64:
2737	movl	4(%esp), %eax
2738	movl	8(%esp), %ecx
2739	movl	12(%esp), %edx
2740	movl	%edx, 4(%eax)
2741	movl	%ecx, (%eax)
2742	ret
2743#endif	/* __lint */
2744#endif	/* notused */
2745
2746#if defined(__lint)
2747
2748/*ARGSUSED*/
2749void
2750scan_memory(caddr_t addr, size_t size)
2751{}
2752
2753#else	/* __lint */
2754
2755#if defined(__amd64)
2756
2757	ENTRY(scan_memory)
2758	shrq	$3, %rsi	/* convert %rsi from byte to quadword count */
2759	jz	.scanm_done
2760	movq	%rsi, %rcx	/* move count into rep control register */
2761	movq	%rdi, %rsi	/* move addr into lodsq control reg. */
2762	rep lodsq		/* scan the memory range */
2763.scanm_done:
2764	rep;	ret	/* use 2 byte return instruction when branch target */
2765			/* AMD Software Optimization Guide - Section 6.2 */
2766	SET_SIZE(scan_memory)
2767
2768#elif defined(__i386)
2769
2770	ENTRY(scan_memory)
2771	pushl	%ecx
2772	pushl	%esi
2773	movl	16(%esp), %ecx	/* move 2nd arg into rep control register */
2774	shrl	$2, %ecx	/* convert from byte count to word count */
2775	jz	.scanm_done
2776	movl	12(%esp), %esi	/* move 1st arg into lodsw control register */
2777	.byte	0xf3		/* rep prefix.  lame assembler.  sigh. */
2778	lodsl
2779.scanm_done:
2780	popl	%esi
2781	popl	%ecx
2782	ret
2783	SET_SIZE(scan_memory)
2784
2785#endif	/* __i386 */
2786#endif	/* __lint */
2787
2788
2789#if defined(__lint)
2790
2791/*ARGSUSED */
2792int
2793lowbit(ulong_t i)
2794{ return (0); }
2795
2796#else	/* __lint */
2797
2798#if defined(__amd64)
2799
2800	ENTRY(lowbit)
2801	movl	$-1, %eax
2802	bsfq	%rdi, %rdi
2803	cmovnz	%edi, %eax
2804	incl	%eax
2805	ret
2806	SET_SIZE(lowbit)
2807
2808#elif defined(__i386)
2809
2810	ENTRY(lowbit)
2811	bsfl	4(%esp), %eax
2812	jz	0f
2813	incl	%eax
2814	ret
28150:
2816	xorl	%eax, %eax
2817	ret
2818	SET_SIZE(lowbit)
2819
2820#endif	/* __i386 */
2821#endif	/* __lint */
2822
2823#if defined(__lint)
2824
2825/*ARGSUSED*/
2826int
2827highbit(ulong_t i)
2828{ return (0); }
2829
2830/*ARGSUSED*/
2831int
2832highbit64(uint64_t i)
2833{ return (0); }
2834
2835#else	/* __lint */
2836
2837#if defined(__amd64)
2838
2839	ENTRY(highbit)
2840	ALTENTRY(highbit64)
2841	movl	$-1, %eax
2842	bsrq	%rdi, %rdi
2843	cmovnz	%edi, %eax
2844	incl	%eax
2845	ret
2846	SET_SIZE(highbit64)
2847	SET_SIZE(highbit)
2848
2849#elif defined(__i386)
2850
2851	ENTRY(highbit)
2852	bsrl	4(%esp), %eax
2853	jz	0f
2854	incl	%eax
2855	ret
28560:
2857	xorl	%eax, %eax
2858	ret
2859	SET_SIZE(highbit)
2860
2861	ENTRY(highbit64)
2862	bsrl	8(%esp), %eax
2863	jz	highbit
2864	addl	$33, %eax
2865	ret
2866	SET_SIZE(highbit64)
2867
2868#endif	/* __i386 */
2869#endif	/* __lint */
2870
2871#if defined(__lint)
2872
2873/*ARGSUSED*/
2874uint64_t
2875rdmsr(uint_t r)
2876{ return (0); }
2877
2878/*ARGSUSED*/
2879void
2880wrmsr(uint_t r, const uint64_t val)
2881{}
2882
2883/*ARGSUSED*/
2884uint64_t
2885xrdmsr(uint_t r)
2886{ return (0); }
2887
2888/*ARGSUSED*/
2889void
2890xwrmsr(uint_t r, const uint64_t val)
2891{}
2892
2893void
2894invalidate_cache(void)
2895{}
2896
2897/*ARGSUSED*/
2898uint64_t
2899get_xcr(uint_t r)
2900{ return (0); }
2901
2902/*ARGSUSED*/
2903void
2904set_xcr(uint_t r, const uint64_t val)
2905{}
2906
2907#else  /* __lint */
2908
2909#define	XMSR_ACCESS_VAL		$0x9c5a203a
2910
2911#if defined(__amd64)
2912
2913	ENTRY(rdmsr)
2914	movl	%edi, %ecx
2915	rdmsr
2916	shlq	$32, %rdx
2917	orq	%rdx, %rax
2918	ret
2919	SET_SIZE(rdmsr)
2920
2921	ENTRY(wrmsr)
2922	movq	%rsi, %rdx
2923	shrq	$32, %rdx
2924	movl	%esi, %eax
2925	movl	%edi, %ecx
2926	wrmsr
2927	ret
2928	SET_SIZE(wrmsr)
2929
2930	ENTRY(xrdmsr)
2931	pushq	%rbp
2932	movq	%rsp, %rbp
2933	movl	%edi, %ecx
2934	movl	XMSR_ACCESS_VAL, %edi	/* this value is needed to access MSR */
2935	rdmsr
2936	shlq	$32, %rdx
2937	orq	%rdx, %rax
2938	leave
2939	ret
2940	SET_SIZE(xrdmsr)
2941
2942	ENTRY(xwrmsr)
2943	pushq	%rbp
2944	movq	%rsp, %rbp
2945	movl	%edi, %ecx
2946	movl	XMSR_ACCESS_VAL, %edi	/* this value is needed to access MSR */
2947	movq	%rsi, %rdx
2948	shrq	$32, %rdx
2949	movl	%esi, %eax
2950	wrmsr
2951	leave
2952	ret
2953	SET_SIZE(xwrmsr)
2954
2955	ENTRY(get_xcr)
2956	movl	%edi, %ecx
2957	#xgetbv
2958	.byte	0x0f,0x01,0xd0
2959	shlq	$32, %rdx
2960	orq	%rdx, %rax
2961	ret
2962	SET_SIZE(get_xcr)
2963
2964	ENTRY(set_xcr)
2965	movq	%rsi, %rdx
2966	shrq	$32, %rdx
2967	movl	%esi, %eax
2968	movl	%edi, %ecx
2969	#xsetbv
2970	.byte	0x0f,0x01,0xd1
2971	ret
2972	SET_SIZE(set_xcr)
2973
2974#elif defined(__i386)
2975
2976	ENTRY(rdmsr)
2977	movl	4(%esp), %ecx
2978	rdmsr
2979	ret
2980	SET_SIZE(rdmsr)
2981
2982	ENTRY(wrmsr)
2983	movl	4(%esp), %ecx
2984	movl	8(%esp), %eax
2985	movl	12(%esp), %edx
2986	wrmsr
2987	ret
2988	SET_SIZE(wrmsr)
2989
2990	ENTRY(xrdmsr)
2991	pushl	%ebp
2992	movl	%esp, %ebp
2993	movl	8(%esp), %ecx
2994	pushl	%edi
2995	movl	XMSR_ACCESS_VAL, %edi	/* this value is needed to access MSR */
2996	rdmsr
2997	popl	%edi
2998	leave
2999	ret
3000	SET_SIZE(xrdmsr)
3001
3002	ENTRY(xwrmsr)
3003	pushl	%ebp
3004	movl	%esp, %ebp
3005	movl	8(%esp), %ecx
3006	movl	12(%esp), %eax
3007	movl	16(%esp), %edx
3008	pushl	%edi
3009	movl	XMSR_ACCESS_VAL, %edi	/* this value is needed to access MSR */
3010	wrmsr
3011	popl	%edi
3012	leave
3013	ret
3014	SET_SIZE(xwrmsr)
3015
3016	ENTRY(get_xcr)
3017	movl	4(%esp), %ecx
3018	#xgetbv
3019	.byte	0x0f,0x01,0xd0
3020	ret
3021	SET_SIZE(get_xcr)
3022
3023	ENTRY(set_xcr)
3024	movl	4(%esp), %ecx
3025	movl	8(%esp), %eax
3026	movl	12(%esp), %edx
3027	#xsetbv
3028	.byte	0x0f,0x01,0xd1
3029	ret
3030	SET_SIZE(set_xcr)
3031
3032#endif	/* __i386 */
3033
3034	ENTRY(invalidate_cache)
3035	wbinvd
3036	ret
3037	SET_SIZE(invalidate_cache)
3038
3039#endif	/* __lint */
3040
3041#if defined(__lint)
3042
3043/*ARGSUSED*/
3044void
3045getcregs(struct cregs *crp)
3046{}
3047
3048#else	/* __lint */
3049
3050#if defined(__amd64)
3051
3052	ENTRY_NP(getcregs)
3053#if defined(__xpv)
3054	/*
3055	 * Only a few of the hardware control registers or descriptor tables
3056	 * are directly accessible to us, so just zero the structure.
3057	 *
3058	 * XXPV	Perhaps it would be helpful for the hypervisor to return
3059	 *	virtualized versions of these for post-mortem use.
3060	 *	(Need to reevaluate - perhaps it already does!)
3061	 */
3062	pushq	%rdi		/* save *crp */
3063	movq	$CREGSZ, %rsi
3064	call	bzero
3065	popq	%rdi
3066
3067	/*
3068	 * Dump what limited information we can
3069	 */
3070	movq	%cr0, %rax
3071	movq	%rax, CREG_CR0(%rdi)	/* cr0 */
3072	movq	%cr2, %rax
3073	movq	%rax, CREG_CR2(%rdi)	/* cr2 */
3074	movq	%cr3, %rax
3075	movq	%rax, CREG_CR3(%rdi)	/* cr3 */
3076	movq	%cr4, %rax
3077	movq	%rax, CREG_CR4(%rdi)	/* cr4 */
3078
3079#else	/* __xpv */
3080
3081#define	GETMSR(r, off, d)	\
3082	movl	$r, %ecx;	\
3083	rdmsr;			\
3084	movl	%eax, off(d);	\
3085	movl	%edx, off+4(d)
3086
3087	xorl	%eax, %eax
3088	movq	%rax, CREG_GDT+8(%rdi)
3089	sgdt	CREG_GDT(%rdi)		/* 10 bytes */
3090	movq	%rax, CREG_IDT+8(%rdi)
3091	sidt	CREG_IDT(%rdi)		/* 10 bytes */
3092	movq	%rax, CREG_LDT(%rdi)
3093	sldt	CREG_LDT(%rdi)		/* 2 bytes */
3094	movq	%rax, CREG_TASKR(%rdi)
3095	str	CREG_TASKR(%rdi)	/* 2 bytes */
3096	movq	%cr0, %rax
3097	movq	%rax, CREG_CR0(%rdi)	/* cr0 */
3098	movq	%cr2, %rax
3099	movq	%rax, CREG_CR2(%rdi)	/* cr2 */
3100	movq	%cr3, %rax
3101	movq	%rax, CREG_CR3(%rdi)	/* cr3 */
3102	movq	%cr4, %rax
3103	movq	%rax, CREG_CR4(%rdi)	/* cr4 */
3104	movq	%cr8, %rax
3105	movq	%rax, CREG_CR8(%rdi)	/* cr8 */
3106	GETMSR(MSR_AMD_KGSBASE, CREG_KGSBASE, %rdi)
3107	GETMSR(MSR_AMD_EFER, CREG_EFER, %rdi)
3108#endif	/* __xpv */
3109	ret
3110	SET_SIZE(getcregs)
3111
3112#undef GETMSR
3113
3114#elif defined(__i386)
3115
3116	ENTRY_NP(getcregs)
3117#if defined(__xpv)
3118	/*
3119	 * Only a few of the hardware control registers or descriptor tables
3120	 * are directly accessible to us, so just zero the structure.
3121	 *
3122	 * XXPV	Perhaps it would be helpful for the hypervisor to return
3123	 *	virtualized versions of these for post-mortem use.
3124	 *	(Need to reevaluate - perhaps it already does!)
3125	 */
3126	movl	4(%esp), %edx
3127	pushl	$CREGSZ
3128	pushl	%edx
3129	call	bzero
3130	addl	$8, %esp
3131	movl	4(%esp), %edx
3132
3133	/*
3134	 * Dump what limited information we can
3135	 */
3136	movl	%cr0, %eax
3137	movl	%eax, CREG_CR0(%edx)	/* cr0 */
3138	movl	%cr2, %eax
3139	movl	%eax, CREG_CR2(%edx)	/* cr2 */
3140	movl	%cr3, %eax
3141	movl	%eax, CREG_CR3(%edx)	/* cr3 */
3142	movl	%cr4, %eax
3143	movl	%eax, CREG_CR4(%edx)	/* cr4 */
3144
3145#else	/* __xpv */
3146
3147	movl	4(%esp), %edx
3148	movw	$0, CREG_GDT+6(%edx)
3149	movw	$0, CREG_IDT+6(%edx)
3150	sgdt	CREG_GDT(%edx)		/* gdt */
3151	sidt	CREG_IDT(%edx)		/* idt */
3152	sldt	CREG_LDT(%edx)		/* ldt */
3153	str	CREG_TASKR(%edx)	/* task */
3154	movl	%cr0, %eax
3155	movl	%eax, CREG_CR0(%edx)	/* cr0 */
3156	movl	%cr2, %eax
3157	movl	%eax, CREG_CR2(%edx)	/* cr2 */
3158	movl	%cr3, %eax
3159	movl	%eax, CREG_CR3(%edx)	/* cr3 */
3160	bt	$X86FSET_LARGEPAGE, x86_featureset
3161	jnc	.nocr4
3162	movl	%cr4, %eax
3163	movl	%eax, CREG_CR4(%edx)	/* cr4 */
3164	jmp	.skip
3165.nocr4:
3166	movl	$0, CREG_CR4(%edx)
3167.skip:
3168#endif
3169	ret
3170	SET_SIZE(getcregs)
3171
3172#endif	/* __i386 */
3173#endif	/* __lint */
3174
3175
3176/*
3177 * A panic trigger is a word which is updated atomically and can only be set
3178 * once.  We atomically store 0xDEFACEDD and load the old value.  If the
3179 * previous value was 0, we succeed and return 1; otherwise return 0.
3180 * This allows a partially corrupt trigger to still trigger correctly.  DTrace
3181 * has its own version of this function to allow it to panic correctly from
3182 * probe context.
3183 */
3184#if defined(__lint)
3185
3186/*ARGSUSED*/
3187int
3188panic_trigger(int *tp)
3189{ return (0); }
3190
3191/*ARGSUSED*/
3192int
3193dtrace_panic_trigger(int *tp)
3194{ return (0); }
3195
3196#else	/* __lint */
3197
3198#if defined(__amd64)
3199
3200	ENTRY_NP(panic_trigger)
3201	xorl	%eax, %eax
3202	movl	$0xdefacedd, %edx
3203	lock
3204	  xchgl	%edx, (%rdi)
3205	cmpl	$0, %edx
3206	je	0f
3207	movl	$0, %eax
3208	ret
32090:	movl	$1, %eax
3210	ret
3211	SET_SIZE(panic_trigger)
3212
3213	ENTRY_NP(dtrace_panic_trigger)
3214	xorl	%eax, %eax
3215	movl	$0xdefacedd, %edx
3216	lock
3217	  xchgl	%edx, (%rdi)
3218	cmpl	$0, %edx
3219	je	0f
3220	movl	$0, %eax
3221	ret
32220:	movl	$1, %eax
3223	ret
3224	SET_SIZE(dtrace_panic_trigger)
3225
3226#elif defined(__i386)
3227
3228	ENTRY_NP(panic_trigger)
3229	movl	4(%esp), %edx		/ %edx = address of trigger
3230	movl	$0xdefacedd, %eax	/ %eax = 0xdefacedd
3231	lock				/ assert lock
3232	xchgl %eax, (%edx)		/ exchange %eax and the trigger
3233	cmpl	$0, %eax		/ if (%eax == 0x0)
3234	je	0f			/   return (1);
3235	movl	$0, %eax		/ else
3236	ret				/   return (0);
32370:	movl	$1, %eax
3238	ret
3239	SET_SIZE(panic_trigger)
3240
3241	ENTRY_NP(dtrace_panic_trigger)
3242	movl	4(%esp), %edx		/ %edx = address of trigger
3243	movl	$0xdefacedd, %eax	/ %eax = 0xdefacedd
3244	lock				/ assert lock
3245	xchgl %eax, (%edx)		/ exchange %eax and the trigger
3246	cmpl	$0, %eax		/ if (%eax == 0x0)
3247	je	0f			/   return (1);
3248	movl	$0, %eax		/ else
3249	ret				/   return (0);
32500:	movl	$1, %eax
3251	ret
3252	SET_SIZE(dtrace_panic_trigger)
3253
3254#endif	/* __i386 */
3255#endif	/* __lint */
3256
3257/*
3258 * The panic() and cmn_err() functions invoke vpanic() as a common entry point
3259 * into the panic code implemented in panicsys().  vpanic() is responsible
3260 * for passing through the format string and arguments, and constructing a
3261 * regs structure on the stack into which it saves the current register
3262 * values.  If we are not dying due to a fatal trap, these registers will
3263 * then be preserved in panicbuf as the current processor state.  Before
3264 * invoking panicsys(), vpanic() activates the first panic trigger (see
3265 * common/os/panic.c) and switches to the panic_stack if successful.  Note that
3266 * DTrace takes a slightly different panic path if it must panic from probe
3267 * context.  Instead of calling panic, it calls into dtrace_vpanic(), which
3268 * sets up the initial stack as vpanic does, calls dtrace_panic_trigger(), and
3269 * branches back into vpanic().
3270 */
3271#if defined(__lint)
3272
3273/*ARGSUSED*/
3274void
3275vpanic(const char *format, va_list alist)
3276{}
3277
3278/*ARGSUSED*/
3279void
3280dtrace_vpanic(const char *format, va_list alist)
3281{}
3282
3283#else	/* __lint */
3284
3285#if defined(__amd64)
3286
3287	ENTRY_NP(vpanic)			/* Initial stack layout: */
3288
3289	pushq	%rbp				/* | %rip |	0x60	*/
3290	movq	%rsp, %rbp			/* | %rbp |	0x58	*/
3291	pushfq					/* | rfl  |	0x50	*/
3292	pushq	%r11				/* | %r11 |	0x48	*/
3293	pushq	%r10				/* | %r10 |	0x40	*/
3294	pushq	%rbx				/* | %rbx |	0x38	*/
3295	pushq	%rax				/* | %rax |	0x30	*/
3296	pushq	%r9				/* | %r9  |	0x28	*/
3297	pushq	%r8				/* | %r8  |	0x20	*/
3298	pushq	%rcx				/* | %rcx |	0x18	*/
3299	pushq	%rdx				/* | %rdx |	0x10	*/
3300	pushq	%rsi				/* | %rsi |	0x8 alist */
3301	pushq	%rdi				/* | %rdi |	0x0 format */
3302
3303	movq	%rsp, %rbx			/* %rbx = current %rsp */
3304
3305	leaq	panic_quiesce(%rip), %rdi	/* %rdi = &panic_quiesce */
3306	call	panic_trigger			/* %eax = panic_trigger() */
3307
3308vpanic_common:
3309	/*
3310	 * The panic_trigger result is in %eax from the call above, and
3311	 * dtrace_panic places it in %eax before branching here.
3312	 * The rdmsr instructions that follow below will clobber %eax so
3313	 * we stash the panic_trigger result in %r11d.
3314	 */
3315	movl	%eax, %r11d
3316	cmpl	$0, %r11d
3317	je	0f
3318
3319	/*
3320	 * If panic_trigger() was successful, we are the first to initiate a
3321	 * panic: we now switch to the reserved panic_stack before continuing.
3322	 */
3323	leaq	panic_stack(%rip), %rsp
3324	addq	$PANICSTKSIZE, %rsp
33250:	subq	$REGSIZE, %rsp
3326	/*
3327	 * Now that we've got everything set up, store the register values as
3328	 * they were when we entered vpanic() to the designated location in
3329	 * the regs structure we allocated on the stack.
3330	 */
3331	movq	0x0(%rbx), %rcx
3332	movq	%rcx, REGOFF_RDI(%rsp)
3333	movq	0x8(%rbx), %rcx
3334	movq	%rcx, REGOFF_RSI(%rsp)
3335	movq	0x10(%rbx), %rcx
3336	movq	%rcx, REGOFF_RDX(%rsp)
3337	movq	0x18(%rbx), %rcx
3338	movq	%rcx, REGOFF_RCX(%rsp)
3339	movq	0x20(%rbx), %rcx
3340
3341	movq	%rcx, REGOFF_R8(%rsp)
3342	movq	0x28(%rbx), %rcx
3343	movq	%rcx, REGOFF_R9(%rsp)
3344	movq	0x30(%rbx), %rcx
3345	movq	%rcx, REGOFF_RAX(%rsp)
3346	movq	0x38(%rbx), %rcx
3347	movq	%rcx, REGOFF_RBX(%rsp)
3348	movq	0x58(%rbx), %rcx
3349
3350	movq	%rcx, REGOFF_RBP(%rsp)
3351	movq	0x40(%rbx), %rcx
3352	movq	%rcx, REGOFF_R10(%rsp)
3353	movq	0x48(%rbx), %rcx
3354	movq	%rcx, REGOFF_R11(%rsp)
3355	movq	%r12, REGOFF_R12(%rsp)
3356
3357	movq	%r13, REGOFF_R13(%rsp)
3358	movq	%r14, REGOFF_R14(%rsp)
3359	movq	%r15, REGOFF_R15(%rsp)
3360
3361	xorl	%ecx, %ecx
3362	movw	%ds, %cx
3363	movq	%rcx, REGOFF_DS(%rsp)
3364	movw	%es, %cx
3365	movq	%rcx, REGOFF_ES(%rsp)
3366	movw	%fs, %cx
3367	movq	%rcx, REGOFF_FS(%rsp)
3368	movw	%gs, %cx
3369	movq	%rcx, REGOFF_GS(%rsp)
3370
3371	movq	$0, REGOFF_TRAPNO(%rsp)
3372
3373	movq	$0, REGOFF_ERR(%rsp)
3374	leaq	vpanic(%rip), %rcx
3375	movq	%rcx, REGOFF_RIP(%rsp)
3376	movw	%cs, %cx
3377	movzwq	%cx, %rcx
3378	movq	%rcx, REGOFF_CS(%rsp)
3379	movq	0x50(%rbx), %rcx
3380	movq	%rcx, REGOFF_RFL(%rsp)
3381	movq	%rbx, %rcx
3382	addq	$0x60, %rcx
3383	movq	%rcx, REGOFF_RSP(%rsp)
3384	movw	%ss, %cx
3385	movzwq	%cx, %rcx
3386	movq	%rcx, REGOFF_SS(%rsp)
3387
3388	/*
3389	 * panicsys(format, alist, rp, on_panic_stack)
3390	 */
3391	movq	REGOFF_RDI(%rsp), %rdi		/* format */
3392	movq	REGOFF_RSI(%rsp), %rsi		/* alist */
3393	movq	%rsp, %rdx			/* struct regs */
3394	movl	%r11d, %ecx			/* on_panic_stack */
3395	call	panicsys
3396	addq	$REGSIZE, %rsp
3397	popq	%rdi
3398	popq	%rsi
3399	popq	%rdx
3400	popq	%rcx
3401	popq	%r8
3402	popq	%r9
3403	popq	%rax
3404	popq	%rbx
3405	popq	%r10
3406	popq	%r11
3407	popfq
3408	leave
3409	ret
3410	SET_SIZE(vpanic)
3411
3412	ENTRY_NP(dtrace_vpanic)			/* Initial stack layout: */
3413
3414	pushq	%rbp				/* | %rip |	0x60	*/
3415	movq	%rsp, %rbp			/* | %rbp |	0x58	*/
3416	pushfq					/* | rfl  |	0x50	*/
3417	pushq	%r11				/* | %r11 |	0x48	*/
3418	pushq	%r10				/* | %r10 |	0x40	*/
3419	pushq	%rbx				/* | %rbx |	0x38	*/
3420	pushq	%rax				/* | %rax |	0x30	*/
3421	pushq	%r9				/* | %r9  |	0x28	*/
3422	pushq	%r8				/* | %r8  |	0x20	*/
3423	pushq	%rcx				/* | %rcx |	0x18	*/
3424	pushq	%rdx				/* | %rdx |	0x10	*/
3425	pushq	%rsi				/* | %rsi |	0x8 alist */
3426	pushq	%rdi				/* | %rdi |	0x0 format */
3427
3428	movq	%rsp, %rbx			/* %rbx = current %rsp */
3429
3430	leaq	panic_quiesce(%rip), %rdi	/* %rdi = &panic_quiesce */
3431	call	dtrace_panic_trigger	/* %eax = dtrace_panic_trigger() */
3432	jmp	vpanic_common
3433
3434	SET_SIZE(dtrace_vpanic)
3435
3436#elif defined(__i386)
3437
3438	ENTRY_NP(vpanic)			/ Initial stack layout:
3439
3440	pushl	%ebp				/ | %eip | 20
3441	movl	%esp, %ebp			/ | %ebp | 16
3442	pushl	%eax				/ | %eax | 12
3443	pushl	%ebx				/ | %ebx |  8
3444	pushl	%ecx				/ | %ecx |  4
3445	pushl	%edx				/ | %edx |  0
3446
3447	movl	%esp, %ebx			/ %ebx = current stack pointer
3448
3449	lea	panic_quiesce, %eax		/ %eax = &panic_quiesce
3450	pushl	%eax				/ push &panic_quiesce
3451	call	panic_trigger			/ %eax = panic_trigger()
3452	addl	$4, %esp			/ reset stack pointer
3453
3454vpanic_common:
3455	cmpl	$0, %eax			/ if (%eax == 0)
3456	je	0f				/   goto 0f;
3457
3458	/*
3459	 * If panic_trigger() was successful, we are the first to initiate a
3460	 * panic: we now switch to the reserved panic_stack before continuing.
3461	 */
3462	lea	panic_stack, %esp		/ %esp  = panic_stack
3463	addl	$PANICSTKSIZE, %esp		/ %esp += PANICSTKSIZE
3464
34650:	subl	$REGSIZE, %esp			/ allocate struct regs
3466
3467	/*
3468	 * Now that we've got everything set up, store the register values as
3469	 * they were when we entered vpanic() to the designated location in
3470	 * the regs structure we allocated on the stack.
3471	 */
3472#if !defined(__GNUC_AS__)
3473	movw	%gs, %edx
3474	movl	%edx, REGOFF_GS(%esp)
3475	movw	%fs, %edx
3476	movl	%edx, REGOFF_FS(%esp)
3477	movw	%es, %edx
3478	movl	%edx, REGOFF_ES(%esp)
3479	movw	%ds, %edx
3480	movl	%edx, REGOFF_DS(%esp)
3481#else	/* __GNUC_AS__ */
3482	mov	%gs, %edx
3483	mov	%edx, REGOFF_GS(%esp)
3484	mov	%fs, %edx
3485	mov	%edx, REGOFF_FS(%esp)
3486	mov	%es, %edx
3487	mov	%edx, REGOFF_ES(%esp)
3488	mov	%ds, %edx
3489	mov	%edx, REGOFF_DS(%esp)
3490#endif	/* __GNUC_AS__ */
3491	movl	%edi, REGOFF_EDI(%esp)
3492	movl	%esi, REGOFF_ESI(%esp)
3493	movl	16(%ebx), %ecx
3494	movl	%ecx, REGOFF_EBP(%esp)
3495	movl	%ebx, %ecx
3496	addl	$20, %ecx
3497	movl	%ecx, REGOFF_ESP(%esp)
3498	movl	8(%ebx), %ecx
3499	movl	%ecx, REGOFF_EBX(%esp)
3500	movl	0(%ebx), %ecx
3501	movl	%ecx, REGOFF_EDX(%esp)
3502	movl	4(%ebx), %ecx
3503	movl	%ecx, REGOFF_ECX(%esp)
3504	movl	12(%ebx), %ecx
3505	movl	%ecx, REGOFF_EAX(%esp)
3506	movl	$0, REGOFF_TRAPNO(%esp)
3507	movl	$0, REGOFF_ERR(%esp)
3508	lea	vpanic, %ecx
3509	movl	%ecx, REGOFF_EIP(%esp)
3510#if !defined(__GNUC_AS__)
3511	movw	%cs, %edx
3512#else	/* __GNUC_AS__ */
3513	mov	%cs, %edx
3514#endif	/* __GNUC_AS__ */
3515	movl	%edx, REGOFF_CS(%esp)
3516	pushfl
3517	popl	%ecx
3518#if defined(__xpv)
3519	/*
3520	 * Synthesize the PS_IE bit from the event mask bit
3521	 */
3522	CURTHREAD(%edx)
3523	KPREEMPT_DISABLE(%edx)
3524	EVENT_MASK_TO_IE(%edx, %ecx)
3525	CURTHREAD(%edx)
3526	KPREEMPT_ENABLE_NOKP(%edx)
3527#endif
3528	movl	%ecx, REGOFF_EFL(%esp)
3529	movl	$0, REGOFF_UESP(%esp)
3530#if !defined(__GNUC_AS__)
3531	movw	%ss, %edx
3532#else	/* __GNUC_AS__ */
3533	mov	%ss, %edx
3534#endif	/* __GNUC_AS__ */
3535	movl	%edx, REGOFF_SS(%esp)
3536
3537	movl	%esp, %ecx			/ %ecx = &regs
3538	pushl	%eax				/ push on_panic_stack
3539	pushl	%ecx				/ push &regs
3540	movl	12(%ebp), %ecx			/ %ecx = alist
3541	pushl	%ecx				/ push alist
3542	movl	8(%ebp), %ecx			/ %ecx = format
3543	pushl	%ecx				/ push format
3544	call	panicsys			/ panicsys();
3545	addl	$16, %esp			/ pop arguments
3546
3547	addl	$REGSIZE, %esp
3548	popl	%edx
3549	popl	%ecx
3550	popl	%ebx
3551	popl	%eax
3552	leave
3553	ret
3554	SET_SIZE(vpanic)
3555
3556	ENTRY_NP(dtrace_vpanic)			/ Initial stack layout:
3557
3558	pushl	%ebp				/ | %eip | 20
3559	movl	%esp, %ebp			/ | %ebp | 16
3560	pushl	%eax				/ | %eax | 12
3561	pushl	%ebx				/ | %ebx |  8
3562	pushl	%ecx				/ | %ecx |  4
3563	pushl	%edx				/ | %edx |  0
3564
3565	movl	%esp, %ebx			/ %ebx = current stack pointer
3566
3567	lea	panic_quiesce, %eax		/ %eax = &panic_quiesce
3568	pushl	%eax				/ push &panic_quiesce
3569	call	dtrace_panic_trigger		/ %eax = dtrace_panic_trigger()
3570	addl	$4, %esp			/ reset stack pointer
3571	jmp	vpanic_common			/ jump back to common code
3572
3573	SET_SIZE(dtrace_vpanic)
3574
3575#endif	/* __i386 */
3576#endif	/* __lint */
3577
3578#if defined(__lint)
3579
3580void
3581hres_tick(void)
3582{}
3583
3584int64_t timedelta;
3585hrtime_t hrtime_base;
3586
3587#else	/* __lint */
3588
3589	DGDEF3(timedelta, 8, 8)
3590	.long	0, 0
3591
3592	/*
3593	 * initialized to a non zero value to make pc_gethrtime()
3594	 * work correctly even before clock is initialized
3595	 */
3596	DGDEF3(hrtime_base, 8, 8)
3597	.long	_MUL(NSEC_PER_CLOCK_TICK, 6), 0
3598
3599	DGDEF3(adj_shift, 4, 4)
3600	.long	ADJ_SHIFT
3601
3602#if defined(__amd64)
3603
3604	ENTRY_NP(hres_tick)
3605	pushq	%rbp
3606	movq	%rsp, %rbp
3607
3608	/*
3609	 * We need to call *gethrtimef before picking up CLOCK_LOCK (obviously,
3610	 * hres_last_tick can only be modified while holding CLOCK_LOCK).
3611	 * At worst, performing this now instead of under CLOCK_LOCK may
3612	 * introduce some jitter in pc_gethrestime().
3613	 */
3614	movq	gethrtimef(%rip), %rsi
3615	INDIRECT_CALL_REG(rsi)
3616	movq	%rax, %r8
3617
3618	leaq	hres_lock(%rip), %rax
3619	movb	$-1, %dl
3620.CL1:
3621	xchgb	%dl, (%rax)
3622	testb	%dl, %dl
3623	jz	.CL3			/* got it */
3624.CL2:
3625	cmpb	$0, (%rax)		/* possible to get lock? */
3626	pause
3627	jne	.CL2
3628	jmp	.CL1			/* yes, try again */
3629.CL3:
3630	/*
3631	 * compute the interval since last time hres_tick was called
3632	 * and adjust hrtime_base and hrestime accordingly
3633	 * hrtime_base is an 8 byte value (in nsec), hrestime is
3634	 * a timestruc_t (sec, nsec)
3635	 */
3636	leaq	hres_last_tick(%rip), %rax
3637	movq	%r8, %r11
3638	subq	(%rax), %r8
3639	addq	%r8, hrtime_base(%rip)	/* add interval to hrtime_base */
3640	addq	%r8, hrestime+8(%rip)	/* add interval to hrestime.tv_nsec */
3641	/*
3642	 * Now that we have CLOCK_LOCK, we can update hres_last_tick
3643	 */
3644	movq	%r11, (%rax)
3645
3646	call	__adj_hrestime
3647
3648	/*
3649	 * release the hres_lock
3650	 */
3651	incl	hres_lock(%rip)
3652	leave
3653	ret
3654	SET_SIZE(hres_tick)
3655
3656#elif defined(__i386)
3657
3658	ENTRY_NP(hres_tick)
3659	pushl	%ebp
3660	movl	%esp, %ebp
3661	pushl	%esi
3662	pushl	%ebx
3663
3664	/*
3665	 * We need to call *gethrtimef before picking up CLOCK_LOCK (obviously,
3666	 * hres_last_tick can only be modified while holding CLOCK_LOCK).
3667	 * At worst, performing this now instead of under CLOCK_LOCK may
3668	 * introduce some jitter in pc_gethrestime().
3669	 */
3670	call	*gethrtimef
3671	movl	%eax, %ebx
3672	movl	%edx, %esi
3673
3674	movl	$hres_lock, %eax
3675	movl	$-1, %edx
3676.CL1:
3677	xchgb	%dl, (%eax)
3678	testb	%dl, %dl
3679	jz	.CL3			/ got it
3680.CL2:
3681	cmpb	$0, (%eax)		/ possible to get lock?
3682	pause
3683	jne	.CL2
3684	jmp	.CL1			/ yes, try again
3685.CL3:
3686	/*
3687	 * compute the interval since last time hres_tick was called
3688	 * and adjust hrtime_base and hrestime accordingly
3689	 * hrtime_base is an 8 byte value (in nsec), hrestime is
3690	 * timestruc_t (sec, nsec)
3691	 */
3692
3693	lea	hres_last_tick, %eax
3694
3695	movl	%ebx, %edx
3696	movl	%esi, %ecx
3697
3698	subl	(%eax), %edx
3699	sbbl	4(%eax), %ecx
3700
3701	addl	%edx, hrtime_base	/ add interval to hrtime_base
3702	adcl	%ecx, hrtime_base+4
3703
3704	addl	%edx, hrestime+4	/ add interval to hrestime.tv_nsec
3705
3706	/
3707	/ Now that we have CLOCK_LOCK, we can update hres_last_tick.
3708	/
3709	movl	%ebx, (%eax)
3710	movl	%esi,  4(%eax)
3711
3712	/ get hrestime at this moment. used as base for pc_gethrestime
3713	/
3714	/ Apply adjustment, if any
3715	/
3716	/ #define HRES_ADJ	(NSEC_PER_CLOCK_TICK >> ADJ_SHIFT)
3717	/ (max_hres_adj)
3718	/
3719	/ void
3720	/ adj_hrestime()
3721	/ {
3722	/	long long adj;
3723	/
3724	/	if (hrestime_adj == 0)
3725	/		adj = 0;
3726	/	else if (hrestime_adj > 0) {
3727	/		if (hrestime_adj < HRES_ADJ)
3728	/			adj = hrestime_adj;
3729	/		else
3730	/			adj = HRES_ADJ;
3731	/	}
3732	/	else {
3733	/		if (hrestime_adj < -(HRES_ADJ))
3734	/			adj = -(HRES_ADJ);
3735	/		else
3736	/			adj = hrestime_adj;
3737	/	}
3738	/
3739	/	timedelta -= adj;
3740	/	hrestime_adj = timedelta;
3741	/	hrestime.tv_nsec += adj;
3742	/
3743	/	while (hrestime.tv_nsec >= NANOSEC) {
3744	/		one_sec++;
3745	/		hrestime.tv_sec++;
3746	/		hrestime.tv_nsec -= NANOSEC;
3747	/	}
3748	/ }
3749__adj_hrestime:
3750	movl	hrestime_adj, %esi	/ if (hrestime_adj == 0)
3751	movl	hrestime_adj+4, %edx
3752	andl	%esi, %esi
3753	jne	.CL4			/ no
3754	andl	%edx, %edx
3755	jne	.CL4			/ no
3756	subl	%ecx, %ecx		/ yes, adj = 0;
3757	subl	%edx, %edx
3758	jmp	.CL5
3759.CL4:
3760	subl	%ecx, %ecx
3761	subl	%eax, %eax
3762	subl	%esi, %ecx
3763	sbbl	%edx, %eax
3764	andl	%eax, %eax		/ if (hrestime_adj > 0)
3765	jge	.CL6
3766
3767	/ In the following comments, HRES_ADJ is used, while in the code
3768	/ max_hres_adj is used.
3769	/
3770	/ The test for "hrestime_adj < HRES_ADJ" is complicated because
3771	/ hrestime_adj is 64-bits, while HRES_ADJ is 32-bits.  We rely
3772	/ on the logical equivalence of:
3773	/
3774	/	!(hrestime_adj < HRES_ADJ)
3775	/
3776	/ and the two step sequence:
3777	/
3778	/	(HRES_ADJ - lsw(hrestime_adj)) generates a Borrow/Carry
3779	/
3780	/ which computes whether or not the least significant 32-bits
3781	/ of hrestime_adj is greater than HRES_ADJ, followed by:
3782	/
3783	/	Previous Borrow/Carry + -1 + msw(hrestime_adj) generates a Carry
3784	/
3785	/ which generates a carry whenever step 1 is true or the most
3786	/ significant long of the longlong hrestime_adj is non-zero.
3787
3788	movl	max_hres_adj, %ecx	/ hrestime_adj is positive
3789	subl	%esi, %ecx
3790	movl	%edx, %eax
3791	adcl	$-1, %eax
3792	jnc	.CL7
3793	movl	max_hres_adj, %ecx	/ adj = HRES_ADJ;
3794	subl	%edx, %edx
3795	jmp	.CL5
3796
3797	/ The following computation is similar to the one above.
3798	/
3799	/ The test for "hrestime_adj < -(HRES_ADJ)" is complicated because
3800	/ hrestime_adj is 64-bits, while HRES_ADJ is 32-bits.  We rely
3801	/ on the logical equivalence of:
3802	/
3803	/	(hrestime_adj > -HRES_ADJ)
3804	/
3805	/ and the two step sequence:
3806	/
3807	/	(HRES_ADJ + lsw(hrestime_adj)) generates a Carry
3808	/
3809	/ which means the least significant 32-bits of hrestime_adj is
3810	/ greater than -HRES_ADJ, followed by:
3811	/
3812	/	Previous Carry + 0 + msw(hrestime_adj) generates a Carry
3813	/
3814	/ which generates a carry only when step 1 is true and the most
3815	/ significant long of the longlong hrestime_adj is -1.
3816
3817.CL6:					/ hrestime_adj is negative
3818	movl	%esi, %ecx
3819	addl	max_hres_adj, %ecx
3820	movl	%edx, %eax
3821	adcl	$0, %eax
3822	jc	.CL7
3823	xor	%ecx, %ecx
3824	subl	max_hres_adj, %ecx	/ adj = -(HRES_ADJ);
3825	movl	$-1, %edx
3826	jmp	.CL5
3827.CL7:
3828	movl	%esi, %ecx		/ adj = hrestime_adj;
3829.CL5:
3830	movl	timedelta, %esi
3831	subl	%ecx, %esi
3832	movl	timedelta+4, %eax
3833	sbbl	%edx, %eax
3834	movl	%esi, timedelta
3835	movl	%eax, timedelta+4	/ timedelta -= adj;
3836	movl	%esi, hrestime_adj
3837	movl	%eax, hrestime_adj+4	/ hrestime_adj = timedelta;
3838	addl	hrestime+4, %ecx
3839
3840	movl	%ecx, %eax		/ eax = tv_nsec
38411:
3842	cmpl	$NANOSEC, %eax		/ if ((unsigned long)tv_nsec >= NANOSEC)
3843	jb	.CL8			/ no
3844	incl	one_sec			/ yes,  one_sec++;
3845	incl	hrestime		/ hrestime.tv_sec++;
3846	addl	$-NANOSEC, %eax		/ tv_nsec -= NANOSEC
3847	jmp	1b			/ check for more seconds
3848
3849.CL8:
3850	movl	%eax, hrestime+4	/ store final into hrestime.tv_nsec
3851	incl	hres_lock		/ release the hres_lock
3852
3853	popl	%ebx
3854	popl	%esi
3855	leave
3856	ret
3857	SET_SIZE(hres_tick)
3858
3859#endif	/* __i386 */
3860#endif	/* __lint */
3861
3862/*
3863 * void prefetch_smap_w(void *)
3864 *
3865 * Prefetch ahead within a linear list of smap structures.
3866 * Not implemented for ia32.  Stub for compatibility.
3867 */
3868
3869#if defined(__lint)
3870
3871/*ARGSUSED*/
3872void prefetch_smap_w(void *smp)
3873{}
3874
3875#else	/* __lint */
3876
3877	ENTRY(prefetch_smap_w)
3878	rep;	ret	/* use 2 byte return instruction when branch target */
3879			/* AMD Software Optimization Guide - Section 6.2 */
3880	SET_SIZE(prefetch_smap_w)
3881
3882#endif	/* __lint */
3883
3884/*
3885 * prefetch_page_r(page_t *)
3886 * issue prefetch instructions for a page_t
3887 */
3888#if defined(__lint)
3889
3890/*ARGSUSED*/
3891void
3892prefetch_page_r(void *pp)
3893{}
3894
3895#else	/* __lint */
3896
3897	ENTRY(prefetch_page_r)
3898	rep;	ret	/* use 2 byte return instruction when branch target */
3899			/* AMD Software Optimization Guide - Section 6.2 */
3900	SET_SIZE(prefetch_page_r)
3901
3902#endif	/* __lint */
3903
3904#if defined(__lint)
3905
3906/*ARGSUSED*/
3907int
3908bcmp(const void *s1, const void *s2, size_t count)
3909{ return (0); }
3910
3911#else   /* __lint */
3912
3913#if defined(__amd64)
3914
3915	ENTRY(bcmp)
3916	pushq	%rbp
3917	movq	%rsp, %rbp
3918#ifdef DEBUG
3919	testq	%rdx,%rdx
3920	je	1f
3921	movq	postbootkernelbase(%rip), %r11
3922	cmpq	%r11, %rdi
3923	jb	0f
3924	cmpq	%r11, %rsi
3925	jnb	1f
39260:	leaq	.bcmp_panic_msg(%rip), %rdi
3927	xorl	%eax, %eax
3928	call	panic
39291:
3930#endif	/* DEBUG */
3931	call	memcmp
3932	testl	%eax, %eax
3933	setne	%dl
3934	leave
3935	movzbl	%dl, %eax
3936	ret
3937	SET_SIZE(bcmp)
3938
3939#elif defined(__i386)
3940
3941#define	ARG_S1		8
3942#define	ARG_S2		12
3943#define	ARG_LENGTH	16
3944
3945	ENTRY(bcmp)
3946	pushl	%ebp
3947	movl	%esp, %ebp	/ create new stack frame
3948#ifdef DEBUG
3949	cmpl	$0, ARG_LENGTH(%ebp)
3950	je	1f
3951	movl    postbootkernelbase, %eax
3952	cmpl    %eax, ARG_S1(%ebp)
3953	jb	0f
3954	cmpl    %eax, ARG_S2(%ebp)
3955	jnb	1f
39560:	pushl   $.bcmp_panic_msg
3957	call    panic
39581:
3959#endif	/* DEBUG */
3960
3961	pushl	%edi		/ save register variable
3962	movl	ARG_S1(%ebp), %eax	/ %eax = address of string 1
3963	movl	ARG_S2(%ebp), %ecx	/ %ecx = address of string 2
3964	cmpl	%eax, %ecx	/ if the same string
3965	je	.equal		/ goto .equal
3966	movl	ARG_LENGTH(%ebp), %edi	/ %edi = length in bytes
3967	cmpl	$4, %edi	/ if %edi < 4
3968	jb	.byte_check	/ goto .byte_check
3969	.align	4
3970.word_loop:
3971	movl	(%ecx), %edx	/ move 1 word from (%ecx) to %edx
3972	leal	-4(%edi), %edi	/ %edi -= 4
3973	cmpl	(%eax), %edx	/ compare 1 word from (%eax) with %edx
3974	jne	.word_not_equal	/ if not equal, goto .word_not_equal
3975	leal	4(%ecx), %ecx	/ %ecx += 4 (next word)
3976	leal	4(%eax), %eax	/ %eax += 4 (next word)
3977	cmpl	$4, %edi	/ if %edi >= 4
3978	jae	.word_loop	/ goto .word_loop
3979.byte_check:
3980	cmpl	$0, %edi	/ if %edi == 0
3981	je	.equal		/ goto .equal
3982	jmp	.byte_loop	/ goto .byte_loop (checks in bytes)
3983.word_not_equal:
3984	leal	4(%edi), %edi	/ %edi += 4 (post-decremented)
3985	.align	4
3986.byte_loop:
3987	movb	(%ecx),	%dl	/ move 1 byte from (%ecx) to %dl
3988	cmpb	%dl, (%eax)	/ compare %dl with 1 byte from (%eax)
3989	jne	.not_equal	/ if not equal, goto .not_equal
3990	incl	%ecx		/ %ecx++ (next byte)
3991	incl	%eax		/ %eax++ (next byte)
3992	decl	%edi		/ %edi--
3993	jnz	.byte_loop	/ if not zero, goto .byte_loop
3994.equal:
3995	xorl	%eax, %eax	/ %eax = 0
3996	popl	%edi		/ restore register variable
3997	leave			/ restore old stack frame
3998	ret			/ return (NULL)
3999	.align	4
4000.not_equal:
4001	movl	$1, %eax	/ return 1
4002	popl	%edi		/ restore register variable
4003	leave			/ restore old stack frame
4004	ret			/ return (NULL)
4005	SET_SIZE(bcmp)
4006
4007#endif	/* __i386 */
4008
4009#ifdef DEBUG
4010	.text
4011.bcmp_panic_msg:
4012	.string "bcmp: arguments below kernelbase"
4013#endif	/* DEBUG */
4014
4015#endif	/* __lint */
4016
4017#if defined(__lint)
4018
4019uint_t
4020bsrw_insn(uint16_t mask)
4021{
4022	uint_t index = sizeof (mask) * NBBY - 1;
4023
4024	while ((mask & (1 << index)) == 0)
4025		index--;
4026	return (index);
4027}
4028
4029#else	/* __lint */
4030
4031#if defined(__amd64)
4032
4033	ENTRY_NP(bsrw_insn)
4034	xorl	%eax, %eax
4035	bsrw	%di, %ax
4036	ret
4037	SET_SIZE(bsrw_insn)
4038
4039#elif defined(__i386)
4040
4041	ENTRY_NP(bsrw_insn)
4042	movw	4(%esp), %cx
4043	xorl	%eax, %eax
4044	bsrw	%cx, %ax
4045	ret
4046	SET_SIZE(bsrw_insn)
4047
4048#endif	/* __i386 */
4049#endif	/* __lint */
4050
4051#if defined(__lint)
4052
4053uint_t
4054atomic_btr32(uint32_t *pending, uint_t pil)
4055{
4056	return (*pending &= ~(1 << pil));
4057}
4058
4059#else	/* __lint */
4060
4061#if defined(__i386)
4062
4063	ENTRY_NP(atomic_btr32)
4064	movl	4(%esp), %ecx
4065	movl	8(%esp), %edx
4066	xorl	%eax, %eax
4067	lock
4068	btrl	%edx, (%ecx)
4069	setc	%al
4070	ret
4071	SET_SIZE(atomic_btr32)
4072
4073#endif	/* __i386 */
4074#endif	/* __lint */
4075
4076#if defined(__lint)
4077
4078/*ARGSUSED*/
4079void
4080switch_sp_and_call(void *newsp, void (*func)(uint_t, uint_t), uint_t arg1,
4081	    uint_t arg2)
4082{}
4083
4084#else	/* __lint */
4085
4086#if defined(__amd64)
4087
4088	ENTRY_NP(switch_sp_and_call)
4089	pushq	%rbp
4090	movq	%rsp, %rbp		/* set up stack frame */
4091	movq	%rdi, %rsp		/* switch stack pointer */
4092	movq	%rdx, %rdi		/* pass func arg 1 */
4093	movq	%rsi, %r11		/* save function to call */
4094	movq	%rcx, %rsi		/* pass func arg 2 */
4095	INDIRECT_CALL_REG(r11)		/* call function */
4096	leave				/* restore stack */
4097	ret
4098	SET_SIZE(switch_sp_and_call)
4099
4100#elif defined(__i386)
4101
4102	ENTRY_NP(switch_sp_and_call)
4103	pushl	%ebp
4104	mov	%esp, %ebp		/* set up stack frame */
4105	movl	8(%ebp), %esp		/* switch stack pointer */
4106	pushl	20(%ebp)		/* push func arg 2 */
4107	pushl	16(%ebp)		/* push func arg 1 */
4108	call	*12(%ebp)		/* call function */
4109	addl	$8, %esp		/* pop arguments */
4110	leave				/* restore stack */
4111	ret
4112	SET_SIZE(switch_sp_and_call)
4113
4114#endif	/* __i386 */
4115#endif	/* __lint */
4116
4117#if defined(__lint)
4118
4119void
4120kmdb_enter(void)
4121{}
4122
4123#else	/* __lint */
4124
4125#if defined(__amd64)
4126
4127	ENTRY_NP(kmdb_enter)
4128	pushq	%rbp
4129	movq	%rsp, %rbp
4130
4131	/*
4132	 * Save flags, do a 'cli' then return the saved flags
4133	 */
4134	call	intr_clear
4135
4136	int	$T_DBGENTR
4137
4138	/*
4139	 * Restore the saved flags
4140	 */
4141	movq	%rax, %rdi
4142	call	intr_restore
4143
4144	leave
4145	ret
4146	SET_SIZE(kmdb_enter)
4147
4148#elif defined(__i386)
4149
4150	ENTRY_NP(kmdb_enter)
4151	pushl	%ebp
4152	movl	%esp, %ebp
4153
4154	/*
4155	 * Save flags, do a 'cli' then return the saved flags
4156	 */
4157	call	intr_clear
4158
4159	int	$T_DBGENTR
4160
4161	/*
4162	 * Restore the saved flags
4163	 */
4164	pushl	%eax
4165	call	intr_restore
4166	addl	$4, %esp
4167
4168	leave
4169	ret
4170	SET_SIZE(kmdb_enter)
4171
4172#endif	/* __i386 */
4173#endif	/* __lint */
4174
4175#if defined(__lint)
4176
4177void
4178return_instr(void)
4179{}
4180
4181#else	/* __lint */
4182
4183	ENTRY_NP(return_instr)
4184	rep;	ret	/* use 2 byte instruction when branch target */
4185			/* AMD Software Optimization Guide - Section 6.2 */
4186	SET_SIZE(return_instr)
4187
4188#endif	/* __lint */
4189
4190#if defined(__lint)
4191
4192ulong_t
4193getflags(void)
4194{
4195	return (0);
4196}
4197
4198#else	/* __lint */
4199
4200#if defined(__amd64)
4201
4202	ENTRY(getflags)
4203	pushfq
4204	popq	%rax
4205#if defined(__xpv)
4206	CURTHREAD(%rdi)
4207	KPREEMPT_DISABLE(%rdi)
4208	/*
4209	 * Synthesize the PS_IE bit from the event mask bit
4210	 */
4211	CURVCPU(%r11)
4212	andq    $_BITNOT(PS_IE), %rax
4213	XEN_TEST_UPCALL_MASK(%r11)
4214	jnz	1f
4215	orq	$PS_IE, %rax
42161:
4217	KPREEMPT_ENABLE_NOKP(%rdi)
4218#endif
4219	ret
4220	SET_SIZE(getflags)
4221
4222#elif defined(__i386)
4223
4224	ENTRY(getflags)
4225	pushfl
4226	popl	%eax
4227#if defined(__xpv)
4228	CURTHREAD(%ecx)
4229	KPREEMPT_DISABLE(%ecx)
4230	/*
4231	 * Synthesize the PS_IE bit from the event mask bit
4232	 */
4233	CURVCPU(%edx)
4234	andl    $_BITNOT(PS_IE), %eax
4235	XEN_TEST_UPCALL_MASK(%edx)
4236	jnz	1f
4237	orl	$PS_IE, %eax
42381:
4239	KPREEMPT_ENABLE_NOKP(%ecx)
4240#endif
4241	ret
4242	SET_SIZE(getflags)
4243
4244#endif	/* __i386 */
4245
4246#endif	/* __lint */
4247
4248#if defined(__lint)
4249
4250ftrace_icookie_t
4251ftrace_interrupt_disable(void)
4252{ return (0); }
4253
4254#else   /* __lint */
4255
4256#if defined(__amd64)
4257
4258	ENTRY(ftrace_interrupt_disable)
4259	pushfq
4260	popq	%rax
4261	CLI(%rdx)
4262	ret
4263	SET_SIZE(ftrace_interrupt_disable)
4264
4265#elif defined(__i386)
4266
4267	ENTRY(ftrace_interrupt_disable)
4268	pushfl
4269	popl	%eax
4270	CLI(%edx)
4271	ret
4272	SET_SIZE(ftrace_interrupt_disable)
4273
4274#endif	/* __i386 */
4275#endif	/* __lint */
4276
4277#if defined(__lint)
4278
4279/*ARGSUSED*/
4280void
4281ftrace_interrupt_enable(ftrace_icookie_t cookie)
4282{}
4283
4284#else	/* __lint */
4285
4286#if defined(__amd64)
4287
4288	ENTRY(ftrace_interrupt_enable)
4289	pushq	%rdi
4290	popfq
4291	ret
4292	SET_SIZE(ftrace_interrupt_enable)
4293
4294#elif defined(__i386)
4295
4296	ENTRY(ftrace_interrupt_enable)
4297	movl	4(%esp), %eax
4298	pushl	%eax
4299	popfl
4300	ret
4301	SET_SIZE(ftrace_interrupt_enable)
4302
4303#endif	/* __i386 */
4304#endif	/* __lint */
4305
4306#if defined (__lint)
4307
4308/*ARGSUSED*/
4309void
4310clflush_insn(caddr_t addr)
4311{}
4312
4313#else /* __lint */
4314
4315#if defined (__amd64)
4316	ENTRY(clflush_insn)
4317	clflush (%rdi)
4318	ret
4319	SET_SIZE(clflush_insn)
4320#elif defined (__i386)
4321	ENTRY(clflush_insn)
4322	movl	4(%esp), %eax
4323	clflush (%eax)
4324	ret
4325	SET_SIZE(clflush_insn)
4326
4327#endif /* __i386 */
4328#endif /* __lint */
4329
4330#if defined (__lint)
4331/*ARGSUSED*/
4332void
4333mfence_insn(void)
4334{}
4335
4336#else /* __lint */
4337
4338#if defined (__amd64)
4339	ENTRY(mfence_insn)
4340	mfence
4341	ret
4342	SET_SIZE(mfence_insn)
4343#elif defined (__i386)
4344	ENTRY(mfence_insn)
4345	mfence
4346	ret
4347	SET_SIZE(mfence_insn)
4348
4349#endif /* __i386 */
4350#endif /* __lint */
4351
4352/*
4353 * VMware implements an I/O port that programs can query to detect if software
4354 * is running in a VMware hypervisor. This hypervisor port behaves differently
4355 * depending on magic values in certain registers and modifies some registers
4356 * as a side effect.
4357 *
4358 * References: http://kb.vmware.com/kb/1009458
4359 */
4360
4361#if defined(__lint)
4362
4363/* ARGSUSED */
4364void
4365vmware_port(int cmd, uint32_t *regs) { return; }
4366
4367#else
4368
4369#if defined(__amd64)
4370
4371	ENTRY(vmware_port)
4372	pushq	%rbx
4373	movl	$VMWARE_HVMAGIC, %eax
4374	movl	$0xffffffff, %ebx
4375	movl	%edi, %ecx
4376	movl	$VMWARE_HVPORT, %edx
4377	inl	(%dx)
4378	movl	%eax, (%rsi)
4379	movl	%ebx, 4(%rsi)
4380	movl	%ecx, 8(%rsi)
4381	movl	%edx, 12(%rsi)
4382	popq	%rbx
4383	ret
4384	SET_SIZE(vmware_port)
4385
4386#elif defined(__i386)
4387
4388	ENTRY(vmware_port)
4389	pushl	%ebx
4390	pushl	%esi
4391	movl	$VMWARE_HVMAGIC, %eax
4392	movl	$0xffffffff, %ebx
4393	movl	12(%esp), %ecx
4394	movl	$VMWARE_HVPORT, %edx
4395	inl	(%dx)
4396	movl	16(%esp), %esi
4397	movl	%eax, (%esi)
4398	movl	%ebx, 4(%esi)
4399	movl	%ecx, 8(%esi)
4400	movl	%edx, 12(%esi)
4401	popl	%esi
4402	popl	%ebx
4403	ret
4404	SET_SIZE(vmware_port)
4405
4406#endif /* __i386 */
4407#endif /* __lint */
4408