xref: /illumos-gate/usr/src/uts/sun4v/ml/trap_table.s (revision 9b0bb795)
1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22/*
23 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
24 * Use is subject to license terms.
25 */
26
27#include "assym.h"
28#include <sys/asm_linkage.h>
29#include <sys/privregs.h>
30#include <sys/sun4asi.h>
31#include <sys/machasi.h>
32#include <sys/hypervisor_api.h>
33#include <sys/machtrap.h>
34#include <sys/machthread.h>
35#include <sys/machbrand.h>
36#include <sys/pcb.h>
37#include <sys/pte.h>
38#include <sys/mmu.h>
39#include <sys/machpcb.h>
40#include <sys/async.h>
41#include <sys/intreg.h>
42#include <sys/scb.h>
43#include <sys/psr_compat.h>
44#include <sys/syscall.h>
45#include <sys/machparam.h>
46#include <sys/traptrace.h>
47#include <vm/hat_sfmmu.h>
48#include <sys/archsystm.h>
49#include <sys/utrap.h>
50#include <sys/clock.h>
51#include <sys/intr.h>
52#include <sys/fpu/fpu_simulator.h>
53#include <vm/seg_spt.h>
54
55/*
56 * WARNING: If you add a fast trap handler which can be invoked by a
57 * non-privileged user, you may have to use the FAST_TRAP_DONE macro
58 * instead of "done" instruction to return back to the user mode. See
59 * comments for the "fast_trap_done" entry point for more information.
60 *
61 * An alternate FAST_TRAP_DONE_CHK_INTR macro should be used for the
62 * cases where you always want to process any pending interrupts before
63 * returning back to the user mode.
64 */
65#define	FAST_TRAP_DONE		\
66	ba,a	fast_trap_done
67
68#define	FAST_TRAP_DONE_CHK_INTR	\
69	ba,a	fast_trap_done_chk_intr
70
71/*
72 * SPARC V9 Trap Table
73 *
74 * Most of the trap handlers are made from common building
75 * blocks, and some are instantiated multiple times within
76 * the trap table. So, I build a bunch of macros, then
77 * populate the table using only the macros.
78 *
79 * Many macros branch to sys_trap.  Its calling convention is:
80 *	%g1		kernel trap handler
81 *	%g2, %g3	args for above
82 *	%g4		desire %pil
83 */
84
85#ifdef	TRAPTRACE
86
87/*
88 * Tracing macro. Adds two instructions if TRAPTRACE is defined.
89 */
90#define	TT_TRACE(label)		\
91	ba	label		;\
92	rd	%pc, %g7
93#define	TT_TRACE_INS	2
94
95#define	TT_TRACE_L(label)	\
96	ba	label		;\
97	rd	%pc, %l4	;\
98	clr	%l4
99#define	TT_TRACE_L_INS	3
100
101#else
102
103#define	TT_TRACE(label)
104#define	TT_TRACE_INS	0
105
106#define	TT_TRACE_L(label)
107#define	TT_TRACE_L_INS	0
108
109#endif
110
111/*
112 * This first set are funneled to trap() with %tt as the type.
113 * Trap will then either panic or send the user a signal.
114 */
115/*
116 * NOT is used for traps that just shouldn't happen.
117 * It comes in both single and quadruple flavors.
118 */
119	.global	trap
120#define	NOT			\
121	TT_TRACE(trace_gen)	;\
122	set	trap, %g1	;\
123	rdpr	%tt, %g3	;\
124	ba,pt	%xcc, sys_trap	;\
125	sub	%g0, 1, %g4	;\
126	.align	32
127#define	NOT4	NOT; NOT; NOT; NOT
128
129#define	NOTP				\
130	TT_TRACE(trace_gen)		;\
131	ba,pt	%xcc, ptl1_panic	;\
132	  mov	PTL1_BAD_TRAP, %g1	;\
133	.align	32
134#define	NOTP4	NOTP; NOTP; NOTP; NOTP
135
136
137/*
138 * BAD is used for trap vectors we don't have a kernel
139 * handler for.
140 * It also comes in single and quadruple versions.
141 */
142#define	BAD	NOT
143#define	BAD4	NOT4
144
145#define	DONE			\
146	done;			\
147	.align	32
148
149/*
150 * TRAP vectors to the trap() function.
151 * It's main use is for user errors.
152 */
153	.global	trap
154#define	TRAP(arg)		\
155	TT_TRACE(trace_gen)	;\
156	set	trap, %g1	;\
157	mov	arg, %g3	;\
158	ba,pt	%xcc, sys_trap	;\
159	sub	%g0, 1, %g4	;\
160	.align	32
161
162/*
163 * SYSCALL is used for unsupported syscall interfaces (with 'which'
164 * set to 'nosys') and legacy support of old SunOS 4.x syscalls (with
165 * 'which' set to 'syscall_trap32').
166 *
167 * The SYSCALL_TRAP* macros are used for syscall entry points.
168 * SYSCALL_TRAP is used to support LP64 syscalls and SYSCALL_TRAP32
169 * is used to support ILP32.  Each macro can only be used once
170 * since they each define a symbol.  The symbols are used as hot patch
171 * points by the brand infrastructure to dynamically enable and disable
172 * brand syscall interposition.  See the comments around BRAND_CALLBACK
173 * and brand_plat_interposition_enable() for more information.
174 */
175#define	SYSCALL_NOTT(which)		\
176	set	(which), %g1		;\
177	ba,pt	%xcc, user_trap		;\
178	sub	%g0, 1, %g4		;\
179	.align	32
180
181#define	SYSCALL(which)			\
182	TT_TRACE(trace_gen)		;\
183	SYSCALL_NOTT(which)
184
185#define	SYSCALL_TRAP32				\
186	TT_TRACE(trace_gen)			;\
187	ALTENTRY(syscall_trap32_patch_point)	\
188	SYSCALL_NOTT(syscall_trap32)
189
190#define	SYSCALL_TRAP				\
191	TT_TRACE(trace_gen)			;\
192	ALTENTRY(syscall_trap_patch_point)	\
193	SYSCALL_NOTT(syscall_trap)
194
195/*
196 * GOTO just jumps to a label.
197 * It's used for things that can be fixed without going thru sys_trap.
198 */
199#define	GOTO(label)		\
200	.global	label		;\
201	ba,a	label		;\
202	.empty			;\
203	.align	32
204
205/*
206 * GOTO_TT just jumps to a label.
207 * correctable ECC error traps at  level 0 and 1 will use this macro.
208 * It's used for things that can be fixed without going thru sys_trap.
209 */
210#define	GOTO_TT(label, ttlabel)		\
211	.global	label		;\
212	TT_TRACE(ttlabel)	;\
213	ba,a	label		;\
214	.empty			;\
215	.align	32
216
217/*
218 * Privileged traps
219 * Takes breakpoint if privileged, calls trap() if not.
220 */
221#define	PRIV(label)			\
222	rdpr	%tstate, %g1		;\
223	btst	TSTATE_PRIV, %g1	;\
224	bnz	label			;\
225	rdpr	%tt, %g3		;\
226	set	trap, %g1		;\
227	ba,pt	%xcc, sys_trap		;\
228	sub	%g0, 1, %g4		;\
229	.align	32
230
231
232/*
233 * DTrace traps.
234 */
235#define	DTRACE_PID			\
236	.global dtrace_pid_probe				;\
237	set	dtrace_pid_probe, %g1				;\
238	ba,pt	%xcc, user_trap					;\
239	sub	%g0, 1, %g4					;\
240	.align	32
241
242#define	DTRACE_RETURN			\
243	.global dtrace_return_probe				;\
244	set	dtrace_return_probe, %g1			;\
245	ba,pt	%xcc, user_trap					;\
246	sub	%g0, 1, %g4					;\
247	.align	32
248
249/*
250 * REGISTER WINDOW MANAGEMENT MACROS
251 */
252
253/*
254 * various convenient units of padding
255 */
256#define	SKIP(n)	.skip 4*(n)
257
258/*
259 * CLEAN_WINDOW is the simple handler for cleaning a register window.
260 */
261#define	CLEAN_WINDOW						\
262	TT_TRACE_L(trace_win)					;\
263	rdpr %cleanwin, %l0; inc %l0; wrpr %l0, %cleanwin	;\
264	clr %l0; clr %l1; clr %l2; clr %l3			;\
265	clr %l4; clr %l5; clr %l6; clr %l7			;\
266	clr %o0; clr %o1; clr %o2; clr %o3			;\
267	clr %o4; clr %o5; clr %o6; clr %o7			;\
268	retry; .align 128
269
270/*
271 * If we get an unresolved tlb miss while in a window handler, the fault
272 * handler will resume execution at the last instruction of the window
273 * hander, instead of delivering the fault to the kernel.  Spill handlers
274 * use this to spill windows into the wbuf.
275 *
276 * The mixed handler works by checking %sp, and branching to the correct
277 * handler.  This is done by branching back to label 1: for 32b frames,
278 * or label 2: for 64b frames; which implies the handler order is: 32b,
279 * 64b, mixed.  The 1: and 2: labels are offset into the routines to
280 * allow the branchs' delay slots to contain useful instructions.
281 */
282
283/*
284 * SPILL_32bit spills a 32-bit-wide kernel register window.  It
285 * assumes that the kernel context and the nucleus context are the
286 * same.  The stack pointer is required to be eight-byte aligned even
287 * though this code only needs it to be four-byte aligned.
288 */
289#define	SPILL_32bit(tail)					\
290	srl	%sp, 0, %sp					;\
2911:	st	%l0, [%sp + 0]					;\
292	st	%l1, [%sp + 4]					;\
293	st	%l2, [%sp + 8]					;\
294	st	%l3, [%sp + 12]					;\
295	st	%l4, [%sp + 16]					;\
296	st	%l5, [%sp + 20]					;\
297	st	%l6, [%sp + 24]					;\
298	st	%l7, [%sp + 28]					;\
299	st	%i0, [%sp + 32]					;\
300	st	%i1, [%sp + 36]					;\
301	st	%i2, [%sp + 40]					;\
302	st	%i3, [%sp + 44]					;\
303	st	%i4, [%sp + 48]					;\
304	st	%i5, [%sp + 52]					;\
305	st	%i6, [%sp + 56]					;\
306	st	%i7, [%sp + 60]					;\
307	TT_TRACE_L(trace_win)					;\
308	saved							;\
309	retry							;\
310	SKIP(31-19-TT_TRACE_L_INS)				;\
311	ba,a,pt	%xcc, fault_32bit_/**/tail			;\
312	.empty
313
314/*
315 * SPILL_32bit_asi spills a 32-bit-wide register window into a 32-bit
316 * wide address space via the designated asi.  It is used to spill
317 * non-kernel windows.  The stack pointer is required to be eight-byte
318 * aligned even though this code only needs it to be four-byte
319 * aligned.
320 */
321#define	SPILL_32bit_asi(asi_num, tail)				\
322	srl	%sp, 0, %sp					;\
3231:	sta	%l0, [%sp + %g0]asi_num				;\
324	mov	4, %g1						;\
325	sta	%l1, [%sp + %g1]asi_num				;\
326	mov	8, %g2						;\
327	sta	%l2, [%sp + %g2]asi_num				;\
328	mov	12, %g3						;\
329	sta	%l3, [%sp + %g3]asi_num				;\
330	add	%sp, 16, %g4					;\
331	sta	%l4, [%g4 + %g0]asi_num				;\
332	sta	%l5, [%g4 + %g1]asi_num				;\
333	sta	%l6, [%g4 + %g2]asi_num				;\
334	sta	%l7, [%g4 + %g3]asi_num				;\
335	add	%g4, 16, %g4					;\
336	sta	%i0, [%g4 + %g0]asi_num				;\
337	sta	%i1, [%g4 + %g1]asi_num				;\
338	sta	%i2, [%g4 + %g2]asi_num				;\
339	sta	%i3, [%g4 + %g3]asi_num				;\
340	add	%g4, 16, %g4					;\
341	sta	%i4, [%g4 + %g0]asi_num				;\
342	sta	%i5, [%g4 + %g1]asi_num				;\
343	sta	%i6, [%g4 + %g2]asi_num				;\
344	sta	%i7, [%g4 + %g3]asi_num				;\
345	TT_TRACE_L(trace_win)					;\
346	saved							;\
347	retry							;\
348	SKIP(31-25-TT_TRACE_L_INS)				;\
349	ba,a,pt %xcc, fault_32bit_/**/tail			;\
350	.empty
351
352#define	SPILL_32bit_tt1(asi_num, tail)				\
353	ba,a,pt	%xcc, fault_32bit_/**/tail			;\
354	.empty							;\
355	.align 128
356
357
358/*
359 * FILL_32bit fills a 32-bit-wide kernel register window.  It assumes
360 * that the kernel context and the nucleus context are the same.  The
361 * stack pointer is required to be eight-byte aligned even though this
362 * code only needs it to be four-byte aligned.
363 */
364#define	FILL_32bit(tail)					\
365	srl	%sp, 0, %sp					;\
3661:	TT_TRACE_L(trace_win)					;\
367	ld	[%sp + 0], %l0					;\
368	ld	[%sp + 4], %l1					;\
369	ld	[%sp + 8], %l2					;\
370	ld	[%sp + 12], %l3					;\
371	ld	[%sp + 16], %l4					;\
372	ld	[%sp + 20], %l5					;\
373	ld	[%sp + 24], %l6					;\
374	ld	[%sp + 28], %l7					;\
375	ld	[%sp + 32], %i0					;\
376	ld	[%sp + 36], %i1					;\
377	ld	[%sp + 40], %i2					;\
378	ld	[%sp + 44], %i3					;\
379	ld	[%sp + 48], %i4					;\
380	ld	[%sp + 52], %i5					;\
381	ld	[%sp + 56], %i6					;\
382	ld	[%sp + 60], %i7					;\
383	restored						;\
384	retry							;\
385	SKIP(31-19-TT_TRACE_L_INS)				;\
386	ba,a,pt	%xcc, fault_32bit_/**/tail			;\
387	.empty
388
389/*
390 * FILL_32bit_asi fills a 32-bit-wide register window from a 32-bit
391 * wide address space via the designated asi.  It is used to fill
392 * non-kernel windows.  The stack pointer is required to be eight-byte
393 * aligned even though this code only needs it to be four-byte
394 * aligned.
395 */
396#define	FILL_32bit_asi(asi_num, tail)				\
397	srl	%sp, 0, %sp					;\
3981:	TT_TRACE_L(trace_win)					;\
399	mov	4, %g1						;\
400	lda	[%sp + %g0]asi_num, %l0				;\
401	mov	8, %g2						;\
402	lda	[%sp + %g1]asi_num, %l1				;\
403	mov	12, %g3						;\
404	lda	[%sp + %g2]asi_num, %l2				;\
405	lda	[%sp + %g3]asi_num, %l3				;\
406	add	%sp, 16, %g4					;\
407	lda	[%g4 + %g0]asi_num, %l4				;\
408	lda	[%g4 + %g1]asi_num, %l5				;\
409	lda	[%g4 + %g2]asi_num, %l6				;\
410	lda	[%g4 + %g3]asi_num, %l7				;\
411	add	%g4, 16, %g4					;\
412	lda	[%g4 + %g0]asi_num, %i0				;\
413	lda	[%g4 + %g1]asi_num, %i1				;\
414	lda	[%g4 + %g2]asi_num, %i2				;\
415	lda	[%g4 + %g3]asi_num, %i3				;\
416	add	%g4, 16, %g4					;\
417	lda	[%g4 + %g0]asi_num, %i4				;\
418	lda	[%g4 + %g1]asi_num, %i5				;\
419	lda	[%g4 + %g2]asi_num, %i6				;\
420	lda	[%g4 + %g3]asi_num, %i7				;\
421	restored						;\
422	retry							;\
423	SKIP(31-25-TT_TRACE_L_INS)				;\
424	ba,a,pt %xcc, fault_32bit_/**/tail			;\
425	.empty
426
427
428/*
429 * SPILL_64bit spills a 64-bit-wide kernel register window.  It
430 * assumes that the kernel context and the nucleus context are the
431 * same.  The stack pointer is required to be eight-byte aligned.
432 */
433#define	SPILL_64bit(tail)					\
4342:	stx	%l0, [%sp + V9BIAS64 + 0]			;\
435	stx	%l1, [%sp + V9BIAS64 + 8]			;\
436	stx	%l2, [%sp + V9BIAS64 + 16]			;\
437	stx	%l3, [%sp + V9BIAS64 + 24]			;\
438	stx	%l4, [%sp + V9BIAS64 + 32]			;\
439	stx	%l5, [%sp + V9BIAS64 + 40]			;\
440	stx	%l6, [%sp + V9BIAS64 + 48]			;\
441	stx	%l7, [%sp + V9BIAS64 + 56]			;\
442	stx	%i0, [%sp + V9BIAS64 + 64]			;\
443	stx	%i1, [%sp + V9BIAS64 + 72]			;\
444	stx	%i2, [%sp + V9BIAS64 + 80]			;\
445	stx	%i3, [%sp + V9BIAS64 + 88]			;\
446	stx	%i4, [%sp + V9BIAS64 + 96]			;\
447	stx	%i5, [%sp + V9BIAS64 + 104]			;\
448	stx	%i6, [%sp + V9BIAS64 + 112]			;\
449	stx	%i7, [%sp + V9BIAS64 + 120]			;\
450	TT_TRACE_L(trace_win)					;\
451	saved							;\
452	retry							;\
453	SKIP(31-18-TT_TRACE_L_INS)				;\
454	ba,a,pt	%xcc, fault_64bit_/**/tail			;\
455	.empty
456
457#define	SPILL_64bit_ktt1(tail)				\
458	ba,a,pt	%xcc, fault_64bit_/**/tail			;\
459	.empty							;\
460	.align 128
461
462#define	SPILL_mixed_ktt1(tail)				\
463	btst	1, %sp						;\
464	bz,a,pt	%xcc, fault_32bit_/**/tail			;\
465	srl	%sp, 0, %sp					;\
466	ba,a,pt	%xcc, fault_64bit_/**/tail			;\
467	.empty							;\
468	.align 128
469
470/*
471 * SPILL_64bit_asi spills a 64-bit-wide register window into a 64-bit
472 * wide address space via the designated asi.  It is used to spill
473 * non-kernel windows.  The stack pointer is required to be eight-byte
474 * aligned.
475 */
476#define	SPILL_64bit_asi(asi_num, tail)				\
477	mov	0 + V9BIAS64, %g1				;\
4782:	stxa	%l0, [%sp + %g1]asi_num				;\
479	mov	8 + V9BIAS64, %g2				;\
480	stxa	%l1, [%sp + %g2]asi_num				;\
481	mov	16 + V9BIAS64, %g3				;\
482	stxa	%l2, [%sp + %g3]asi_num				;\
483	mov	24 + V9BIAS64, %g4				;\
484	stxa	%l3, [%sp + %g4]asi_num				;\
485	add	%sp, 32, %g5					;\
486	stxa	%l4, [%g5 + %g1]asi_num				;\
487	stxa	%l5, [%g5 + %g2]asi_num				;\
488	stxa	%l6, [%g5 + %g3]asi_num				;\
489	stxa	%l7, [%g5 + %g4]asi_num				;\
490	add	%g5, 32, %g5					;\
491	stxa	%i0, [%g5 + %g1]asi_num				;\
492	stxa	%i1, [%g5 + %g2]asi_num				;\
493	stxa	%i2, [%g5 + %g3]asi_num				;\
494	stxa	%i3, [%g5 + %g4]asi_num				;\
495	add	%g5, 32, %g5					;\
496	stxa	%i4, [%g5 + %g1]asi_num				;\
497	stxa	%i5, [%g5 + %g2]asi_num				;\
498	stxa	%i6, [%g5 + %g3]asi_num				;\
499	stxa	%i7, [%g5 + %g4]asi_num				;\
500	TT_TRACE_L(trace_win)					;\
501	saved							;\
502	retry							;\
503	SKIP(31-25-TT_TRACE_L_INS)				;\
504	ba,a,pt %xcc, fault_64bit_/**/tail			;\
505	.empty
506
507#define	SPILL_64bit_tt1(asi_num, tail)				\
508	ba,a,pt	%xcc, fault_64bit_/**/tail			;\
509	.empty							;\
510	.align 128
511
512/*
513 * FILL_64bit fills a 64-bit-wide kernel register window.  It assumes
514 * that the kernel context and the nucleus context are the same.  The
515 * stack pointer is required to be eight-byte aligned.
516 */
517#define	FILL_64bit(tail)					\
5182:	TT_TRACE_L(trace_win)					;\
519	ldx	[%sp + V9BIAS64 + 0], %l0			;\
520	ldx	[%sp + V9BIAS64 + 8], %l1			;\
521	ldx	[%sp + V9BIAS64 + 16], %l2			;\
522	ldx	[%sp + V9BIAS64 + 24], %l3			;\
523	ldx	[%sp + V9BIAS64 + 32], %l4			;\
524	ldx	[%sp + V9BIAS64 + 40], %l5			;\
525	ldx	[%sp + V9BIAS64 + 48], %l6			;\
526	ldx	[%sp + V9BIAS64 + 56], %l7			;\
527	ldx	[%sp + V9BIAS64 + 64], %i0			;\
528	ldx	[%sp + V9BIAS64 + 72], %i1			;\
529	ldx	[%sp + V9BIAS64 + 80], %i2			;\
530	ldx	[%sp + V9BIAS64 + 88], %i3			;\
531	ldx	[%sp + V9BIAS64 + 96], %i4			;\
532	ldx	[%sp + V9BIAS64 + 104], %i5			;\
533	ldx	[%sp + V9BIAS64 + 112], %i6			;\
534	ldx	[%sp + V9BIAS64 + 120], %i7			;\
535	restored						;\
536	retry							;\
537	SKIP(31-18-TT_TRACE_L_INS)				;\
538	ba,a,pt	%xcc, fault_64bit_/**/tail			;\
539	.empty
540
541/*
542 * FILL_64bit_asi fills a 64-bit-wide register window from a 64-bit
543 * wide address space via the designated asi.  It is used to fill
544 * non-kernel windows.  The stack pointer is required to be eight-byte
545 * aligned.
546 */
547#define	FILL_64bit_asi(asi_num, tail)				\
548	mov	V9BIAS64 + 0, %g1				;\
5492:	TT_TRACE_L(trace_win)					;\
550	ldxa	[%sp + %g1]asi_num, %l0				;\
551	mov	V9BIAS64 + 8, %g2				;\
552	ldxa	[%sp + %g2]asi_num, %l1				;\
553	mov	V9BIAS64 + 16, %g3				;\
554	ldxa	[%sp + %g3]asi_num, %l2				;\
555	mov	V9BIAS64 + 24, %g4				;\
556	ldxa	[%sp + %g4]asi_num, %l3				;\
557	add	%sp, 32, %g5					;\
558	ldxa	[%g5 + %g1]asi_num, %l4				;\
559	ldxa	[%g5 + %g2]asi_num, %l5				;\
560	ldxa	[%g5 + %g3]asi_num, %l6				;\
561	ldxa	[%g5 + %g4]asi_num, %l7				;\
562	add	%g5, 32, %g5					;\
563	ldxa	[%g5 + %g1]asi_num, %i0				;\
564	ldxa	[%g5 + %g2]asi_num, %i1				;\
565	ldxa	[%g5 + %g3]asi_num, %i2				;\
566	ldxa	[%g5 + %g4]asi_num, %i3				;\
567	add	%g5, 32, %g5					;\
568	ldxa	[%g5 + %g1]asi_num, %i4				;\
569	ldxa	[%g5 + %g2]asi_num, %i5				;\
570	ldxa	[%g5 + %g3]asi_num, %i6				;\
571	ldxa	[%g5 + %g4]asi_num, %i7				;\
572	restored						;\
573	retry							;\
574	SKIP(31-25-TT_TRACE_L_INS)				;\
575	ba,a,pt	%xcc, fault_64bit_/**/tail			;\
576	.empty
577
578
579/*
580 * SPILL_mixed spills either size window, depending on
581 * whether %sp is even or odd, to a 32-bit address space.
582 * This may only be used in conjunction with SPILL_32bit/
583 * FILL_64bit.
584 * Clear upper 32 bits of %sp if it is odd.
585 * We won't need to clear them in 64 bit kernel.
586 */
587#define	SPILL_mixed						\
588	btst	1, %sp						;\
589	bz,a,pt	%xcc, 1b					;\
590	srl	%sp, 0, %sp					;\
591	ba,pt	%xcc, 2b					;\
592	nop							;\
593	.align	128
594
595/*
596 * FILL_mixed(ASI) fills either size window, depending on
597 * whether %sp is even or odd, from a 32-bit address space.
598 * This may only be used in conjunction with FILL_32bit/
599 * FILL_64bit. New versions of FILL_mixed_{tt1,asi} would be
600 * needed for use with FILL_{32,64}bit_{tt1,asi}. Particular
601 * attention should be paid to the instructions that belong
602 * in the delay slots of the branches depending on the type
603 * of fill handler being branched to.
604 * Clear upper 32 bits of %sp if it is odd.
605 * We won't need to clear them in 64 bit kernel.
606 */
607#define	FILL_mixed						\
608	btst	1, %sp						;\
609	bz,a,pt	%xcc, 1b					;\
610	srl	%sp, 0, %sp					;\
611	ba,pt	%xcc, 2b					;\
612	nop							;\
613	.align	128
614
615
616/*
617 * SPILL_32clean/SPILL_64clean spill 32-bit and 64-bit register windows,
618 * respectively, into the address space via the designated asi.  The
619 * unbiased stack pointer is required to be eight-byte aligned (even for
620 * the 32-bit case even though this code does not require such strict
621 * alignment).
622 *
623 * With SPARC v9 the spill trap takes precedence over the cleanwin trap
624 * so when cansave == 0, canrestore == 6, and cleanwin == 6 the next save
625 * will cause cwp + 2 to be spilled but will not clean cwp + 1.  That
626 * window may contain kernel data so in user_rtt we set wstate to call
627 * these spill handlers on the first user spill trap.  These handler then
628 * spill the appropriate window but also back up a window and clean the
629 * window that didn't get a cleanwin trap.
630 */
631#define	SPILL_32clean(asi_num, tail)				\
632	srl	%sp, 0, %sp					;\
633	sta	%l0, [%sp + %g0]asi_num				;\
634	mov	4, %g1						;\
635	sta	%l1, [%sp + %g1]asi_num				;\
636	mov	8, %g2						;\
637	sta	%l2, [%sp + %g2]asi_num				;\
638	mov	12, %g3						;\
639	sta	%l3, [%sp + %g3]asi_num				;\
640	add	%sp, 16, %g4					;\
641	sta	%l4, [%g4 + %g0]asi_num				;\
642	sta	%l5, [%g4 + %g1]asi_num				;\
643	sta	%l6, [%g4 + %g2]asi_num				;\
644	sta	%l7, [%g4 + %g3]asi_num				;\
645	add	%g4, 16, %g4					;\
646	sta	%i0, [%g4 + %g0]asi_num				;\
647	sta	%i1, [%g4 + %g1]asi_num				;\
648	sta	%i2, [%g4 + %g2]asi_num				;\
649	sta	%i3, [%g4 + %g3]asi_num				;\
650	add	%g4, 16, %g4					;\
651	sta	%i4, [%g4 + %g0]asi_num				;\
652	sta	%i5, [%g4 + %g1]asi_num				;\
653	sta	%i6, [%g4 + %g2]asi_num				;\
654	sta	%i7, [%g4 + %g3]asi_num				;\
655	TT_TRACE_L(trace_win)					;\
656	b	.spill_clean					;\
657	  mov	WSTATE_USER32, %g7				;\
658	SKIP(31-25-TT_TRACE_L_INS)				;\
659	ba,a,pt	%xcc, fault_32bit_/**/tail			;\
660	.empty
661
662#define	SPILL_64clean(asi_num, tail)				\
663	mov	0 + V9BIAS64, %g1				;\
664	stxa	%l0, [%sp + %g1]asi_num				;\
665	mov	8 + V9BIAS64, %g2				;\
666	stxa	%l1, [%sp + %g2]asi_num				;\
667	mov	16 + V9BIAS64, %g3				;\
668	stxa	%l2, [%sp + %g3]asi_num				;\
669	mov	24 + V9BIAS64, %g4				;\
670	stxa	%l3, [%sp + %g4]asi_num				;\
671	add	%sp, 32, %g5					;\
672	stxa	%l4, [%g5 + %g1]asi_num				;\
673	stxa	%l5, [%g5 + %g2]asi_num				;\
674	stxa	%l6, [%g5 + %g3]asi_num				;\
675	stxa	%l7, [%g5 + %g4]asi_num				;\
676	add	%g5, 32, %g5					;\
677	stxa	%i0, [%g5 + %g1]asi_num				;\
678	stxa	%i1, [%g5 + %g2]asi_num				;\
679	stxa	%i2, [%g5 + %g3]asi_num				;\
680	stxa	%i3, [%g5 + %g4]asi_num				;\
681	add	%g5, 32, %g5					;\
682	stxa	%i4, [%g5 + %g1]asi_num				;\
683	stxa	%i5, [%g5 + %g2]asi_num				;\
684	stxa	%i6, [%g5 + %g3]asi_num				;\
685	stxa	%i7, [%g5 + %g4]asi_num				;\
686	TT_TRACE_L(trace_win)					;\
687	b	.spill_clean					;\
688	  mov	WSTATE_USER64, %g7				;\
689	SKIP(31-25-TT_TRACE_L_INS)				;\
690	ba,a,pt	%xcc, fault_64bit_/**/tail			;\
691	.empty
692
693
694/*
695 * Floating point disabled.
696 */
697#define	FP_DISABLED_TRAP		\
698	TT_TRACE(trace_gen)		;\
699	ba,pt	%xcc,.fp_disabled	;\
700	nop				;\
701	.align	32
702
703/*
704 * Floating point exceptions.
705 */
706#define	FP_IEEE_TRAP			\
707	TT_TRACE(trace_gen)		;\
708	ba,pt	%xcc,.fp_ieee_exception	;\
709	nop				;\
710	.align	32
711
712#define	FP_TRAP				\
713	TT_TRACE(trace_gen)		;\
714	ba,pt	%xcc,.fp_exception	;\
715	nop				;\
716	.align	32
717
718/*
719 * ECACHE_ECC error traps at level 0 and level 1
720 */
721#define	ECACHE_ECC(table_name)		\
722	.global	table_name		;\
723table_name:				;\
724	membar	#Sync			;\
725	set	trap, %g1		;\
726	rdpr	%tt, %g3		;\
727	ba,pt	%xcc, sys_trap		;\
728	sub	%g0, 1, %g4		;\
729	.align	32
730
731/*
732 * illegal instruction trap
733 */
734#define	ILLTRAP_INSTR			  \
735	membar	#Sync			  ;\
736	TT_TRACE(trace_gen)		  ;\
737	or	%g0, P_UTRAP4, %g2	  ;\
738	or	%g0, T_UNIMP_INSTR, %g3   ;\
739	sethi	%hi(.check_v9utrap), %g4  ;\
740	jmp	%g4 + %lo(.check_v9utrap) ;\
741	nop				  ;\
742	.align	32
743
744/*
745 * tag overflow trap
746 */
747#define	TAG_OVERFLOW			  \
748	TT_TRACE(trace_gen)		  ;\
749	or	%g0, P_UTRAP10, %g2	  ;\
750	or	%g0, T_TAG_OVERFLOW, %g3  ;\
751	sethi	%hi(.check_v9utrap), %g4  ;\
752	jmp	%g4 + %lo(.check_v9utrap) ;\
753	nop				  ;\
754	.align	32
755
756/*
757 * divide by zero trap
758 */
759#define	DIV_BY_ZERO			  \
760	TT_TRACE(trace_gen)		  ;\
761	or	%g0, P_UTRAP11, %g2	  ;\
762	or	%g0, T_IDIV0, %g3	  ;\
763	sethi	%hi(.check_v9utrap), %g4  ;\
764	jmp	%g4 + %lo(.check_v9utrap) ;\
765	nop				  ;\
766	.align	32
767
768/*
769 * trap instruction for V9 user trap handlers
770 */
771#define	TRAP_INSTR			  \
772	TT_TRACE(trace_gen)		  ;\
773	or	%g0, T_SOFTWARE_TRAP, %g3 ;\
774	sethi	%hi(.check_v9utrap), %g4  ;\
775	jmp	%g4 + %lo(.check_v9utrap) ;\
776	nop				  ;\
777	.align	32
778#define	TRP4	TRAP_INSTR; TRAP_INSTR; TRAP_INSTR; TRAP_INSTR
779
780/*
781 * LEVEL_INTERRUPT is for level N interrupts.
782 * VECTOR_INTERRUPT is for the vector trap.
783 */
784#define	LEVEL_INTERRUPT(level)		\
785	.global	tt_pil/**/level		;\
786tt_pil/**/level:			;\
787	ba,pt	%xcc, pil_interrupt	;\
788	mov	level, %g4		;\
789	.align	32
790
791#define	LEVEL14_INTERRUPT			\
792	ba	pil14_interrupt			;\
793	mov	PIL_14, %g4			;\
794	.align	32
795
796#define        LEVEL15_INTERRUPT                       \
797       ba      pil15_interrupt                 ;\
798       mov     PIL_15, %g4                     ;\
799       .align  32
800
801#define CPU_MONDO			\
802	ba,a,pt	%xcc, cpu_mondo		;\
803	.align	32
804
805#define DEV_MONDO			\
806	ba,a,pt	%xcc, dev_mondo		;\
807	.align	32
808
809/*
810 * We take over the rtba after we set our trap table and
811 * fault status area. The watchdog reset trap is now handled by the OS.
812 */
813#define WATCHDOG_RESET			\
814	mov	PTL1_BAD_WATCHDOG, %g1	;\
815	ba,a,pt	%xcc, .watchdog_trap	;\
816	.align	32
817
818/*
819 * RED is for traps that use the red mode handler.
820 * We should never see these either.
821 */
822#define RED			\
823	mov	PTL1_BAD_RED, %g1	;\
824	ba,a,pt	%xcc, .watchdog_trap	;\
825	.align	32
826
827
828/*
829 * MMU Trap Handlers.
830 */
831
832/*
833 * synthesize for trap(): SFSR in %g3
834 */
835#define	IMMU_EXCEPTION							\
836	MMU_FAULT_STATUS_AREA(%g3)					;\
837	rdpr	%tpc, %g2						;\
838	ldx	[%g3 + MMFSA_I_TYPE], %g1				;\
839	ldx	[%g3 + MMFSA_I_CTX], %g3				;\
840	sllx	%g3, SFSR_CTX_SHIFT, %g3				;\
841	or	%g3, %g1, %g3						;\
842	ba,pt	%xcc, .mmu_exception_end				;\
843	mov	T_INSTR_EXCEPTION, %g1					;\
844	.align	32
845
846/*
847 * synthesize for trap(): TAG_ACCESS in %g2, SFSR in %g3
848 */
849#define	DMMU_EXCEPTION							\
850	ba,a,pt	%xcc, .dmmu_exception					;\
851	.align	32
852
853/*
854 * synthesize for trap(): SFAR in %g2, SFSR in %g3
855 */
856#define	DMMU_EXC_AG_PRIV						\
857	MMU_FAULT_STATUS_AREA(%g3)					;\
858	ldx	[%g3 + MMFSA_D_ADDR], %g2				;\
859	/* Fault type not available in MMU fault status area */		;\
860	mov	MMFSA_F_PRVACT, %g1					;\
861	ldx	[%g3 + MMFSA_D_CTX], %g3				;\
862	sllx	%g3, SFSR_CTX_SHIFT, %g3				;\
863	ba,pt	%xcc, .mmu_priv_exception				;\
864	or	%g3, %g1, %g3						;\
865	.align	32
866
867/*
868 * synthesize for trap(): SFAR in %g2, SFSR in %g3
869 */
870#define	DMMU_EXC_AG_NOT_ALIGNED						\
871	MMU_FAULT_STATUS_AREA(%g3)					;\
872	ldx	[%g3 + MMFSA_D_ADDR], %g2				;\
873	/* Fault type not available in MMU fault status area */		;\
874	mov	MMFSA_F_UNALIGN, %g1					;\
875	ldx	[%g3 + MMFSA_D_CTX], %g3				;\
876	sllx	%g3, SFSR_CTX_SHIFT, %g3				;\
877	ba,pt	%xcc, .mmu_exception_not_aligned			;\
878	or	%g3, %g1, %g3			/* SFSR */		;\
879	.align	32
880/*
881 * SPARC V9 IMPL. DEP. #109(1) and (2) and #110(1) and (2)
882 */
883
884/*
885 * synthesize for trap(): SFAR in %g2, SFSR in %g3
886 */
887#define	DMMU_EXC_LDDF_NOT_ALIGNED					\
888	ba,a,pt	%xcc, .dmmu_exc_lddf_not_aligned			;\
889	.align	32
890/*
891 * synthesize for trap(): SFAR in %g2, SFSR in %g3
892 */
893#define	DMMU_EXC_STDF_NOT_ALIGNED					\
894	ba,a,pt	%xcc, .dmmu_exc_stdf_not_aligned			;\
895	.align	32
896
897#if defined(cscope)
898/*
899 * Define labels to direct cscope quickly to labels that
900 * are generated by macro expansion of DTLB_MISS().
901 */
902	.global	tt0_dtlbmiss
903tt0_dtlbmiss:
904	.global	tt1_dtlbmiss
905tt1_dtlbmiss:
906	nop
907#endif
908
909/*
910 * Data miss handler (must be exactly 32 instructions)
911 *
912 * This handler is invoked only if the hypervisor has been instructed
913 * not to do any TSB walk.
914 *
915 * Kernel and invalid context cases are handled by the sfmmu_kdtlb_miss
916 * handler.
917 *
918 * User TLB miss handling depends upon whether a user process has one or
919 * two TSBs. User TSB information (physical base and size code) is kept
920 * in two dedicated scratchpad registers. Absence of a user TSB (primarily
921 * second TSB) is indicated by a negative value (-1) in that register.
922 */
923
924/*
925 * synthesize for miss handler: pseudo-tag access in %g2 (with context "type"
926 * (0=kernel, 1=invalid, or 2=user) rather than context ID)
927 */
928#define	DTLB_MISS(table_name)						;\
929	.global	table_name/**/_dtlbmiss					;\
930table_name/**/_dtlbmiss:						;\
931	GET_MMU_D_PTAGACC_CTXTYPE(%g2, %g3)	/* 8 instr */		;\
932	cmp	%g3, INVALID_CONTEXT					;\
933	ble,pn	%xcc, sfmmu_kdtlb_miss					;\
934	  srlx	%g2, TAG_VALO_SHIFT, %g7	/* g7 = tsb tag */	;\
935	mov	SCRATCHPAD_UTSBREG2, %g1				;\
936	ldxa	[%g1]ASI_SCRATCHPAD, %g1	/* get 2nd tsbreg */	;\
937	brgez,pn %g1, sfmmu_udtlb_slowpath	/* branch if 2 TSBs */	;\
938	  nop								;\
939	GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5)	/* 11 instr */		;\
940	ba,pt	%xcc, sfmmu_udtlb_fastpath	/* no 4M TSB, miss */	;\
941	  srlx	%g2, TAG_VALO_SHIFT, %g7	/* g7 = tsb tag */	;\
942	.align 128
943
944
945#if defined(cscope)
946/*
947 * Define labels to direct cscope quickly to labels that
948 * are generated by macro expansion of ITLB_MISS().
949 */
950	.global	tt0_itlbmiss
951tt0_itlbmiss:
952	.global	tt1_itlbmiss
953tt1_itlbmiss:
954	nop
955#endif
956
957/*
958 * Instruction miss handler.
959 *
960 * This handler is invoked only if the hypervisor has been instructed
961 * not to do any TSB walk.
962 *
963 * ldda instructions will have their ASI patched
964 * by sfmmu_patch_ktsb at runtime.
965 * MUST be EXACTLY 32 instructions or we'll break.
966 */
967
968/*
969 * synthesize for miss handler: TAG_ACCESS in %g2 (with context "type"
970 * (0=kernel, 1=invalid, or 2=user) rather than context ID)
971 */
972#define	ITLB_MISS(table_name)						 \
973	.global	table_name/**/_itlbmiss					;\
974table_name/**/_itlbmiss:						;\
975	GET_MMU_I_PTAGACC_CTXTYPE(%g2, %g3)	/* 8 instr */		;\
976	cmp	%g3, INVALID_CONTEXT					;\
977	ble,pn	%xcc, sfmmu_kitlb_miss					;\
978	  srlx	%g2, TAG_VALO_SHIFT, %g7	/* g7 = tsb tag */	;\
979	mov	SCRATCHPAD_UTSBREG2, %g1				;\
980	ldxa	[%g1]ASI_SCRATCHPAD, %g1	/* get 2nd tsbreg */	;\
981	brgez,pn %g1, sfmmu_uitlb_slowpath	/* branch if 2 TSBs */	;\
982	  nop								;\
983	GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5)	/* 11 instr */		;\
984	ba,pt	%xcc, sfmmu_uitlb_fastpath	/* no 4M TSB, miss */	;\
985	  srlx	%g2, TAG_VALO_SHIFT, %g7	/* g7 = tsb tag */	;\
986	.align 128
987
988#define	DTSB_MISS \
989	GOTO_TT(sfmmu_slow_dmmu_miss,trace_dmmu)
990
991#define	ITSB_MISS \
992	GOTO_TT(sfmmu_slow_immu_miss,trace_immu)
993
994/*
995 * This macro is the first level handler for fast protection faults.
996 * It first demaps the tlb entry which generated the fault and then
997 * attempts to set the modify bit on the hash.  It needs to be
998 * exactly 32 instructions.
999 */
1000/*
1001 * synthesize for miss handler: TAG_ACCESS in %g2 (with context "type"
1002 * (0=kernel, 1=invalid, or 2=user) rather than context ID)
1003 */
1004#define	DTLB_PROT							 \
1005	GET_MMU_D_PTAGACC_CTXTYPE(%g2, %g3)	/* 8 instr */		;\
1006	/*								;\
1007	 *   g2 = pseudo-tag access register (ctx type rather than ctx ID) ;\
1008	 *   g3 = ctx type (0, 1, or 2)					;\
1009	 */								;\
1010	TT_TRACE(trace_dataprot)	/* 2 instr ifdef TRAPTRACE */	;\
1011					/* clobbers g1 and g6 XXXQ? */	;\
1012	brnz,pt %g3, sfmmu_uprot_trap		/* user trap */		;\
1013	  nop								;\
1014	ba,a,pt	%xcc, sfmmu_kprot_trap		/* kernel trap */	;\
1015	.align 128
1016
1017#define	DMMU_EXCEPTION_TL1						;\
1018	ba,a,pt	%xcc, mmu_trap_tl1					;\
1019	.align 32
1020
1021#define	MISALIGN_ADDR_TL1						;\
1022	ba,a,pt	%xcc, mmu_trap_tl1					;\
1023	.align 32
1024
1025/*
1026 * Trace a tsb hit
1027 * g1 = tsbe pointer (in/clobbered)
1028 * g2 = tag access register (in)
1029 * g3 - g4 = scratch (clobbered)
1030 * g5 = tsbe data (in)
1031 * g6 = scratch (clobbered)
1032 * g7 = pc we jumped here from (in)
1033 * ttextra = value to OR in to trap type (%tt) (in)
1034 */
1035#ifdef TRAPTRACE
1036#define TRACE_TSBHIT(ttextra)						 \
1037	membar	#Sync							;\
1038	sethi	%hi(FLUSH_ADDR), %g6					;\
1039	flush	%g6							;\
1040	TRACE_PTR(%g3, %g6)						;\
1041	GET_TRACE_TICK(%g6, %g4)					;\
1042	stxa	%g6, [%g3 + TRAP_ENT_TICK]%asi				;\
1043	stna	%g2, [%g3 + TRAP_ENT_SP]%asi	/* tag access */	;\
1044	stna	%g5, [%g3 + TRAP_ENT_F1]%asi	/* tsb data */		;\
1045	rdpr	%tnpc, %g6						;\
1046	stna	%g6, [%g3 + TRAP_ENT_F2]%asi				;\
1047	stna	%g1, [%g3 + TRAP_ENT_F3]%asi	/* tsb pointer */	;\
1048	stna	%g0, [%g3 + TRAP_ENT_F4]%asi				;\
1049	rdpr	%tpc, %g6						;\
1050	stna	%g6, [%g3 + TRAP_ENT_TPC]%asi				;\
1051	TRACE_SAVE_TL_GL_REGS(%g3, %g6)					;\
1052	rdpr	%tt, %g6						;\
1053	or	%g6, (ttextra), %g1					;\
1054	stha	%g1, [%g3 + TRAP_ENT_TT]%asi				;\
1055	MMU_FAULT_STATUS_AREA(%g4)					;\
1056	mov	MMFSA_D_ADDR, %g1					;\
1057	cmp	%g6, FAST_IMMU_MISS_TT					;\
1058	move	%xcc, MMFSA_I_ADDR, %g1					;\
1059	cmp	%g6, T_INSTR_MMU_MISS					;\
1060	move	%xcc, MMFSA_I_ADDR, %g1					;\
1061	ldx	[%g4 + %g1], %g1					;\
1062	stxa	%g1, [%g3 + TRAP_ENT_TSTATE]%asi /* fault addr */	;\
1063	mov	MMFSA_D_CTX, %g1					;\
1064	cmp	%g6, FAST_IMMU_MISS_TT					;\
1065	move	%xcc, MMFSA_I_CTX, %g1					;\
1066	cmp	%g6, T_INSTR_MMU_MISS					;\
1067	move	%xcc, MMFSA_I_CTX, %g1					;\
1068	ldx	[%g4 + %g1], %g1					;\
1069	stna	%g1, [%g3 + TRAP_ENT_TR]%asi				;\
1070	TRACE_NEXT(%g3, %g4, %g6)
1071#else
1072#define TRACE_TSBHIT(ttextra)
1073#endif
1074
1075
1076/*
1077 * =======================================================================
1078 *		SPARC V9 TRAP TABLE
1079 *
1080 * The trap table is divided into two halves: the first half is used when
1081 * taking traps when TL=0; the second half is used when taking traps from
1082 * TL>0. Note that handlers in the second half of the table might not be able
1083 * to make the same assumptions as handlers in the first half of the table.
1084 *
1085 * Worst case trap nesting so far:
1086 *
1087 *	at TL=0 client issues software trap requesting service
1088 *	at TL=1 nucleus wants a register window
1089 *	at TL=2 register window clean/spill/fill takes a TLB miss
1090 *	at TL=3 processing TLB miss
1091 *	at TL=4 handle asynchronous error
1092 *
1093 * Note that a trap from TL=4 to TL=5 places Spitfire in "RED mode".
1094 *
1095 * =======================================================================
1096 */
1097	.section ".text"
1098	.align	4
1099	.global trap_table, scb, trap_table0, trap_table1, etrap_table
1100	.type	trap_table, #object
1101	.type	trap_table0, #object
1102	.type	trap_table1, #object
1103	.type	scb, #object
1104trap_table:
1105scb:
1106trap_table0:
1107	/* hardware traps */
1108	NOT;				/* 000	reserved */
1109	RED;				/* 001	power on reset */
1110	WATCHDOG_RESET;			/* 002	watchdog reset */
1111	RED;				/* 003	externally initiated reset */
1112	RED;				/* 004	software initiated reset */
1113	RED;				/* 005	red mode exception */
1114	NOT; NOT;			/* 006 - 007 reserved */
1115	IMMU_EXCEPTION;			/* 008	instruction access exception */
1116	ITSB_MISS;			/* 009	instruction access MMU miss */
1117 	NOT;				/* 00A  reserved */
1118	NOT; NOT4;			/* 00B - 00F reserved */
1119	ILLTRAP_INSTR;			/* 010	illegal instruction */
1120	TRAP(T_PRIV_INSTR);		/* 011	privileged opcode */
1121	TRAP(T_UNIMP_LDD);		/* 012	unimplemented LDD */
1122	TRAP(T_UNIMP_STD);		/* 013	unimplemented STD */
1123	NOT4; NOT4; NOT4;		/* 014 - 01F reserved */
1124	FP_DISABLED_TRAP;		/* 020	fp disabled */
1125	FP_IEEE_TRAP;			/* 021	fp exception ieee 754 */
1126	FP_TRAP;			/* 022	fp exception other */
1127	TAG_OVERFLOW;			/* 023	tag overflow */
1128	CLEAN_WINDOW;			/* 024 - 027 clean window */
1129	DIV_BY_ZERO;			/* 028	division by zero */
1130	NOT;				/* 029	internal processor error */
1131	NOT; NOT; NOT4;			/* 02A - 02F reserved */
1132	DMMU_EXCEPTION;			/* 030	data access exception */
1133	DTSB_MISS;			/* 031	data access MMU miss */
1134	NOT;				/* 032  reserved */
1135	NOT;				/* 033	data access protection */
1136	DMMU_EXC_AG_NOT_ALIGNED;	/* 034	mem address not aligned */
1137	DMMU_EXC_LDDF_NOT_ALIGNED;	/* 035	LDDF mem address not aligned */
1138	DMMU_EXC_STDF_NOT_ALIGNED;	/* 036	STDF mem address not aligned */
1139	DMMU_EXC_AG_PRIV;		/* 037	privileged action */
1140	NOT;				/* 038	LDQF mem address not aligned */
1141	NOT;				/* 039	STQF mem address not aligned */
1142	NOT; NOT; NOT4;			/* 03A - 03F reserved */
1143	NOT;				/* 040	async data error */
1144	LEVEL_INTERRUPT(1);		/* 041	interrupt level 1 */
1145	LEVEL_INTERRUPT(2);		/* 042	interrupt level 2 */
1146	LEVEL_INTERRUPT(3);		/* 043	interrupt level 3 */
1147	LEVEL_INTERRUPT(4);		/* 044	interrupt level 4 */
1148	LEVEL_INTERRUPT(5);		/* 045	interrupt level 5 */
1149	LEVEL_INTERRUPT(6);		/* 046	interrupt level 6 */
1150	LEVEL_INTERRUPT(7);		/* 047	interrupt level 7 */
1151	LEVEL_INTERRUPT(8);		/* 048	interrupt level 8 */
1152	LEVEL_INTERRUPT(9);		/* 049	interrupt level 9 */
1153	LEVEL_INTERRUPT(10);		/* 04A	interrupt level 10 */
1154	LEVEL_INTERRUPT(11);		/* 04B	interrupt level 11 */
1155	LEVEL_INTERRUPT(12);		/* 04C	interrupt level 12 */
1156	LEVEL_INTERRUPT(13);		/* 04D	interrupt level 13 */
1157	LEVEL14_INTERRUPT;		/* 04E	interrupt level 14 */
1158	LEVEL15_INTERRUPT;		/* 04F	interrupt level 15 */
1159	NOT4; NOT4; NOT4; NOT4;		/* 050 - 05F reserved */
1160	NOT;				/* 060	interrupt vector */
1161	GOTO(kmdb_trap);		/* 061	PA watchpoint */
1162	GOTO(kmdb_trap);		/* 062	VA watchpoint */
1163	NOT;				/* 063	reserved */
1164	ITLB_MISS(tt0);			/* 064	instruction access MMU miss */
1165	DTLB_MISS(tt0);			/* 068	data access MMU miss */
1166	DTLB_PROT;			/* 06C	data access protection */
1167	NOT;				/* 070  reserved */
1168	NOT;				/* 071  reserved */
1169	NOT;				/* 072  reserved */
1170	NOT;				/* 073  reserved */
1171	NOT4; NOT4			/* 074 - 07B reserved */
1172	CPU_MONDO;			/* 07C	cpu_mondo */
1173	DEV_MONDO;			/* 07D	dev_mondo */
1174	GOTO_TT(resumable_error, trace_gen);	/* 07E  resumable error */
1175	GOTO_TT(nonresumable_error, trace_gen);	/* 07F  non-reasumable error */
1176	NOT4;				/* 080	spill 0 normal */
1177	SPILL_32bit_asi(ASI_AIUP,sn0);	/* 084	spill 1 normal */
1178	SPILL_64bit_asi(ASI_AIUP,sn0);	/* 088	spill 2 normal */
1179	SPILL_32clean(ASI_AIUP,sn0);	/* 08C	spill 3 normal */
1180	SPILL_64clean(ASI_AIUP,sn0);	/* 090	spill 4 normal */
1181	SPILL_32bit(not);		/* 094	spill 5 normal */
1182	SPILL_64bit(not);		/* 098	spill 6 normal */
1183	SPILL_mixed;			/* 09C	spill 7 normal */
1184	NOT4;				/* 0A0	spill 0 other */
1185	SPILL_32bit_asi(ASI_AIUS,so0);	/* 0A4	spill 1 other */
1186	SPILL_64bit_asi(ASI_AIUS,so0);	/* 0A8	spill 2 other */
1187	SPILL_32bit_asi(ASI_AIUS,so0);	/* 0AC	spill 3 other */
1188	SPILL_64bit_asi(ASI_AIUS,so0);	/* 0B0	spill 4 other */
1189	NOT4;				/* 0B4	spill 5 other */
1190	NOT4;				/* 0B8	spill 6 other */
1191	NOT4;				/* 0BC	spill 7 other */
1192	NOT4;				/* 0C0	fill 0 normal */
1193	FILL_32bit_asi(ASI_AIUP,fn0);	/* 0C4	fill 1 normal */
1194	FILL_64bit_asi(ASI_AIUP,fn0);	/* 0C8	fill 2 normal */
1195	FILL_32bit_asi(ASI_AIUP,fn0);	/* 0CC	fill 3 normal */
1196	FILL_64bit_asi(ASI_AIUP,fn0);	/* 0D0	fill 4 normal */
1197	FILL_32bit(not);		/* 0D4	fill 5 normal */
1198	FILL_64bit(not);		/* 0D8	fill 6 normal */
1199	FILL_mixed;			/* 0DC	fill 7 normal */
1200	NOT4;				/* 0E0	fill 0 other */
1201	NOT4;				/* 0E4	fill 1 other */
1202	NOT4;				/* 0E8	fill 2 other */
1203	NOT4;				/* 0EC	fill 3 other */
1204	NOT4;				/* 0F0	fill 4 other */
1205	NOT4;				/* 0F4	fill 5 other */
1206	NOT4;				/* 0F8	fill 6 other */
1207	NOT4;				/* 0FC	fill 7 other */
1208	/* user traps */
1209	GOTO(syscall_trap_4x);		/* 100	old system call */
1210	TRAP(T_BREAKPOINT);		/* 101	user breakpoint */
1211	TRAP(T_DIV0);			/* 102	user divide by zero */
1212	GOTO(.flushw);			/* 103	flush windows */
1213	GOTO(.clean_windows);		/* 104	clean windows */
1214	BAD;				/* 105	range check ?? */
1215	GOTO(.fix_alignment);		/* 106	do unaligned references */
1216	BAD;				/* 107	unused */
1217	SYSCALL_TRAP32;			/* 108	ILP32 system call on LP64 */
1218	GOTO(set_trap0_addr);		/* 109	set trap0 address */
1219	BAD; BAD; BAD4;			/* 10A - 10F unused */
1220	TRP4; TRP4; TRP4; TRP4;		/* 110 - 11F V9 user trap handlers */
1221	GOTO(.getcc);			/* 120	get condition codes */
1222	GOTO(.setcc);			/* 121	set condition codes */
1223	GOTO(.getpsr);			/* 122	get psr */
1224	GOTO(.setpsr);			/* 123	set psr (some fields) */
1225	GOTO(get_timestamp);		/* 124	get timestamp */
1226	GOTO(get_virtime);		/* 125	get lwp virtual time */
1227	PRIV(self_xcall);		/* 126	self xcall */
1228	GOTO(get_hrestime);		/* 127	get hrestime */
1229	BAD;				/* 128	ST_SETV9STACK */
1230	GOTO(.getlgrp);			/* 129  get lgrpid */
1231	BAD; BAD; BAD4;			/* 12A - 12F unused */
1232	BAD4; BAD4; 			/* 130 - 137 unused */
1233	DTRACE_PID;			/* 138  dtrace pid tracing provider */
1234	BAD;				/* 139  unused */
1235	DTRACE_RETURN;			/* 13A	dtrace pid return probe */
1236	BAD; BAD4;			/* 13B - 13F unused */
1237	SYSCALL_TRAP;			/* 140  LP64 system call */
1238	SYSCALL(nosys);			/* 141  unused system call trap */
1239#ifdef DEBUG_USER_TRAPTRACECTL
1240	GOTO(.traptrace_freeze);	/* 142  freeze traptrace */
1241	GOTO(.traptrace_unfreeze);	/* 143  unfreeze traptrace */
1242#else
1243	SYSCALL(nosys);			/* 142  unused system call trap */
1244	SYSCALL(nosys);			/* 143  unused system call trap */
1245#endif
1246	BAD4; BAD4; BAD4;		/* 144 - 14F unused */
1247	BAD4; BAD4; BAD4; BAD4;		/* 150 - 15F unused */
1248	BAD4; BAD4; BAD4; BAD4;		/* 160 - 16F unused */
1249	BAD;				/* 170 - unused */
1250	BAD;				/* 171 - unused */
1251	BAD; BAD;			/* 172 - 173 unused */
1252	BAD4; BAD4;			/* 174 - 17B unused */
1253#ifdef	PTL1_PANIC_DEBUG
1254	mov PTL1_BAD_DEBUG, %g1; GOTO(ptl1_panic);
1255					/* 17C	test ptl1_panic */
1256#else
1257	BAD;				/* 17C  unused */
1258#endif	/* PTL1_PANIC_DEBUG */
1259	PRIV(kmdb_trap);		/* 17D	kmdb enter (L1-A) */
1260	PRIV(kmdb_trap);		/* 17E	kmdb breakpoint */
1261	PRIV(obp_bpt);			/* 17F	obp breakpoint */
1262	/* reserved */
1263	NOT4; NOT4; NOT4; NOT4;		/* 180 - 18F reserved */
1264	NOT4; NOT4; NOT4; NOT4;		/* 190 - 19F reserved */
1265	NOT4; NOT4; NOT4; NOT4;		/* 1A0 - 1AF reserved */
1266	NOT4; NOT4; NOT4; NOT4;		/* 1B0 - 1BF reserved */
1267	NOT4; NOT4; NOT4; NOT4;		/* 1C0 - 1CF reserved */
1268	NOT4; NOT4; NOT4; NOT4;		/* 1D0 - 1DF reserved */
1269	NOT4; NOT4; NOT4; NOT4;		/* 1E0 - 1EF reserved */
1270	NOT4; NOT4; NOT4; NOT4;		/* 1F0 - 1FF reserved */
1271	.size	trap_table0, (.-trap_table0)
1272trap_table1:
1273	NOT4; NOT4;			/* 000 - 007 unused */
1274	NOT;				/* 008	instruction access exception */
1275	ITSB_MISS;			/* 009	instruction access MMU miss */
1276 	NOT;				/* 00A  reserved */
1277	NOT; NOT4;			/* 00B - 00F unused */
1278	NOT4; NOT4; NOT4; NOT4;		/* 010 - 01F unused */
1279	NOT4;				/* 020 - 023 unused */
1280	CLEAN_WINDOW;			/* 024 - 027 clean window */
1281	NOT4; NOT4;			/* 028 - 02F unused */
1282	DMMU_EXCEPTION_TL1;		/* 030 	data access exception */
1283	DTSB_MISS;			/* 031  data access MMU miss */
1284	NOT;				/* 032  reserved */
1285	NOT;				/* 033	unused */
1286	MISALIGN_ADDR_TL1;		/* 034	mem address not aligned */
1287	NOT; NOT; NOT; NOT4; NOT4	/* 035 - 03F unused */
1288	NOT4; NOT4; NOT4; NOT4;		/* 040 - 04F unused */
1289	NOT4; NOT4; NOT4; NOT4;		/* 050 - 05F unused */
1290	NOT;				/* 060	unused */
1291	GOTO(kmdb_trap_tl1);		/* 061	PA watchpoint */
1292	GOTO(kmdb_trap_tl1);		/* 062	VA watchpoint */
1293	NOT;				/* 063	reserved */
1294	ITLB_MISS(tt1);			/* 064	instruction access MMU miss */
1295	DTLB_MISS(tt1);			/* 068	data access MMU miss */
1296	DTLB_PROT;			/* 06C	data access protection */
1297	NOT;				/* 070  reserved */
1298	NOT;				/* 071  reserved */
1299	NOT;				/* 072  reserved */
1300	NOT;				/* 073  reserved */
1301	NOT4; NOT4;			/* 074 - 07B reserved */
1302	NOT;				/* 07C  reserved */
1303	NOT;				/* 07D  reserved */
1304	NOT;				/* 07E  resumable error */
1305	GOTO_TT(nonresumable_error, trace_gen);	/* 07F  nonresumable error */
1306	NOTP4;				/* 080	spill 0 normal */
1307	SPILL_32bit_tt1(ASI_AIUP,sn1);	/* 084	spill 1 normal */
1308	SPILL_64bit_tt1(ASI_AIUP,sn1);	/* 088	spill 2 normal */
1309	SPILL_32bit_tt1(ASI_AIUP,sn1);	/* 08C	spill 3 normal */
1310	SPILL_64bit_tt1(ASI_AIUP,sn1);	/* 090	spill 4 normal */
1311	NOTP4;				/* 094	spill 5 normal */
1312	SPILL_64bit_ktt1(sk);		/* 098	spill 6 normal */
1313	SPILL_mixed_ktt1(sk);		/* 09C	spill 7 normal */
1314	NOTP4;				/* 0A0	spill 0 other */
1315	SPILL_32bit_tt1(ASI_AIUS,so1);	/* 0A4  spill 1 other */
1316	SPILL_64bit_tt1(ASI_AIUS,so1);	/* 0A8	spill 2 other */
1317	SPILL_32bit_tt1(ASI_AIUS,so1);	/* 0AC	spill 3 other */
1318	SPILL_64bit_tt1(ASI_AIUS,so1);	/* 0B0  spill 4 other */
1319	NOTP4;				/* 0B4  spill 5 other */
1320	NOTP4;				/* 0B8  spill 6 other */
1321	NOTP4;				/* 0BC  spill 7 other */
1322	NOT4;				/* 0C0	fill 0 normal */
1323	NOT4;				/* 0C4	fill 1 normal */
1324	NOT4;				/* 0C8	fill 2 normal */
1325	NOT4;				/* 0CC	fill 3 normal */
1326	NOT4;				/* 0D0	fill 4 normal */
1327	NOT4;				/* 0D4	fill 5 normal */
1328	NOT4;				/* 0D8	fill 6 normal */
1329	NOT4;				/* 0DC	fill 7 normal */
1330	NOT4; NOT4; NOT4; NOT4;		/* 0E0 - 0EF unused */
1331	NOT4; NOT4; NOT4; NOT4;		/* 0F0 - 0FF unused */
1332/*
1333 * Code running at TL>0 does not use soft traps, so
1334 * we can truncate the table here.
1335 * However:
1336 * sun4v uses (hypervisor) ta instructions at TL > 0, so
1337 * provide a safety net for now.
1338 */
1339	/* soft traps */
1340	BAD4; BAD4; BAD4; BAD4;		/* 100 - 10F unused */
1341	BAD4; BAD4; BAD4; BAD4;		/* 110 - 11F unused */
1342	BAD4; BAD4; BAD4; BAD4;		/* 120 - 12F unused */
1343	BAD4; BAD4; BAD4; BAD4;		/* 130 - 13F unused */
1344	BAD4; BAD4; BAD4; BAD4;		/* 140 - 14F unused */
1345	BAD4; BAD4; BAD4; BAD4;		/* 150 - 15F unused */
1346	BAD4; BAD4; BAD4; BAD4;		/* 160 - 16F unused */
1347	BAD4; BAD4; BAD4; BAD4;		/* 170 - 17F unused */
1348	/* reserved */
1349	NOT4; NOT4; NOT4; NOT4;		/* 180 - 18F reserved */
1350	NOT4; NOT4; NOT4; NOT4;		/* 190 - 19F reserved */
1351	NOT4; NOT4; NOT4; NOT4;		/* 1A0 - 1AF reserved */
1352	NOT4; NOT4; NOT4; NOT4;		/* 1B0 - 1BF reserved */
1353	NOT4; NOT4; NOT4; NOT4;		/* 1C0 - 1CF reserved */
1354	NOT4; NOT4; NOT4; NOT4;		/* 1D0 - 1DF reserved */
1355	NOT4; NOT4; NOT4; NOT4;		/* 1E0 - 1EF reserved */
1356	NOT4; NOT4; NOT4; NOT4;		/* 1F0 - 1FF reserved */
1357etrap_table:
1358	.size	trap_table1, (.-trap_table1)
1359	.size	trap_table, (.-trap_table)
1360	.size	scb, (.-scb)
1361
1362/*
1363 * We get to exec_fault in the case of an instruction miss and tte
1364 * has no execute bit set.  We go to tl0 to handle it.
1365 *
1366 * g1 = tsbe pointer (in/clobbered)
1367 * g2 = tag access register (in)
1368 * g3 - g4 = scratch (clobbered)
1369 * g5 = tsbe data (in)
1370 * g6 = scratch (clobbered)
1371 * g7 = pc we jumped here from (in)
1372 */
1373/*
1374 * synthesize for miss handler: TAG_ACCESS in %g2 (with context "type"
1375 * (0=kernel, 1=invalid, or 2=user) rather than context ID)
1376 */
1377	ALTENTRY(exec_fault)
1378	TRACE_TSBHIT(TT_MMU_EXEC)
1379	MMU_FAULT_STATUS_AREA(%g4)
1380	ldx	[%g4 + MMFSA_I_ADDR], %g2	/* g2 = address */
1381	ldx	[%g4 + MMFSA_I_CTX], %g3	/* g3 = ctx */
1382	srlx	%g2, MMU_PAGESHIFT, %g2		! align address to page boundry
1383	cmp	%g3, USER_CONTEXT_TYPE
1384	sllx	%g2, MMU_PAGESHIFT, %g2
1385	movgu	%icc, USER_CONTEXT_TYPE, %g3
1386	or	%g2, %g3, %g2			/* TAG_ACCESS */
1387	mov	T_INSTR_MMU_MISS, %g3		! arg2 = traptype
1388	set	trap, %g1
1389	ba,pt	%xcc, sys_trap
1390	  mov	-1, %g4
1391
1392.mmu_exception_not_aligned:
1393	/* %g2 = sfar, %g3 = sfsr */
1394	rdpr	%tstate, %g1
1395	btst	TSTATE_PRIV, %g1
1396	bnz,pn	%icc, 2f
1397	nop
1398	CPU_ADDR(%g1, %g4)				! load CPU struct addr
1399	ldn	[%g1 + CPU_THREAD], %g1			! load thread pointer
1400	ldn	[%g1 + T_PROCP], %g1			! load proc pointer
1401	ldn	[%g1 + P_UTRAPS], %g5			! are there utraps?
1402	brz,pt	%g5, 2f
1403	nop
1404	ldn	[%g5 + P_UTRAP15], %g5			! unaligned utrap?
1405	brz,pn	%g5, 2f
1406	nop
1407	btst	1, %sp
1408	bz,pt	%xcc, 1f				! 32 bit user program
1409	nop
1410	ba,pt	%xcc, .setup_v9utrap			! 64 bit user program
1411	nop
14121:
1413	ba,pt	%xcc, .setup_utrap
1414	or	%g2, %g0, %g7
14152:
1416	ba,pt	%xcc, .mmu_exception_end
1417	mov	T_ALIGNMENT, %g1
1418
1419.mmu_priv_exception:
1420	rdpr	%tstate, %g1
1421	btst	TSTATE_PRIV, %g1
1422	bnz,pn	%icc, 1f
1423	nop
1424	CPU_ADDR(%g1, %g4)				! load CPU struct addr
1425	ldn	[%g1 + CPU_THREAD], %g1			! load thread pointer
1426	ldn	[%g1 + T_PROCP], %g1			! load proc pointer
1427	ldn	[%g1 + P_UTRAPS], %g5			! are there utraps?
1428	brz,pt	%g5, 1f
1429	nop
1430	ldn	[%g5 + P_UTRAP16], %g5
1431	brnz,pt	%g5, .setup_v9utrap
1432	nop
14331:
1434	mov	T_PRIV_INSTR, %g1
1435
1436.mmu_exception_end:
1437	CPU_INDEX(%g4, %g5)
1438	set	cpu_core, %g5
1439	sllx	%g4, CPU_CORE_SHIFT, %g4
1440	add	%g4, %g5, %g4
1441	lduh	[%g4 + CPUC_DTRACE_FLAGS], %g5
1442	andcc	%g5, CPU_DTRACE_NOFAULT, %g0
1443	bz	1f
1444	or	%g5, CPU_DTRACE_BADADDR, %g5
1445	stuh	%g5, [%g4 + CPUC_DTRACE_FLAGS]
1446	done
1447
14481:
1449	sllx	%g3, 32, %g3
1450	or	%g3, %g1, %g3
1451	set	trap, %g1
1452	ba,pt	%xcc, sys_trap
1453	sub	%g0, 1, %g4
1454
1455.fp_disabled:
1456	CPU_ADDR(%g1, %g4)				! load CPU struct addr
1457	ldn	[%g1 + CPU_THREAD], %g1			! load thread pointer
1458	rdpr	%tstate, %g4
1459	btst	TSTATE_PRIV, %g4
1460	bnz,a,pn %icc, ptl1_panic
1461	  mov	PTL1_BAD_FPTRAP, %g1
1462
1463	ldn	[%g1 + T_PROCP], %g1			! load proc pointer
1464	ldn	[%g1 + P_UTRAPS], %g5			! are there utraps?
1465	brz,a,pt %g5, 2f
1466	  nop
1467	ldn	[%g5 + P_UTRAP7], %g5			! fp_disabled utrap?
1468	brz,a,pn %g5, 2f
1469	  nop
1470	btst	1, %sp
1471	bz,a,pt	%xcc, 1f				! 32 bit user program
1472	  nop
1473	ba,a,pt	%xcc, .setup_v9utrap			! 64 bit user program
1474	  nop
14751:
1476	ba,pt	%xcc, .setup_utrap
1477	  or	%g0, %g0, %g7
14782:
1479	set	fp_disabled, %g1
1480	ba,pt	%xcc, sys_trap
1481	  sub	%g0, 1, %g4
1482
1483.fp_ieee_exception:
1484	rdpr	%tstate, %g1
1485	btst	TSTATE_PRIV, %g1
1486	bnz,a,pn %icc, ptl1_panic
1487	  mov	PTL1_BAD_FPTRAP, %g1
1488	CPU_ADDR(%g1, %g4)				! load CPU struct addr
1489	stx	%fsr, [%g1 + CPU_TMP1]
1490	ldx	[%g1 + CPU_TMP1], %g2
1491	ldn	[%g1 + CPU_THREAD], %g1			! load thread pointer
1492	ldn	[%g1 + T_PROCP], %g1			! load proc pointer
1493	ldn	[%g1 + P_UTRAPS], %g5			! are there utraps?
1494	brz,a,pt %g5, 1f
1495	  nop
1496	ldn	[%g5 + P_UTRAP8], %g5
1497	brnz,a,pt %g5, .setup_v9utrap
1498	  nop
14991:
1500	set	_fp_ieee_exception, %g1
1501	ba,pt	%xcc, sys_trap
1502	  sub	%g0, 1, %g4
1503
1504/*
1505 * Register Inputs:
1506 *	%g5		user trap handler
1507 *	%g7		misaligned addr - for alignment traps only
1508 */
1509.setup_utrap:
1510	set	trap, %g1			! setup in case we go
1511	mov	T_FLUSH_PCB, %g3		! through sys_trap on
1512	sub	%g0, 1, %g4			! the save instruction below
1513
1514	/*
1515	 * If the DTrace pid provider is single stepping a copied-out
1516	 * instruction, t->t_dtrace_step will be set. In that case we need
1517	 * to abort the single-stepping (since execution of the instruction
1518	 * was interrupted) and use the value of t->t_dtrace_npc as the %npc.
1519	 */
1520	save	%sp, -SA(MINFRAME32), %sp	! window for trap handler
1521	CPU_ADDR(%g1, %g4)			! load CPU struct addr
1522	ldn	[%g1 + CPU_THREAD], %g1		! load thread pointer
1523	ldub	[%g1 + T_DTRACE_STEP], %g2	! load t->t_dtrace_step
1524	rdpr	%tnpc, %l2			! arg1 == tnpc
1525	brz,pt	%g2, 1f
1526	rdpr	%tpc, %l1			! arg0 == tpc
1527
1528	ldub	[%g1 + T_DTRACE_AST], %g2	! load t->t_dtrace_ast
1529	ldn	[%g1 + T_DTRACE_NPC], %l2	! arg1 = t->t_dtrace_npc (step)
1530	brz,pt	%g2, 1f
1531	st	%g0, [%g1 + T_DTRACE_FT]	! zero all pid provider flags
1532	stub	%g2, [%g1 + T_ASTFLAG]		! aston(t) if t->t_dtrace_ast
15331:
1534	mov	%g7, %l3			! arg2 == misaligned address
1535
1536	rdpr	%tstate, %g1			! cwp for trap handler
1537	rdpr	%cwp, %g4
1538	bclr	TSTATE_CWP_MASK, %g1
1539	wrpr	%g1, %g4, %tstate
1540	wrpr	%g0, %g5, %tnpc			! trap handler address
1541	FAST_TRAP_DONE
1542	/* NOTREACHED */
1543
1544.check_v9utrap:
1545	rdpr	%tstate, %g1
1546	btst	TSTATE_PRIV, %g1
1547	bnz,a,pn %icc, 3f
1548	  nop
1549	CPU_ADDR(%g4, %g1)				! load CPU struct addr
1550	ldn	[%g4 + CPU_THREAD], %g5			! load thread pointer
1551	ldn	[%g5 + T_PROCP], %g5			! load proc pointer
1552	ldn	[%g5 + P_UTRAPS], %g5			! are there utraps?
1553
1554	cmp	%g3, T_SOFTWARE_TRAP
1555	bne,a,pt %icc, 1f
1556	  nop
1557
1558	brz,pt %g5, 3f			! if p_utraps == NULL goto trap()
1559	  rdpr	%tt, %g3		! delay - get actual hw trap type
1560
1561	sub	%g3, 254, %g1		! UT_TRAP_INSTRUCTION_16 = p_utraps[18]
1562	ba,pt	%icc, 2f
1563	  smul	%g1, CPTRSIZE, %g2
15641:
1565	brz,a,pt %g5, 3f		! if p_utraps == NULL goto trap()
1566	  nop
1567
1568	cmp	%g3, T_UNIMP_INSTR
1569	bne,a,pt %icc, 2f
1570	  nop
1571
1572	mov	1, %g1
1573	st	%g1, [%g4 + CPU_TL1_HDLR] ! set CPU_TL1_HDLR
1574	rdpr	%tpc, %g1		! ld trapping instruction using
1575	lduwa	[%g1]ASI_AIUP, %g1	! "AS IF USER" ASI which could fault
1576	st	%g0, [%g4 + CPU_TL1_HDLR] ! clr CPU_TL1_HDLR
1577
1578	sethi	%hi(0xc1c00000), %g4	! setup mask for illtrap instruction
1579	andcc	%g1, %g4, %g4		! and instruction with mask
1580	bnz,a,pt %icc, 3f		! if %g4 == zero, %g1 is an ILLTRAP
1581	  nop				! fall thru to setup
15822:
1583	ldn	[%g5 + %g2], %g5
1584	brnz,a,pt %g5, .setup_v9utrap
1585	  nop
15863:
1587	set	trap, %g1
1588	ba,pt	%xcc, sys_trap
1589	  sub	%g0, 1, %g4
1590	/* NOTREACHED */
1591
1592/*
1593 * Register Inputs:
1594 *	%g5		user trap handler
1595 */
1596.setup_v9utrap:
1597	set	trap, %g1			! setup in case we go
1598	mov	T_FLUSH_PCB, %g3		! through sys_trap on
1599	sub	%g0, 1, %g4			! the save instruction below
1600
1601	/*
1602	 * If the DTrace pid provider is single stepping a copied-out
1603	 * instruction, t->t_dtrace_step will be set. In that case we need
1604	 * to abort the single-stepping (since execution of the instruction
1605	 * was interrupted) and use the value of t->t_dtrace_npc as the %npc.
1606	 */
1607	save	%sp, -SA(MINFRAME64), %sp	! window for trap handler
1608	CPU_ADDR(%g1, %g4)			! load CPU struct addr
1609	ldn	[%g1 + CPU_THREAD], %g1		! load thread pointer
1610	ldub	[%g1 + T_DTRACE_STEP], %g2	! load t->t_dtrace_step
1611	rdpr	%tnpc, %l7			! arg1 == tnpc
1612	brz,pt	%g2, 1f
1613	rdpr	%tpc, %l6			! arg0 == tpc
1614
1615	ldub	[%g1 + T_DTRACE_AST], %g2	! load t->t_dtrace_ast
1616	ldn	[%g1 + T_DTRACE_NPC], %l7	! arg1 == t->t_dtrace_npc (step)
1617	brz,pt	%g2, 1f
1618	st	%g0, [%g1 + T_DTRACE_FT]	! zero all pid provider flags
1619	stub	%g2, [%g1 + T_ASTFLAG]		! aston(t) if t->t_dtrace_ast
16201:
1621	rdpr	%tstate, %g2			! cwp for trap handler
1622	rdpr	%cwp, %g4
1623	bclr	TSTATE_CWP_MASK, %g2
1624	wrpr	%g2, %g4, %tstate
1625
1626	ldn	[%g1 + T_PROCP], %g4		! load proc pointer
1627	ldn	[%g4 + P_AS], %g4		! load as pointer
1628	ldn	[%g4 + A_USERLIMIT], %g4	! load as userlimit
1629	cmp	%l7, %g4			! check for single-step set
1630	bne,pt	%xcc, 4f
1631	  nop
1632	ldn	[%g1 + T_LWP], %g1		! load klwp pointer
1633	ld	[%g1 + PCB_STEP], %g4		! load single-step flag
1634	cmp	%g4, STEP_ACTIVE		! step flags set in pcb?
1635	bne,pt	%icc, 4f
1636	  nop
1637	stn	%g5, [%g1 + PCB_TRACEPC]	! save trap handler addr in pcb
1638	mov	%l7, %g4			! on entry to precise user trap
1639	add	%l6, 4, %l7			! handler, %l6 == pc, %l7 == npc
1640						! at time of trap
1641	wrpr	%g0, %g4, %tnpc			! generate FLTBOUNDS,
1642						! %g4 == userlimit
1643	FAST_TRAP_DONE
1644	/* NOTREACHED */
16454:
1646	wrpr	%g0, %g5, %tnpc			! trap handler address
1647	FAST_TRAP_DONE_CHK_INTR
1648	/* NOTREACHED */
1649
1650.fp_exception:
1651	CPU_ADDR(%g1, %g4)
1652	stx	%fsr, [%g1 + CPU_TMP1]
1653	ldx	[%g1 + CPU_TMP1], %g2
1654
1655	/*
1656	 * Cheetah takes unfinished_FPop trap for certain range of operands
1657	 * to the "fitos" instruction. Instead of going through the slow
1658	 * software emulation path, we try to simulate the "fitos" instruction
1659	 * via "fitod" and "fdtos" provided the following conditions are met:
1660	 *
1661	 *	fpu_exists is set (if DEBUG)
1662	 *	not in privileged mode
1663	 *	ftt is unfinished_FPop
1664	 *	NXM IEEE trap is not enabled
1665	 *	instruction at %tpc is "fitos"
1666	 *
1667	 *  Usage:
1668	 *	%g1	per cpu address
1669	 *	%g2	%fsr
1670	 *	%g6	user instruction
1671	 *
1672	 * Note that we can take a memory access related trap while trying
1673	 * to fetch the user instruction. Therefore, we set CPU_TL1_HDLR
1674	 * flag to catch those traps and let the SFMMU code deal with page
1675	 * fault and data access exception.
1676	 */
1677#if defined(DEBUG) || defined(NEED_FPU_EXISTS)
1678	sethi	%hi(fpu_exists), %g7
1679	ld	[%g7 + %lo(fpu_exists)], %g7
1680	brz,pn %g7, .fp_exception_cont
1681	  nop
1682#endif
1683	rdpr	%tstate, %g7			! branch if in privileged mode
1684	btst	TSTATE_PRIV, %g7
1685	bnz,pn	%xcc, .fp_exception_cont
1686	srl	%g2, FSR_FTT_SHIFT, %g7		! extract ftt from %fsr
1687	and	%g7, (FSR_FTT>>FSR_FTT_SHIFT), %g7
1688	cmp	%g7, FTT_UNFIN
1689	set	FSR_TEM_NX, %g5
1690	bne,pn	%xcc, .fp_exception_cont	! branch if NOT unfinished_FPop
1691	  andcc	%g2, %g5, %g0
1692	bne,pn	%xcc, .fp_exception_cont	! branch if FSR_TEM_NX enabled
1693	  rdpr	%tpc, %g5			! get faulting PC
1694
1695	or	%g0, 1, %g7
1696	st	%g7, [%g1 + CPU_TL1_HDLR]	! set tl1_hdlr flag
1697	lda	[%g5]ASI_USER, %g6		! get user's instruction
1698	st	%g0, [%g1 + CPU_TL1_HDLR]	! clear tl1_hdlr flag
1699
1700	set	FITOS_INSTR_MASK, %g7
1701	and	%g6, %g7, %g7
1702	set	FITOS_INSTR, %g5
1703	cmp	%g7, %g5
1704	bne,pn	%xcc, .fp_exception_cont	! branch if not FITOS_INSTR
1705	 nop
1706
1707	/*
1708	 * This is unfinished FPops trap for "fitos" instruction. We
1709	 * need to simulate "fitos" via "fitod" and "fdtos" instruction
1710	 * sequence.
1711	 *
1712	 * We need a temporary FP register to do the conversion. Since
1713	 * both source and destination operands for the "fitos" instruction
1714	 * have to be within %f0-%f31, we use an FP register from the upper
1715	 * half to guarantee that it won't collide with the source or the
1716	 * dest operand. However, we do have to save and restore its value.
1717	 *
1718	 * We use %d62 as a temporary FP register for the conversion and
1719	 * branch to appropriate instruction within the conversion tables
1720	 * based upon the rs2 and rd values.
1721	 */
1722
1723	std	%d62, [%g1 + CPU_TMP1]		! save original value
1724
1725	srl	%g6, FITOS_RS2_SHIFT, %g7
1726	and	%g7, FITOS_REG_MASK, %g7
1727	set	_fitos_fitod_table, %g4
1728	sllx	%g7, 2, %g7
1729	jmp	%g4 + %g7
1730	  ba,pt	%xcc, _fitos_fitod_done
1731	.empty
1732
1733_fitos_fitod_table:
1734	  fitod	%f0, %d62
1735	  fitod	%f1, %d62
1736	  fitod	%f2, %d62
1737	  fitod	%f3, %d62
1738	  fitod	%f4, %d62
1739	  fitod	%f5, %d62
1740	  fitod	%f6, %d62
1741	  fitod	%f7, %d62
1742	  fitod	%f8, %d62
1743	  fitod	%f9, %d62
1744	  fitod	%f10, %d62
1745	  fitod	%f11, %d62
1746	  fitod	%f12, %d62
1747	  fitod	%f13, %d62
1748	  fitod	%f14, %d62
1749	  fitod	%f15, %d62
1750	  fitod	%f16, %d62
1751	  fitod	%f17, %d62
1752	  fitod	%f18, %d62
1753	  fitod	%f19, %d62
1754	  fitod	%f20, %d62
1755	  fitod	%f21, %d62
1756	  fitod	%f22, %d62
1757	  fitod	%f23, %d62
1758	  fitod	%f24, %d62
1759	  fitod	%f25, %d62
1760	  fitod	%f26, %d62
1761	  fitod	%f27, %d62
1762	  fitod	%f28, %d62
1763	  fitod	%f29, %d62
1764	  fitod	%f30, %d62
1765	  fitod	%f31, %d62
1766_fitos_fitod_done:
1767
1768	/*
1769	 * Now convert data back into single precision
1770	 */
1771	srl	%g6, FITOS_RD_SHIFT, %g7
1772	and	%g7, FITOS_REG_MASK, %g7
1773	set	_fitos_fdtos_table, %g4
1774	sllx	%g7, 2, %g7
1775	jmp	%g4 + %g7
1776	  ba,pt	%xcc, _fitos_fdtos_done
1777	.empty
1778
1779_fitos_fdtos_table:
1780	  fdtos	%d62, %f0
1781	  fdtos	%d62, %f1
1782	  fdtos	%d62, %f2
1783	  fdtos	%d62, %f3
1784	  fdtos	%d62, %f4
1785	  fdtos	%d62, %f5
1786	  fdtos	%d62, %f6
1787	  fdtos	%d62, %f7
1788	  fdtos	%d62, %f8
1789	  fdtos	%d62, %f9
1790	  fdtos	%d62, %f10
1791	  fdtos	%d62, %f11
1792	  fdtos	%d62, %f12
1793	  fdtos	%d62, %f13
1794	  fdtos	%d62, %f14
1795	  fdtos	%d62, %f15
1796	  fdtos	%d62, %f16
1797	  fdtos	%d62, %f17
1798	  fdtos	%d62, %f18
1799	  fdtos	%d62, %f19
1800	  fdtos	%d62, %f20
1801	  fdtos	%d62, %f21
1802	  fdtos	%d62, %f22
1803	  fdtos	%d62, %f23
1804	  fdtos	%d62, %f24
1805	  fdtos	%d62, %f25
1806	  fdtos	%d62, %f26
1807	  fdtos	%d62, %f27
1808	  fdtos	%d62, %f28
1809	  fdtos	%d62, %f29
1810	  fdtos	%d62, %f30
1811	  fdtos	%d62, %f31
1812_fitos_fdtos_done:
1813
1814	ldd	[%g1 + CPU_TMP1], %d62		! restore %d62
1815
1816#if DEBUG
1817	/*
1818	 * Update FPop_unfinished trap kstat
1819	 */
1820	set	fpustat+FPUSTAT_UNFIN_KSTAT, %g7
1821	ldx	[%g7], %g5
18221:
1823	add	%g5, 1, %g6
1824
1825	casxa	[%g7] ASI_N, %g5, %g6
1826	cmp	%g5, %g6
1827	bne,a,pn %xcc, 1b
1828	  or	%g0, %g6, %g5
1829
1830	/*
1831	 * Update fpu_sim_fitos kstat
1832	 */
1833	set	fpuinfo+FPUINFO_FITOS_KSTAT, %g7
1834	ldx	[%g7], %g5
18351:
1836	add	%g5, 1, %g6
1837
1838	casxa	[%g7] ASI_N, %g5, %g6
1839	cmp	%g5, %g6
1840	bne,a,pn %xcc, 1b
1841	  or	%g0, %g6, %g5
1842#endif /* DEBUG */
1843
1844	FAST_TRAP_DONE
1845
1846.fp_exception_cont:
1847	/*
1848	 * Let _fp_exception deal with simulating FPop instruction.
1849	 * Note that we need to pass %fsr in %g2 (already read above).
1850	 */
1851
1852	set	_fp_exception, %g1
1853	ba,pt	%xcc, sys_trap
1854	sub	%g0, 1, %g4
1855
1856
1857/*
1858 * Register windows
1859 */
1860.flushw:
1861.clean_windows:
1862	rdpr	%tnpc, %g1
1863	wrpr	%g1, %tpc
1864	add	%g1, 4, %g1
1865	wrpr	%g1, %tnpc
1866	set	trap, %g1
1867	mov	T_FLUSH_PCB, %g3
1868	ba,pt	%xcc, sys_trap
1869	sub	%g0, 1, %g4
1870
1871/*
1872 * .spill_clean: clean the previous window, restore the wstate, and
1873 * "done".
1874 *
1875 * Entry: %g7 contains new wstate
1876 */
1877.spill_clean:
1878	sethi	%hi(nwin_minus_one), %g5
1879	ld	[%g5 + %lo(nwin_minus_one)], %g5 ! %g5 = nwin - 1
1880	rdpr	%cwp, %g6			! %g6 = %cwp
1881	deccc	%g6				! %g6--
1882	movneg	%xcc, %g5, %g6			! if (%g6<0) %g6 = nwin-1
1883	wrpr	%g6, %cwp
1884	TT_TRACE_L(trace_win)
1885	clr	%l0
1886	clr	%l1
1887	clr	%l2
1888	clr	%l3
1889	clr	%l4
1890	clr	%l5
1891	clr	%l6
1892	clr	%l7
1893	wrpr	%g0, %g7, %wstate
1894	saved
1895	retry			! restores correct %cwp
1896
1897.fix_alignment:
1898	CPU_ADDR(%g1, %g2)		! load CPU struct addr to %g1 using %g2
1899	ldn	[%g1 + CPU_THREAD], %g1	! load thread pointer
1900	ldn	[%g1 + T_PROCP], %g1
1901	mov	1, %g2
1902	stb	%g2, [%g1 + P_FIXALIGNMENT]
1903	FAST_TRAP_DONE
1904
1905#define	STDF_REG(REG, ADDR, TMP)		\
1906	sll	REG, 3, REG			;\
1907mark1:	set	start1, TMP			;\
1908	jmp	REG + TMP			;\
1909	  nop					;\
1910start1:	ba,pt	%xcc, done1			;\
1911	  std	%f0, [ADDR + CPU_TMP1]		;\
1912	ba,pt	%xcc, done1			;\
1913	  std	%f32, [ADDR + CPU_TMP1]		;\
1914	ba,pt	%xcc, done1			;\
1915	  std	%f2, [ADDR + CPU_TMP1]		;\
1916	ba,pt	%xcc, done1			;\
1917	  std	%f34, [ADDR + CPU_TMP1]		;\
1918	ba,pt	%xcc, done1			;\
1919	  std	%f4, [ADDR + CPU_TMP1]		;\
1920	ba,pt	%xcc, done1			;\
1921	  std	%f36, [ADDR + CPU_TMP1]		;\
1922	ba,pt	%xcc, done1			;\
1923	  std	%f6, [ADDR + CPU_TMP1]		;\
1924	ba,pt	%xcc, done1			;\
1925	  std	%f38, [ADDR + CPU_TMP1]		;\
1926	ba,pt	%xcc, done1			;\
1927	  std	%f8, [ADDR + CPU_TMP1]		;\
1928	ba,pt	%xcc, done1			;\
1929	  std	%f40, [ADDR + CPU_TMP1]		;\
1930	ba,pt	%xcc, done1			;\
1931	  std	%f10, [ADDR + CPU_TMP1]		;\
1932	ba,pt	%xcc, done1			;\
1933	  std	%f42, [ADDR + CPU_TMP1]		;\
1934	ba,pt	%xcc, done1			;\
1935	  std	%f12, [ADDR + CPU_TMP1]		;\
1936	ba,pt	%xcc, done1			;\
1937	  std	%f44, [ADDR + CPU_TMP1]		;\
1938	ba,pt	%xcc, done1			;\
1939	  std	%f14, [ADDR + CPU_TMP1]		;\
1940	ba,pt	%xcc, done1			;\
1941	  std	%f46, [ADDR + CPU_TMP1]		;\
1942	ba,pt	%xcc, done1			;\
1943	  std	%f16, [ADDR + CPU_TMP1]		;\
1944	ba,pt	%xcc, done1			;\
1945	  std	%f48, [ADDR + CPU_TMP1]		;\
1946	ba,pt	%xcc, done1			;\
1947	  std	%f18, [ADDR + CPU_TMP1]		;\
1948	ba,pt	%xcc, done1			;\
1949	  std	%f50, [ADDR + CPU_TMP1]		;\
1950	ba,pt	%xcc, done1			;\
1951	  std	%f20, [ADDR + CPU_TMP1]		;\
1952	ba,pt	%xcc, done1			;\
1953	  std	%f52, [ADDR + CPU_TMP1]		;\
1954	ba,pt	%xcc, done1			;\
1955	  std	%f22, [ADDR + CPU_TMP1]		;\
1956	ba,pt	%xcc, done1			;\
1957	  std	%f54, [ADDR + CPU_TMP1]		;\
1958	ba,pt	%xcc, done1			;\
1959	  std	%f24, [ADDR + CPU_TMP1]		;\
1960	ba,pt	%xcc, done1			;\
1961	  std	%f56, [ADDR + CPU_TMP1]		;\
1962	ba,pt	%xcc, done1			;\
1963	  std	%f26, [ADDR + CPU_TMP1]		;\
1964	ba,pt	%xcc, done1			;\
1965	  std	%f58, [ADDR + CPU_TMP1]		;\
1966	ba,pt	%xcc, done1			;\
1967	  std	%f28, [ADDR + CPU_TMP1]		;\
1968	ba,pt	%xcc, done1			;\
1969	  std	%f60, [ADDR + CPU_TMP1]		;\
1970	ba,pt	%xcc, done1			;\
1971	  std	%f30, [ADDR + CPU_TMP1]		;\
1972	ba,pt	%xcc, done1			;\
1973	  std	%f62, [ADDR + CPU_TMP1]		;\
1974done1:
1975
1976#define	LDDF_REG(REG, ADDR, TMP)		\
1977	sll	REG, 3, REG			;\
1978mark2:	set	start2, TMP			;\
1979	jmp	REG + TMP			;\
1980	  nop					;\
1981start2:	ba,pt	%xcc, done2			;\
1982	  ldd	[ADDR + CPU_TMP1], %f0		;\
1983	ba,pt	%xcc, done2			;\
1984	  ldd	[ADDR + CPU_TMP1], %f32		;\
1985	ba,pt	%xcc, done2			;\
1986	  ldd	[ADDR + CPU_TMP1], %f2		;\
1987	ba,pt	%xcc, done2			;\
1988	  ldd	[ADDR + CPU_TMP1], %f34		;\
1989	ba,pt	%xcc, done2			;\
1990	  ldd	[ADDR + CPU_TMP1], %f4		;\
1991	ba,pt	%xcc, done2			;\
1992	  ldd	[ADDR + CPU_TMP1], %f36		;\
1993	ba,pt	%xcc, done2			;\
1994	  ldd	[ADDR + CPU_TMP1], %f6		;\
1995	ba,pt	%xcc, done2			;\
1996	  ldd	[ADDR + CPU_TMP1], %f38		;\
1997	ba,pt	%xcc, done2			;\
1998	  ldd	[ADDR + CPU_TMP1], %f8		;\
1999	ba,pt	%xcc, done2			;\
2000	  ldd	[ADDR + CPU_TMP1], %f40		;\
2001	ba,pt	%xcc, done2			;\
2002	  ldd	[ADDR + CPU_TMP1], %f10		;\
2003	ba,pt	%xcc, done2			;\
2004	  ldd	[ADDR + CPU_TMP1], %f42		;\
2005	ba,pt	%xcc, done2			;\
2006	  ldd	[ADDR + CPU_TMP1], %f12		;\
2007	ba,pt	%xcc, done2			;\
2008	  ldd	[ADDR + CPU_TMP1], %f44		;\
2009	ba,pt	%xcc, done2			;\
2010	  ldd	[ADDR + CPU_TMP1], %f14		;\
2011	ba,pt	%xcc, done2			;\
2012	  ldd	[ADDR + CPU_TMP1], %f46		;\
2013	ba,pt	%xcc, done2			;\
2014	  ldd	[ADDR + CPU_TMP1], %f16		;\
2015	ba,pt	%xcc, done2			;\
2016	  ldd	[ADDR + CPU_TMP1], %f48		;\
2017	ba,pt	%xcc, done2			;\
2018	  ldd	[ADDR + CPU_TMP1], %f18		;\
2019	ba,pt	%xcc, done2			;\
2020	  ldd	[ADDR + CPU_TMP1], %f50		;\
2021	ba,pt	%xcc, done2			;\
2022	  ldd	[ADDR + CPU_TMP1], %f20		;\
2023	ba,pt	%xcc, done2			;\
2024	  ldd	[ADDR + CPU_TMP1], %f52		;\
2025	ba,pt	%xcc, done2			;\
2026	  ldd	[ADDR + CPU_TMP1], %f22		;\
2027	ba,pt	%xcc, done2			;\
2028	  ldd	[ADDR + CPU_TMP1], %f54		;\
2029	ba,pt	%xcc, done2			;\
2030	  ldd	[ADDR + CPU_TMP1], %f24		;\
2031	ba,pt	%xcc, done2			;\
2032	  ldd	[ADDR + CPU_TMP1], %f56		;\
2033	ba,pt	%xcc, done2			;\
2034	  ldd	[ADDR + CPU_TMP1], %f26		;\
2035	ba,pt	%xcc, done2			;\
2036	  ldd	[ADDR + CPU_TMP1], %f58		;\
2037	ba,pt	%xcc, done2			;\
2038	  ldd	[ADDR + CPU_TMP1], %f28		;\
2039	ba,pt	%xcc, done2			;\
2040	  ldd	[ADDR + CPU_TMP1], %f60		;\
2041	ba,pt	%xcc, done2			;\
2042	  ldd	[ADDR + CPU_TMP1], %f30		;\
2043	ba,pt	%xcc, done2			;\
2044	  ldd	[ADDR + CPU_TMP1], %f62		;\
2045done2:
2046
2047.lddf_exception_not_aligned:
2048	/* %g2 = sfar, %g3 = sfsr */
2049	mov	%g2, %g5		! stash sfar
2050#if defined(DEBUG) || defined(NEED_FPU_EXISTS)
2051	sethi	%hi(fpu_exists), %g2	! check fpu_exists
2052	ld	[%g2 + %lo(fpu_exists)], %g2
2053	brz,a,pn %g2, 4f
2054	  nop
2055#endif
2056	CPU_ADDR(%g1, %g4)
2057	or	%g0, 1, %g4
2058	st	%g4, [%g1 + CPU_TL1_HDLR] ! set tl1_hdlr flag
2059
2060	rdpr	%tpc, %g2
2061	lda	[%g2]ASI_AIUP, %g6	! get the user's lddf instruction
2062	srl	%g6, 23, %g1		! using ldda or not?
2063	and	%g1, 1, %g1
2064	brz,a,pt %g1, 2f		! check for ldda instruction
2065	  nop
2066	srl	%g6, 13, %g1		! check immflag
2067	and	%g1, 1, %g1
2068	rdpr	%tstate, %g2		! %tstate in %g2
2069	brnz,a,pn %g1, 1f
2070	  srl	%g2, 31, %g1		! get asi from %tstate
2071	srl	%g6, 5, %g1		! get asi from instruction
2072	and	%g1, 0xFF, %g1		! imm_asi field
20731:
2074	cmp	%g1, ASI_P		! primary address space
2075	be,a,pt %icc, 2f
2076	  nop
2077	cmp	%g1, ASI_PNF		! primary no fault address space
2078	be,a,pt %icc, 2f
2079	  nop
2080	cmp	%g1, ASI_S		! secondary address space
2081	be,a,pt %icc, 2f
2082	  nop
2083	cmp	%g1, ASI_SNF		! secondary no fault address space
2084	bne,a,pn %icc, 3f
2085	  nop
20862:
2087	lduwa	[%g5]ASI_USER, %g7	! get first half of misaligned data
2088	add	%g5, 4, %g5		! increment misaligned data address
2089	lduwa	[%g5]ASI_USER, %g5	! get second half of misaligned data
2090
2091	sllx	%g7, 32, %g7
2092	or	%g5, %g7, %g5		! combine data
2093	CPU_ADDR(%g7, %g1)		! save data on a per-cpu basis
2094	stx	%g5, [%g7 + CPU_TMP1]	! save in cpu_tmp1
2095
2096	srl	%g6, 25, %g3		! %g6 has the instruction
2097	and	%g3, 0x1F, %g3		! %g3 has rd
2098	LDDF_REG(%g3, %g7, %g4)
2099
2100	CPU_ADDR(%g1, %g4)
2101	st	%g0, [%g1 + CPU_TL1_HDLR] ! clear tl1_hdlr flag
2102	FAST_TRAP_DONE
21033:
2104	CPU_ADDR(%g1, %g4)
2105	st	%g0, [%g1 + CPU_TL1_HDLR] ! clear tl1_hdlr flag
21064:
2107	set	T_USER, %g3		! trap type in %g3
2108	or	%g3, T_LDDF_ALIGN, %g3
2109	mov	%g5, %g2		! misaligned vaddr in %g2
2110	set	fpu_trap, %g1		! goto C for the little and
2111	ba,pt	%xcc, sys_trap		! no fault little asi's
2112	  sub	%g0, 1, %g4
2113
2114.stdf_exception_not_aligned:
2115	/* %g2 = sfar, %g3 = sfsr */
2116	mov	%g2, %g5
2117
2118#if defined(DEBUG) || defined(NEED_FPU_EXISTS)
2119	sethi	%hi(fpu_exists), %g7		! check fpu_exists
2120	ld	[%g7 + %lo(fpu_exists)], %g3
2121	brz,a,pn %g3, 4f
2122	  nop
2123#endif
2124	CPU_ADDR(%g1, %g4)
2125	or	%g0, 1, %g4
2126	st	%g4, [%g1 + CPU_TL1_HDLR] ! set tl1_hdlr flag
2127
2128	rdpr	%tpc, %g2
2129	lda	[%g2]ASI_AIUP, %g6	! get the user's stdf instruction
2130
2131	srl	%g6, 23, %g1		! using stda or not?
2132	and	%g1, 1, %g1
2133	brz,a,pt %g1, 2f		! check for stda instruction
2134	  nop
2135	srl	%g6, 13, %g1		! check immflag
2136	and	%g1, 1, %g1
2137	rdpr	%tstate, %g2		! %tstate in %g2
2138	brnz,a,pn %g1, 1f
2139	  srl	%g2, 31, %g1		! get asi from %tstate
2140	srl	%g6, 5, %g1		! get asi from instruction
2141	and	%g1, 0xff, %g1		! imm_asi field
21421:
2143	cmp	%g1, ASI_P		! primary address space
2144	be,a,pt %icc, 2f
2145	  nop
2146	cmp	%g1, ASI_S		! secondary address space
2147	bne,a,pn %icc, 3f
2148	  nop
21492:
2150	srl	%g6, 25, %g6
2151	and	%g6, 0x1F, %g6		! %g6 has rd
2152	CPU_ADDR(%g7, %g1)
2153	STDF_REG(%g6, %g7, %g4)		! STDF_REG(REG, ADDR, TMP)
2154
2155	ldx	[%g7 + CPU_TMP1], %g6
2156	srlx	%g6, 32, %g7
2157	stuwa	%g7, [%g5]ASI_USER	! first half
2158	add	%g5, 4, %g5		! increment misaligned data address
2159	stuwa	%g6, [%g5]ASI_USER	! second half
2160
2161	CPU_ADDR(%g1, %g4)
2162	st	%g0, [%g1 + CPU_TL1_HDLR] ! clear tl1_hdlr flag
2163	FAST_TRAP_DONE
21643:
2165	CPU_ADDR(%g1, %g4)
2166	st	%g0, [%g1 + CPU_TL1_HDLR] ! clear tl1_hdlr flag
21674:
2168	set	T_USER, %g3		! trap type in %g3
2169	or	%g3, T_STDF_ALIGN, %g3
2170	mov	%g5, %g2		! misaligned vaddr in %g2
2171	set	fpu_trap, %g1		! goto C for the little and
2172	ba,pt	%xcc, sys_trap		! nofault little asi's
2173	  sub	%g0, 1, %g4
2174
2175#ifdef DEBUG_USER_TRAPTRACECTL
2176
2177.traptrace_freeze:
2178	mov	%l0, %g1 ; mov	%l1, %g2 ; mov	%l2, %g3 ; mov	%l4, %g4
2179	TT_TRACE_L(trace_win)
2180	mov	%g4, %l4 ; mov	%g3, %l2 ; mov	%g2, %l1 ; mov	%g1, %l0
2181	set	trap_freeze, %g1
2182	mov	1, %g2
2183	st	%g2, [%g1]
2184	FAST_TRAP_DONE
2185
2186.traptrace_unfreeze:
2187	set	trap_freeze, %g1
2188	st	%g0, [%g1]
2189	mov	%l0, %g1 ; mov	%l1, %g2 ; mov	%l2, %g3 ; mov	%l4, %g4
2190	TT_TRACE_L(trace_win)
2191	mov	%g4, %l4 ; mov	%g3, %l2 ; mov	%g2, %l1 ; mov	%g1, %l0
2192	FAST_TRAP_DONE
2193
2194#endif /* DEBUG_USER_TRAPTRACECTL */
2195
2196.getcc:
2197	CPU_ADDR(%g1, %g2)
2198	stx	%o0, [%g1 + CPU_TMP1]		! save %o0
2199	rdpr	%tstate, %g3			! get tstate
2200	srlx	%g3, PSR_TSTATE_CC_SHIFT, %o0	! shift ccr to V8 psr
2201	set	PSR_ICC, %g2
2202	and	%o0, %g2, %o0			! mask out the rest
2203	srl	%o0, PSR_ICC_SHIFT, %o0		! right justify
2204	wrpr	%g0, 0, %gl
2205	mov	%o0, %g1			! move ccr to normal %g1
2206	wrpr	%g0, 1, %gl
2207	! cannot assume globals retained their values after increasing %gl
2208	CPU_ADDR(%g1, %g2)
2209	ldx	[%g1 + CPU_TMP1], %o0		! restore %o0
2210	FAST_TRAP_DONE
2211
2212.setcc:
2213	CPU_ADDR(%g1, %g2)
2214	stx	%o0, [%g1 + CPU_TMP1]		! save %o0
2215	wrpr	%g0, 0, %gl
2216	mov	%g1, %o0
2217	wrpr	%g0, 1, %gl
2218	! cannot assume globals retained their values after increasing %gl
2219	CPU_ADDR(%g1, %g2)
2220	sll	%o0, PSR_ICC_SHIFT, %g2
2221	set	PSR_ICC, %g3
2222	and	%g2, %g3, %g2			! mask out rest
2223	sllx	%g2, PSR_TSTATE_CC_SHIFT, %g2
2224	rdpr	%tstate, %g3			! get tstate
2225	srl	%g3, 0, %g3			! clear upper word
2226	or	%g3, %g2, %g3			! or in new bits
2227	wrpr	%g3, %tstate
2228	ldx	[%g1 + CPU_TMP1], %o0		! restore %o0
2229	FAST_TRAP_DONE
2230
2231/*
2232 * getpsr(void)
2233 * Note that the xcc part of the ccr is not provided.
2234 * The V8 code shows why the V9 trap is not faster:
2235 * #define GETPSR_TRAP() \
2236 *      mov %psr, %i0; jmp %l2; rett %l2+4; nop;
2237 */
2238
2239	.type	.getpsr, #function
2240.getpsr:
2241	rdpr	%tstate, %g1			! get tstate
2242	srlx	%g1, PSR_TSTATE_CC_SHIFT, %o0	! shift ccr to V8 psr
2243	set	PSR_ICC, %g2
2244	and	%o0, %g2, %o0			! mask out the rest
2245
2246	rd	%fprs, %g1			! get fprs
2247	and	%g1, FPRS_FEF, %g2		! mask out dirty upper/lower
2248	sllx	%g2, PSR_FPRS_FEF_SHIFT, %g2	! shift fef to V8 psr.ef
2249	or	%o0, %g2, %o0			! or result into psr.ef
2250
2251	set	V9_PSR_IMPLVER, %g2		! SI assigned impl/ver: 0xef
2252	or	%o0, %g2, %o0			! or psr.impl/ver
2253	FAST_TRAP_DONE
2254	SET_SIZE(.getpsr)
2255
2256/*
2257 * setpsr(newpsr)
2258 * Note that there is no support for ccr.xcc in the V9 code.
2259 */
2260
2261	.type	.setpsr, #function
2262.setpsr:
2263	rdpr	%tstate, %g1			! get tstate
2264!	setx	TSTATE_V8_UBITS, %g2
2265	or 	%g0, CCR_ICC, %g3
2266	sllx	%g3, TSTATE_CCR_SHIFT, %g2
2267
2268	andn	%g1, %g2, %g1			! zero current user bits
2269	set	PSR_ICC, %g2
2270	and	%g2, %o0, %g2			! clear all but psr.icc bits
2271	sllx	%g2, PSR_TSTATE_CC_SHIFT, %g3	! shift to tstate.ccr.icc
2272	wrpr	%g1, %g3, %tstate		! write tstate
2273
2274	set	PSR_EF, %g2
2275	and	%g2, %o0, %g2			! clear all but fp enable bit
2276	srlx	%g2, PSR_FPRS_FEF_SHIFT, %g4	! shift ef to V9 fprs.fef
2277	wr	%g0, %g4, %fprs			! write fprs
2278
2279	CPU_ADDR(%g1, %g2)			! load CPU struct addr to %g1
2280	ldn	[%g1 + CPU_THREAD], %g2		! load thread pointer
2281	ldn	[%g2 + T_LWP], %g3		! load klwp pointer
2282	ldn	[%g3 + LWP_FPU], %g2		! get lwp_fpu pointer
2283	stuw	%g4, [%g2 + FPU_FPRS]		! write fef value to fpu_fprs
2284	srlx	%g4, 2, %g4			! shift fef value to bit 0
2285	stub	%g4, [%g2 + FPU_EN]		! write fef value to fpu_en
2286	FAST_TRAP_DONE
2287	SET_SIZE(.setpsr)
2288
2289/*
2290 * getlgrp
2291 * get home lgrpid on which the calling thread is currently executing.
2292 */
2293	.type	.getlgrp, #function
2294.getlgrp:
2295	! Thanks for the incredibly helpful comments
2296	CPU_ADDR(%g1, %g2)		! load CPU struct addr to %g1 using %g2
2297	ld	[%g1 + CPU_ID], %o0	! load cpu_id
2298	ldn	[%g1 + CPU_THREAD], %g2	! load thread pointer
2299	ldn	[%g2 + T_LPL], %g2	! load lpl pointer
2300	ld	[%g2 + LPL_LGRPID], %g1	! load lpl_lgrpid
2301	sra	%g1, 0, %o1
2302	FAST_TRAP_DONE
2303	SET_SIZE(.getlgrp)
2304
2305/*
2306 * Entry for old 4.x trap (trap 0).
2307 */
2308	ENTRY_NP(syscall_trap_4x)
2309	CPU_ADDR(%g1, %g2)		! load CPU struct addr to %g1 using %g2
2310	ldn	[%g1 + CPU_THREAD], %g2	! load thread pointer
2311	ldn	[%g2 + T_LWP], %g2	! load klwp pointer
2312	ld	[%g2 + PCB_TRAP0], %g2	! lwp->lwp_pcb.pcb_trap0addr
2313	brz,pn	%g2, 1f			! has it been set?
2314	st	%l0, [%g1 + CPU_TMP1]	! delay - save some locals
2315	st	%l1, [%g1 + CPU_TMP2]
2316	rdpr	%tnpc, %l1		! save old tnpc
2317	wrpr	%g0, %g2, %tnpc		! setup tnpc
2318
2319	mov	%g1, %l0		! save CPU struct addr
2320	wrpr	%g0, 0, %gl
2321	mov	%l1, %g6		! pass tnpc to user code in %g6
2322	wrpr	%g0, 1, %gl
2323	ld	[%l0 + CPU_TMP2], %l1	! restore locals
2324	ld	[%l0 + CPU_TMP1], %l0
2325	FAST_TRAP_DONE_CHK_INTR
23261:
2327	!
2328	! check for old syscall mmap which is the only different one which
2329	! must be the same.  Others are handled in the compatibility library.
2330	!
2331	mov	%g1, %l0		! save CPU struct addr
2332	wrpr	%g0, 0, %gl
2333	cmp	%g1, OSYS_mmap		! compare to old 4.x mmap
2334	movz	%icc, SYS_mmap, %g1
2335	wrpr	%g0, 1, %gl
2336	ld	[%l0 + CPU_TMP1], %l0
2337	SYSCALL(syscall_trap32)
2338	SET_SIZE(syscall_trap_4x)
2339
2340/*
2341 * Handler for software trap 9.
2342 * Set trap0 emulation address for old 4.x system call trap.
2343 * XXX - this should be a system call.
2344 */
2345	ENTRY_NP(set_trap0_addr)
2346	CPU_ADDR(%g1, %g2)		! load CPU struct addr to %g1 using %g2
2347	st	%l0, [%g1 + CPU_TMP1]	! save some locals
2348	st	%l1, [%g1 + CPU_TMP2]
2349	mov	%g1, %l0	! preserve CPU addr
2350	wrpr	%g0, 0, %gl
2351	mov	%g1, %l1
2352	wrpr	%g0, 1, %gl
2353	! cannot assume globals retained their values after increasing %gl
2354	ldn	[%l0 + CPU_THREAD], %g2	! load thread pointer
2355	ldn	[%g2 + T_LWP], %g2	! load klwp pointer
2356	andn	%l1, 3, %l1		! force alignment
2357	st	%l1, [%g2 + PCB_TRAP0]	! lwp->lwp_pcb.pcb_trap0addr
2358	ld	[%l0 + CPU_TMP2], %l1	! restore locals
2359	ld	[%l0 + CPU_TMP1], %l0
2360	FAST_TRAP_DONE
2361	SET_SIZE(set_trap0_addr)
2362
2363/*
2364 * mmu_trap_tl1
2365 * trap handler for unexpected mmu traps.
2366 * simply checks if the trap was a user lddf/stdf alignment trap, in which
2367 * case we go to fpu_trap or a user trap from the window handler, in which
2368 * case we go save the state on the pcb.  Otherwise, we go to ptl1_panic.
2369 */
2370	.type	mmu_trap_tl1, #function
2371mmu_trap_tl1:
2372#ifdef	TRAPTRACE
2373	TRACE_PTR(%g5, %g6)
2374	GET_TRACE_TICK(%g6, %g7)
2375	stxa	%g6, [%g5 + TRAP_ENT_TICK]%asi
2376	TRACE_SAVE_TL_GL_REGS(%g5, %g6)
2377	rdpr	%tt, %g6
2378	stha	%g6, [%g5 + TRAP_ENT_TT]%asi
2379	rdpr	%tstate, %g6
2380	stxa	%g6, [%g5 + TRAP_ENT_TSTATE]%asi
2381	stna	%sp, [%g5 + TRAP_ENT_SP]%asi
2382	stna	%g0, [%g5 + TRAP_ENT_TR]%asi
2383	rdpr	%tpc, %g6
2384	stna	%g6, [%g5 + TRAP_ENT_TPC]%asi
2385	MMU_FAULT_STATUS_AREA(%g6)
2386	ldx	[%g6 + MMFSA_D_ADDR], %g6
2387	stna	%g6, [%g5 + TRAP_ENT_F1]%asi !  MMU fault address
2388	CPU_PADDR(%g7, %g6);
2389	add	%g7, CPU_TL1_HDLR, %g7
2390	lda	[%g7]ASI_MEM, %g6
2391	stna	%g6, [%g5 + TRAP_ENT_F2]%asi
2392	MMU_FAULT_STATUS_AREA(%g6)
2393	ldx	[%g6 + MMFSA_D_TYPE], %g7 ! XXXQ should be a MMFSA_F_ constant?
2394	ldx	[%g6 + MMFSA_D_CTX], %g6
2395	sllx	%g6, SFSR_CTX_SHIFT, %g6
2396	or	%g6, %g7, %g6
2397	stna	%g6, [%g5 + TRAP_ENT_F3]%asi ! MMU context/type
2398	set	0xdeadbeef, %g6
2399	stna	%g6, [%g5 + TRAP_ENT_F4]%asi
2400	TRACE_NEXT(%g5, %g6, %g7)
2401#endif /* TRAPTRACE */
2402	CPU_PADDR(%g7, %g6);
2403	add     %g7, CPU_TL1_HDLR, %g7		! %g7 = &cpu_m.tl1_hdlr (PA)
2404	lda	[%g7]ASI_MEM, %g6
2405	brz,a,pt %g6, 1f
2406	  nop
2407	sta     %g0, [%g7]ASI_MEM
2408	! XXXQ need to setup registers for sfmmu_mmu_trap?
2409	ba,a,pt	%xcc, sfmmu_mmu_trap		! handle page faults
24101:
2411	rdpr	%tpc, %g7
2412	/* in user_rtt? */
2413	set	rtt_fill_start, %g6
2414	cmp	%g7, %g6
2415	blu,pn	%xcc, 6f
2416	 .empty
2417	set	rtt_fill_end, %g6
2418	cmp	%g7, %g6
2419	bgeu,pn %xcc, 6f
2420	 nop
2421	set	fault_rtt_fn1, %g7
2422	ba,a	7f
24236:
2424	! check to see if the trap pc is in a window spill/fill handling
2425	rdpr	%tpc, %g7
2426	/* tpc should be in the trap table */
2427	set	trap_table, %g6
2428	cmp	%g7, %g6
2429	blu,a,pn %xcc, ptl1_panic
2430	  mov	PTL1_BAD_MMUTRAP, %g1
2431	set	etrap_table, %g6
2432	cmp	%g7, %g6
2433	bgeu,a,pn %xcc, ptl1_panic
2434	  mov	PTL1_BAD_MMUTRAP, %g1
2435	! pc is inside the trap table, convert to trap type
2436	srl	%g7, 5, %g6		! XXXQ need #define
2437	and	%g6, 0x1ff, %g6		! XXXQ need #define
2438	! and check for a window trap type
2439	and	%g6, WTRAP_TTMASK, %g6
2440	cmp	%g6, WTRAP_TYPE
2441	bne,a,pn %xcc, ptl1_panic
2442	  mov	PTL1_BAD_MMUTRAP, %g1
2443	andn	%g7, WTRAP_ALIGN, %g7	/* 128 byte aligned */
2444	add	%g7, WTRAP_FAULTOFF, %g7
2445
24467:
2447	! Arguments are passed in the global set active after the
2448	! 'done' instruction. Before switching sets, must save
2449	! the calculated next pc
2450	wrpr	%g0, %g7, %tnpc
2451	wrpr	%g0, 1, %gl
2452	rdpr	%tt, %g5
2453	MMU_FAULT_STATUS_AREA(%g7)
2454	cmp	%g5, T_ALIGNMENT
2455	be,pn	%xcc, 1f
2456	ldx	[%g7 + MMFSA_D_ADDR], %g6
2457	ldx	[%g7 + MMFSA_D_CTX], %g7
2458	srlx	%g6, MMU_PAGESHIFT, %g6		/* align address */
2459	cmp	%g7, USER_CONTEXT_TYPE
2460	sllx	%g6, MMU_PAGESHIFT, %g6
2461	movgu	%icc, USER_CONTEXT_TYPE, %g7
2462	or	%g6, %g7, %g6			/* TAG_ACCESS */
24631:
2464	done
2465	SET_SIZE(mmu_trap_tl1)
2466
2467/*
2468 * Several traps use kmdb_trap and kmdb_trap_tl1 as their handlers.  These
2469 * traps are valid only when kmdb is loaded.  When the debugger is active,
2470 * the code below is rewritten to transfer control to the appropriate
2471 * debugger entry points.
2472 */
2473	.global	kmdb_trap
2474	.align	8
2475kmdb_trap:
2476	ba,a	trap_table0
2477	jmp	%g1 + 0
2478	nop
2479
2480	.global	kmdb_trap_tl1
2481	.align	8
2482kmdb_trap_tl1:
2483	ba,a	trap_table0
2484	jmp	%g1 + 0
2485	nop
2486
2487/*
2488 * This entry is copied from OBP's trap table during boot.
2489 */
2490	.global	obp_bpt
2491	.align	8
2492obp_bpt:
2493	NOT
2494
2495
2496
2497#ifdef	TRAPTRACE
2498/*
2499 * TRAPTRACE support.
2500 * labels here are branched to with "rd %pc, %g7" in the delay slot.
2501 * Return is done by "jmp %g7 + 4".
2502 */
2503
2504trace_dmmu:
2505	TRACE_PTR(%g3, %g6)
2506	GET_TRACE_TICK(%g6, %g5)
2507	stxa	%g6, [%g3 + TRAP_ENT_TICK]%asi
2508	TRACE_SAVE_TL_GL_REGS(%g3, %g6)
2509	rdpr	%tt, %g6
2510	stha	%g6, [%g3 + TRAP_ENT_TT]%asi
2511	rdpr	%tstate, %g6
2512	stxa	%g6, [%g3 + TRAP_ENT_TSTATE]%asi
2513	stna	%sp, [%g3 + TRAP_ENT_SP]%asi
2514	rdpr	%tpc, %g6
2515	stna	%g6, [%g3 + TRAP_ENT_TPC]%asi
2516	MMU_FAULT_STATUS_AREA(%g6)
2517	ldx	[%g6 + MMFSA_D_ADDR], %g4
2518	stxa	%g4, [%g3 + TRAP_ENT_TR]%asi
2519	ldx	[%g6 + MMFSA_D_CTX], %g4
2520	stxa	%g4, [%g3 + TRAP_ENT_F1]%asi
2521	ldx	[%g6 + MMFSA_D_TYPE], %g4
2522	stxa	%g4, [%g3 + TRAP_ENT_F2]%asi
2523	stxa	%g6, [%g3 + TRAP_ENT_F3]%asi
2524	stna	%g0, [%g3 + TRAP_ENT_F4]%asi
2525	TRACE_NEXT(%g3, %g4, %g5)
2526	jmp	%g7 + 4
2527	nop
2528
2529trace_immu:
2530	TRACE_PTR(%g3, %g6)
2531	GET_TRACE_TICK(%g6, %g5)
2532	stxa	%g6, [%g3 + TRAP_ENT_TICK]%asi
2533	TRACE_SAVE_TL_GL_REGS(%g3, %g6)
2534	rdpr	%tt, %g6
2535	stha	%g6, [%g3 + TRAP_ENT_TT]%asi
2536	rdpr	%tstate, %g6
2537	stxa	%g6, [%g3 + TRAP_ENT_TSTATE]%asi
2538	stna	%sp, [%g3 + TRAP_ENT_SP]%asi
2539	rdpr	%tpc, %g6
2540	stna	%g6, [%g3 + TRAP_ENT_TPC]%asi
2541	MMU_FAULT_STATUS_AREA(%g6)
2542	ldx	[%g6 + MMFSA_I_ADDR], %g4
2543	stxa	%g4, [%g3 + TRAP_ENT_TR]%asi
2544	ldx	[%g6 + MMFSA_I_CTX], %g4
2545	stxa	%g4, [%g3 + TRAP_ENT_F1]%asi
2546	ldx	[%g6 + MMFSA_I_TYPE], %g4
2547	stxa	%g4, [%g3 + TRAP_ENT_F2]%asi
2548	stxa	%g6, [%g3 + TRAP_ENT_F3]%asi
2549	stna	%g0, [%g3 + TRAP_ENT_F4]%asi
2550	TRACE_NEXT(%g3, %g4, %g5)
2551	jmp	%g7 + 4
2552	nop
2553
2554trace_gen:
2555	TRACE_PTR(%g3, %g6)
2556	GET_TRACE_TICK(%g6, %g5)
2557	stxa	%g6, [%g3 + TRAP_ENT_TICK]%asi
2558	TRACE_SAVE_TL_GL_REGS(%g3, %g6)
2559	rdpr	%tt, %g6
2560	stha	%g6, [%g3 + TRAP_ENT_TT]%asi
2561	rdpr	%tstate, %g6
2562	stxa	%g6, [%g3 + TRAP_ENT_TSTATE]%asi
2563	stna	%sp, [%g3 + TRAP_ENT_SP]%asi
2564	rdpr	%tpc, %g6
2565	stna	%g6, [%g3 + TRAP_ENT_TPC]%asi
2566	stna	%g0, [%g3 + TRAP_ENT_TR]%asi
2567	stna	%g0, [%g3 + TRAP_ENT_F1]%asi
2568	stna	%g0, [%g3 + TRAP_ENT_F2]%asi
2569	stna	%g0, [%g3 + TRAP_ENT_F3]%asi
2570	stna	%g0, [%g3 + TRAP_ENT_F4]%asi
2571	TRACE_NEXT(%g3, %g4, %g5)
2572	jmp	%g7 + 4
2573	nop
2574
2575trace_win:
2576	TRACE_WIN_INFO(0, %l0, %l1, %l2)
2577	! Keep the locals as clean as possible, caller cleans %l4
2578	clr	%l2
2579	clr	%l1
2580	jmp	%l4 + 4
2581	  clr	%l0
2582
2583/*
2584 * Trace a tsb hit
2585 * g1 = tsbe pointer (in/clobbered)
2586 * g2 = tag access register (in)
2587 * g3 - g4 = scratch (clobbered)
2588 * g5 = tsbe data (in)
2589 * g6 = scratch (clobbered)
2590 * g7 = pc we jumped here from (in)
2591 */
2592
2593	! Do not disturb %g5, it will be used after the trace
2594	ALTENTRY(trace_tsbhit)
2595	TRACE_TSBHIT(0)
2596	jmp	%g7 + 4
2597	nop
2598
2599/*
2600 * Trace a TSB miss
2601 *
2602 * g1 = tsb8k pointer (in)
2603 * g2 = tag access register (in)
2604 * g3 = tsb4m pointer (in)
2605 * g4 = tsbe tag (in/clobbered)
2606 * g5 - g6 = scratch (clobbered)
2607 * g7 = pc we jumped here from (in)
2608 */
2609	.global	trace_tsbmiss
2610trace_tsbmiss:
2611	membar	#Sync
2612	sethi	%hi(FLUSH_ADDR), %g6
2613	flush	%g6
2614	TRACE_PTR(%g5, %g6)
2615	stna	%g2, [%g5 + TRAP_ENT_SP]%asi		! tag access
2616	stna	%g4, [%g5 + TRAP_ENT_F1]%asi		! XXX? tsb tag
2617	GET_TRACE_TICK(%g6, %g4)
2618	stxa	%g6, [%g5 + TRAP_ENT_TICK]%asi
2619	rdpr	%tnpc, %g6
2620	stna	%g6, [%g5 + TRAP_ENT_F2]%asi
2621	stna	%g1, [%g5 + TRAP_ENT_F3]%asi		! tsb8k pointer
2622	rdpr	%tpc, %g6
2623	stna	%g6, [%g5 + TRAP_ENT_TPC]%asi
2624	TRACE_SAVE_TL_GL_REGS(%g5, %g6)
2625	rdpr	%tt, %g6
2626	or	%g6, TT_MMU_MISS, %g4
2627	stha	%g4, [%g5 + TRAP_ENT_TT]%asi
2628	mov	MMFSA_D_ADDR, %g4
2629	cmp	%g6, FAST_IMMU_MISS_TT
2630	move	%xcc, MMFSA_I_ADDR, %g4
2631	cmp	%g6, T_INSTR_MMU_MISS
2632	move	%xcc, MMFSA_I_ADDR, %g4
2633	MMU_FAULT_STATUS_AREA(%g6)
2634	ldx	[%g6 + %g4], %g6
2635	stxa	%g6, [%g5 + TRAP_ENT_TSTATE]%asi	! tag target
2636	cmp	%g4, MMFSA_D_ADDR
2637	move	%xcc, MMFSA_D_CTX, %g4
2638	movne	%xcc, MMFSA_I_CTX, %g4
2639	MMU_FAULT_STATUS_AREA(%g6)
2640	ldx	[%g6 + %g4], %g6
2641	stxa	%g6, [%g5 + TRAP_ENT_F4]%asi		! context ID
2642	stna	%g3, [%g5 + TRAP_ENT_TR]%asi		! tsb4m pointer
2643	TRACE_NEXT(%g5, %g4, %g6)
2644	jmp	%g7 + 4
2645	nop
2646
2647/*
2648 * g2 = tag access register (in)
2649 * g3 = ctx type (0, 1 or 2) (in) (not used)
2650 */
2651trace_dataprot:
2652	membar	#Sync
2653	sethi	%hi(FLUSH_ADDR), %g6
2654	flush	%g6
2655	TRACE_PTR(%g1, %g6)
2656	GET_TRACE_TICK(%g6, %g4)
2657	stxa	%g6, [%g1 + TRAP_ENT_TICK]%asi
2658	rdpr	%tpc, %g6
2659	stna	%g6, [%g1 + TRAP_ENT_TPC]%asi
2660	rdpr	%tstate, %g6
2661	stxa	%g6, [%g1 + TRAP_ENT_TSTATE]%asi
2662	stna	%g2, [%g1 + TRAP_ENT_SP]%asi		! tag access reg
2663	stna	%g0, [%g1 + TRAP_ENT_F1]%asi
2664	stna	%g0, [%g1 + TRAP_ENT_F2]%asi
2665	stna	%g0, [%g1 + TRAP_ENT_F3]%asi
2666	stna	%g0, [%g1 + TRAP_ENT_F4]%asi
2667	TRACE_SAVE_TL_GL_REGS(%g1, %g6)
2668	rdpr	%tt, %g6
2669	stha	%g6, [%g1 + TRAP_ENT_TT]%asi
2670	mov	MMFSA_D_CTX, %g4
2671	cmp	%g6, FAST_IMMU_MISS_TT
2672	move	%xcc, MMFSA_I_CTX, %g4
2673	cmp	%g6, T_INSTR_MMU_MISS
2674	move	%xcc, MMFSA_I_CTX, %g4
2675	MMU_FAULT_STATUS_AREA(%g6)
2676	ldx	[%g6 + %g4], %g6
2677	stxa	%g6, [%g1 + TRAP_ENT_TR]%asi	! context ID
2678	TRACE_NEXT(%g1, %g4, %g5)
2679	jmp	%g7 + 4
2680	nop
2681
2682#endif /* TRAPTRACE */
2683
2684/*
2685 * Handle watchdog reset trap. Enable the MMU using the MMU_ENABLE
2686 * HV service, which requires the return target to be specified as a VA
2687 * since we are enabling the MMU. We set the target to ptl1_panic.
2688 */
2689
2690	.type	.watchdog_trap, #function
2691.watchdog_trap:
2692	mov	1, %o0
2693	setx	ptl1_panic, %g2, %o1
2694	mov	MMU_ENABLE, %o5
2695	ta	FAST_TRAP
2696	done
2697	SET_SIZE(.watchdog_trap)
2698/*
2699 * synthesize for trap(): SFAR in %g2, SFSR in %g3
2700 */
2701	.type	.dmmu_exc_lddf_not_aligned, #function
2702.dmmu_exc_lddf_not_aligned:
2703	MMU_FAULT_STATUS_AREA(%g3)
2704	ldx	[%g3 + MMFSA_D_ADDR], %g2
2705	/* Fault type not available in MMU fault status area */
2706	mov	MMFSA_F_UNALIGN, %g1
2707	ldx	[%g3 + MMFSA_D_CTX], %g3
2708	sllx	%g3, SFSR_CTX_SHIFT, %g3
2709	btst	1, %sp
2710	bnz,pt	%xcc, .lddf_exception_not_aligned
2711	or	%g3, %g1, %g3			/* SFSR */
2712	ba,a,pt	%xcc, .mmu_exception_not_aligned
2713	SET_SIZE(.dmmu_exc_lddf_not_aligned)
2714
2715/*
2716 * synthesize for trap(): SFAR in %g2, SFSR in %g3
2717 */
2718	.type	.dmmu_exc_stdf_not_aligned, #function
2719.dmmu_exc_stdf_not_aligned:
2720	MMU_FAULT_STATUS_AREA(%g3)
2721	ldx	[%g3 + MMFSA_D_ADDR], %g2
2722	/* Fault type not available in MMU fault status area */
2723	mov	MMFSA_F_UNALIGN, %g1
2724	ldx	[%g3 + MMFSA_D_CTX], %g3
2725	sllx	%g3, SFSR_CTX_SHIFT, %g3
2726	btst	1, %sp
2727	bnz,pt	%xcc, .stdf_exception_not_aligned
2728	or	%g3, %g1, %g3			/* SFSR */
2729	ba,a,pt	%xcc, .mmu_exception_not_aligned
2730	SET_SIZE(.dmmu_exc_stdf_not_aligned)
2731
2732	.type	.dmmu_exception, #function
2733.dmmu_exception:
2734	MMU_FAULT_STATUS_AREA(%g3)
2735	ldx	[%g3 + MMFSA_D_ADDR], %g2
2736	ldx	[%g3 + MMFSA_D_TYPE], %