17c478bdstevel@tonic-gate/*
27c478bdstevel@tonic-gate * CDDL HEADER START
37c478bdstevel@tonic-gate *
47c478bdstevel@tonic-gate * The contents of this file are subject to the terms of the
525cf1a3jl * Common Development and Distribution License (the "License").
625cf1a3jl * You may not use this file except in compliance with the License.
77c478bdstevel@tonic-gate *
87c478bdstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bdstevel@tonic-gate * or http://www.opensolaris.org/os/licensing.
107c478bdstevel@tonic-gate * See the License for the specific language governing permissions
117c478bdstevel@tonic-gate * and limitations under the License.
127c478bdstevel@tonic-gate *
137c478bdstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each
147c478bdstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bdstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the
167c478bdstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying
177c478bdstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bdstevel@tonic-gate *
197c478bdstevel@tonic-gate * CDDL HEADER END
207c478bdstevel@tonic-gate */
217c478bdstevel@tonic-gate/*
22bd28a47Prashanth Sreenivasa * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
237c478bdstevel@tonic-gate */
247c478bdstevel@tonic-gate
257c478bdstevel@tonic-gate#include "assym.h"
267c478bdstevel@tonic-gate
277c478bdstevel@tonic-gate#include <sys/cmn_err.h>
287c478bdstevel@tonic-gate#include <sys/ftrace.h>
297c478bdstevel@tonic-gate#include <sys/asm_linkage.h>
307c478bdstevel@tonic-gate#include <sys/machthread.h>
317c478bdstevel@tonic-gate#include <sys/machcpuvar.h>
327c478bdstevel@tonic-gate#include <sys/intreg.h>
33b0fc0e7govinda#include <sys/ivintr.h>
347c478bdstevel@tonic-gate
357c478bdstevel@tonic-gate#ifdef TRAPTRACE
367c478bdstevel@tonic-gate#include <sys/traptrace.h>
377c478bdstevel@tonic-gate#endif /* TRAPTRACE */
387c478bdstevel@tonic-gate
397c478bdstevel@tonic-gate
407c478bdstevel@tonic-gate/*
417c478bdstevel@tonic-gate * (TT 0x40..0x4F, TL>0) Interrupt Level N Handler (N == 1..15)
427c478bdstevel@tonic-gate * 	Register passed from LEVEL_INTERRUPT(level)
437c478bdstevel@tonic-gate *	%g4 - interrupt request level
447c478bdstevel@tonic-gate */
457c478bdstevel@tonic-gate	ENTRY_NP(pil_interrupt)
467c478bdstevel@tonic-gate	!
477c478bdstevel@tonic-gate	! Register usage
487c478bdstevel@tonic-gate	!	%g1 - cpu
49b0fc0e7govinda	!	%g2 - pointer to intr_vec_t (iv)
507c478bdstevel@tonic-gate	!	%g4 - pil
51b0fc0e7govinda	!	%g3, %g5, %g6, %g7 - temps
527c478bdstevel@tonic-gate	!
53b0fc0e7govinda	! Grab the first or list head intr_vec_t off the intr_head[pil]
54b0fc0e7govinda	! and panic immediately if list head is NULL. Otherwise, update
55b0fc0e7govinda	! intr_head[pil] to next intr_vec_t on the list and clear softint
56b0fc0e7govinda	! %clear_softint, if next intr_vec_t is NULL.
577c478bdstevel@tonic-gate	!
58b0fc0e7govinda	CPU_ADDR(%g1, %g5)		! %g1 = cpu
597c478bdstevel@tonic-gate	!
607c478bdstevel@tonic-gate	ALTENTRY(pil_interrupt_common)
61b0fc0e7govinda	sll	%g4, CPTRSHIFT, %g5	! %g5 = offset to the pil entry
62b0fc0e7govinda	add	%g1, INTR_HEAD, %g6	! %g6 = &cpu->m_cpu.intr_head
63b0fc0e7govinda	add	%g6, %g5, %g6		! %g6 = &cpu->m_cpu.intr_head[pil]
64b0fc0e7govinda	ldn	[%g6], %g2		! %g2 = cpu->m_cpu.intr_head[pil]
65b0fc0e7govinda	brnz,pt	%g2, 0f			! check list head (iv) is NULL
667c478bdstevel@tonic-gate	nop
67b0fc0e7govinda	ba	ptl1_panic		! panic, list head (iv) is NULL
68b0fc0e7govinda	mov	PTL1_BAD_INTR_VEC, %g1
69b0fc0e7govinda0:
70b0fc0e7govinda	lduh	[%g2 + IV_FLAGS], %g7	! %g7 = iv->iv_flags
71b0fc0e7govinda	and	%g7, IV_SOFTINT_MT, %g3 ! %g3 = iv->iv_flags & IV_SOFTINT_MT
72b0fc0e7govinda	brz,pt	%g3, 1f			! check for multi target softint
73b0fc0e7govinda	add	%g2, IV_PIL_NEXT, %g7	! g7% = &iv->iv_pil_next
74b0fc0e7govinda	ld	[%g1 + CPU_ID], %g3	! for multi target softint, use cpuid
75b0fc0e7govinda	sll	%g3, CPTRSHIFT, %g3	! convert cpuid to offset address
76b0fc0e7govinda	add	%g7, %g3, %g7		! %g5 = &iv->iv_xpil_next[cpuid]
777c478bdstevel@tonic-gate1:
78b0fc0e7govinda	ldn	[%g7], %g3		! %g3 = next intr_vec_t
79b0fc0e7govinda	brnz,pn	%g3, 2f			! branch if next intr_vec_t non NULL
80b0fc0e7govinda	stn	%g3, [%g6]		! update cpu->m_cpu.intr_head[pil]
81b0fc0e7govinda	add	%g1, INTR_TAIL, %g6	! %g6 =  &cpu->m_cpu.intr_tail
82b0fc0e7govinda	stn	%g0, [%g5 + %g6]	! clear cpu->m_cpu.intr_tail[pil]
83b0fc0e7govinda	mov	1, %g5			! %g5 = 1
84b0fc0e7govinda	sll	%g5, %g4, %g5		! %g5 = 1 << pil
85b0fc0e7govinda	wr	%g5, CLEAR_SOFTINT	! clear interrupt on this pil
86b0fc0e7govinda2:
877c478bdstevel@tonic-gate#ifdef TRAPTRACE
887c478bdstevel@tonic-gate	TRACE_PTR(%g5, %g6)
897c478bdstevel@tonic-gate	TRACE_SAVE_TL_GL_REGS(%g5, %g6)
907c478bdstevel@tonic-gate	rdpr	%tt, %g6
91b0fc0e7govinda	stha	%g6, [%g5 + TRAP_ENT_TT]%asi	! trap_type = %tt
927c478bdstevel@tonic-gate	rdpr	%tpc, %g6
93b0fc0e7govinda	stna	%g6, [%g5 + TRAP_ENT_TPC]%asi	! trap_pc = %tpc
947c478bdstevel@tonic-gate	rdpr	%tstate, %g6
95b0fc0e7govinda	stxa	%g6, [%g5 + TRAP_ENT_TSTATE]%asi ! trap_tstate = %tstate
96b0fc0e7govinda	stna	%sp, [%g5 + TRAP_ENT_SP]%asi	! trap_sp = %sp
97b0fc0e7govinda	stna	%g2, [%g5 + TRAP_ENT_TR]%asi	! trap_tr = first intr_vec
98b0fc0e7govinda	stna	%g3, [%g5 + TRAP_ENT_F1]%asi	! trap_f1 = next intr_vec
99023e71dHaik Aftandilian	GET_TRACE_TICK(%g6, %g3)
100023e71dHaik Aftandilian	stxa	%g6, [%g5 + TRAP_ENT_TICK]%asi	! trap_tick = %tick
1017c478bdstevel@tonic-gate	sll	%g4, CPTRSHIFT, %g3
1027c478bdstevel@tonic-gate	add	%g1, INTR_HEAD, %g6
103b0fc0e7govinda	ldn	[%g6 + %g3], %g6		! %g6=cpu->m_cpu.intr_head[pil]
104b0fc0e7govinda	stna	%g6, [%g5 + TRAP_ENT_F2]%asi	! trap_f2 = intr_head[pil]
1057c478bdstevel@tonic-gate	add	%g1, INTR_TAIL, %g6
106b0fc0e7govinda	ldn	[%g6 + %g3], %g6		! %g6=cpu->m_cpu.intr_tail[pil]
107b0fc0e7govinda	stna	%g6, [%g5 + TRAP_ENT_F3]%asi	! trap_f3 = intr_tail[pil]
108b0fc0e7govinda	stna	%g4, [%g5 + TRAP_ENT_F4]%asi	! trap_f4 = pil
1097c478bdstevel@tonic-gate	TRACE_NEXT(%g5, %g6, %g3)
1107c478bdstevel@tonic-gate#endif /* TRAPTRACE */
1117c478bdstevel@tonic-gate	!
112b0fc0e7govinda	! clear the iv_pending flag for this interrupt request
1137c478bdstevel@tonic-gate	!
114b0fc0e7govinda	lduh	[%g2 + IV_FLAGS], %g3		! %g3 = iv->iv_flags
115b0fc0e7govinda	andn	%g3, IV_SOFTINT_PEND, %g3	! %g3 = !(iv->iv_flags & PEND)
116b0fc0e7govinda	sth	%g3, [%g2 + IV_FLAGS]		! clear IV_SOFTINT_PEND flag
117b0fc0e7govinda	stn	%g0, [%g7]			! clear iv->iv_pil_next or
118b0fc0e7govinda						!       iv->iv_pil_xnext
1197c478bdstevel@tonic-gate
1207c478bdstevel@tonic-gate	!
1217c478bdstevel@tonic-gate	! Prepare for sys_trap()
1227c478bdstevel@tonic-gate	!
1237c478bdstevel@tonic-gate	! Registers passed to sys_trap()
1247c478bdstevel@tonic-gate	!	%g1 - interrupt handler at TL==0
125b0fc0e7govinda	!	%g2 - pointer to current intr_vec_t (iv),
126b0fc0e7govinda	!	      job queue for intr_thread or current_thread
1277c478bdstevel@tonic-gate	!	%g3 - pil
1287c478bdstevel@tonic-gate	!	%g4 - initial pil for handler
1297c478bdstevel@tonic-gate	!
1307c478bdstevel@tonic-gate	! figure which handler to run and which %pil it starts at
1317c478bdstevel@tonic-gate	! intr_thread starts at DISP_LEVEL to prevent preemption
1327c478bdstevel@tonic-gate	! current_thread starts at PIL_MAX to protect cpu_intr_actv
1337c478bdstevel@tonic-gate	!
134b0fc0e7govinda	mov	%g4, %g3		! %g3 = %g4, pil
1357c478bdstevel@tonic-gate	cmp	%g4, LOCK_LEVEL
136b0fc0e7govinda	bg,a,pt	%xcc, 3f		! branch if pil > LOCK_LEVEL
137b0fc0e7govinda	mov	PIL_MAX, %g4		! %g4 = PIL_MAX (15)
138b0fc0e7govinda	sethi	%hi(intr_thread), %g1	! %g1 = intr_thread
139b0fc0e7govinda	mov	DISP_LEVEL, %g4		! %g4 = DISP_LEVEL (11)
1407c478bdstevel@tonic-gate	ba,pt	%xcc, sys_trap
1417c478bdstevel@tonic-gate	or	%g1, %lo(intr_thread), %g1
142b0fc0e7govinda3:
143b0fc0e7govinda	sethi	%hi(current_thread), %g1 ! %g1 = current_thread
1447c478bdstevel@tonic-gate	ba,pt	%xcc, sys_trap
1457c478bdstevel@tonic-gate	or	%g1, %lo(current_thread), %g1
1467c478bdstevel@tonic-gate	SET_SIZE(pil_interrupt_common)
1477c478bdstevel@tonic-gate	SET_SIZE(pil_interrupt)
1487c478bdstevel@tonic-gate
1497c478bdstevel@tonic-gate
1507c478bdstevel@tonic-gate_spurious:
1517c478bdstevel@tonic-gate	.asciz	"!interrupt 0x%x at level %d not serviced"
152b0fc0e7govinda
1537c478bdstevel@tonic-gate/*
1547c478bdstevel@tonic-gate * SERVE_INTR_PRE is called once, just before the first invocation
1557c478bdstevel@tonic-gate * of SERVE_INTR.
1567c478bdstevel@tonic-gate *
1577c478bdstevel@tonic-gate * Registers on entry:
1587c478bdstevel@tonic-gate *
159b0fc0e7govinda * iv_p, cpu, regs: may be out-registers
1607c478bdstevel@tonic-gate * ls1, ls2: local scratch registers
1617c478bdstevel@tonic-gate * os1, os2, os3: scratch registers, may be out
1627c478bdstevel@tonic-gate */
1637c478bdstevel@tonic-gate
164b0fc0e7govinda#define SERVE_INTR_PRE(iv_p, cpu, ls1, ls2, os1, os2, os3, regs)	\
165b0fc0e7govinda	mov	iv_p, ls1;						\
166b0fc0e7govinda	mov	iv_p, ls2;						\
167b0fc0e7govinda	SERVE_INTR_TRACE(iv_p, os1, os2, os3, regs);
1687c478bdstevel@tonic-gate
1697c478bdstevel@tonic-gate/*
1707c478bdstevel@tonic-gate * SERVE_INTR is called immediately after either SERVE_INTR_PRE or
1717c478bdstevel@tonic-gate * SERVE_INTR_NEXT, without intervening code. No register values
1727c478bdstevel@tonic-gate * may be modified.
1737c478bdstevel@tonic-gate *
1747c478bdstevel@tonic-gate * After calling SERVE_INTR, the caller must check if os3 is set. If
1757c478bdstevel@tonic-gate * so, there is another interrupt to process. The caller must call
1767c478bdstevel@tonic-gate * SERVE_INTR_NEXT, immediately followed by SERVE_INTR.
1777c478bdstevel@tonic-gate *
1787c478bdstevel@tonic-gate * Before calling SERVE_INTR_NEXT, the caller may perform accounting
1797c478bdstevel@tonic-gate * and other actions which need to occur after invocation of an interrupt
1807c478bdstevel@tonic-gate * handler. However, the values of ls1 and os3 *must* be preserved and
1817c478bdstevel@tonic-gate * passed unmodified into SERVE_INTR_NEXT.
1827c478bdstevel@tonic-gate *
1837c478bdstevel@tonic-gate * Registers on return from SERVE_INTR:
1847c478bdstevel@tonic-gate *
1857c478bdstevel@tonic-gate * ls1 - the pil just processed
186b0fc0e7govinda * ls2 - the pointer to intr_vec_t (iv) just processed
1877c478bdstevel@tonic-gate * os3 - if set, another interrupt needs to be processed
1887c478bdstevel@tonic-gate * cpu, ls1, os3 - must be preserved if os3 is set
1897c478bdstevel@tonic-gate */
1907c478bdstevel@tonic-gate
1917c478bdstevel@tonic-gate#define	SERVE_INTR(os5, cpu, ls1, ls2, os1, os2, os3, os4)		\
1927c478bdstevel@tonic-gate	ldn	[ls1 + IV_HANDLER], os2;				\
193b0fc0e7govinda	ldn	[ls1 + IV_ARG1], %o0;					\
194b0fc0e7govinda	ldn	[ls1 + IV_ARG2], %o1;					\
1957c478bdstevel@tonic-gate	call	os2;							\
1967c478bdstevel@tonic-gate	lduh	[ls1 + IV_PIL], ls1;					\
1977c478bdstevel@tonic-gate	brnz,pt	%o0, 2f;						\
1987c478bdstevel@tonic-gate	mov	CE_WARN, %o0;						\
1997c478bdstevel@tonic-gate	set	_spurious, %o1;						\
2007c478bdstevel@tonic-gate	mov	ls2, %o2;						\
2017c478bdstevel@tonic-gate	call	cmn_err;						\
2027c478bdstevel@tonic-gate	rdpr	%pil, %o3;						\
2037c478bdstevel@tonic-gate2:	ldn	[THREAD_REG + T_CPU], cpu;				\
2047c478bdstevel@tonic-gate	sll	ls1, 3, os1;						\
2057c478bdstevel@tonic-gate	add	os1, CPU_STATS_SYS_INTR - 8, os2;			\
2067c478bdstevel@tonic-gate	ldx	[cpu + os2], os3;					\
2077c478bdstevel@tonic-gate	inc	os3;							\
2087c478bdstevel@tonic-gate	stx	os3, [cpu + os2];					\
2097c478bdstevel@tonic-gate	sll	ls1, CPTRSHIFT, os2;					\
2107c478bdstevel@tonic-gate	add	cpu,  INTR_HEAD, os1;					\
2117c478bdstevel@tonic-gate	add	os1, os2, os1;						\
2127c478bdstevel@tonic-gate	ldn	[os1], os3;
2137c478bdstevel@tonic-gate
2147c478bdstevel@tonic-gate/*
2157c478bdstevel@tonic-gate * Registers on entry:
2167c478bdstevel@tonic-gate *
2177c478bdstevel@tonic-gate * cpu			- cpu pointer (clobbered, set to cpu upon completion)
2187c478bdstevel@tonic-gate * ls1, os3		- preserved from prior call to SERVE_INTR
2197c478bdstevel@tonic-gate * ls2			- local scratch reg (not preserved)
2207c478bdstevel@tonic-gate * os1, os2, os4, os5	- scratch reg, can be out (not preserved)
2217c478bdstevel@tonic-gate */
2227c478bdstevel@tonic-gate#define SERVE_INTR_NEXT(os5, cpu, ls1, ls2, os1, os2, os3, os4)		\
2237c478bdstevel@tonic-gate	sll	ls1, CPTRSHIFT, os4;					\
2247c478bdstevel@tonic-gate	add	cpu, INTR_HEAD, os1;					\
2257c478bdstevel@tonic-gate	rdpr	%pstate, ls2;						\
2267c478bdstevel@tonic-gate	wrpr	ls2, PSTATE_IE, %pstate;				\
227b0fc0e7govinda	lduh	[os3 + IV_FLAGS], os2;					\
228b0fc0e7govinda	and	os2, IV_SOFTINT_MT, os2;				\
229b0fc0e7govinda	brz,pt	os2, 4f;						\
230b0fc0e7govinda	add	os3, IV_PIL_NEXT, os2;					\
231b0fc0e7govinda	ld	[cpu + CPU_ID], os5;					\
232b0fc0e7govinda	sll	os5, CPTRSHIFT, os5;					\
233b0fc0e7govinda	add	os2, os5, os2;						\
234b0fc0e7govinda4:	ldn	[os2], os5;						\
235b0fc0e7govinda	brnz,pn	os5, 5f;						\
236b0fc0e7govinda	stn	os5, [os1 + os4];					\
2377c478bdstevel@tonic-gate	add	cpu, INTR_TAIL, os1;					\
2387c478bdstevel@tonic-gate	stn	%g0, [os1 + os4];					\
2397c478bdstevel@tonic-gate	mov	1, os1;							\
2407c478bdstevel@tonic-gate	sll	os1, ls1, os1;						\
2417c478bdstevel@tonic-gate	wr	os1, CLEAR_SOFTINT;					\
242b0fc0e7govinda5:	lduh	[os3 + IV_FLAGS], ls1;                                  \
243b0fc0e7govinda	andn	ls1, IV_SOFTINT_PEND, ls1;				\
244b0fc0e7govinda	sth	ls1, [os3 + IV_FLAGS];				        \
245b0fc0e7govinda	stn	%g0, [os2];						\
2467c478bdstevel@tonic-gate	wrpr	%g0, ls2, %pstate;					\
247b0fc0e7govinda	mov	os3, ls1;						\
248b0fc0e7govinda	mov	os3, ls2;						\
249b0fc0e7govinda	SERVE_INTR_TRACE2(os5, os1, os2, os3, os4);
2507c478bdstevel@tonic-gate
2517c478bdstevel@tonic-gate#ifdef TRAPTRACE
2527c478bdstevel@tonic-gate/*
2537c478bdstevel@tonic-gate * inum - not modified, _spurious depends on it.
2547c478bdstevel@tonic-gate */
2557c478bdstevel@tonic-gate#define	SERVE_INTR_TRACE(inum, os1, os2, os3, os4)			\
2567c478bdstevel@tonic-gate	rdpr	%pstate, os3;						\
2577c478bdstevel@tonic-gate	andn	os3, PSTATE_IE | PSTATE_AM, os2;			\
2587c478bdstevel@tonic-gate	wrpr	%g0, os2, %pstate;					\
259023e71dHaik Aftandilian	TRACE_PTR(os1, os2); 						\
2607c478bdstevel@tonic-gate	ldn	[os4 + PC_OFF], os2;					\
2617c478bdstevel@tonic-gate	stna	os2, [os1 + TRAP_ENT_TPC]%asi;				\
2627c478bdstevel@tonic-gate	ldx	[os4 + TSTATE_OFF], os2;				\
2637c478bdstevel@tonic-gate	stxa	os2, [os1 + TRAP_ENT_TSTATE]%asi;			\
2647c478bdstevel@tonic-gate	mov	os3, os4;						\
265023e71dHaik Aftandilian	GET_TRACE_TICK(os2, os3);					\
2667c478bdstevel@tonic-gate	stxa	os2, [os1 + TRAP_ENT_TICK]%asi;				\
2677c478bdstevel@tonic-gate	TRACE_SAVE_TL_GL_REGS(os1, os2);				\
2687c478bdstevel@tonic-gate	set	TT_SERVE_INTR, os2;					\
2697c478bdstevel@tonic-gate	rdpr	%pil, os3;						\
2707c478bdstevel@tonic-gate	or	os2, os3, os2;						\
2717c478bdstevel@tonic-gate	stha	os2, [os1 + TRAP_ENT_TT]%asi;				\
2727c478bdstevel@tonic-gate	stna	%sp, [os1 + TRAP_ENT_SP]%asi;				\
2737c478bdstevel@tonic-gate	stna	inum, [os1 + TRAP_ENT_TR]%asi;				\
2747c478bdstevel@tonic-gate	stna	%g0, [os1 + TRAP_ENT_F1]%asi;				\
2757c478bdstevel@tonic-gate	stna	%g0, [os1 + TRAP_ENT_F2]%asi;				\
2767c478bdstevel@tonic-gate	stna	%g0, [os1 + TRAP_ENT_F3]%asi;				\
2777c478bdstevel@tonic-gate	stna	%g0, [os1 + TRAP_ENT_F4]%asi;				\
2787c478bdstevel@tonic-gate	TRACE_NEXT(os1, os2, os3);					\
2797c478bdstevel@tonic-gate	wrpr	%g0, os4, %pstate
2807c478bdstevel@tonic-gate#else	/* TRAPTRACE */
2817c478bdstevel@tonic-gate#define SERVE_INTR_TRACE(inum, os1, os2, os3, os4)
2827c478bdstevel@tonic-gate#endif	/* TRAPTRACE */
2837c478bdstevel@tonic-gate
2847c478bdstevel@tonic-gate#ifdef TRAPTRACE
2857c478bdstevel@tonic-gate/*
2867c478bdstevel@tonic-gate * inum - not modified, _spurious depends on it.
2877c478bdstevel@tonic-gate */
2887c478bdstevel@tonic-gate#define	SERVE_INTR_TRACE2(inum, os1, os2, os3, os4)			\
2897c478bdstevel@tonic-gate	rdpr	%pstate, os3;						\
2907c478bdstevel@tonic-gate	andn	os3, PSTATE_IE | PSTATE_AM, os2;			\
2917c478bdstevel@tonic-gate	wrpr	%g0, os2, %pstate;					\
292023e71dHaik Aftandilian	TRACE_PTR(os1, os2); 						\
2937c478bdstevel@tonic-gate	stna	%g0, [os1 + TRAP_ENT_TPC]%asi;				\
2947c478bdstevel@tonic-gate	stxa	%g0, [os1 + TRAP_ENT_TSTATE]%asi;			\
2957c478bdstevel@tonic-gate	mov	os3, os4;						\
296023e71dHaik Aftandilian	GET_TRACE_TICK(os2, os3);					\
2977c478bdstevel@tonic-gate	stxa	os2, [os1 + TRAP_ENT_TICK]%asi;				\
2987c478bdstevel@tonic-gate	TRACE_SAVE_TL_GL_REGS(os1, os2);				\
2997c478bdstevel@tonic-gate	set	TT_SERVE_INTR, os2;					\
3007c478bdstevel@tonic-gate	rdpr	%pil, os3;						\
3017c478bdstevel@tonic-gate	or	os2, os3, os2;						\
3027c478bdstevel@tonic-gate	stha	os2, [os1 + TRAP_ENT_TT]%asi;				\
3037c478bdstevel@tonic-gate	stna	%sp, [os1 + TRAP_ENT_SP]%asi;				\
3047c478bdstevel@tonic-gate	stna	inum, [os1 + TRAP_ENT_TR]%asi;				\
3057c478bdstevel@tonic-gate	stna	%g0, [os1 + TRAP_ENT_F1]%asi;				\
3067c478bdstevel@tonic-gate	stna	%g0, [os1 + TRAP_ENT_F2]%asi;				\
3077c478bdstevel@tonic-gate	stna	%g0, [os1 + TRAP_ENT_F3]%asi;				\
3087c478bdstevel@tonic-gate	stna	%g0, [os1 + TRAP_ENT_F4]%asi;				\
3097c478bdstevel@tonic-gate	TRACE_NEXT(os1, os2, os3);					\
3107c478bdstevel@tonic-gate	wrpr	%g0, os4, %pstate
3117c478bdstevel@tonic-gate#else	/* TRAPTRACE */
3127c478bdstevel@tonic-gate#define SERVE_INTR_TRACE2(inum, os1, os2, os3, os4)
3137c478bdstevel@tonic-gate#endif	/* TRAPTRACE */
3147c478bdstevel@tonic-gate
3157c478bdstevel@tonic-gate#define	INTRCNT_LIMIT 16
3167c478bdstevel@tonic-gate
3177c478bdstevel@tonic-gate/*
3187c478bdstevel@tonic-gate * Handle an interrupt in a new thread.
3197c478bdstevel@tonic-gate *	Entry:
3207c478bdstevel@tonic-gate *		%o0       = pointer to regs structure
321b0fc0e7govinda *		%o1       = pointer to current intr_vec_t (iv) to be processed
3227c478bdstevel@tonic-gate *		%o2       = pil
3237c478bdstevel@tonic-gate *		%sp       = on current thread's kernel stack
3247c478bdstevel@tonic-gate *		%o7       = return linkage to trap code
3257c478bdstevel@tonic-gate *		%g7       = current thread
3267c478bdstevel@tonic-gate *		%pstate   = normal globals, interrupts enabled,
3277c478bdstevel@tonic-gate *		            privileged, fp disabled
3287c478bdstevel@tonic-gate *		%pil      = DISP_LEVEL
3297c478bdstevel@tonic-gate *
3307c478bdstevel@tonic-gate *	Register Usage
3317c478bdstevel@tonic-gate *		%l0       = return linkage
3327c478bdstevel@tonic-gate *		%l1       = pil
3337c478bdstevel@tonic-gate *		%l2 - %l3 = scratch
3347c478bdstevel@tonic-gate *		%l4 - %l7 = reserved for sys_trap
3357c478bdstevel@tonic-gate *		%o2       = cpu
3367c478bdstevel@tonic-gate *		%o3       = intr thread
3377c478bdstevel@tonic-gate *		%o0       = scratch
3387c478bdstevel@tonic-gate *		%o4 - %o5 = scratch
3397c478bdstevel@tonic-gate */
3407c478bdstevel@tonic-gate	ENTRY_NP(intr_thread)
3417c478bdstevel@tonic-gate	mov	%o7, %l0
3427c478bdstevel@tonic-gate	mov	%o2, %l1
3437c478bdstevel@tonic-gate	!
3447c478bdstevel@tonic-gate	! See if we are interrupting another interrupt thread.
3457c478bdstevel@tonic-gate	!
3467c478bdstevel@tonic-gate	lduh	[THREAD_REG + T_FLAGS], %o3
3477c478bdstevel@tonic-gate	andcc	%o3, T_INTR_THREAD, %g0
3487c478bdstevel@tonic-gate	bz,pt	%xcc, 1f
3497c478bdstevel@tonic-gate	ldn	[THREAD_REG + T_CPU], %o2	! delay - load CPU pointer
3507c478bdstevel@tonic-gate
3517c478bdstevel@tonic-gate	! We have interrupted an interrupt thread. Take a timestamp,
3527c478bdstevel@tonic-gate	! compute its interval, and update its cumulative counter.
3537c478bdstevel@tonic-gate	add	THREAD_REG, T_INTR_START, %o5
3547c478bdstevel@tonic-gate0:
3557c478bdstevel@tonic-gate	ldx	[%o5], %o3
3567c478bdstevel@tonic-gate	brz,pn	%o3, 1f
3577c478bdstevel@tonic-gate	! We came in on top of an interrupt thread that had no timestamp.
3587c478bdstevel@tonic-gate	! This could happen if, for instance, an interrupt thread which had
3597c478bdstevel@tonic-gate	! previously blocked is being set up to run again in resume(), but
3607c478bdstevel@tonic-gate	! resume() hasn't yet stored a timestamp for it. Or, it could be in
3617c478bdstevel@tonic-gate	! swtch() after its slice has been accounted for.
3627c478bdstevel@tonic-gate	! Only account for the time slice if the starting timestamp is non-zero.
363bd28a47Prashanth Sreenivasa	RD_CLOCK_TICK(%o4,%l2,%l3,__LINE__)
3647c478bdstevel@tonic-gate	sub	%o4, %o3, %o4			! o4 has interval
3657c478bdstevel@tonic-gate
3667c478bdstevel@tonic-gate	! A high-level interrupt in current_thread() interrupting here
3677c478bdstevel@tonic-gate	! will account for the interrupted thread's time slice, but
3687c478bdstevel@tonic-gate	! only if t_intr_start is non-zero. Since this code is going to account
3697c478bdstevel@tonic-gate	! for the time slice, we want to "atomically" load the thread's
3707c478bdstevel@tonic-gate	! starting timestamp, calculate the interval with %tick, and zero
3717c478bdstevel@tonic-gate	! its starting timestamp.
3727c478bdstevel@tonic-gate	! To do this, we do a casx on the t_intr_start field, and store 0 to it.
3737c478bdstevel@tonic-gate	! If it has changed since we loaded it above, we need to re-compute the
3747c478bdstevel@tonic-gate	! interval, since a changed t_intr_start implies current_thread placed
3757c478bdstevel@tonic-gate	! a new, later timestamp there after running a high-level interrupt,
3767c478bdstevel@tonic-gate	! and the %tick val in %o4 had become stale.
3777c478bdstevel@tonic-gate	mov	%g0, %l2
3787c478bdstevel@tonic-gate	casx	[%o5], %o3, %l2
3797c478bdstevel@tonic-gate
3807c478bdstevel@tonic-gate	! If %l2 == %o3, our casx was successful. If not, the starting timestamp
3817c478bdstevel@tonic-gate	! changed between loading it (after label 0b) and computing the
3827c478bdstevel@tonic-gate	! interval above.
3837c478bdstevel@tonic-gate	cmp	%l2, %o3
3847c478bdstevel@tonic-gate	bne,pn	%xcc, 0b
3857c478bdstevel@tonic-gate
3867c478bdstevel@tonic-gate	! Check for Energy Star mode
3877c478bdstevel@tonic-gate	lduh	[%o2 + CPU_DIVISOR], %l2	! delay -- %l2 = clock divisor
3887c478bdstevel@tonic-gate	cmp	%l2, 1
3897c478bdstevel@tonic-gate	bg,a,pn	%xcc, 2f
3907c478bdstevel@tonic-gate	mulx	%o4, %l2, %o4	! multiply interval by clock divisor iff > 1
3917c478bdstevel@tonic-gate2:
3927c478bdstevel@tonic-gate	! We now know that a valid interval for the interrupted interrupt
3937c478bdstevel@tonic-gate	! thread is in %o4. Update its cumulative counter.
3947c478bdstevel@tonic-gate	ldub	[THREAD_REG + T_PIL], %l3	! load PIL
3957c478bdstevel@tonic-gate	sllx	%l3, 4, %l3		! convert PIL index to byte offset
3967c478bdstevel@tonic-gate	add	%l3, CPU_MCPU, %l3	! CPU_INTRSTAT is too big for use
3977c478bdstevel@tonic-gate	add	%l3, MCPU_INTRSTAT, %l3	! as const, add offsets separately
3987c478bdstevel@tonic-gate	ldx	[%o2 + %l3], %o5	! old counter in o5
3997c478bdstevel@tonic-gate	add	%o5, %o4, %o5		! new counter in o5
4007c478bdstevel@tonic-gate	stx	%o5, [%o2 + %l3]	! store new counter
4017c478bdstevel@tonic-gate
402eda8946esolom	! Also update intracct[]
403eda8946esolom	lduh	[%o2 + CPU_MSTATE], %l3
404eda8946esolom	sllx	%l3, 3, %l3
405eda8946esolom	add	%l3, CPU_INTRACCT, %l3
406eda8946esolom	add	%l3, %o2, %l3
407eda8946esolom0:
408eda8946esolom	ldx	[%l3], %o5
409eda8946esolom	add	%o5, %o4, %o3
410eda8946esolom	casx	[%l3], %o5, %o3
411eda8946esolom	cmp	%o5, %o3
412eda8946esolom	bne,pn	%xcc, 0b
413eda8946esolom	nop
414eda8946esolom
4157c478bdstevel@tonic-gate1:
4167c478bdstevel@tonic-gate	!
4177c478bdstevel@tonic-gate	! Get set to run interrupt thread.
4187c478bdstevel@tonic-gate	! There should always be an interrupt thread since we allocate one
4197c478bdstevel@tonic-gate	! for each level on the CPU.
4207c478bdstevel@tonic-gate	!
4217c478bdstevel@tonic-gate	! Note that the code in kcpc_overflow_intr -relies- on the ordering
4227c478bdstevel@tonic-gate	! of events here -- in particular that t->t_lwp of the interrupt thread
4237c478bdstevel@tonic-gate	! is set to the pinned thread *before* curthread is changed.
4247c478bdstevel@tonic-gate	!
4257c478bdstevel@tonic-gate	ldn	[%o2 + CPU_INTR_THREAD], %o3	! interrupt thread pool
4267c478bdstevel@tonic-gate	ldn	[%o3 + T_LINK], %o4		! unlink thread from CPU's list
4277c478bdstevel@tonic-gate	stn	%o4, [%o2 + CPU_INTR_THREAD]
4287c478bdstevel@tonic-gate	!
4297c478bdstevel@tonic-gate	! Set bit for this level in CPU's active interrupt bitmask.
4307c478bdstevel@tonic-gate	!
4317c478bdstevel@tonic-gate	ld	[%o2 + CPU_INTR_ACTV], %o5
4327c478bdstevel@tonic-gate	mov	1, %o4
4337c478bdstevel@tonic-gate	sll	%o4, %l1, %o4
4347c478bdstevel@tonic-gate#ifdef DEBUG
4357c478bdstevel@tonic-gate	!
4367c478bdstevel@tonic-gate	! ASSERT(!(CPU->cpu_intr_actv & (1 << PIL)))
4377c478bdstevel@tonic-gate	!
4387c478bdstevel@tonic-gate	andcc	%o5, %o4, %g0
4397c478bdstevel@tonic-gate	bz,pt	%xcc, 0f
4407c478bdstevel@tonic-gate	nop
4417c478bdstevel@tonic-gate	! Do not call panic if a panic is already in progress.
4427c478bdstevel@tonic-gate	sethi	%hi(panic_quiesce), %l2
4437c478bdstevel@tonic-gate	ld	[%l2 + %lo(panic_quiesce)], %l2
4447c478bdstevel@tonic-gate	brnz,pn	%l2, 0f
4457c478bdstevel@tonic-gate	nop
4467c478bdstevel@tonic-gate	sethi	%hi(intr_thread_actv_bit_set), %o0
4477c478bdstevel@tonic-gate	call	panic
4487c478bdstevel@tonic-gate	or	%o0, %lo(intr_thread_actv_bit_set), %o0
4497c478bdstevel@tonic-gate0:
4507c478bdstevel@tonic-gate#endif /* DEBUG */
4517c478bdstevel@tonic-gate	or	%o5, %o4, %o5
4527c478bdstevel@tonic-gate	st	%o5, [%o2 + CPU_INTR_ACTV]
4537c478bdstevel@tonic-gate	!
4547c478bdstevel@tonic-gate	! Consider the new thread part of the same LWP so that
4557c478bdstevel@tonic-gate	! window overflow code can find the PCB.
4567c478bdstevel@tonic-gate	!
4577c478bdstevel@tonic-gate	ldn	[THREAD_REG + T_LWP], %o4
4587c478bdstevel@tonic-gate	stn	%o4, [%o3 + T_LWP]
4597c478bdstevel@tonic-gate	!
4607c478bdstevel@tonic-gate	! Threads on the interrupt thread free list could have state already
4617c478bdstevel@tonic-gate	! set to TS_ONPROC, but it helps in debugging if they're TS_FREE
4627c478bdstevel@tonic-gate	! Could eliminate the next two instructions with a little work.
4637c478bdstevel@tonic-gate	!
4647c478bdstevel@tonic-gate	mov	TS_ONPROC, %o4
4657c478bdstevel@tonic-gate	st	%o4, [%o3 + T_STATE]
4667c478bdstevel@tonic-gate	!
4677c478bdstevel@tonic-gate	! Push interrupted thread onto list from new thread.
4687c478bdstevel@tonic-gate	! Set the new thread as the current one.
4697c478bdstevel@tonic-gate	! Set interrupted thread's T_SP because if it is the idle thread,
4707c478bdstevel@tonic-gate	! resume may use that stack between threads.
4717c478bdstevel@tonic-gate	!
4727c478bdstevel@tonic-gate	stn	%o7, [THREAD_REG + T_PC]	! mark pc for resume
4737c478bdstevel@tonic-gate	stn	%sp, [THREAD_REG + T_SP]	! mark stack for resume
4747c478bdstevel@tonic-gate	stn	THREAD_REG, [%o3 + T_INTR]	! push old thread
4757c478bdstevel@tonic-gate	stn	%o3, [%o2 + CPU_THREAD]		! set new thread
4767c478bdstevel@tonic-gate	mov	%o3, THREAD_REG			! set global curthread register
4777c478bdstevel@tonic-gate	ldn	[%o3 + T_STACK], %o4		! interrupt stack pointer
4787c478bdstevel@tonic-gate	sub	%o4, STACK_BIAS, %sp
4797c478bdstevel@tonic-gate	!
4807c478bdstevel@tonic-gate	! Initialize thread priority level from intr_pri
4817c478bdstevel@tonic-gate	!
4827c478bdstevel@tonic-gate	sethi	%hi(intr_pri), %o4
4837c478bdstevel@tonic-gate	ldsh	[%o4 + %lo(intr_pri)], %o4	! grab base interrupt priority
4847c478bdstevel@tonic-gate	add	%l1, %o4, %o4		! convert level to dispatch priority
4857c478bdstevel@tonic-gate	sth	%o4, [THREAD_REG + T_PRI]
4867c478bdstevel@tonic-gate	stub	%l1, [THREAD_REG + T_PIL]	! save pil for intr_passivate
4877c478bdstevel@tonic-gate
4887c478bdstevel@tonic-gate	! Store starting timestamp in thread structure.
4897c478bdstevel@tonic-gate	add	THREAD_REG, T_INTR_START, %o3
4907c478bdstevel@tonic-gate1:
4917c478bdstevel@tonic-gate	ldx	[%o3], %o5
492bd28a47Prashanth Sreenivasa	RD_CLOCK_TICK(%o4,%l2,%l3,__LINE__)
4937c478bdstevel@tonic-gate	casx	[%o3], %o5, %o4
4947c478bdstevel@tonic-gate	cmp	%o4, %o5
4957c478bdstevel@tonic-gate	! If a high-level interrupt occurred while we were attempting to store
4967c478bdstevel@tonic-gate	! the timestamp, try again.
4977c478bdstevel@tonic-gate	bne,pn	%xcc, 1b
4987c478bdstevel@tonic-gate	nop
4997c478bdstevel@tonic-gate
5007c478bdstevel@tonic-gate	wrpr	%g0, %l1, %pil			! lower %pil to new level
5017c478bdstevel@tonic-gate	!
5027c478bdstevel@tonic-gate	! Fast event tracing.
5037c478bdstevel@tonic-gate	!
5047c478bdstevel@tonic-gate	ld	[%o2 + CPU_FTRACE_STATE], %o4	! %o2 = curthread->t_cpu
5057c478bdstevel@tonic-gate	btst	FTRACE_ENABLED, %o4
5067c478bdstevel@tonic-gate	be,pt	%icc, 1f			! skip if ftrace disabled
5077c478bdstevel@tonic-gate	  mov	%l1, %o5
5087c478bdstevel@tonic-gate	!
5097c478bdstevel@tonic-gate	! Tracing is enabled - write the trace entry.
5107c478bdstevel@tonic-gate	!
5117c478bdstevel@tonic-gate	save	%sp, -SA(MINFRAME), %sp
5127c478bdstevel@tonic-gate	set	ftrace_intr_thread_format_str, %o0
5137c478bdstevel@tonic-gate	mov	%i0, %o1
5147c478bdstevel@tonic-gate	mov	%i1, %o2
5157c478bdstevel@tonic-gate	mov	%i5, %o3
5164df4bd6bs	call	ftrace_3
5174df4bd6bs	ldn	[%i0 + PC_OFF], %o4
5187c478bdstevel@tonic-gate	restore
5197c478bdstevel@tonic-gate1:
5207c478bdstevel@tonic-gate	!
5217c478bdstevel@tonic-gate	! call the handler
5227c478bdstevel@tonic-gate	!
5237c478bdstevel@tonic-gate	SERVE_INTR_PRE(%o1, %o2, %l1, %l3, %o4, %o5, %o3, %o0)
5247c478bdstevel@tonic-gate	!
5257c478bdstevel@tonic-gate	! %o0 and %o1 are now available as scratch registers.
5267c478bdstevel@tonic-gate	!
5277c478bdstevel@tonic-gate0:
5287c478bdstevel@tonic-gate	SERVE_INTR(%o1, %o2, %l1, %l3, %o4, %o5, %o3, %o0)
5297c478bdstevel@tonic-gate	!
530b0fc0e7govinda	! If %o3 is set, we must call serve_intr_next, and both %l1 and %o3
5317c478bdstevel@tonic-gate	! must be preserved. %l1 holds our pil, %l3 holds our inum.
5327c478bdstevel@tonic-gate	!
5337c478bdstevel@tonic-gate	! Note: %l1 is the pil level we're processing, but we may have a
5347c478bdstevel@tonic-gate	! higher effective pil because a higher-level interrupt may have
5357c478bdstevel@tonic-gate	! blocked.
5367c478bdstevel@tonic-gate	!
5377c478bdstevel@tonic-gate	wrpr	%g0, DISP_LEVEL, %pil
5387c478bdstevel@tonic-gate	!
5397c478bdstevel@tonic-gate	! Take timestamp, compute interval, update cumulative counter.
5407c478bdstevel@tonic-gate	!
5417c478bdstevel@tonic-gate	add	THREAD_REG, T_INTR_START, %o5
5427c478bdstevel@tonic-gate1:
5437c478bdstevel@tonic-gate	ldx	[%o5], %o0
5447c478bdstevel@tonic-gate#ifdef DEBUG
5457c478bdstevel@tonic-gate	brnz	%o0, 9f
5467c478bdstevel@tonic-gate	nop
5477c478bdstevel@tonic-gate	! Do not call panic if a panic is already in progress.
5487c478bdstevel@tonic-gate	sethi	%hi(panic_quiesce), %o1
5497c478bdstevel@tonic-gate	ld	[%o1 + %lo(panic_quiesce)], %o1
5507c478bdstevel@tonic-gate	brnz,pn	%o1, 9f
5517c478bdstevel@tonic-gate	nop
5527c478bdstevel@tonic-gate	sethi	%hi(intr_thread_t_intr_start_zero), %o0
5537c478bdstevel@tonic-gate	call	panic
5547c478bdstevel@tonic-gate	or	%o0, %lo(intr_thread_t_intr_start_zero), %o0
5557c478bdstevel@tonic-gate9:
5567c478bdstevel@tonic-gate#endif /* DEBUG */
557bd28a47Prashanth Sreenivasa	RD_CLOCK_TICK(%o1,%l2,%l3,__LINE__)
5587c478bdstevel@tonic-gate	sub	%o1, %o0, %l2			! l2 has interval
5597c478bdstevel@tonic-gate	!
5607c478bdstevel@tonic-gate	! The general outline of what the code here does is:
5617c478bdstevel@tonic-gate	! 1. load t_intr_start, %tick, and calculate the delta
5627c478bdstevel@tonic-gate	! 2. replace t_intr_start with %tick (if %o3 is set) or 0.
5637c478bdstevel@tonic-gate	!
5647c478bdstevel@tonic-gate	! The problem is that a high-level interrupt could arrive at any time.
5657c478bdstevel@tonic-gate	! It will account for (%tick - t_intr_start) for us when it starts,
5667c478bdstevel@tonic-gate	! unless we have set t_intr_start to zero, and then set t_intr_start
5677c478bdstevel@tonic-gate	! to a new %tick when it finishes. To account for this, our first step
5687c478bdstevel@tonic-gate	! is to load t_intr_start and the last is to use casx to store the new
5697c478bdstevel@tonic-gate	! t_intr_start. This guarantees atomicity in reading t_intr_start,
5707c478bdstevel@tonic-gate	! reading %tick, and updating t_intr_start.
5717c478bdstevel@tonic-gate	!
5727c478bdstevel@tonic-gate	movrz	%o3, %g0, %o1
5737c478bdstevel@tonic-gate	casx	[%o5], %o0, %o1
5747c478bdstevel@tonic-gate	cmp	%o0, %o1
5757c478bdstevel@tonic-gate	bne,pn	%xcc, 1b
5767c478bdstevel@tonic-gate	!
5777c478bdstevel@tonic-gate	! Check for Energy Star mode
5787c478bdstevel@tonic-gate	!
5797c478bdstevel@tonic-gate	lduh	[%o2 + CPU_DIVISOR], %o0	! delay -- %o0 = clock divisor
5807c478bdstevel@tonic-gate	cmp	%o0, 1
5817c478bdstevel@tonic-gate	bg,a,pn	%xcc, 2f
5827c478bdstevel@tonic-gate	mulx	%l2, %o0, %l2	! multiply interval by clock divisor iff > 1
5837c478bdstevel@tonic-gate2:
5847c478bdstevel@tonic-gate	!
5857c478bdstevel@tonic-gate	! Update cpu_intrstat. If o3 is set then we will be processing another
5867c478bdstevel@tonic-gate	! interrupt. Above we have set t_intr_start to %tick, not 0. This
5877c478bdstevel@tonic-gate	! means a high-level interrupt can arrive and update the same stats
5887c478bdstevel@tonic-gate	! we're updating. Need to use casx.
5897c478bdstevel@tonic-gate	!
5907c478bdstevel@tonic-gate	sllx	%l1, 4, %o1			! delay - PIL as byte offset
5917c478bdstevel@tonic-gate	add	%o1, CPU_MCPU, %o1		! CPU_INTRSTAT const too big
5927c478bdstevel@tonic-gate	add	%o1, MCPU_INTRSTAT, %o1		! add parts separately
5937c478bdstevel@tonic-gate	add	%o1, %o2, %o1
5947c478bdstevel@tonic-gate1:
5957c478bdstevel@tonic-gate	ldx	[%o1], %o5			! old counter in o5
5967c478bdstevel@tonic-gate	add	%o5, %l2, %o0			! new counter in o0
5977c478bdstevel@tonic-gate 	stx	%o0, [%o1 + 8]			! store into intrstat[pil][1]
5987c478bdstevel@tonic-gate	casx	[%o1], %o5, %o0			! and into intrstat[pil][0]
5997c478bdstevel@tonic-gate	cmp	%o5, %o0
6007c478bdstevel@tonic-gate	bne,pn	%xcc, 1b
6017c478bdstevel@tonic-gate	nop
602eda8946esolom
603eda8946esolom	! Also update intracct[]
604eda8946esolom	lduh	[%o2 + CPU_MSTATE], %o1
605eda8946esolom	sllx	%o1, 3, %o1
606eda8946esolom	add	%o1, CPU_INTRACCT, %o1
607eda8946esolom	add	%o1, %o2, %o1
608eda8946esolom1:
609eda8946esolom	ldx	[%o1], %o5
610eda8946esolom	add	%o5, %l2, %o0
611eda8946esolom	casx	[%o1], %o5, %o0
612eda8946esolom	cmp	%o5, %o0
613eda8946esolom	bne,pn	%xcc, 1b
614eda8946esolom	nop
615eda8946esolom
6167c478bdstevel@tonic-gate	!
6177c478bdstevel@tonic-gate	! Don't keep a pinned process pinned indefinitely. Bump cpu_intrcnt
6187c478bdstevel@tonic-gate	! for each interrupt handler we invoke. If we hit INTRCNT_LIMIT, then
6197c478bdstevel@tonic-gate	! we've crossed the threshold and we should unpin the pinned threads
6207c478bdstevel@tonic-gate	! by preempt()ing ourselves, which will bubble up the t_intr chain
6217c478bdstevel@tonic-gate	! until hitting the non-interrupt thread, which will then in turn
6227c478bdstevel@tonic-gate	! preempt itself allowing the interrupt processing to resume. Finally,
6237c478bdstevel@tonic-gate	! the scheduler takes over and picks the next thread to run.
6247c478bdstevel@tonic-gate	!
6257c478bdstevel@tonic-gate	! If our CPU is quiesced, we cannot preempt because the idle thread
6267c478bdstevel@tonic-gate	! won't ever re-enter the scheduler, and the interrupt will be forever
6277c478bdstevel@tonic-gate	! blocked.
6287c478bdstevel@tonic-gate	!
6297c478bdstevel@tonic-gate	! If t_intr is NULL, we're not pinning anyone, so we use a simpler
6307c478bdstevel@tonic-gate	! algorithm. Just check for cpu_kprunrun, and if set then preempt.
6317c478bdstevel@tonic-gate	! This insures we enter the scheduler if a higher-priority thread
6327c478bdstevel@tonic-gate	! has become runnable.
6337c478bdstevel@tonic-gate	!
6347c478bdstevel@tonic-gate	lduh	[%o2 + CPU_FLAGS], %o5		! don't preempt if quiesced
6357c478bdstevel@tonic-gate	andcc	%o5, CPU_QUIESCED, %g0
6367c478bdstevel@tonic-gate	bnz,pn	%xcc, 1f
6377c478bdstevel@tonic-gate
6387c478bdstevel@tonic-gate	ldn     [THREAD_REG + T_INTR], %o5      ! pinning anything?
6397c478bdstevel@tonic-gate	brz,pn  %o5, 3f				! if not, don't inc intrcnt
6407c478bdstevel@tonic-gate
6417c478bdstevel@tonic-gate	ldub	[%o2 + CPU_INTRCNT], %o5	! delay - %o5 = cpu_intrcnt
6427c478bdstevel@tonic-gate	inc	%o5
6437c478bdstevel@tonic-gate	cmp	%o5, INTRCNT_LIMIT		! have we hit the limit?
6447c478bdstevel@tonic-gate	bl,a,pt	%xcc, 1f			! no preempt if < INTRCNT_LIMIT
6457c478bdstevel@tonic-gate	stub	%o5, [%o2 + CPU_INTRCNT]	! delay annul - inc CPU_INTRCNT
6467c478bdstevel@tonic-gate	bg,pn	%xcc, 2f			! don't inc stats again
6477c478bdstevel@tonic-gate	!
6487c478bdstevel@tonic-gate	! We've reached the limit. Set cpu_intrcnt and cpu_kprunrun, and do
6497c478bdstevel@tonic-gate	! CPU_STATS_ADDQ(cp, sys, intrunpin, 1). Then call preempt.
6507c478bdstevel@tonic-gate	!
6517c478bdstevel@tonic-gate	mov	1, %o4				! delay
6527c478bdstevel@tonic-gate	stub	%o4, [%o2 + CPU_KPRUNRUN]
6537c478bdstevel@tonic-gate	ldx	[%o2 + CPU_STATS_SYS_INTRUNPIN], %o4
6547c478bdstevel@tonic-gate	inc	%o4
6557c478bdstevel@tonic-gate	stx	%o4, [%o2 + CPU_STATS_SYS_INTRUNPIN]
6567c478bdstevel@tonic-gate	ba	2f
6577c478bdstevel@tonic-gate	stub	%o5, [%o2 + CPU_INTRCNT]	! delay
6587c478bdstevel@tonic-gate3:
6597c478bdstevel@tonic-gate	! Code for t_intr == NULL
6607c478bdstevel@tonic-gate	ldub	[%o2 + CPU_KPRUNRUN], %o5
6617c478bdstevel@tonic-gate	brz,pt	%o5, 1f				! don't preempt unless kprunrun
6627c478bdstevel@tonic-gate2:
6637c478bdstevel@tonic-gate	! Time to call preempt
6647c478bdstevel@tonic-gate	mov	%o2, %l3			! delay - save %o2
6657c478bdstevel@tonic-gate	call	preempt
6667c478bdstevel@tonic-gate	mov	%o3, %l2			! delay - save %o3.
6677c478bdstevel@tonic-gate	mov	%l3, %o2			! restore %o2
6687c478bdstevel@tonic-gate	mov	%l2, %o3			! restore %o3
6697c478bdstevel@tonic-gate	wrpr	%g0, DISP_LEVEL, %pil		! up from cpu_base_spl
6707c478bdstevel@tonic-gate1:
6717c478bdstevel@tonic-gate	!
672b0fc0e7govinda	! Do we need to call serve_intr_next and do this again?
6737c478bdstevel@tonic-gate	!
6747c478bdstevel@tonic-gate	brz,a,pt %o3, 0f
6757c478bdstevel@tonic-gate	ld	[%o2 + CPU_INTR_ACTV], %o5	! delay annulled
6767c478bdstevel@tonic-gate	!
6777c478bdstevel@tonic-gate	! Restore %pil before calling serve_intr() again. We must check
6787c478bdstevel@tonic-gate	! CPU_BASE_SPL and set %pil to max(our-pil, CPU_BASE_SPL)
6797c478bdstevel@tonic-gate	!
6807c478bdstevel@tonic-gate	ld	[%o2 + CPU_BASE_SPL], %o4
6817c478bdstevel@tonic-gate	cmp	%o4, %l1
6827c478bdstevel@tonic-gate	movl	%xcc, %l1, %o4
6837c478bdstevel@tonic-gate	wrpr	%g0, %o4, %pil
6847c478bdstevel@tonic-gate	SERVE_INTR_NEXT(%o1, %o2, %l1, %l3, %o4, %o5, %o3, %o0)
6857c478bdstevel@tonic-gate	ba	0b				! compute new stats
6867c478bdstevel@tonic-gate	nop
6877c478bdstevel@tonic-gate0:
6887c478bdstevel@tonic-gate	!
6897c478bdstevel@tonic-gate	! Clear bit for this level in CPU's interrupt active bitmask.
6907c478bdstevel@tonic-gate	!
6917c478bdstevel@tonic-gate	mov	1, %o4
6927c478bdstevel@tonic-gate	sll	%o4, %l1, %o4
6937c478bdstevel@tonic-gate#ifdef DEBUG
6947c478bdstevel@tonic-gate	!
6957c478bdstevel@tonic-gate	! ASSERT(CPU->cpu_intr_actv & (1 << PIL))
6967c478bdstevel@tonic-gate	!
6977c478bdstevel@tonic-gate	andcc	%o4, %o5, %g0
6987c478bdstevel@tonic-gate	bnz,pt	%xcc, 0f
6997c478bdstevel@tonic-gate	nop
7007c478bdstevel@tonic-gate	! Do not call panic if a panic is already in progress.
7017c478bdstevel@tonic-gate	sethi	%hi(panic_quiesce), %l2
7027c478bdstevel@tonic-gate	ld	[%l2 + %lo(panic_quiesce)], %l2
7037c478bdstevel@tonic-gate	brnz,pn	%l2, 0f
7047c478bdstevel@tonic-gate	nop
7057c478bdstevel@tonic-gate	sethi	%hi(intr_thread_actv_bit_not_set), %o0
7067c478bdstevel@tonic-gate	call	panic
7077c478bdstevel@tonic-gate	or	%o0, %lo(intr_thread_actv_bit_not_set), %o0
7087c478bdstevel@tonic-gate0:
7097c478bdstevel@tonic-gate#endif /* DEBUG */
7107c478bdstevel@tonic-gate	andn	%o5, %o4, %o5
7117c478bdstevel@tonic-gate	st	%o5, [%o2 + CPU_INTR_ACTV]
7127c478bdstevel@tonic-gate	!
7137c478bdstevel@tonic-gate	! If there is still an interrupted thread underneath this one,
7147c478bdstevel@tonic-gate	! then the interrupt was never blocked and the return is fairly
7157c478bdstevel@tonic-gate	! simple.  Otherwise jump to intr_thread_exit.
7167c478bdstevel@tonic-gate	!
7177c478bdstevel@tonic-gate	ldn	[THREAD_REG + T_INTR], %o4	! pinned thread
7187c478bdstevel@tonic-gate	brz,pn	%o4, intr_thread_exit		! branch if none
7197c478bdstevel@tonic-gate	nop
7207c478bdstevel@tonic-gate	!
7217c478bdstevel@tonic-gate	! link the thread back onto the interrupt thread pool
7227c478bdstevel@tonic-gate	!
7237c478bdstevel@tonic-gate	ldn	[%o2 + CPU_INTR_THREAD], %o3
7247c478bdstevel@tonic-gate	stn	%o3, [THREAD_REG + T_LINK]
7257c478bdstevel@tonic-gate	stn	THREAD_REG, [%o2 + CPU_INTR_THREAD]
7267c478bdstevel@tonic-gate	!
7277c478bdstevel@tonic-gate	! set the thread state to free so kernel debuggers don't see it
7287c478bdstevel@tonic-gate	!
7297c478bdstevel@tonic-gate	mov	TS_FREE, %o5
7307c478bdstevel@tonic-gate	st	%o5, [THREAD_REG + T_STATE]
7317c478bdstevel@tonic-gate	!
7327c478bdstevel@tonic-gate	! Switch back to the interrupted thread and return
7337c478bdstevel@tonic-gate	!
7347c478bdstevel@tonic-gate	stn	%o4, [%o2 + CPU_THREAD]
73525cf1a3jl	membar	#StoreLoad			! sync with mutex_exit()
7367c478bdstevel@tonic-gate	mov	%o4, THREAD_REG
7377c478bdstevel@tonic-gate
7387c478bdstevel@tonic-gate	! If we pinned an interrupt thread, store its starting timestamp.
7397c478bdstevel@tonic-gate	lduh	[THREAD_REG + T_FLAGS], %o5
7407c478bdstevel@tonic-gate	andcc	%o5, T_INTR_THREAD, %g0
7417c478bdstevel@tonic-gate	bz,pt	%xcc, 1f
7427c478bdstevel@tonic-gate	ldn	[THREAD_REG + T_SP], %sp	! delay - restore %sp
7437c478bdstevel@tonic-gate
7447c478bdstevel@tonic-gate	add	THREAD_REG, T_INTR_START, %o3	! o3 has &curthread->t_intr_star
7457c478bdstevel@tonic-gate0:
7467c478bdstevel@tonic-gate	ldx	[%o3], %o4			! o4 = t_intr_start before
747bd28a47Prashanth Sreenivasa	RD_CLOCK_TICK(%o5,%l2,%l3,__LINE__)
7487c478bdstevel@tonic-gate	casx	[%o3], %o4, %o5			! put o5 in ts if o4 == ts after
7497c478bdstevel@tonic-gate	cmp	%o4, %o5
7507c478bdstevel@tonic-gate	! If a high-level interrupt occurred while we were attempting to store
7517c478bdstevel@tonic-gate	! the timestamp, try again.
7527c478bdstevel@tonic-gate	bne,pn	%xcc, 0b
7537c478bdstevel@tonic-gate	ldn	[THREAD_REG + T_SP], %sp	! delay - restore %sp
7547c478bdstevel@tonic-gate1:
7557c478bdstevel@tonic-gate	! If the thread being restarted isn't pinning anyone, and no interrupts
7567c478bdstevel@tonic-gate	! are pending, zero out cpu_intrcnt
7577c478bdstevel@tonic-gate	ldn	[THREAD_REG + T_INTR], %o4
7587c478bdstevel@tonic-gate	brnz,pn	%o4, 2f
7597c478bdstevel@tonic-gate	rd	SOFTINT, %o4			! delay
7607c478bdstevel@tonic-gate	set	SOFTINT_MASK, %o5
7617c478bdstevel@tonic-gate	andcc	%o4, %o5, %g0
7627c478bdstevel@tonic-gate	bz,a,pt	%xcc, 2f
7637c478bdstevel@tonic-gate	stub	%g0, [%o2 + CPU_INTRCNT]	! delay annul
7647c478bdstevel@tonic-gate2:
7657c478bdstevel@tonic-gate	jmp	%l0 + 8
7667c478bdstevel@tonic-gate	nop
7677c478bdstevel@tonic-gate	SET_SIZE(intr_thread)
7687c478bdstevel@tonic-gate	/* Not Reached */
7697c478bdstevel@tonic-gate
7707c478bdstevel@tonic-gate	!
7717c478bdstevel@tonic-gate	! An interrupt returned on what was once (and still might be)
7727c478bdstevel@tonic-gate	! an interrupt thread stack, but the interrupted process is no longer
7737c478bdstevel@tonic-gate	! there.  This means the interrupt must have blocked.
7747c478bdstevel@tonic-gate	!
7757c478bdstevel@tonic-gate	! There is no longer a thread under this one, so put this thread back
7767c478bdstevel@tonic-gate	! on the CPU's free list and resume the idle thread which will dispatch
7777c478bdstevel@tonic-gate	! the next thread to run.
7787c478bdstevel@tonic-gate	!
7797c478bdstevel@tonic-gate	! All traps below DISP_LEVEL are disabled here, but the mondo interrupt
7807c478bdstevel@tonic-gate	! is enabled.
7817c478bdstevel@tonic-gate	!
7827c478bdstevel@tonic-gate	ENTRY_NP(intr_thread_exit)
7837c478bdstevel@tonic-gate#ifdef TRAPTRACE
7847c478bdstevel@tonic-gate	rdpr	%pstate, %l2
7857c478bdstevel@tonic-gate	andn	%l2, PSTATE_IE | PSTATE_AM, %o4
7867c478bdstevel@tonic-gate	wrpr	%g0, %o4, %pstate			! cpu to known state
7877c478bdstevel@tonic-gate	TRACE_PTR(%o4, %o5)
788023e71dHaik Aftandilian	GET_TRACE_TICK(%o5, %o0)
7897c478bdstevel@tonic-gate	stxa	%o5, [%o4 + TRAP_ENT_TICK]%asi
7907c478bdstevel@tonic-gate	TRACE_SAVE_TL_GL_REGS(%o4, %o5)
7917c478bdstevel@tonic-gate	set	TT_INTR_EXIT, %o5
7927c478bdstevel@tonic-gate	stha	%o5, [%o4 + TRAP_ENT_TT]%asi
7937c478bdstevel@tonic-gate	stna	%g0, [%o4 + TRAP_ENT_TPC]%asi
7947c478bdstevel@tonic-gate	stxa	%g0, [%o4 + TRAP_ENT_TSTATE]%asi
7957c478bdstevel@tonic-gate	stna	%sp, [%o4 + TRAP_ENT_SP]%asi
7967c478bdstevel@tonic-gate	stna	THREAD_REG, [%o4 + TRAP_ENT_TR]%asi
7977c478bdstevel@tonic-gate	ld	[%o2 + CPU_BASE_SPL], %o5
7987c478bdstevel@tonic-gate	stna	%o5, [%o4 + TRAP_ENT_F1]%asi
7997c478bdstevel@tonic-gate	stna	%g0, [%o4 + TRAP_ENT_F2]%asi
8007c478bdstevel@tonic-gate	stna	%g0, [%o4 + TRAP_ENT_F3]%asi
8017c478bdstevel@tonic-gate	stna	%g0, [%o4 + TRAP_ENT_F4]%asi
8027c478bdstevel@tonic-gate	TRACE_NEXT(%o4, %o5, %o0)
8037c478bdstevel@tonic-gate	wrpr	%g0, %l2, %pstate
8047c478bdstevel@tonic-gate#endif /* TRAPTRACE */
8057c478bdstevel@tonic-gate	! cpu_stats.sys.intrblk++
8067c478bdstevel@tonic-gate        ldx	[%o2 + CPU_STATS_SYS_INTRBLK], %o4
8077c478bdstevel@tonic-gate        inc     %o4
8087c478bdstevel@tonic-gate        stx	%o4, [%o2 + CPU_STATS_SYS_INTRBLK]
8097c478bdstevel@tonic-gate	!
8107c478bdstevel@tonic-gate	! Put thread back on the interrupt thread list.
8117c478bdstevel@tonic-gate	!
8127c478bdstevel@tonic-gate
8137c478bdstevel@tonic-gate	!
8147c478bdstevel@tonic-gate	! Set the CPU's base SPL level.
8157c478bdstevel@tonic-gate	!
8167c478bdstevel@tonic-gate#ifdef DEBUG
8177c478bdstevel@tonic-gate	!
8187c478bdstevel@tonic-gate	! ASSERT(!(CPU->cpu_intr_actv & (1 << PIL)))
8197c478bdstevel@tonic-gate	!
8207c478bdstevel@tonic-gate	ld	[%o2 + CPU_INTR_ACTV], %o5
8217c478bdstevel@tonic-gate	mov	1, %o4
8227c478bdstevel@tonic-gate	sll	%o4, %l1, %o4
8237c478bdstevel@tonic-gate	and	%o5, %o4, %o4
8247c478bdstevel@tonic-gate	brz,pt	%o4, 0f
8257c478bdstevel@tonic-gate	nop
8267c478bdstevel@tonic-gate	! Do not call panic if a panic is already in progress.
8277c478bdstevel@tonic-gate	sethi	%hi(panic_quiesce), %l2
8287c478bdstevel@tonic-gate	ld	[%l2 + %lo(panic_quiesce)], %l2
8297c478bdstevel@tonic-gate	brnz,pn	%l2, 0f
8307c478bdstevel@tonic-gate	nop
8317c478bdstevel@tonic-gate	sethi	%hi(intr_thread_exit_actv_bit_set), %o0
8327c478bdstevel@tonic-gate	call	panic
8337c478bdstevel@tonic-gate	or	%o0, %lo(intr_thread_exit_actv_bit_set), %o0
8347c478bdstevel@tonic-gate0:
8357c478bdstevel@tonic-gate#endif /* DEBUG */
8367c478bdstevel@tonic-gate	call	_intr_set_spl			! set CPU's base SPL level
8377c478bdstevel@tonic-gate	ld	[%o2 + CPU_INTR_ACTV], %o5	! delay - load active mask
8387c478bdstevel@tonic-gate	!
8397c478bdstevel@tonic-gate	! set the thread state to free so kernel debuggers don't see it
8407c478bdstevel@tonic-gate	!
8417c478bdstevel@tonic-gate	mov	TS_FREE, %o4
8427c478bdstevel@tonic-gate	st	%o4, [THREAD_REG + T_STATE]
8437c478bdstevel@tonic-gate	!
8447c478bdstevel@tonic-gate	! Put thread on either the interrupt pool or the free pool and
8457c478bdstevel@tonic-gate	! call swtch() to resume another thread.
8467c478bdstevel@tonic-gate	!
8477c478bdstevel@tonic-gate	ldn	[%o2 + CPU_INTR_THREAD], %o5	! get list pointer
8487c478bdstevel@tonic-gate	stn	%o5, [THREAD_REG + T_LINK]
8497c478bdstevel@tonic-gate	call	swtch				! switch to best thread
8507c478bdstevel@tonic-gate	stn	THREAD_REG, [%o2 + CPU_INTR_THREAD] ! delay - put thread on list
8517c478bdstevel@tonic-gate	ba,a,pt	%xcc, .				! swtch() shouldn't return
8527c478bdstevel@tonic-gate	SET_SIZE(intr_thread_exit)
8537c478bdstevel@tonic-gate
8547c478bdstevel@tonic-gate	.global ftrace_intr_thread_format_str
8557c478bdstevel@tonic-gateftrace_intr_thread_format_str:
8567c478bdstevel@tonic-gate	.asciz	"intr_thread(): regs=0x%lx, int=0x%lx, pil=0x%lx"
8577c478bdstevel@tonic-gate#ifdef DEBUG
8587c478bdstevel@tonic-gateintr_thread_actv_bit_set:
8597c478bdstevel@tonic-gate	.asciz	"intr_thread():	cpu_intr_actv bit already set for PIL"
8607c478bdstevel@tonic-gateintr_thread_actv_bit_not_set:
8617c478bdstevel@tonic-gate	.asciz	"intr_thread():	cpu_intr_actv bit not set for PIL"
8627c478bdstevel@tonic-gateintr_thread_exit_actv_bit_set:
8637c478bdstevel@tonic-gate	.asciz	"intr_thread_exit(): cpu_intr_actv bit erroneously set for PIL"
8647c478bdstevel@tonic-gateintr_thread_t_intr_start_zero:
8657c478bdstevel@tonic-gate	.asciz	"intr_thread():	t_intr_start zero upon handler return"
8667c478bdstevel@tonic-gate#endif /* DEBUG */
8677c478bdstevel@tonic-gate
8687c478bdstevel@tonic-gate/*
8697c478bdstevel@tonic-gate * Handle an interrupt in the current thread
8707c478bdstevel@tonic-gate *	Entry:
8717c478bdstevel@tonic-gate *		%o0       = pointer to regs structure
872b0fc0e7govinda *		%o1       = pointer to current intr_vec_t (iv) to be processed
8737c478bdstevel@tonic-gate *		%o2       = pil
8747c478bdstevel@tonic-gate *		%sp       = on current thread's kernel stack
8757c478bdstevel@tonic-gate *		%o7       = return linkage to trap code
8767c478bdstevel@tonic-gate *		%g7       = current thread
8779b0bb79John Levon *		%pstate   = normal globals, interrupts enabled,
8787c478bdstevel@tonic-gate *		            privileged, fp disabled
8797c478bdstevel@tonic-gate *		%pil      = PIL_MAX
8807c478bdstevel@tonic-gate *
8817c478bdstevel@tonic-gate *	Register Usage
8827c478bdstevel@tonic-gate *		%l0       = return linkage
8837c478bdstevel@tonic-gate *		%l1       = old stack
8847c478bdstevel@tonic-gate *		%l2 - %l3 = scratch
8857c478bdstevel@tonic-gate *		%l4 - %l7 = reserved for sys_trap
8867c478bdstevel@tonic-gate *		%o3       = cpu
8877c478bdstevel@tonic-gate *		%o0       = scratch
8887c478bdstevel@tonic-gate *		%o4 - %o5 = scratch
8897c478bdstevel@tonic-gate */
8907c478bdstevel@tonic-gate	ENTRY_NP(current_thread)
8917c478bdstevel@tonic-gate
8927c478bdstevel@tonic-gate	mov	%o7, %l0
8937c478bdstevel@tonic-gate	ldn	[THREAD_REG + T_CPU], %o3
8942604657vb
8952604657vb	ldn	[THREAD_REG + T_ONFAULT], %l2
8962604657vb	brz,pt	%l2, no_onfault		! branch if no onfault label set
8972604657vb	nop
8982604657vb	stn	%g0, [THREAD_REG + T_ONFAULT]! clear onfault label
8992604657vb	ldn	[THREAD_REG + T_LOFAULT], %l3
9002604657vb	stn	%g0, [THREAD_REG + T_LOFAULT]! clear lofault data
9012604657vb
9022604657vb	sub	%o2, LOCK_LEVEL + 1, %o5
9032604657vb	sll	%o5, CPTRSHIFT, %o5
9042604657vb	add	%o5, CPU_OFD, %o4	! %o4 has on_fault data offset
9052604657vb	stn	%l2, [%o3 + %o4]	! save onfault label for pil %o2
9062604657vb	add	%o5, CPU_LFD, %o4	! %o4 has lofault data offset
9072604657vb	stn	%l3, [%o3 + %o4]	! save lofault data for pil %o2
9082604657vb
9092604657vbno_onfault:
9102604657vb	ldn	[THREAD_REG + T_ONTRAP], %l2
9112604657vb	brz,pt	%l2, 6f			! branch if no on_trap protection
9122604657vb	nop
9132604657vb	stn	%g0, [THREAD_REG + T_ONTRAP]! clear on_trap protection
9142604657vb	sub	%o2, LOCK_LEVEL + 1, %o5
9152604657vb	sll	%o5, CPTRSHIFT, %o5
9162604657vb	add	%o5, CPU_OTD, %o4	! %o4 has on_trap data offset
9172604657vb	stn	%l2, [%o3 + %o4]	! save on_trap label for pil %o2
9182604657vb
9197c478bdstevel@tonic-gate	!
9207c478bdstevel@tonic-gate	! Set bit for this level in CPU's active interrupt bitmask.
9217c478bdstevel@tonic-gate	!
9222604657vb6:	ld	[%o3 + CPU_INTR_ACTV], %o5	! o5 has cpu_intr_actv b4 chng
9237c478bdstevel@tonic-gate	mov	1, %o4
9247c478bdstevel@tonic-gate	sll	%o4, %o2, %o4			! construct mask for level
9257c478bdstevel@tonic-gate#ifdef DEBUG
9267c478bdstevel@tonic-gate	!
9277c478bdstevel@tonic-gate	! ASSERT(!(CPU->cpu_intr_actv & (1 << PIL)))
9287c478bdstevel@tonic-gate	!
9297c478bdstevel@tonic-gate	andcc	%o5, %o4, %g0
9307c478bdstevel@tonic-gate	bz,pt	%xcc, 0f
9317c478bdstevel@tonic-gate	nop
9327c478bdstevel@tonic-gate	! Do not call panic if a panic is already in progress.
9337c478bdstevel@tonic-gate	sethi	%hi(panic_quiesce), %l2
9347c478bdstevel@tonic-gate	ld	[%l2 + %lo(panic_quiesce)], %l2
9357c478bdstevel@tonic-gate	brnz,pn	%l2, 0f
9367c478bdstevel@tonic-gate	nop
9377c478bdstevel@tonic-gate	sethi	%hi(current_thread_actv_bit_set), %o0
9387c478bdstevel@tonic-gate	call	panic
9397c478bdstevel@tonic-gate	or	%o0, %lo(current_thread_actv_bit_set), %o0
9407c478bdstevel@tonic-gate0:
9417c478bdstevel@tonic-gate#endif /* DEBUG */
9427c478bdstevel@tonic-gate	or	%o5, %o4, %o4
9437c478bdstevel@tonic-gate	!
9447c478bdstevel@tonic-gate	! See if we are interrupting another high-level interrupt.
9457c478bdstevel@tonic-gate	!
9467c478bdstevel@tonic-gate	srl	%o5, LOCK_LEVEL + 1, %o5	! only look at high-level bits
9477c478bdstevel@tonic-gate	brz,pt	%o5, 1f
9487c478bdstevel@tonic-gate	st	%o4, [%o3 + CPU_INTR_ACTV]	! delay - store active mask
9497c478bdstevel@tonic-gate	!
9507c478bdstevel@tonic-gate	! We have interrupted another high-level interrupt. Find its PIL,
9517c478bdstevel@tonic-gate	! compute the interval it ran for, and update its cumulative counter.
9527c478bdstevel@tonic-gate	!
9537c478bdstevel@tonic-gate	! Register usage:
9547c478bdstevel@tonic-gate
9557c478bdstevel@tonic-gate	! o2 = PIL of this interrupt
9567c478bdstevel@tonic-gate	! o5 = high PIL bits of INTR_ACTV (not including this PIL)
9577c478bdstevel@tonic-gate	! l1 = bitmask used to find other active high-level PIL
9587c478bdstevel@tonic-gate	! o4 = index of bit set in l1
9597c478bdstevel@tonic-gate	! Use cpu_intr_actv to find the cpu_pil_high_start[] offset for the
9607c478bdstevel@tonic-gate	! interrupted high-level interrupt.
9617c478bdstevel@tonic-gate	! Create mask for cpu_intr_actv. Begin by looking for bits set
9627c478bdstevel@tonic-gate	! at one level below the current PIL. Since %o5 contains the active
9637c478bdstevel@tonic-gate	! mask already shifted right by (LOCK_LEVEL + 1), we start by looking
9647c478bdstevel@tonic-gate	! at bit (current_pil - (LOCK_LEVEL + 2)).
9657c478bdstevel@tonic-gate	sub	%o2, LOCK_LEVEL + 2, %o4
9667c478bdstevel@tonic-gate	mov	1, %l1
9677c478bdstevel@tonic-gate	sll	%l1, %o4, %l1
9687c478bdstevel@tonic-gate2:
9697c478bdstevel@tonic-gate#ifdef DEBUG
9707c478bdstevel@tonic-gate	! ASSERT(%l1 != 0) (we didn't shift the bit off the right edge)
9717c478bdstevel@tonic-gate	brnz,pt	%l1, 9f
9727c478bdstevel@tonic-gate	nop
9737c478bdstevel@tonic-gate
9747c478bdstevel@tonic-gate	! Don't panic if a panic is already in progress.
9757c478bdstevel@tonic-gate	sethi	%hi(panic_quiesce), %l3
9767c478bdstevel@tonic-gate	ld	[%l3 + %lo(panic_quiesce)], %l3
9777c478bdstevel@tonic-gate	brnz,pn	%l3, 9f
9787c478bdstevel@tonic-gate	nop
9797c478bdstevel@tonic-gate	sethi	%hi(current_thread_nested_PIL_not_found), %o0
9807c478bdstevel@tonic-gate	call	panic
9817c478bdstevel@tonic-gate	or	%o0, %lo(current_thread_nested_PIL_not_found), %o0
9827c478bdstevel@tonic-gate9:
9837c478bdstevel@tonic-gate#endif /* DEBUG */
9847c478bdstevel@tonic-gate	andcc	%l1, %o5, %g0		! test mask against high-level bits of
9857c478bdstevel@tonic-gate	bnz	%xcc, 3f		! cpu_intr_actv
9867c478bdstevel@tonic-gate	nop
9877c478bdstevel@tonic-gate	srl	%l1, 1, %l1		! No match. Try next lower PIL.
9887c478bdstevel@tonic-gate	ba,pt	%xcc, 2b
9897c478bdstevel@tonic-gate	sub	%o4, 1, %o4		! delay - decrement PIL
9907c478bdstevel@tonic-gate3:
9917c478bdstevel@tonic-gate	sll	%o4, 3, %o4			! index to byte offset
9927c478bdstevel@tonic-gate	add	%o4, CPU_MCPU, %l1	! CPU_PIL_HIGH_START is too large
9937c478bdstevel@tonic-gate	add	%l1, MCPU_PIL_HIGH_START, %l1
9947c478bdstevel@tonic-gate	ldx	[%o3 + %l1], %l3		! load starting timestamp
9957c478bdstevel@tonic-gate#ifdef DEBUG
9967c478bdstevel@tonic-gate	brnz,pt	%l3, 9f
9977c478bdstevel@tonic-gate	nop
9987c478bdstevel@tonic-gate	! Don't panic if a panic is already in progress.
9997c478bdstevel@tonic-gate	sethi	%hi(panic_quiesce), %l1
10007c478bdstevel@tonic-gate	ld	[%l1 + %lo(panic_quiesce)], %l1
10017c478bdstevel@tonic-gate	brnz,pn	%l1, 9f
10027c478bdstevel@tonic-gate	nop
10037c478bdstevel@tonic-gate	srl	%o4, 3, %o1			! Find interrupted PIL for panic
10047c478bdstevel@tonic-gate	add	%o1, LOCK_LEVEL + 1, %o1
10057c478bdstevel@tonic-gate	sethi	%hi(current_thread_nested_pil_zero), %o0
10067c478bdstevel@tonic-gate	call	panic
10077c478bdstevel@tonic-gate	or	%o0, %lo(current_thread_nested_pil_zero), %o0
10087c478bdstevel@tonic-gate9:
10097c478bdstevel@tonic-gate#endif /* DEBUG */
1010bd28a47Prashanth Sreenivasa	RD_CLOCK_TICK_NO_SUSPEND_CHECK(%l1, %l2)
10117c478bdstevel@tonic-gate	sub	%l1, %l3, %l3			! interval in %l3
10127c478bdstevel@tonic-gate	!
10137c478bdstevel@tonic-gate	! Check for Energy Star mode
10147c478bdstevel@tonic-gate	!
10157c478bdstevel@tonic-gate	lduh	[%o3 + CPU_DIVISOR], %l1	! %l1 = clock divisor
10167c478bdstevel@tonic-gate	cmp	%l1, 1
10177c478bdstevel@tonic-gate	bg,a,pn	%xcc, 2f
10187c478bdstevel@tonic-gate	mulx	%l3, %l1, %l3	! multiply interval by clock divisor iff > 1
10197c478bdstevel@tonic-gate2:
10207c478bdstevel@tonic-gate	!
10217c478bdstevel@tonic-gate	! We need to find the CPU offset of the cumulative counter. We start
10227c478bdstevel@tonic-gate	! with %o4, which has (PIL - (LOCK_LEVEL + 1)) * 8. We need PIL * 16,
10237c478bdstevel@tonic-gate	! so we shift left 1, then add (LOCK_LEVEL + 1) * 16, which is
10247c478bdstevel@tonic-gate	! CPU_INTRSTAT_LOW_PIL_OFFSET.
10257c478bdstevel@tonic-gate	!
10267c478bdstevel@tonic-gate	sll	%o4, 1, %o4
10277c478bdstevel@tonic-gate	add	%o4, CPU_MCPU, %o4		! CPU_INTRSTAT const too large
10287c478bdstevel@tonic-gate	add	%o4, MCPU_INTRSTAT, %o4		! add parts separately
10297c478bdstevel@tonic-gate	add	%o4, CPU_INTRSTAT_LOW_PIL_OFFSET, %o4
10307c478bdstevel@tonic-gate	ldx	[%o3 + %o4], %l1		! old counter in l1
10317c478bdstevel@tonic-gate	add	%l1, %l3, %l1			! new counter in l1
1032eda8946esolom	stx	%l1, [%o3 + %o4]		! store new counter
1033eda8946esolom
1034eda8946esolom	! Also update intracct[]
1035eda8946esolom	lduh	[%o3 + CPU_MSTATE], %o4
1036eda8946esolom	sllx	%o4, 3, %o4
1037eda8946esolom	add	%o4, CPU_INTRACCT, %o4
1038eda8946esolom	ldx	[%o3 + %o4], %l1
1039eda8946esolom	add	%l1, %l3, %l1
10407c478bdstevel@tonic-gate	! Another high-level interrupt is active below this one, so
10417c478bdstevel@tonic-gate	! there is no need to check for an interrupt thread. That will be
10427c478bdstevel@tonic-gate	! done by the lowest priority high-level interrupt active.
10437c478bdstevel@tonic-gate	ba,pt	%xcc, 5f
10447c478bdstevel@tonic-gate	stx	%l1, [%o3 + %o4]		! delay - store new counter
10457c478bdstevel@tonic-gate1:
10467c478bdstevel@tonic-gate	! If we haven't interrupted another high-level interrupt, we may be
10477c478bdstevel@tonic-gate	! interrupting a low level interrupt thread. If so, compute its interval
10487c478bdstevel@tonic-gate	! and update its cumulative counter.
10497c478bdstevel@tonic-gate	lduh	[THREAD_REG + T_FLAGS], %o4
10507c478bdstevel@tonic-gate	andcc	%o4, T_INTR_THREAD, %g0
10517c478bdstevel@tonic-gate	bz,pt	%xcc, 4f
10527c478bdstevel@tonic-gate	nop
10537c478bdstevel@tonic-gate
10547c478bdstevel@tonic-gate	! We have interrupted an interrupt thread. Take timestamp, compute
10557c478bdstevel@tonic-gate	! interval, update cumulative counter.
10567c478bdstevel@tonic-gate
10577c478bdstevel@tonic-gate	! Check t_intr_start. If it is zero, either intr_thread() or
10587c478bdstevel@tonic-gate	! current_thread() (at a lower PIL, of course) already did
10597c478bdstevel@tonic-gate	! the accounting for the underlying interrupt thread.
10607c478bdstevel@tonic-gate	ldx	[THREAD_REG + T_INTR_START], %o5
10617c478bdstevel@tonic-gate	brz,pn	%o5, 4f
10627c478bdstevel@tonic-gate	nop
10637c478bdstevel@tonic-gate
10647c478bdstevel@tonic-gate	stx	%g0, [THREAD_REG + T_INTR_START]
1065bd28a47Prashanth Sreenivasa	RD_CLOCK_TICK_NO_SUSPEND_CHECK(%o4, %l2)
10667c478bdstevel@tonic-gate	sub	%o4, %o5, %o5			! o5 has the interval
10677c478bdstevel@tonic-gate
10687c478bdstevel@tonic-gate	! Check for Energy Star mode
10697c478bdstevel@tonic-gate	lduh	[%o3 + CPU_DIVISOR], %o4	! %o4 = clock divisor
10707c478bdstevel@tonic-gate	cmp	%o4, 1
10717c478bdstevel@tonic-gate	bg,a,pn	%xcc, 2f
10727c478bdstevel@tonic-gate	mulx	%o5, %o4, %o5	! multiply interval by clock divisor iff > 1
10737c478bdstevel@tonic-gate2:
10747c478bdstevel@tonic-gate	ldub	[THREAD_REG + T_PIL], %o4
10757c478bdstevel@tonic-gate	sllx	%o4, 4, %o4			! PIL index to byte offset
10767c478bdstevel@tonic-gate	add	%o4, CPU_MCPU, %o4		! CPU_INTRSTAT const too large
10777c478bdstevel@tonic-gate	add	%o4, MCPU_INTRSTAT, %o4		! add parts separately
10787c478bdstevel@tonic-gate	ldx	[%o3 + %o4], %l2		! old counter in l2
10797c478bdstevel@tonic-gate	add	%l2, %o5, %l2			! new counter in l2
10807c478bdstevel@tonic-gate	stx	%l2, [%o3 + %o4]		! store new counter
1081eda8946esolom
1082eda8946esolom	! Also update intracct[]
1083eda8946esolom	lduh	[%o3 + CPU_MSTATE], %o4
1084eda8946esolom	sllx	%o4, 3, %o4
1085eda8946esolom	add	%o4, CPU_INTRACCT, %o4
1086eda8946esolom	ldx	[%o3 + %o4], %l2
1087eda8946esolom	add	%l2, %o5, %l2
1088eda8946esolom	stx	%l2, [%o3 + %o4]
10897c478bdstevel@tonic-gate4:
10907c478bdstevel@tonic-gate	!
10917c478bdstevel@tonic-gate	! Handle high-level interrupts on separate interrupt stack.
10927c478bdstevel@tonic-gate	! No other high-level interrupts are active, so switch to int stack.
10937c478bdstevel@tonic-gate	!
10947c478bdstevel@tonic-gate	mov	%sp, %l1
10957c478bdstevel@tonic-gate	ldn	[%o3 + CPU_INTR_STACK], %l3
10967c478bdstevel@tonic-gate	sub	%l3, STACK_BIAS, %sp
10977c478bdstevel@tonic-gate
10987c478bdstevel@tonic-gate5:
10997c478bdstevel@tonic-gate#ifdef DEBUG
11007c478bdstevel@tonic-gate	!
11017c478bdstevel@tonic-gate	! ASSERT(%o2 > LOCK_LEVEL)
11027c478bdstevel@tonic-gate	!
11037c478bdstevel@tonic-gate	cmp	%o2, LOCK_LEVEL
11047c478bdstevel@tonic-gate	bg,pt	%xcc, 3f
11057c478bdstevel@tonic-gate	nop
11067c478bdstevel@tonic-gate	mov	CE_PANIC, %o0
11077c478bdstevel@tonic-gate	sethi	%hi(current_thread_wrong_pil), %o1
11087c478bdstevel@tonic-gate	call	cmn_err				! %o2 has the %pil already
11097c478bdstevel@tonic-gate	or	%o1, %lo(current_thread_wrong_pil), %o1
11107c478bdstevel@tonic-gate#endif
11117c478bdstevel@tonic-gate3:
11127c478bdstevel@tonic-gate	! Store starting timestamp for this PIL in CPU structure at
11137c478bdstevel@tonic-gate	! cpu.cpu_m.pil_high_start[PIL - (LOCK_LEVEL + 1)]
11147c478bdstevel@tonic-gate        sub     %o2, LOCK_LEVEL + 1, %o4	! convert PIL to array index
11157c478bdstevel@tonic-gate	sllx    %o4, 3, %o4			! index to byte offset
11167c478bdstevel@tonic-gate	add	%o4, CPU_MCPU, %o4	! CPU_PIL_HIGH_START is too large
11177c478bdstevel@tonic-gate	add	%o4, MCPU_PIL_HIGH_START, %o4
1118bd28a47Prashanth Sreenivasa	RD_CLOCK_TICK_NO_SUSPEND_CHECK(%o5, %l2)
11197c478bdstevel@tonic-gate        stx     %o5, [%o3 + %o4]
11207c478bdstevel@tonic-gate
11217c478bdstevel@tonic-gate	wrpr	%g0, %o2, %pil			! enable interrupts
11227c478bdstevel@tonic-gate
11237c478bdstevel@tonic-gate	!
11247c478bdstevel@tonic-gate	! call the handler
11257c478bdstevel@tonic-gate	!
11267c478bdstevel@tonic-gate	SERVE_INTR_PRE(%o1, %o3, %l2, %l3, %o4, %o5, %o2, %o0)
11277c478bdstevel@tonic-gate1:
11287c478bdstevel@tonic-gate	SERVE_INTR(%o1, %o3, %l2, %l3, %o4, %o5, %o2, %o0)
11297c478bdstevel@tonic-gate
11307c478bdstevel@tonic-gate	brz,a,pt %o2, 0f			! if %o2, more intrs await
11317c478bdstevel@tonic-gate	rdpr	%pil, %o2			! delay annulled
11327c478bdstevel@tonic-gate	SERVE_INTR_NEXT(%o1, %o3, %l2, %l3, %o4, %o5, %o2, %o0)
11337c478bdstevel@tonic-gate	ba	1b
11347c478bdstevel@tonic-gate	nop
11357c478bdstevel@tonic-gate0:
11367c478bdstevel@tonic-gate	wrpr	%g0, PIL_MAX, %pil		! disable interrupts (1-15)
11377c478bdstevel@tonic-gate
11387c478bdstevel@tonic-gate	cmp	%o2, PIL_15
11397c478bdstevel@tonic-gate	bne,pt	%xcc, 3f
11407c478bdstevel@tonic-gate	nop
11417c478bdstevel@tonic-gate
11427c478bdstevel@tonic-gate	sethi	%hi(cpc_level15_inum), %o1
1143b0fc0e7govinda	ldx	[%o1 + %lo(cpc_level15_inum)], %o1 ! arg for intr_enqueue_req
11447c478bdstevel@tonic-gate	brz	%o1, 3f
11457c478bdstevel@tonic-gate	nop
11467c478bdstevel@tonic-gate
11477c478bdstevel@tonic-gate	rdpr 	%pstate, %g5
11487c478bdstevel@tonic-gate	andn	%g5, PSTATE_IE, %g1
11497c478bdstevel@tonic-gate	wrpr	%g0, %g1, %pstate		! Disable vec interrupts
11507c478bdstevel@tonic-gate
11517c478bdstevel@tonic-gate	call	intr_enqueue_req		! preserves %g5
11527c478bdstevel@tonic-gate	mov	PIL_15, %o0
11537c478bdstevel@tonic-gate
11547c478bdstevel@tonic-gate	! clear perfcntr overflow
11557c478bdstevel@tonic-gate	mov	1, %o0
11567c478bdstevel@tonic-gate	sllx	%o0, PIL_15, %o0
11577c478bdstevel@tonic-gate	wr	%o0, CLEAR_SOFTINT
11587c478bdstevel@tonic-gate
11597c478bdstevel@tonic-gate	wrpr	%g0, %g5, %pstate		! Enable vec interrupts
11607c478bdstevel@tonic-gate
11617c478bdstevel@tonic-gate3:
11627c478bdstevel@tonic-gate	cmp	%o2, PIL_14
11637c478bdstevel@tonic-gate	be	tick_rtt			!  cpu-specific tick processing
11647c478bdstevel@tonic-gate	nop
11657c478bdstevel@tonic-gate	.global	current_thread_complete
11667c478bdstevel@tonic-gatecurrent_thread_complete:
11677c478bdstevel@tonic-gate	!
11687c478bdstevel@tonic-gate	! Register usage:
11697c478bdstevel@tonic-gate	!
11707c478bdstevel@tonic-gate	! %l1 = stack pointer
11717c478bdstevel@tonic-gate	! %l2 = CPU_INTR_ACTV >> (LOCK_LEVEL + 1)
11727c478bdstevel@tonic-gate	! %o2 = PIL
11737c478bdstevel@tonic-gate	! %o3 = CPU pointer
11747c478bdstevel@tonic-gate	! %o4, %o5, %l3, %l4, %l5 = scratch
11757c478bdstevel@tonic-gate	!
11767c478bdstevel@tonic-gate	ldn	[THREAD_REG + T_CPU], %o3
11777c478bdstevel@tonic-gate	!
11787c478bdstevel@tonic-gate	! Clear bit for this level in CPU's interrupt active bitmask.
11797c478bdstevel@tonic-gate	!
11807c478bdstevel@tonic-gate	ld	[%o3 + CPU_INTR_ACTV], %l2
11817c478bdstevel@tonic-gate	mov	1, %o5
11827c478bdstevel@tonic-gate	sll	%o5, %o2, %o5
11837c478bdstevel@tonic-gate#ifdef DEBUG
11847c478bdstevel@tonic-gate	!
11857c478bdstevel@tonic-gate	! ASSERT(CPU->cpu_intr_actv & (1 << PIL))
11867c478bdstevel@tonic-gate	!
11877c478bdstevel@tonic-gate	andcc	%l2, %o5, %g0
11887c478bdstevel@tonic-gate	bnz,pt	%xcc, 0f
11897c478bdstevel@tonic-gate	nop
11907c478bdstevel@tonic-gate	! Do not call panic if a panic is already in progress.
11917c478bdstevel@tonic-gate	sethi	%hi(panic_quiesce), %l2
11927c478bdstevel@tonic-gate	ld	[%l2 + %lo(panic_quiesce)], %l2
11937c478bdstevel@tonic-gate	brnz,pn	%l2, 0f
11947c478bdstevel@tonic-gate	nop
11957c478bdstevel@tonic-gate	sethi	%hi(current_thread_actv_bit_not_set), %o0
11967c478bdstevel@tonic-gate	call	panic
11977c478bdstevel@tonic-gate	or	%o0, %lo(current_thread_actv_bit_not_set), %o0
11987c478bdstevel@tonic-gate0:
11997c478bdstevel@tonic-gate#endif /* DEBUG */
12007c478bdstevel@tonic-gate	andn	%l2, %o5, %l2
12017c478bdstevel@tonic-gate	st	%l2, [%o3 + CPU_INTR_ACTV]
12027c478bdstevel@tonic-gate
12037c478bdstevel@tonic-gate	! Take timestamp, compute interval, update cumulative counter.
12047c478bdstevel@tonic-gate        sub     %o2, LOCK_LEVEL + 1, %o4	! PIL to array index
12057c478bdstevel@tonic-gate	sllx    %o4, 3, %o4			! index to byte offset
12067c478bdstevel@tonic-gate	add	%o4, CPU_MCPU, %o4	! CPU_PIL_HIGH_START is too large
12077c478bdstevel@tonic-gate	add	%o4, MCPU_PIL_HIGH_START, %o4
1208bd28a47Prashanth Sreenivasa	RD_CLOCK_TICK_NO_SUSPEND_CHECK(%o5, %o0)
12097c478bdstevel@tonic-gate	ldx     [%o3 + %o4], %o0
12107c478bdstevel@tonic-gate#ifdef DEBUG
12117c478bdstevel@tonic-gate	! ASSERT(cpu.cpu_m.pil_high_start[pil - (LOCK_LEVEL + 1)] != 0)
12127c478bdstevel@tonic-gate	brnz,pt	%o0, 9f
12137c478bdstevel@tonic-gate	nop
12147c478bdstevel@tonic-gate	! Don't panic if a panic is already in progress.
12157c478bdstevel@tonic-gate	sethi	%hi(panic_quiesce), %l2
12167c478bdstevel@tonic-gate	ld	[%l2 + %lo(panic_quiesce)], %l2
12177c478bdstevel@tonic-gate	brnz,pn	%l2, 9f
12187c478bdstevel@tonic-gate	nop
12197c478bdstevel@tonic-gate	sethi	%hi(current_thread_timestamp_zero), %o0
12207c478bdstevel@tonic-gate	call	panic
12217c478bdstevel@tonic-gate	or	%o0, %lo(current_thread_timestamp_zero), %o0
12227c478bdstevel@tonic-gate9:
12237c478bdstevel@tonic-gate#endif /* DEBUG */
12247c478bdstevel@tonic-gate	stx	%g0, [%o3 + %o4]
12257c478bdstevel@tonic-gate	sub	%o5, %o0, %o5			! interval in o5
12267c478bdstevel@tonic-gate
12277c478bdstevel@tonic-gate	! Check for Energy Star mode
12287c478bdstevel@tonic-gate	lduh	[%o3 + CPU_DIVISOR], %o4	! %o4 = clock divisor
12297c478bdstevel@tonic-gate	cmp	%o4, 1
12307c478bdstevel@tonic-gate	bg,a,pn	%xcc, 2f
12317c478bdstevel@tonic-gate	mulx	%o5, %o4, %o5	! multiply interval by clock divisor iff > 1
12327c478bdstevel@tonic-gate2:
12337c478bdstevel@tonic-gate	sllx	%o2, 4, %o4			! PIL index to byte offset
12347c478bdstevel@tonic-gate	add	%o4, CPU_MCPU, %o4		! CPU_INTRSTAT too large
12357c478bdstevel@tonic-gate	add	%o4, MCPU_INTRSTAT, %o4		! add parts separately
12367c478bdstevel@tonic-gate	ldx	[%o3 + %o4], %o0		! old counter in o0
12377c478bdstevel@tonic-gate	add	%o0, %o5, %o0			! new counter in o0
12387c478bdstevel@tonic-gate	stx	%o0, [%o3 + %o4]		! store new counter
1239eda8946esolom
1240eda8946esolom	! Also update intracct[]
1241eda8946esolom	lduh	[%o3 + CPU_MSTATE], %o4
1242eda8946esolom	sllx	%o4, 3, %o4
1243eda8946esolom	add	%o4, CPU_INTRACCT, %o4
1244eda8946esolom	ldx	[%o3 + %o4], %o0
1245eda8946esolom	add	%o0, %o5, %o0
1246eda8946esolom	stx	%o0, [%o3 + %o4]
12477c478bdstevel@tonic-gate
12487c478bdstevel@tonic-gate	!
12497c478bdstevel@tonic-gate	! get back on current thread's stack
12507c478bdstevel@tonic-gate	!
12517c478bdstevel@tonic-gate	srl	%l2, LOCK_LEVEL + 1, %l2
12527c478bdstevel@tonic-gate	tst	%l2				! any more high-level ints?
12537c478bdstevel@tonic-gate	movz	%xcc, %l1, %sp
12547c478bdstevel@tonic-gate	!
12557c478bdstevel@tonic-gate	! Current register usage:
12567c478bdstevel@tonic-gate	! o2 = PIL
12577c478bdstevel@tonic-gate	! o3 = CPU pointer
12587c478bdstevel@tonic-gate	! l0 = return address
12597c478bdstevel@tonic-gate	! l2 = intr_actv shifted right
12607c478bdstevel@tonic-gate	!
12617c478bdstevel@tonic-gate	bz,pt	%xcc, 3f			! if l2 was zero, no more ints
12627c478bdstevel@tonic-gate	nop
12637c478bdstevel@tonic-gate	!
12647c478bdstevel@tonic-gate	! We found another high-level interrupt active below the one that just
12657c478bdstevel@tonic-gate	! returned. Store a starting timestamp for it in the CPU structure.
12667c478bdstevel@tonic-gate	!
12677c478bdstevel@tonic-gate	! Use cpu_intr_actv to find the cpu_pil_high_start[] offset for the
12687c478bdstevel@tonic-gate	! interrupted high-level interrupt.
12697c478bdstevel@tonic-gate	! Create mask for cpu_intr_actv. Begin by looking for bits set
12707c478bdstevel@tonic-gate	! at one level below the current PIL. Since %l2 contains the active
12717c478bdstevel@tonic-gate	! mask already shifted right by (LOCK_LEVEL + 1), we start by looking
12727c478bdstevel@tonic-gate	! at bit (current_pil - (LOCK_LEVEL + 2)).
12737c478bdstevel@tonic-gate	! %l1 = mask, %o5 = index of bit set in mask
12747c478bdstevel@tonic-gate	!
12757c478bdstevel@tonic-gate	mov	1, %l1
12767c478bdstevel@tonic-gate	sub	%o2, LOCK_LEVEL + 2, %o5
12777c478bdstevel@tonic-gate	sll	%l1, %o5, %l1			! l1 = mask for level
12787c478bdstevel@tonic-gate1:
12797c478bdstevel@tonic-gate#ifdef DEBUG
12807c478bdstevel@tonic-gate	! ASSERT(%l1 != 0) (we didn't shift the bit off the right edge)
12817c478bdstevel@tonic-gate	brnz,pt	%l1, 9f
12827c478bdstevel@tonic-gate	nop
12837c478bdstevel@tonic-gate	sethi	%hi(current_thread_nested_PIL_not_found), %o0
12847c478bdstevel@tonic-gate	call	panic
12857c478bdstevel@tonic-gate	or	%o0, %lo(current_thread_nested_PIL_not_found), %o0
12867c478bdstevel@tonic-gate9:
12877c478bdstevel@tonic-gate#endif /* DEBUG */
12887c478bdstevel@tonic-gate	andcc	%l1, %l2, %g0		! test mask against high-level bits of
12897c478bdstevel@tonic-gate	bnz	%xcc, 2f		! cpu_intr_actv
12907c478bdstevel@tonic-gate	nop
12917c478bdstevel@tonic-gate	srl	%l1, 1, %l1		! No match. Try next lower PIL.
12927c478bdstevel@tonic-gate	ba,pt	%xcc, 1b
12937c478bdstevel@tonic-gate	sub	%o5, 1, %o5		! delay - decrement PIL
12947c478bdstevel@tonic-gate2:
12957c478bdstevel@tonic-gate	sll	%o5, 3, %o5		! convert array index to byte offset
12967c478bdstevel@tonic-gate	add	%o5, CPU_MCPU, %o5	! CPU_PIL_HIGH_START is too large
12977c478bdstevel@tonic-gate	add	%o5, MCPU_PIL_HIGH_START, %o5
1298bd28a47Prashanth Sreenivasa	RD_CLOCK_TICK_NO_SUSPEND_CHECK(%o4, %l2)
12997c478bdstevel@tonic-gate	! Another high-level interrupt is active below this one, so
13007c478bdstevel@tonic-gate	! there is no need to check for an interrupt thread. That will be
13017c478bdstevel@tonic-gate	! done by the lowest priority high-level interrupt active.
13022604657vb	ba,pt	%xcc, 7f
13037c478bdstevel@tonic-gate	stx	%o4, [%o3 + %o5]	! delay - store timestamp
13047c478bdstevel@tonic-gate3:
13057c478bdstevel@tonic-gate	! If we haven't interrupted another high-level interrupt, we may have
13067c478bdstevel@tonic-gate	! interrupted a low level interrupt thread. If so, store a starting
13077c478bdstevel@tonic-gate	! timestamp in its thread structure.
13087c478bdstevel@tonic-gate	lduh	[THREAD_REG + T_FLAGS], %o4
13097c478bdstevel@tonic-gate	andcc	%o4, T_INTR_THREAD, %g0
13102604657vb	bz,pt	%xcc, 7f
13117c478bdstevel@tonic-gate	nop
13127c478bdstevel@tonic-gate
1313bd28a47Prashanth Sreenivasa	RD_CLOCK_TICK_NO_SUSPEND_CHECK(%o4, %l2)
13147c478bdstevel@tonic-gate	stx	%o4, [THREAD_REG + T_INTR_START]
13152604657vb
13162604657vb7:
13172604657vb	sub	%o2, LOCK_LEVEL + 1, %o4
13182604657vb	sll	%o4, CPTRSHIFT, %o5
13192604657vb
13202604657vb	! Check on_trap saved area and restore as needed
13212604657vb	add	%o5, CPU_OTD, %o4
13222604657vb	ldn	[%o3 + %o4], %l2
13232604657vb	brz,pt %l2, no_ontrp_restore
13242604657vb	nop
13252604657vb	stn	%l2, [THREAD_REG + T_ONTRAP] ! restore
13262604657vb	stn	%g0, [%o3 + %o4]	! clear
13272604657vb
13282604657vbno_ontrp_restore:
13292604657vb	! Check on_fault saved area and restore as needed
13302604657vb	add	%o5, CPU_OFD, %o4
13312604657vb	ldn	[%o3 + %o4], %l2
13322604657vb	brz,pt %l2, 8f
13332604657vb	nop
13342604657vb	stn	%l2, [THREAD_REG + T_ONFAULT] ! restore
13352604657vb	stn	%g0, [%o3 + %o4]	! clear
13362604657vb	add	%o5, CPU_LFD, %o4
13372604657vb	ldn	[%o3 + %o4], %l2
13382604657vb	stn	%l2, [THREAD_REG + T_LOFAULT] ! restore
13392604657vb	stn	%g0, [%o3 + %o4]	! clear
13402604657vb
13412604657vb
13422604657vb8:
13437c478bdstevel@tonic-gate	! Enable interrupts and return
13447c478bdstevel@tonic-gate	jmp	%l0 + 8
13457c478bdstevel@tonic-gate	wrpr	%g0, %o2, %pil			! enable interrupts
13467c478bdstevel@tonic-gate	SET_SIZE(current_thread)
13477c478bdstevel@tonic-gate
13487c478bdstevel@tonic-gate
13497c478bdstevel@tonic-gate#ifdef DEBUG
13507c478bdstevel@tonic-gatecurrent_thread_wrong_pil:
13517c478bdstevel@tonic-gate	.asciz	"current_thread: unexpected pil level: %d"
13527c478bdstevel@tonic-gatecurrent_thread_actv_bit_set:
13537c478bdstevel@tonic-gate	.asciz	"current_thread(): cpu_intr_actv bit already set for PIL"
13547c478bdstevel@tonic-gatecurrent_thread_actv_bit_not_set:
13557c478bdstevel@tonic-gate	.asciz	"current_thread(): cpu_intr_actv bit not set for PIL"
13567c478bdstevel@tonic-gatecurrent_thread_nested_pil_zero:
13577c478bdstevel@tonic-gate	.asciz	"current_thread(): timestamp zero for nested PIL %d"
13587c478bdstevel@tonic-gatecurrent_thread_timestamp_zero:
13597c478bdstevel@tonic-gate	.asciz	"current_thread(): timestamp zero upon handler return"
13607c478bdstevel@tonic-gatecurrent_thread_nested_PIL_not_found:
13617c478bdstevel@tonic-gate	.asciz	"current_thread: couldn't find nested high-level PIL"
13627c478bdstevel@tonic-gate#endif /* DEBUG */
13637c478bdstevel@tonic-gate
13647c478bdstevel@tonic-gate/*
13657c478bdstevel@tonic-gate * Return a thread's interrupt level.
13667c478bdstevel@tonic-gate * Since this isn't saved anywhere but in %l4 on interrupt entry, we
13677c478bdstevel@tonic-gate * must dig it out of the save area.
13687c478bdstevel@tonic-gate *
13697c478bdstevel@tonic-gate * Caller 'swears' that this really is an interrupt thread.
13707c478bdstevel@tonic-gate *
13717c478bdstevel@tonic-gate * int
13727c478bdstevel@tonic-gate * intr_level(t)
13737c478bdstevel@tonic-gate *	kthread_id_t	t;
13747c478bdstevel@tonic-gate */
13757c478bdstevel@tonic-gate
13767c478bdstevel@tonic-gate	ENTRY_NP(intr_level)
13777c478bdstevel@tonic-gate	retl
13787c478bdstevel@tonic-gate	ldub	[%o0 + T_PIL], %o0		! return saved pil
13797c478bdstevel@tonic-gate	SET_SIZE(intr_level)
13807c478bdstevel@tonic-gate
13817c478bdstevel@tonic-gate	ENTRY_NP(disable_pil_intr)
13827c478bdstevel@tonic-gate	rdpr	%pil, %o0
13837c478bdstevel@tonic-gate	retl
13847c478bdstevel@tonic-gate	wrpr	%g0, PIL_MAX, %pil		! disable interrupts (1-15)
13857c478bdstevel@tonic-gate	SET_SIZE(disable_pil_intr)
13867c478bdstevel@tonic-gate
13877c478bdstevel@tonic-gate	ENTRY_NP(enable_pil_intr)
13887c478bdstevel@tonic-gate	retl
13897c478bdstevel@tonic-gate	wrpr	%o0, %pil
13907c478bdstevel@tonic-gate	SET_SIZE(enable_pil_intr)
13917c478bdstevel@tonic-gate
13927c478bdstevel@tonic-gate	ENTRY_NP(disable_vec_intr)
13937c478bdstevel@tonic-gate	rdpr	%pstate, %o0
13947c478bdstevel@tonic-gate	andn	%o0, PSTATE_IE, %g1
13957c478bdstevel@tonic-gate	retl
13967c478bdstevel@tonic-gate	wrpr	%g0, %g1, %pstate		! disable interrupt
13977c478bdstevel@tonic-gate	SET_SIZE(disable_vec_intr)
13987c478bdstevel@tonic-gate
13997c478bdstevel@tonic-gate	ENTRY_NP(enable_vec_intr)
14007c478bdstevel@tonic-gate	retl
14017c478bdstevel@tonic-gate	wrpr	%g0, %o0, %pstate
14027c478bdstevel@tonic-gate	SET_SIZE(enable_vec_intr)
14037c478bdstevel@tonic-gate
14047c478bdstevel@tonic-gate	ENTRY_NP(cbe_level14)
14057c478bdstevel@tonic-gate	save    %sp, -SA(MINFRAME), %sp ! get a new window
14067c478bdstevel@tonic-gate	!
14077c478bdstevel@tonic-gate	! Make sure that this is from TICK_COMPARE; if not just return
14087c478bdstevel@tonic-gate	!
14097c478bdstevel@tonic-gate	rd	SOFTINT, %l1
14107c478bdstevel@tonic-gate	set	(TICK_INT_MASK | STICK_INT_MASK), %o2
14117c478bdstevel@tonic-gate	andcc	%l1, %o2, %g0
14127c478bdstevel@tonic-gate	bz,pn	%icc, 2f
14137c478bdstevel@tonic-gate	nop
14147c478bdstevel@tonic-gate
14157c478bdstevel@tonic-gate	CPU_ADDR(%o1, %o2)
14167c478bdstevel@tonic-gate	call	cyclic_fire
14177c478bdstevel@tonic-gate	mov	%o1, %o0
14187c478bdstevel@tonic-gate2:
14197c478bdstevel@tonic-gate	ret
14207c478bdstevel@tonic-gate	restore	%g0, 1, %o0
14217c478bdstevel@tonic-gate	SET_SIZE(cbe_level14)
14227c478bdstevel@tonic-gate
14237c478bdstevel@tonic-gate
1424a1af7bacwb	ENTRY_NP(kdi_setsoftint)
14257c478bdstevel@tonic-gate	save	%sp, -SA(MINFRAME), %sp	! get a new window
14267c478bdstevel@tonic-gate	rdpr	%pstate, %l5
14277c478bdstevel@tonic-gate	andn	%l5, PSTATE_IE, %l1
14287c478bdstevel@tonic-gate	wrpr	%l1, %pstate		! disable interrupt
14297c478bdstevel@tonic-gate	!
1430b0fc0e7govinda	! We have a pointer to an interrupt vector data structure.
1431b0fc0e7govinda	! Put the request on the cpu's softint priority list and
1432b0fc0e7govinda	! set %set_softint.
14337c478bdstevel@tonic-gate	!
14347c478bdstevel@tonic-gate	! Register usage
1435b0fc0e7govinda	! 	%i0 - pointer to intr_vec_t (iv)
14367c478bdstevel@tonic-gate	!	%l2 - requested pil
1437b0fc0e7govinda	!	%l4 - cpu
1438b0fc0e7govinda	!	%l5 - pstate
1439b0fc0e7govinda	!	%l1, %l3, %l6 - temps
14407c478bdstevel@tonic-gate	!
1441b0fc0e7govinda	! check if a softint is pending for this softint,
1442b0fc0e7govinda	! if one is pending, don't bother queuing another.
14437c478bdstevel@tonic-gate	!
1444b0fc0e7govinda	lduh	[%i0 + IV_FLAGS], %l1	! %l1 = iv->iv_flags
1445b0fc0e7govinda	and	%l1, IV_SOFTINT_PEND, %l6 ! %l6 = iv->iv_flags & IV_SOFTINT_PEND
1446b0fc0e7govinda	brnz,pn	%l6, 4f			! branch if softint is already pending
1447b0fc0e7govinda	or	%l1, IV_SOFTINT_PEND, %l2
1448b0fc0e7govinda	sth	%l2, [%i0 + IV_FLAGS]	! Set IV_SOFTINT_PEND flag
1449b0fc0e7govinda
1450b0fc0e7govinda	CPU_ADDR(%l4, %l2)		! %l4 = cpu
1451b0fc0e7govinda	lduh	[%i0 + IV_PIL], %l2	! %l2 = iv->iv_pil
1452b0fc0e7govinda
14537c478bdstevel@tonic-gate	!
1454b0fc0e7govinda	! Insert intr_vec_t (iv) to appropriate cpu's softint priority list
14557c478bdstevel@tonic-gate	!
1456b0fc0e7govinda	sll	%l2, CPTRSHIFT, %l0	! %l0 = offset to pil entry
1457b0fc0e7govinda	add	%l4, INTR_TAIL, %l6	! %l6 = &cpu->m_cpu.intr_tail
1458b0fc0e7govinda	ldn	[%l6 + %l0], %l1	! %l1 = cpu->m_cpu.intr_tail[pil]
1459b0fc0e7govinda					!       current tail (ct)
1460b0fc0e7govinda	brz,pt	%l1, 2f			! branch if current tail is NULL
1461b0fc0e7govinda	stn	%i0, [%l6 + %l0]	! make intr_vec_t (iv) as new tail
14627c478bdstevel@tonic-gate	!
1463b0fc0e7govinda	! there's pending intr_vec_t already
14647c478bdstevel@tonic-gate	!
1465b0fc0e7govinda	lduh	[%l1 + IV_FLAGS], %l6	! %l6 = ct->iv_flags
1466b0fc0e7govinda	and	%l6, IV_SOFTINT_MT, %l6	! %l6 = ct->iv_flags & IV_SOFTINT_MT
1467b0fc0e7govinda	brz,pt	%l6, 1f			! check for Multi target softint flag
1468b0fc0e7govinda	add	%l1, IV_PIL_NEXT, %l3	! %l3 = &ct->iv_pil_next
1469b0fc0e7govinda	ld	[%l4 + CPU_ID], %l6	! for multi target softint, use cpuid
1470b0fc0e7govinda	sll	%l6, CPTRSHIFT, %l6	! calculate offset address from cpuid
1471b0fc0e7govinda	add	%l3, %l6, %l3		! %l3 =  &ct->iv_xpil_next[cpuid]
1472b0fc0e7govinda1:
14737c478bdstevel@tonic-gate	!
1474b0fc0e7govinda	! update old tail
14757c478bdstevel@tonic-gate	!
14767c478bdstevel@tonic-gate	ba,pt	%xcc, 3f
1477b0fc0e7govinda	stn	%i0, [%l3]		! [%l3] = iv, set pil_next field
14787c478bdstevel@tonic-gate2:
14797c478bdstevel@tonic-gate	!
1480b0fc0e7govinda	! no pending intr_vec_t; make intr_vec_t as new head
14817c478bdstevel@tonic-gate	!
1482b0fc0e7govinda	add	%l4, INTR_HEAD, %l6	! %l6 = &cpu->m_cpu.intr_head[pil]
1483b0fc0e7govinda	stn	%i0, [%l6 + %l0]	! cpu->m_cpu.intr_head[pil] = iv
14847c478bdstevel@tonic-gate3:
14857c478bdstevel@tonic-gate	!
14867c478bdstevel@tonic-gate	! Write %set_softint with (1<<pil) to cause a "pil" level trap
14877c478bdstevel@tonic-gate	!
1488b0fc0e7govinda	mov	1, %l1			! %l1 = 1
1489b0fc0e7govinda	sll	%l1, %l2, %l1		! %l1 = 1 << pil
1490b0fc0e7govinda	wr	%l1, SET_SOFTINT	! trigger required pil softint
14917c478bdstevel@tonic-gate4:
1492b0fc0e7govinda	wrpr	%g0, %l5, %pstate	! %pstate = saved %pstate (in %l5)
14937c478bdstevel@tonic-gate	ret
14947c478bdstevel@tonic-gate	restore
1495a1af7bacwb	SET_SIZE(kdi_setsoftint)
1496b0fc0e7govinda
14977c478bdstevel@tonic-gate	!
14987c478bdstevel@tonic-gate	! Register usage
1499b0fc0e7govinda	!	Arguments:
1500b0fc0e7govinda	! 	%g1 - Pointer to intr_vec_t (iv)
15017c478bdstevel@tonic-gate	!
1502b0fc0e7govinda	!	Internal:
1503b0fc0e7govinda	!	%g2 - pil
1504b0fc0e7govinda	!	%g4 - cpu
1505b0fc0e7govinda	!	%g3,%g5-g7 - temps
15067c478bdstevel@tonic-gate	!
15077c478bdstevel@tonic-gate	ENTRY_NP(setsoftint_tl1)
15087c478bdstevel@tonic-gate	!
1509b0fc0e7govinda	! We have a pointer to an interrupt vector data structure.
1510b0fc0e7govinda	! Put the request on the cpu's softint priority list and
1511b0fc0e7govinda	! set %set_softint.
15127c478bdstevel@tonic-gate	!
1513b0fc0e7govinda	CPU_ADDR(%g4, %g2)		! %g4 = cpu
1514b0fc0e7govinda	lduh	[%g1 + IV_PIL], %g2	! %g2 = iv->iv_pil
1515b0fc0e7govinda
15167c478bdstevel@tonic-gate	!
1517b0fc0e7govinda	! Insert intr_vec_t (iv) to appropriate cpu's softint priority list
15187c478bdstevel@tonic-gate	!
1519b0fc0e7govinda	sll	%g2, CPTRSHIFT, %g7	! %g7 = offset to pil entry
1520b0fc0e7govinda	add	%g4, INTR_TAIL, %g6	! %g6 = &cpu->m_cpu.intr_tail
1521b0fc0e7govinda	ldn	[%g6 + %g7], %g5	! %g5 = cpu->m_cpu.intr_tail[pil]
1522b0fc0e7govinda					!       current tail (ct)
1523b0fc0e7govinda	brz,pt	%g5, 1f			! branch if current tail is NULL
1524b0fc0e7govinda	stn	%g1, [%g6 + %g7]	! make intr_rec_t (iv) as new tail
15257c478bdstevel@tonic-gate	!
1526b0fc0e7govinda	! there's pending intr_vec_t already
1527b0fc0e7govinda	!
1528b0fc0e7govinda	lduh	[%g5 + IV_FLAGS], %g6	! %g6 = ct->iv_flags
1529b0fc0e7govinda	and	%g6, IV_SOFTINT_MT, %g6	! %g6 = ct->iv_flags & IV_SOFTINT_MT
1530b0fc0e7govinda	brz,pt	%g6, 0f			! check for Multi target softint flag
1531b0fc0e7govinda	add	%g5, IV_PIL_NEXT, %g3	! %g3 = &ct->iv_pil_next
1532b0fc0e7govinda	ld	[%g4 + CPU_ID], %g6	! for multi target softint, use cpuid
1533b0fc0e7govinda	sll	%g6, CPTRSHIFT, %g6	! calculate offset address from cpuid
1534b0fc0e7govinda	add	%g3, %g6, %g3		! %g3 = &ct->iv_xpil_next[cpuid]
1535b0fc0e7govinda0:
1536b0fc0e7govinda	!
1537b0fc0e7govinda	! update old tail
1538b0fc0e7govinda	!
1539b0fc0e7govinda	ba,pt	%xcc, 2f
1540b0fc0e7govinda	stn	%g1, [%g3]		! [%g3] = iv, set pil_next field
1541b0fc0e7govinda1:
1542b0fc0e7govinda	!
1543b0fc0e7govinda	! no pending intr_vec_t; make intr_vec_t as new head
15447c478bdstevel@tonic-gate	!
1545b0fc0e7govinda	add	%g4, INTR_HEAD, %g6	! %g6 = &cpu->m_cpu.intr_head[pil]
1546b0fc0e7govinda	stn	%g1, [%g6 + %g7]	! cpu->m_cpu.intr_head[pil] = iv
1547b0fc0e7govinda2:
1548b0fc0e7govinda#ifdef TRAPTRACE
1549b0fc0e7govinda	TRACE_PTR(%g5, %g6)
1550023e71dHaik Aftandilian	GET_TRACE_TICK(%g6, %g3)
1551b0fc0e7govinda	stxa	%g6, [%g5 + TRAP_ENT_TICK]%asi	! trap_tick = %tick
1552b0fc0e7govinda	TRACE_SAVE_TL_GL_REGS(%g5, %g6)
1553b0fc0e7govinda	rdpr	%tt, %g6
1554b0fc0e7govinda	stha	%g6, [%g5 + TRAP_ENT_TT]%asi	! trap_type = %tt
1555b0fc0e7govinda	rdpr	%tpc, %g6
1556b0fc0e7govinda	stna	%g6, [%g5 + TRAP_ENT_TPC]%asi	! trap_pc = %tpc
1557b0fc0e7govinda	rdpr	%tstate, %g6
1558b0fc0e7govinda	stxa	%g6, [%g5 + TRAP_ENT_TSTATE]%asi ! trap_tstate = %tstate
1559b0fc0e7govinda	stna	%sp, [%g5 + TRAP_ENT_SP]%asi	! trap_sp = %sp
1560b0fc0e7govinda	stna	%g1, [%g5 + TRAP_ENT_TR]%asi	! trap_tr = iv
1561b0fc0e7govinda	ldn	[%g1 + IV_PIL_NEXT], %g6	!
1562b0fc0e7govinda	stna	%g6, [%g5 + TRAP_ENT_F1]%asi	! trap_f1 = iv->iv_pil_next
1563b0fc0e7govinda	add	%g4, INTR_HEAD, %g6
1564b0fc0e7govinda	ldn	[%g6 + %g7], %g6		! %g6=cpu->m_cpu.intr_head[pil]
1565b0fc0e7govinda	stna	%g6, [%g5 + TRAP_ENT_F2]%asi	! trap_f2 = intr_head[pil]
1566b0fc0e7govinda	add	%g4, INTR_TAIL, %g6
1567b0fc0e7govinda	ldn	[%g6 + %g7], %g6		! %g6=cpu->m_cpu.intr_tail[pil]
1568b0fc0e7govinda	stna	%g6, [%g5 + TRAP_ENT_F3]%asi	! trap_f3 = intr_tail[pil]
1569b0fc0e7govinda	stna	%g2, [%g5 + TRAP_ENT_F4]%asi	! trap_f4 = pil
1570b0fc0e7govinda	TRACE_NEXT(%g5, %g6, %g3)
1571b0fc0e7govinda#endif /* TRAPTRACE */
15727c478bdstevel@tonic-gate	!
1573b0fc0e7govinda	! Write %set_softint with (1<<pil) to cause a "pil" level trap
1574b0fc0e7govinda	!
1575b0fc0e7govinda	mov	1, %g5			! %g5 = 1
1576b0fc0e7govinda	sll	%g5, %g2, %g5		! %g5 = 1 << pil
1577b0fc0e7govinda	wr	%g5, SET_SOFTINT	! trigger required pil softint
1578b0fc0e7govinda	retry
1579b0fc0e7govinda	SET_SIZE(setsoftint_tl1)
15807c478bdstevel@tonic-gate
1581b0fc0e7govinda	!
1582b0fc0e7govinda	! Register usage
1583b0fc0e7govinda	!	Arguments:
1584b0fc0e7govinda	! 	%g1 - inumber
1585b0fc0e7govinda	!
1586b0fc0e7govinda	!	Internal:
1587b0fc0e7govinda	! 	%g1 - softint pil mask
1588b0fc0e7govinda	!	%g2 - pil of intr_vec_t
1589b0fc0e7govinda	!	%g3 - pointer to current intr_vec_t (iv)
1590b0fc0e7govinda	!	%g4 - cpu
1591b0fc0e7govinda	!	%g5, %g6,%g7 - temps
1592b0fc0e7govinda	!
1593b0fc0e7govinda	ENTRY_NP(setvecint_tl1)
1594b0fc0e7govinda	!
1595b0fc0e7govinda	! Verify the inumber received (should be inum < MAXIVNUM).
1596b0fc0e7govinda	!
1597b0fc0e7govinda	set	MAXIVNUM, %g2
1598b0fc0e7govinda	cmp	%g1, %g2
1599b0fc0e7govinda	bgeu,pn	%xcc, .no_ivintr
1600b0fc0e7govinda	clr	%g2			! expected in .no_ivintr
16017c478bdstevel@tonic-gate
16027c478bdstevel@tonic-gate	!
1603b0fc0e7govinda	! Fetch data from intr_vec_table according to the inum.
1604b0fc0e7govinda	!
1605b0fc0e7govinda	! We have an interrupt number. Fetch the interrupt vector requests
1606b0fc0e7govinda	! from the interrupt vector table for a given interrupt number and
1607b0fc0e7govinda	! insert them into cpu's softint priority lists and set %set_softint.
16087c478bdstevel@tonic-gate	!
1609b0fc0e7govinda	set	intr_vec_table, %g5	! %g5 = intr_vec_table
1610b0fc0e7govinda	sll	%g1, CPTRSHIFT, %g6	! %g6 = offset to inum entry in table
1611b0fc0e7govinda	add	%g5, %g6, %g5		! %g5 = &intr_vec_table[inum]
1612b0fc0e7govinda	ldn	[%g5], %g3		! %g3 = pointer to first entry of
1613b0fc0e7govinda					!       intr_vec_t list
1614b0fc0e7govinda
1615b0fc0e7govinda	! Verify the first intr_vec_t pointer for a given inum and it should
1616b0fc0e7govinda	! not be NULL. This used to be guarded by DEBUG but broken drivers can
1617b0fc0e7govinda	! cause spurious tick interrupts when the softint register is programmed
1618b0fc0e7govinda	! with 1 << 0 at the end of this routine. Now we always check for a
1619b0fc0e7govinda	! valid intr_vec_t pointer.
1620b0fc0e7govinda	brz,pn	%g3, .no_ivintr
1621b0fc0e7govinda	nop
16227c478bdstevel@tonic-gate
16237c478bdstevel@tonic-gate	!
1624b0fc0e7govinda	! Traverse the intr_vec_t link list, put each item on to corresponding
1625b0fc0e7govinda	! CPU softint priority queue, and compose the final softint pil mask.
16267c478bdstevel@tonic-gate	!
1627b0fc0e7govinda	! At this point:
1628b0fc0e7govinda	!	%g3 = intr_vec_table[inum]
16297c478bdstevel@tonic-gate	!
1630b0fc0e7govinda	CPU_ADDR(%g4, %g2)		! %g4 = cpu
1631b0fc0e7govinda	mov	%g0, %g1		! %g1 = 0, initialize pil mask to 0
1632b0fc0e7govinda0:
16337c478bdstevel@tonic-gate	!
1634b0fc0e7govinda	! Insert next intr_vec_t (iv) to appropriate cpu's softint priority list
1635b0fc0e7govinda	!
1636b0fc0e7govinda	! At this point:
1637b0fc0e7govinda	!	%g1 = softint pil mask
1638b0fc0e7govinda	!	%g3 = pointer to next intr_vec_t (iv)
1639b0fc0e7govinda	!	%g4 = cpu
1640b0fc0e7govinda	!
1641b0fc0e7govinda	lduh	[%g3 + IV_PIL], %g2	! %g2 = iv->iv_pil
1642b0fc0e7govinda	sll	%g2, CPTRSHIFT, %g7	! %g7 = offset to pil entry
1643b0fc0e7govinda	add	%g4, INTR_TAIL, %g6	! %g6 = &cpu->m_cpu.intr_tail
1644b0fc0e7govinda	ldn	[%g6 + %g7], %g5	! %g5 = cpu->m_cpu.intr_tail[pil]
1645b0fc0e7govinda					! 	current tail (ct)
1646b0fc0e7govinda	brz,pt	%g5, 2f			! branch if current tail is NULL
1647b0fc0e7govinda	stn	%g3, [%g6 + %g7]	! make intr_vec_t (iv) as new tail
1648b0fc0e7govinda					! cpu->m_cpu.intr_tail[pil] = iv
1649b0fc0e7govinda	!
1650b0fc0e7govinda	! there's pending intr_vec_t already
1651b0fc0e7govinda	!
1652b0fc0e7govinda	lduh	[%g5 + IV_FLAGS], %g6	! %g6 = ct->iv_flags
1653b0fc0e7govinda	and	%g6, IV_SOFTINT_MT, %g6	! %g6 = ct->iv_flags & IV_SOFTINT_MT
1654b0fc0e7govinda	brz,pt	%g6, 1f			! check for Multi target softint flag
1655b0fc0e7govinda	add	%g5, IV_PIL_NEXT, %g5	! %g5 = &ct->iv_pil_next
1656b0fc0e7govinda	ld	[%g4 + CPU_ID], %g6	! for multi target softint, use cpuid
1657b0fc0e7govinda	sll	%g6, CPTRSHIFT, %g6	! calculate offset address from cpuid
1658b0fc0e7govinda	add	%g5, %g6, %g5		! %g5 = &ct->iv_xpil_next[cpuid]
1659b0fc0e7govinda1:
16607c478bdstevel@tonic-gate	!
1661b0fc0e7govinda	! update old tail
16627c478bdstevel@tonic-gate	!
16637c478bdstevel@tonic-gate	ba,pt	%xcc, 3f
1664b0fc0e7govinda	stn	%g3, [%g5]		! [%g5] = iv, set pil_next field
16657c478bdstevel@tonic-gate2:
16667c478bdstevel@tonic-gate	!
1667b0fc0e7govinda	! no pending intr_vec_t; make intr_vec_t as new head
16687c478bdstevel@tonic-gate	!
1669b0fc0e7govinda	add	%g4, INTR_HEAD, %g6	!  %g6 = &cpu->m_cpu.intr_head[pil]
1670b0fc0e7govinda	stn	%g3, [%g6 + %g7]	!  cpu->m_cpu.intr_head[pil] = iv
16717c478bdstevel@tonic-gate3:
16727c478bdstevel@tonic-gate#ifdef TRAPTRACE
1673b0fc0e7govinda	TRACE_PTR(%g5, %g6)
1674b0fc0e7govinda	TRACE_SAVE_TL_GL_REGS(%g5, %g6)
16757c478bdstevel@tonic-gate	rdpr	%tt, %g6
1676b0fc0e7govinda	stha	%g6, [%g5 + TRAP_ENT_TT]%asi	! trap_type = %tt`
16777c478bdstevel@tonic-gate	rdpr	%tpc, %g6
1678b0fc0e7govinda	stna	%g6, [%g5 + TRAP_ENT_TPC]%asi	! trap_pc = %tpc
16797c478bdstevel@tonic-gate	rdpr	%tstate, %g6
1680b0fc0e7govinda	stxa	%g6, [%g5 + TRAP_ENT_TSTATE]%asi ! trap_tstate = %tstate
1681b0fc0e7govinda	stna	%sp, [%g5 + TRAP_ENT_SP]%asi	! trap_sp = %sp
1682b0fc0e7govinda	stna	%g3, [%g5 + TRAP_ENT_TR]%asi	! trap_tr = iv
1683b0fc0e7govinda	stna	%g1, [%g5 + TRAP_ENT_F1]%asi	! trap_f1 = pil mask
16847c478bdstevel@tonic-gate	add	%g4, INTR_HEAD, %g6
1685b0fc0e7govinda	ldn	[%g6 + %g7], %g6		! %g6=cpu->m_cpu.intr_head[pil]
1686b0fc0e7govinda	stna	%g6, [%g5 + TRAP_ENT_F2]%asi	! trap_f2 = intr_head[pil]
16877c478bdstevel@tonic-gate	add	%g4, INTR_TAIL, %g6
1688b0fc0e7govinda	ldn	[%g6 + %g7], %g6		! %g6=cpu->m_cpu.intr_tail[pil]
1689b0fc0e7govinda	stna	%g6, [%g5 + TRAP_ENT_F3]%asi	! trap_f3 = intr_tail[pil]
1690b0fc0e7govinda	stna	%g2, [%g5 + TRAP_ENT_F4]%asi	! trap_f4 = pil
1691023e71dHaik Aftandilian	GET_TRACE_TICK(%g6, %g7)
1692023e71dHaik Aftandilian	stxa	%g6, [%g5 + TRAP_ENT_TICK]%asi	! trap_tick = %tick
1693b0fc0e7govinda	TRACE_NEXT(%g5, %g6, %g7)
16947c478bdstevel@tonic-gate#endif /* TRAPTRACE */
1695b0fc0e7govinda	mov	1, %g6			! %g6 = 1
1696b0fc0e7govinda	sll	%g6, %g2, %g6		! %g6 = 1 << pil
1697b0fc0e7govinda	or	%g1, %g6, %g1		! %g1 |= (1 << pil), pil mask
1698b0fc0e7govinda	ldn	[%g3 + IV_VEC_NEXT], %g3 ! %g3 = pointer to next intr_vec_t (iv)
1699b0fc0e7govinda	brnz,pn	%g3, 0b			! iv->iv_vec_next is non NULL, goto 0b
1700b0fc0e7govinda	nop
1701b0fc0e7govinda	wr	%g1, SET_SOFTINT	! triggered one or more pil softints
17027c478bdstevel@tonic-gate	retry
17037c478bdstevel@tonic-gate
17047c478bdstevel@tonic-gate.no_ivintr:
17057c478bdstevel@tonic-gate	! no_ivintr: arguments: rp, inum (%g1), pil (%g2 == 0)
17067c478bdstevel@tonic-gate	mov	%g2, %g3
17077c478bdstevel@tonic-gate	mov	%g1, %g2
17087c478bdstevel@tonic-gate	set	no_ivintr, %g1
17097c478bdstevel@tonic-gate	ba,pt	%xcc, sys_trap
17107c478bdstevel@tonic-gate	mov	PIL_15, %g4
1711b0fc0e7govinda	SET_SIZE(setvecint_tl1)
17127c478bdstevel@tonic-gate
17137c478bdstevel@tonic-gate	ENTRY_NP(wr_clr_softint)
17147c478bdstevel@tonic-gate	retl
17157c478bdstevel@tonic-gate	wr	%o0, CLEAR_SOFTINT
17167c478bdstevel@tonic-gate	SET_SIZE(wr_clr_softint)
17177c478bdstevel@tonic-gate
17187c478bdstevel@tonic-gate/*
17197c478bdstevel@tonic-gate * intr_enqueue_req
17207c478bdstevel@tonic-gate *
17217c478bdstevel@tonic-gate * %o0 - pil
1722b0fc0e7govinda * %o1 - pointer to intr_vec_t (iv)
17237c478bdstevel@tonic-gate * %o5 - preserved
17247c478bdstevel@tonic-gate * %g5 - preserved
17257c478bdstevel@tonic-gate */
17267c478bdstevel@tonic-gate	ENTRY_NP(intr_enqueue_req)
1727b0fc0e7govinda	!
1728b0fc0e7govinda	CPU_ADDR(%g4, %g1)		! %g4 = cpu
17297c478bdstevel@tonic-gate
1730b0fc0e7govinda	!
1731b0fc0e7govinda	! Insert intr_vec_t (iv) to appropriate cpu's softint priority list
1732b0fc0e7govinda	!
1733b0fc0e7govinda	sll	%o0, CPTRSHIFT, %o0	! %o0 = offset to pil entry
1734b0fc0e7govinda	add	%g4, INTR_TAIL, %g6	! %g6 = &cpu->m_cpu.intr_tail
1735b0fc0e7govinda	ldn	[%o0 + %g6], %g1	! %g1 = cpu->m_cpu.intr_tail[pil]
1736b0fc0e7govinda					!       current tail (ct)
1737b0fc0e7govinda	brz,pt	%g1, 2f			! branch if current tail is NULL
1738b0fc0e7govinda	stn	%o1, [%g6 + %o0]	! make intr_vec_t (iv) as new tail
17397c478bdstevel@tonic-gate
1740b0fc0e7govinda	!
1741b0fc0e7govinda	! there's pending intr_vec_t already
1742b0fc0e7govinda	!
1743b0fc0e7govinda	lduh	[%g1 + IV_FLAGS], %g6	! %g6 = ct->iv_flags
1744b0fc0e7govinda	and	%g6, IV_SOFTINT_MT, %g6	! %g6 = ct->iv_flags & IV_SOFTINT_MT
1745b0fc0e7govinda	brz,pt	%g6, 1f			! check for Multi target softint flag
1746b0fc0e7govinda	add	%g1, IV_PIL_NEXT, %g3	! %g3 = &ct->iv_pil_next
1747b0fc0e7govinda	ld	[%g4 + CPU_ID], %g6	! for multi target softint, use cpuid
1748b0fc0e7govinda	sll	%g6, CPTRSHIFT, %g6	! calculate offset address from cpuid
1749b0fc0e7govinda	add	%g3, %g6, %g3		! %g3 = &ct->iv_xpil_next[cpuid]
1750b0fc0e7govinda1:
1751b0fc0e7govinda	!
1752b0fc0e7govinda	! update old tail
1753b0fc0e7govinda	!
17547c478bdstevel@tonic-gate	ba,pt	%xcc, 3f
1755b0fc0e7govinda	stn	%o1, [%g3]		! {%g5] = iv, set pil_next field
17567c478bdstevel@tonic-gate2:
1757b0fc0e7govinda	!
1758b0fc0e7govinda	! no intr_vec_t's queued so make intr_vec_t as new head
1759b0fc0e7govinda	!
1760b0fc0e7govinda	add	%g4, INTR_HEAD, %g6	! %g6 = &cpu->m_cpu.intr_head[pil]
1761b0fc0e7govinda	stn	%o1, [%g6 + %o0]	! cpu->m_cpu.intr_head[pil] = iv
17627c478bdstevel@tonic-gate3:
17637c478bdstevel@tonic-gate	retl
17647c478bdstevel@tonic-gate	nop
17657c478bdstevel@tonic-gate	SET_SIZE(intr_enqueue_req)
17667c478bdstevel@tonic-gate
17677c478bdstevel@tonic-gate/*
17687c478bdstevel@tonic-gate * Set CPU's base SPL level, based on which interrupt levels are active.
17697c478bdstevel@tonic-gate * 	Called at spl7 or above.
17707c478bdstevel@tonic-gate */
17717c478bdstevel@tonic-gate
17727c478bdstevel@tonic-gate	ENTRY_NP(set_base_spl)
17737c478bdstevel@tonic-gate	ldn	[THREAD_REG + T_CPU], %o2	! load CPU pointer
17747c478bdstevel@tonic-gate	ld	[%o2 + CPU_INTR_ACTV], %o5	! load active interrupts mask
17757c478bdstevel@tonic-gate
17767c478bdstevel@tonic-gate/*
17777c478bdstevel@tonic-gate * WARNING: non-standard callinq sequence; do not call from C
17787c478bdstevel@tonic-gate *	%o2 = pointer to CPU
17797c478bdstevel@tonic-gate *	%o5 = updated CPU_INTR_ACTV
17807c478bdstevel@tonic-gate */
17817c478bdstevel@tonic-gate_intr_set_spl:					! intr_thread_exit enters here
17827c478bdstevel@tonic-gate	!
17837c478bdstevel@tonic-gate	! Determine highest interrupt level active.  Several could be blocked
17847c478bdstevel@tonic-gate	! at higher levels than this one, so must convert flags to a PIL
17857c478bdstevel@tonic-gate	! Normally nothing will be blocked, so test this first.
17867c478bdstevel@tonic-gate	!
17877c478bdstevel@tonic-gate	brz,pt	%o5, 1f				! nothing active
17887c478bdstevel@tonic-gate	sra	%o5, 11, %o3			! delay - set %o3 to bits 15-11
17897c478bdstevel@tonic-gate	set	_intr_flag_table, %o1
17907c478bdstevel@tonic-gate	tst	%o3				! see if any of the bits set
17917c478bdstevel@tonic-gate	ldub	[%o1 + %o3], %o3		! load bit number
17927c478bdstevel@tonic-gate	bnz,a,pn %xcc, 1f			! yes, add 10 and we're done
17937c478bdstevel@tonic-gate	add	%o3, 11-1, %o3			! delay - add bit number - 1
17947c478bdstevel@tonic-gate
17957c478bdstevel@tonic-gate	sra	%o5, 6, %o3			! test bits 10-6
17967c478bdstevel@tonic-gate	tst	%o3
17977c478bdstevel@tonic-gate	ldub	[%o1 + %o3], %o3
17987c478bdstevel@tonic-gate	bnz,a,pn %xcc, 1f
17997c478bdstevel@tonic-gate	add	%o3, 6-1, %o3
18007c478bdstevel@tonic-gate
18017c478bdstevel@tonic-gate	sra	%o5, 1, %o3			! test bits 5-1
18027c478bdstevel@tonic-gate	ldub	[%o1 + %o3], %o3
18037c478bdstevel@tonic-gate
18047c478bdstevel@tonic-gate	!
18057c478bdstevel@tonic-gate	! highest interrupt level number active is in %l6
18067c478bdstevel@tonic-gate	!
18077c478bdstevel@tonic-gate1:
18087c478bdstevel@tonic-gate	retl
18097c478bdstevel@tonic-gate	st	%o3, [%o2 + CPU_BASE_SPL]	! delay - store base priority
18107c478bdstevel@tonic-gate	SET_SIZE(set_base_spl)
18117c478bdstevel@tonic-gate
18127c478bdstevel@tonic-gate/*
18137c478bdstevel@tonic-gate * Table that finds the most significant bit set in a five bit field.
18147c478bdstevel@tonic-gate * Each entry is the high-order bit number + 1 of it's index in the table.
18157c478bdstevel@tonic-gate * This read-only data is in the text segment.
18167c478bdstevel@tonic-gate */
18177c478bdstevel@tonic-gate_intr_flag_table:
18187c478bdstevel@tonic-gate	.byte	0, 1, 2, 2,	3, 3, 3, 3,	4, 4, 4, 4,	4, 4, 4, 4
18197c478bdstevel@tonic-gate	.byte	5, 5, 5, 5,	5, 5, 5, 5,	5, 5, 5, 5,	5, 5, 5, 5
18207c478bdstevel@tonic-gate	.align	4
18217c478bdstevel@tonic-gate
18227c478bdstevel@tonic-gate/*
18237c478bdstevel@tonic-gate * int
18247c478bdstevel@tonic-gate * intr_passivate(from, to)
18257c478bdstevel@tonic-gate *	kthread_id_t	from;		interrupt thread
18267c478bdstevel@tonic-gate *	kthread_id_t	to;		interrupted thread
18277c478bdstevel@tonic-gate */
18287c478bdstevel@tonic-gate
18297c478bdstevel@tonic-gate	ENTRY_NP(intr_passivate)
18307c478bdstevel@tonic-gate	save	%sp, -SA(MINFRAME), %sp	! get a new window
18317c478bdstevel@tonic-gate
18327c478bdstevel@tonic-gate	flushw				! force register windows to stack
18337c478bdstevel@tonic-gate	!
18347c478bdstevel@tonic-gate	! restore registers from the base of the stack of the interrupt thread.
18357c478bdstevel@tonic-gate	!
18367c478bdstevel@tonic-gate	ldn	[%i0 + T_STACK], %i2	! get stack save area pointer
18377c478bdstevel@tonic-gate	ldn	[%i2 + (0*GREGSIZE)], %l0	! load locals
18387c478bdstevel@tonic-gate	ldn	[%i2 + (1*GREGSIZE)], %l1
18397c478bdstevel@tonic-gate	ldn	[%i2 + (2*GREGSIZE)], %l2
18407c478bdstevel@tonic-gate	ldn	[%i2 + (3*GREGSIZE)], %l3
18417c478bdstevel@tonic-gate	ldn	[%i2 + (4*GREGSIZE)], %l4
18427c478bdstevel@tonic-gate	ldn	[%i2 + (5*GREGSIZE)], %l5
18437c478bdstevel@tonic-gate	ldn	[%i2 + (6*GREGSIZE)], %l6
18447c478bdstevel@tonic-gate	ldn	[%i2 + (7*GREGSIZE)], %l7
18457c478bdstevel@tonic-gate	ldn	[%i2 + (8*GREGSIZE)], %o0	! put ins from stack in outs
18467c478bdstevel@tonic-gate	ldn	[%i2 + (9*GREGSIZE)], %o1
18477c478bdstevel@tonic-gate	ldn	[%i2 + (10*GREGSIZE)], %o2
18487c478bdstevel@tonic-gate	ldn	[%i2 + (11*GREGSIZE)], %o3
18497c478bdstevel@tonic-gate	ldn	[%i2 + (12*GREGSIZE)], %o4
18507c478bdstevel@tonic-gate	ldn	[%i2 + (13*GREGSIZE)], %o5
18517c478bdstevel@tonic-gate	ldn	[%i2 + (14*GREGSIZE)], %i4
18527c478bdstevel@tonic-gate					! copy stack/pointer without using %sp
18537c478bdstevel@tonic-gate	ldn	[%i2 + (15*GREGSIZE)], %i5
18547c478bdstevel@tonic-gate	!
18557c478bdstevel@tonic-gate	! put registers into the save area at the top of the interrupted
18567c478bdstevel@tonic-gate	! thread's stack, pointed to by %l7 in the save area just loaded.
18577c478bdstevel@tonic-gate	!
18587c478bdstevel@tonic-gate	ldn	[%i1 + T_SP], %i3	! get stack save area pointer
18597c478bdstevel@tonic-gate	stn	%l0, [%i3 + STACK_BIAS + (0*GREGSIZE)]	! save locals
18607c478bdstevel@tonic-gate	stn	%l1, [%i3 + STACK_BIAS + (1*GREGSIZE)]
18617c478bdstevel@tonic-gate	stn	%l2, [%i3 + STACK_BIAS + (2*GREGSIZE)]
18627c478bdstevel@tonic-gate	stn	%l3, [%i3 + STACK_BIAS + (3*GREGSIZE)]
18637c478bdstevel@tonic-gate	stn	%l4, [%i3 + STACK_BIAS + (4*GREGSIZE)]
18647c478bdstevel@tonic-gate	stn	%l5, [%i3 + STACK_BIAS + (5*GREGSIZE)]
18657c478bdstevel@tonic-gate	stn	%l6, [%i3 + STACK_BIAS + (6*GREGSIZE)]
18667c478bdstevel@tonic-gate	stn	%l7, [%i3 + STACK_BIAS + (7*GREGSIZE)]
18677c478bdstevel@tonic-gate	stn	%o0, [%i3 + STACK_BIAS + (8*GREGSIZE)]	! save ins using outs
18687c478bdstevel@tonic-gate	stn	%o1, [%i3 + STACK_BIAS + (9*GREGSIZE)]
18697c478bdstevel@tonic-gate	stn	%o2, [%i3 + STACK_BIAS + (10*GREGSIZE)]
18707c478bdstevel@tonic-gate	stn	%o3, [%i3 + STACK_BIAS + (11*GREGSIZE)]
18717c478bdstevel@tonic-gate	stn	%o4, [%i3 + STACK_BIAS + (12*GREGSIZE)]
18727c478bdstevel@tonic-gate	stn	%o5, [%i3 + STACK_BIAS + (13*GREGSIZE)]
18737c478bdstevel@tonic-gate	stn	%i4, [%i3 + STACK_BIAS + (14*GREGSIZE)]
18747c478bdstevel@tonic-gate						! fp, %i7 copied using %i4
18757c478bdstevel@tonic-gate	stn	%i5, [%i3 + STACK_BIAS + (15*GREGSIZE)]
18767c478bdstevel@tonic-gate	stn	%g0, [%i2 + ((8+6)*GREGSIZE)]
18777c478bdstevel@tonic-gate						! clear fp in save area
18787c478bdstevel@tonic-gate
18797c478bdstevel@tonic-gate	! load saved pil for return
18807c478bdstevel@tonic-gate	ldub	[%i0 + T_PIL], %i0
18817c478bdstevel@tonic-gate	ret
18827c478bdstevel@tonic-gate	restore
18837c478bdstevel@tonic-gate	SET_SIZE(intr_passivate)
18847c478bdstevel@tonic-gate
18857c478bdstevel@tonic-gate/*
18867c478bdstevel@tonic-gate * intr_get_time() is a resource for interrupt handlers to determine how
18877c478bdstevel@tonic-gate * much time has been spent handling the current interrupt. Such a function
18887c478bdstevel@tonic-gate * is needed because higher level interrupts can arrive during the
18897c478bdstevel@tonic-gate * processing of an interrupt, thus making direct comparisons of %tick by
18907c478bdstevel@tonic-gate * the handler inaccurate. intr_get_time() only returns time spent in the
18917c478bdstevel@tonic-gate * current interrupt handler.
18927c478bdstevel@tonic-gate *
18937c478bdstevel@tonic-gate * The caller must be calling from an interrupt handler running at a pil
18947c478bdstevel@tonic-gate * below or at lock level. Timings are not provided for high-level
18957c478bdstevel@tonic-gate * interrupts.
18967c478bdstevel@tonic-gate *
18977c478bdstevel@tonic-gate * The first time intr_get_time() is called while handling an interrupt,
18987c478bdstevel@tonic-gate * it returns the time since the interrupt handler was invoked. Subsequent
18997c478bdstevel@tonic-gate * calls will return the time since the prior call to intr_get_time(). Time
19009b0bb79John Levon * is returned as ticks, adjusted for any clock divisor due to power
19019b0bb79John Levon * management. Use tick2ns() to convert ticks to nsec. Warning: ticks may
19027c478bdstevel@tonic-gate * not be the same across CPUs.
19037c478bdstevel@tonic-gate *
19047c478bdstevel@tonic-gate * Theory Of Intrstat[][]:
19057c478bdstevel@tonic-gate *
19067c478bdstevel@tonic-gate * uint64_t intrstat[pil][0..1] is an array indexed by pil level, with two
19077c478bdstevel@tonic-gate * uint64_ts per pil.
19087c478bdstevel@tonic-gate *
19097c478bdstevel@tonic-gate * intrstat[pil][0] is a cumulative count of the number of ticks spent
19107c478bdstevel@tonic-gate * handling all interrupts at the specified pil on this CPU. It is
19117c478bdstevel@tonic-gate * exported via kstats to the user.
19127c478bdstevel@tonic-gate *
19137c478bdstevel@tonic-gate * intrstat[pil][1] is always a count of ticks less than or equal to the
19147c478bdstevel@tonic-gate * value in [0]. The difference between [1] and [0] is the value returned
19157c478bdstevel@tonic-gate * by a call to intr_get_time(). At the start of interrupt processing,
19167c478bdstevel@tonic-gate * [0] and [1] will be equal (or nearly so). As the interrupt consumes
19177c478bdstevel@tonic-gate * time, [0] will increase, but [1] will remain the same. A call to
19187c478bdstevel@tonic-gate * intr_get_time() will return the difference, then update [1] to be the
19197c478bdstevel@tonic-gate * same as [0]. Future calls will return the time since the last call.
19207c478bdstevel@tonic-gate * Finally, when the interrupt completes, [1] is updated to the same as [0].
19217c478bdstevel@tonic-gate *
19227c478bdstevel@tonic-gate * Implementation:
19237c478bdstevel@tonic-gate *
19247c478bdstevel@tonic-gate * intr_get_time() works much like a higher level interrupt arriving. It
19257c478bdstevel@tonic-gate * "checkpoints" the timing information by incrementing intrstat[pil][0]
19267c478bdstevel@tonic-gate * to include elapsed running time, and by setting t_intr_start to %tick.
19277c478bdstevel@tonic-gate * It then sets the return value to intrstat[pil][0] - intrstat[pil][1],
19287c478bdstevel@tonic-gate * and updates intrstat[pil][1] to be the same as the new value of
19297c478bdstevel@tonic-gate * intrstat[pil][0].
19307c478bdstevel@tonic-gate *
19317c478bdstevel@tonic-gate * In the normal handling of interrupts, after an interrupt handler returns
19327c478bdstevel@tonic-gate * and the code in intr_thread() updates intrstat[pil][0], it then sets
19337c478bdstevel@tonic-gate * intrstat[pil][1] to the new value of intrstat[pil][0]. When [0] == [1],
19347c478bdstevel@tonic-gate * the timings are reset, i.e. intr_get_time() will return [0] - [1] which
19357c478bdstevel@tonic-gate * is 0.
19367c478bdstevel@tonic-gate *
19377c478bdstevel@tonic-gate * Whenever interrupts arrive on a CPU which is handling a lower pil
19387c478bdstevel@tonic-gate * interrupt, they update the lower pil's [0] to show time spent in the
19397c478bdstevel@tonic-gate * handler that they've interrupted. This results in a growing discrepancy
19407c478bdstevel@tonic-gate * between [0] and [1], which is returned the next time intr_get_time() is
19417c478bdstevel@tonic-gate * called. Time spent in the higher-pil interrupt will not be returned in
19427c478bdstevel@tonic-gate * the next intr_get_time() call from the original interrupt, because
19437c478bdstevel@tonic-gate * the higher-pil interrupt's time is accumulated in intrstat[higherpil][].
19447c478bdstevel@tonic-gate */
19457c478bdstevel@tonic-gate	ENTRY_NP(intr_get_time)
19467c478bdstevel@tonic-gate#ifdef DEBUG
19477c478bdstevel@tonic-gate	!
19487c478bdstevel@tonic-gate	! Lots of asserts, but just check panic_quiesce first.
19497c478bdstevel@tonic-gate	! Don't bother with lots of tests if we're just ignoring them.
19507c478bdstevel@tonic-gate	!
19517c478bdstevel@tonic-gate	sethi	%hi(panic_quiesce), %o0
19527c478bdstevel@tonic-gate	ld	[%o0 + %lo(panic_quiesce)], %o0
19537c478bdstevel@tonic-gate	brnz,pn	%o0, 2f
19547c478bdstevel@tonic-gate	nop
19557c478bdstevel@tonic-gate	!
19567c478bdstevel@tonic-gate	! ASSERT(%pil <= LOCK_LEVEL)
19577c478bdstevel@tonic-gate	!
19587c478bdstevel@tonic-gate	rdpr	%pil, %o1
19597c478bdstevel@tonic-gate	cmp	%o1, LOCK_LEVEL
19607c478bdstevel@tonic-gate	ble,pt	%xcc, 0f
19617c478bdstevel@tonic-gate	sethi	%hi(intr_get_time_high_pil), %o0	! delay
19627c478bdstevel@tonic-gate	call	panic
19637c478bdstevel@tonic-gate	or	%o0, %lo(intr_get_time_high_pil), %o0
19647c478bdstevel@tonic-gate0:
19657c478bdstevel@tonic-gate	!
19667c478bdstevel@tonic-gate	! ASSERT((t_flags & T_INTR_THREAD) != 0 && t_pil > 0)
19677c478bdstevel@tonic-gate	!
19687c478bdstevel@tonic-gate	lduh	[THREAD_REG + T_FLAGS], %o2
19697c478bdstevel@tonic-gate	andcc	%o2, T_INTR_THREAD, %g0
19707c478bdstevel@tonic-gate	bz,pn	%xcc, 1f
19717c478bdstevel@tonic-gate	ldub	[THREAD_REG + T_PIL], %o1		! delay
19727c478bdstevel@tonic-gate	brnz,pt	%o1, 0f
19737c478bdstevel@tonic-gate1:
19747c478bdstevel@tonic-gate	sethi	%hi(intr_get_time_not_intr), %o0
19757c478bdstevel@tonic-gate	call	panic
19767c478bdstevel@tonic-gate	or	%o0, %lo(intr_get_time_not_intr), %o0
19777c478bdstevel@tonic-gate0:
19787c478bdstevel@tonic-gate	!
19797c478bdstevel@tonic-gate	! ASSERT(t_intr_start != 0)
19807c478bdstevel@tonic-gate	!
19817c478bdstevel@tonic-gate	ldx	[THREAD_REG + T_INTR_START], %o1
19827c478bdstevel@tonic-gate	brnz,pt	%o1, 2f
19837c478bdstevel@tonic-gate	sethi	%hi(intr_get_time_no_start_time), %o0	! delay
19847c478bdstevel@tonic-gate	call	panic
19857c478bdstevel@tonic-gate	or	%o0, %lo(intr_get_time_no_start_time), %o0
19867c478bdstevel@tonic-gate2:
19877c478bdstevel@tonic-gate#endif /* DEBUG */
19887c478bdstevel@tonic-gate	!
19897c478bdstevel@tonic-gate	! %o0 = elapsed time and return value
19907c478bdstevel@tonic-gate	! %o1 = pil
19917c478bdstevel@tonic-gate	! %o2 = scratch
19927c478bdstevel@tonic-gate	! %o3 = scratch
19937c478bdstevel@tonic-gate	! %o4 = scratch
19947c478bdstevel@tonic-gate	! %o5 = cpu
19957c478bdstevel@tonic-gate	!
19967c478bdstevel@tonic-gate	wrpr	%g0, PIL_MAX, %pil	! make this easy -- block normal intrs
19977c478bdstevel@tonic-gate	ldn	[THREAD_REG + T_CPU], %o5
19987c478bdstevel@tonic-gate	ldub	[THREAD_REG + T_PIL], %o1
19997c478bdstevel@tonic-gate	ldx	[THREAD_REG + T_INTR_START], %o3 ! %o3 = t_intr_start
20007c478bdstevel@tonic-gate	!
20017c478bdstevel@tonic-gate	! Calculate elapsed time since t_intr_start. Update t_intr_start,
20027c478bdstevel@tonic-gate	! get delta, and multiply by cpu_divisor if necessary.
20037c478bdstevel@tonic-gate	!
2004bd28a47Prashanth Sreenivasa	RD_CLOCK_TICK_NO_SUSPEND_CHECK(%o2, %o0)
20057c478bdstevel@tonic-gate	stx	%o2, [THREAD_REG + T_INTR_START]
20067c478bdstevel@tonic-gate	sub	%o2, %o3, %o0
20077c478bdstevel@tonic-gate
20087c478bdstevel@tonic-gate	lduh	[%o5 + CPU_DIVISOR], %o4
20097c478bdstevel@tonic-gate	cmp	%o4, 1
20107c478bdstevel@tonic-gate	bg,a,pn	%xcc, 1f
20117c478bdstevel@tonic-gate	mulx	%o0, %o4, %o0	! multiply interval by clock divisor iff > 1
20127c478bdstevel@tonic-gate1:
2013eda8946esolom	! Update intracct[]
2014eda8946esolom	lduh	[%o5 + CPU_MSTATE], %o4
2015eda8946esolom	sllx	%o4, 3, %o4
2016eda8946esolom	add	%o4, CPU_INTRACCT, %o4
2017eda8946esolom	ldx	[%o5 + %o4], %o2
2018eda8946esolom	add	%o2, %o0, %o2
2019eda8946esolom	stx	%o2, [%o5 + %o4]
2020eda8946esolom
20217c478bdstevel@tonic-gate	!
20227c478bdstevel@tonic-gate	! Increment cpu_m.intrstat[pil][0]. Calculate elapsed time since
20237c478bdstevel@tonic-gate	! cpu_m.intrstat[pil][1], which is either when the interrupt was
20247c478bdstevel@tonic-gate	! first entered, or the last time intr_get_time() was invoked. Then
20257c478bdstevel@tonic-gate	! update cpu_m.intrstat[pil][1] to match [0].
20267c478bdstevel@tonic-gate	!
20277c478bdstevel@tonic-gate	sllx	%o1, 4, %o3
20287c478bdstevel@tonic-gate	add	%o3, CPU_MCPU, %o3
20297c478bdstevel@tonic-gate	add	%o3, MCPU_INTRSTAT, %o3
20307c478bdstevel@tonic-gate	add	%o3, %o5, %o3		! %o3 = cpu_m.intrstat[pil][0]
20317c478bdstevel@tonic-gate	ldx	[%o3], %o2
20327c478bdstevel@tonic-gate	add	%o2, %o0, %o2		! %o2 = new value for intrstat
20337c478bdstevel@tonic-gate	stx	%o2, [%o3]
20347c478bdstevel@tonic-gate	ldx	[%o3 + 8], %o4		! %o4 = cpu_m.intrstat[pil][1]
20357c478bdstevel@tonic-gate	sub	%o2, %o4, %o0		! %o0 is elapsed time since %o4
20367c478bdstevel@tonic-gate	stx	%o2, [%o3 + 8]		! make [1] match [0], resetting time
20377c478bdstevel@tonic-gate
20387c478bdstevel@tonic-gate	ld	[%o5 + CPU_BASE_SPL], %o2	! restore %pil to the greater
20397c478bdstevel@tonic-gate	cmp	%o2, %