17c478bdstevel@tonic-gate/*
27c478bdstevel@tonic-gate * CDDL HEADER START
37c478bdstevel@tonic-gate *
47c478bdstevel@tonic-gate * The contents of this file are subject to the terms of the
525cf1a3jl * Common Development and Distribution License (the "License").
625cf1a3jl * You may not use this file except in compliance with the License.
77c478bdstevel@tonic-gate *
87c478bdstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bdstevel@tonic-gate * or http://www.opensolaris.org/os/licensing.
107c478bdstevel@tonic-gate * See the License for the specific language governing permissions
117c478bdstevel@tonic-gate * and limitations under the License.
127c478bdstevel@tonic-gate *
137c478bdstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each
147c478bdstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bdstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the
167c478bdstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying
177c478bdstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bdstevel@tonic-gate *
197c478bdstevel@tonic-gate * CDDL HEADER END
207c478bdstevel@tonic-gate */
217c478bdstevel@tonic-gate/*
22bd28a47Prashanth Sreenivasa * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
237c478bdstevel@tonic-gate */
247c478bdstevel@tonic-gate
257c478bdstevel@tonic-gate#if defined(lint)
267c478bdstevel@tonic-gate#include <sys/types.h>
277c478bdstevel@tonic-gate#include <sys/thread.h>
287c478bdstevel@tonic-gate#else	/* lint */
297c478bdstevel@tonic-gate#include "assym.h"
307c478bdstevel@tonic-gate#endif	/* lint */
317c478bdstevel@tonic-gate
327c478bdstevel@tonic-gate#include <sys/cmn_err.h>
337c478bdstevel@tonic-gate#include <sys/ftrace.h>
347c478bdstevel@tonic-gate#include <sys/asm_linkage.h>
357c478bdstevel@tonic-gate#include <sys/machthread.h>
367c478bdstevel@tonic-gate#include <sys/machcpuvar.h>
377c478bdstevel@tonic-gate#include <sys/intreg.h>
38b0fc0e7govinda#include <sys/ivintr.h>
397c478bdstevel@tonic-gate
407c478bdstevel@tonic-gate#ifdef TRAPTRACE
417c478bdstevel@tonic-gate#include <sys/traptrace.h>
427c478bdstevel@tonic-gate#endif /* TRAPTRACE */
437c478bdstevel@tonic-gate
447c478bdstevel@tonic-gate#if defined(lint)
457c478bdstevel@tonic-gate
467c478bdstevel@tonic-gate/* ARGSUSED */
477c478bdstevel@tonic-gatevoid
487c478bdstevel@tonic-gatepil_interrupt(int level)
497c478bdstevel@tonic-gate{}
507c478bdstevel@tonic-gate
517c478bdstevel@tonic-gate#else	/* lint */
527c478bdstevel@tonic-gate
537c478bdstevel@tonic-gate
547c478bdstevel@tonic-gate/*
557c478bdstevel@tonic-gate * (TT 0x40..0x4F, TL>0) Interrupt Level N Handler (N == 1..15)
567c478bdstevel@tonic-gate * 	Register passed from LEVEL_INTERRUPT(level)
577c478bdstevel@tonic-gate *	%g4 - interrupt request level
587c478bdstevel@tonic-gate */
597c478bdstevel@tonic-gate	ENTRY_NP(pil_interrupt)
607c478bdstevel@tonic-gate	!
617c478bdstevel@tonic-gate	! Register usage
627c478bdstevel@tonic-gate	!	%g1 - cpu
63b0fc0e7govinda	!	%g2 - pointer to intr_vec_t (iv)
647c478bdstevel@tonic-gate	!	%g4 - pil
65b0fc0e7govinda	!	%g3, %g5, %g6, %g7 - temps
667c478bdstevel@tonic-gate	!
67b0fc0e7govinda	! Grab the first or list head intr_vec_t off the intr_head[pil]
68b0fc0e7govinda	! and panic immediately if list head is NULL. Otherwise, update
69b0fc0e7govinda	! intr_head[pil] to next intr_vec_t on the list and clear softint
70b0fc0e7govinda	! %clear_softint, if next intr_vec_t is NULL.
717c478bdstevel@tonic-gate	!
72b0fc0e7govinda	CPU_ADDR(%g1, %g5)		! %g1 = cpu
737c478bdstevel@tonic-gate	!
747c478bdstevel@tonic-gate	ALTENTRY(pil_interrupt_common)
75b0fc0e7govinda	sll	%g4, CPTRSHIFT, %g5	! %g5 = offset to the pil entry
76b0fc0e7govinda	add	%g1, INTR_HEAD, %g6	! %g6 = &cpu->m_cpu.intr_head
77b0fc0e7govinda	add	%g6, %g5, %g6		! %g6 = &cpu->m_cpu.intr_head[pil]
78b0fc0e7govinda	ldn	[%g6], %g2		! %g2 = cpu->m_cpu.intr_head[pil]
79b0fc0e7govinda	brnz,pt	%g2, 0f			! check list head (iv) is NULL
807c478bdstevel@tonic-gate	nop
81b0fc0e7govinda	ba	ptl1_panic		! panic, list head (iv) is NULL
82b0fc0e7govinda	mov	PTL1_BAD_INTR_VEC, %g1
83b0fc0e7govinda0:
84b0fc0e7govinda	lduh	[%g2 + IV_FLAGS], %g7	! %g7 = iv->iv_flags
85b0fc0e7govinda	and	%g7, IV_SOFTINT_MT, %g3 ! %g3 = iv->iv_flags & IV_SOFTINT_MT
86b0fc0e7govinda	brz,pt	%g3, 1f			! check for multi target softint
87b0fc0e7govinda	add	%g2, IV_PIL_NEXT, %g7	! g7% = &iv->iv_pil_next
88b0fc0e7govinda	ld	[%g1 + CPU_ID], %g3	! for multi target softint, use cpuid
89b0fc0e7govinda	sll	%g3, CPTRSHIFT, %g3	! convert cpuid to offset address
90b0fc0e7govinda	add	%g7, %g3, %g7		! %g5 = &iv->iv_xpil_next[cpuid]
917c478bdstevel@tonic-gate1:
92b0fc0e7govinda	ldn	[%g7], %g3		! %g3 = next intr_vec_t
93b0fc0e7govinda	brnz,pn	%g3, 2f			! branch if next intr_vec_t non NULL
94b0fc0e7govinda	stn	%g3, [%g6]		! update cpu->m_cpu.intr_head[pil]
95b0fc0e7govinda	add	%g1, INTR_TAIL, %g6	! %g6 =  &cpu->m_cpu.intr_tail
96b0fc0e7govinda	stn	%g0, [%g5 + %g6]	! clear cpu->m_cpu.intr_tail[pil]
97b0fc0e7govinda	mov	1, %g5			! %g5 = 1
98b0fc0e7govinda	sll	%g5, %g4, %g5		! %g5 = 1 << pil
99b0fc0e7govinda	wr	%g5, CLEAR_SOFTINT	! clear interrupt on this pil
100b0fc0e7govinda2:
1017c478bdstevel@tonic-gate#ifdef TRAPTRACE
1027c478bdstevel@tonic-gate	TRACE_PTR(%g5, %g6)
1037c478bdstevel@tonic-gate	TRACE_SAVE_TL_GL_REGS(%g5, %g6)
1047c478bdstevel@tonic-gate	rdpr	%tt, %g6
105b0fc0e7govinda	stha	%g6, [%g5 + TRAP_ENT_TT]%asi	! trap_type = %tt
1067c478bdstevel@tonic-gate	rdpr	%tpc, %g6
107b0fc0e7govinda	stna	%g6, [%g5 + TRAP_ENT_TPC]%asi	! trap_pc = %tpc
1087c478bdstevel@tonic-gate	rdpr	%tstate, %g6
109b0fc0e7govinda	stxa	%g6, [%g5 + TRAP_ENT_TSTATE]%asi ! trap_tstate = %tstate
110b0fc0e7govinda	stna	%sp, [%g5 + TRAP_ENT_SP]%asi	! trap_sp = %sp
111b0fc0e7govinda	stna	%g2, [%g5 + TRAP_ENT_TR]%asi	! trap_tr = first intr_vec
112b0fc0e7govinda	stna	%g3, [%g5 + TRAP_ENT_F1]%asi	! trap_f1 = next intr_vec
113023e71dHaik Aftandilian	GET_TRACE_TICK(%g6, %g3)
114023e71dHaik Aftandilian	stxa	%g6, [%g5 + TRAP_ENT_TICK]%asi	! trap_tick = %tick
1157c478bdstevel@tonic-gate	sll	%g4, CPTRSHIFT, %g3
1167c478bdstevel@tonic-gate	add	%g1, INTR_HEAD, %g6
117b0fc0e7govinda	ldn	[%g6 + %g3], %g6		! %g6=cpu->m_cpu.intr_head[pil]
118b0fc0e7govinda	stna	%g6, [%g5 + TRAP_ENT_F2]%asi	! trap_f2 = intr_head[pil]
1197c478bdstevel@tonic-gate	add	%g1, INTR_TAIL, %g6
120b0fc0e7govinda	ldn	[%g6 + %g3], %g6		! %g6=cpu->m_cpu.intr_tail[pil]
121b0fc0e7govinda	stna	%g6, [%g5 + TRAP_ENT_F3]%asi	! trap_f3 = intr_tail[pil]
122b0fc0e7govinda	stna	%g4, [%g5 + TRAP_ENT_F4]%asi	! trap_f4 = pil
1237c478bdstevel@tonic-gate	TRACE_NEXT(%g5, %g6, %g3)
1247c478bdstevel@tonic-gate#endif /* TRAPTRACE */
1257c478bdstevel@tonic-gate	!
126b0fc0e7govinda	! clear the iv_pending flag for this interrupt request
1277c478bdstevel@tonic-gate	!
128b0fc0e7govinda	lduh	[%g2 + IV_FLAGS], %g3		! %g3 = iv->iv_flags
129b0fc0e7govinda	andn	%g3, IV_SOFTINT_PEND, %g3	! %g3 = !(iv->iv_flags & PEND)
130b0fc0e7govinda	sth	%g3, [%g2 + IV_FLAGS]		! clear IV_SOFTINT_PEND flag
131b0fc0e7govinda	stn	%g0, [%g7]			! clear iv->iv_pil_next or
132b0fc0e7govinda						!       iv->iv_pil_xnext
1337c478bdstevel@tonic-gate
1347c478bdstevel@tonic-gate	!
1357c478bdstevel@tonic-gate	! Prepare for sys_trap()
1367c478bdstevel@tonic-gate	!
1377c478bdstevel@tonic-gate	! Registers passed to sys_trap()
1387c478bdstevel@tonic-gate	!	%g1 - interrupt handler at TL==0
139b0fc0e7govinda	!	%g2 - pointer to current intr_vec_t (iv),
140b0fc0e7govinda	!	      job queue for intr_thread or current_thread
1417c478bdstevel@tonic-gate	!	%g3 - pil
1427c478bdstevel@tonic-gate	!	%g4 - initial pil for handler
1437c478bdstevel@tonic-gate	!
1447c478bdstevel@tonic-gate	! figure which handler to run and which %pil it starts at
1457c478bdstevel@tonic-gate	! intr_thread starts at DISP_LEVEL to prevent preemption
1467c478bdstevel@tonic-gate	! current_thread starts at PIL_MAX to protect cpu_intr_actv
1477c478bdstevel@tonic-gate	!
148b0fc0e7govinda	mov	%g4, %g3		! %g3 = %g4, pil
1497c478bdstevel@tonic-gate	cmp	%g4, LOCK_LEVEL
150b0fc0e7govinda	bg,a,pt	%xcc, 3f		! branch if pil > LOCK_LEVEL
151b0fc0e7govinda	mov	PIL_MAX, %g4		! %g4 = PIL_MAX (15)
152b0fc0e7govinda	sethi	%hi(intr_thread), %g1	! %g1 = intr_thread
153b0fc0e7govinda	mov	DISP_LEVEL, %g4		! %g4 = DISP_LEVEL (11)
1547c478bdstevel@tonic-gate	ba,pt	%xcc, sys_trap
1557c478bdstevel@tonic-gate	or	%g1, %lo(intr_thread), %g1
156b0fc0e7govinda3:
157b0fc0e7govinda	sethi	%hi(current_thread), %g1 ! %g1 = current_thread
1587c478bdstevel@tonic-gate	ba,pt	%xcc, sys_trap
1597c478bdstevel@tonic-gate	or	%g1, %lo(current_thread), %g1
1607c478bdstevel@tonic-gate	SET_SIZE(pil_interrupt_common)
1617c478bdstevel@tonic-gate	SET_SIZE(pil_interrupt)
1627c478bdstevel@tonic-gate
1637c478bdstevel@tonic-gate#endif	/* lint */
1647c478bdstevel@tonic-gate
1657c478bdstevel@tonic-gate
1667c478bdstevel@tonic-gate#ifndef	lint
1677c478bdstevel@tonic-gate_spurious:
1687c478bdstevel@tonic-gate	.asciz	"!interrupt 0x%x at level %d not serviced"
169b0fc0e7govinda
1707c478bdstevel@tonic-gate/*
1717c478bdstevel@tonic-gate * SERVE_INTR_PRE is called once, just before the first invocation
1727c478bdstevel@tonic-gate * of SERVE_INTR.
1737c478bdstevel@tonic-gate *
1747c478bdstevel@tonic-gate * Registers on entry:
1757c478bdstevel@tonic-gate *
176b0fc0e7govinda * iv_p, cpu, regs: may be out-registers
1777c478bdstevel@tonic-gate * ls1, ls2: local scratch registers
1787c478bdstevel@tonic-gate * os1, os2, os3: scratch registers, may be out
1797c478bdstevel@tonic-gate */
1807c478bdstevel@tonic-gate
181b0fc0e7govinda#define SERVE_INTR_PRE(iv_p, cpu, ls1, ls2, os1, os2, os3, regs)	\
182b0fc0e7govinda	mov	iv_p, ls1;						\
183b0fc0e7govinda	mov	iv_p, ls2;						\
184b0fc0e7govinda	SERVE_INTR_TRACE(iv_p, os1, os2, os3, regs);
1857c478bdstevel@tonic-gate
1867c478bdstevel@tonic-gate/*
1877c478bdstevel@tonic-gate * SERVE_INTR is called immediately after either SERVE_INTR_PRE or
1887c478bdstevel@tonic-gate * SERVE_INTR_NEXT, without intervening code. No register values
1897c478bdstevel@tonic-gate * may be modified.
1907c478bdstevel@tonic-gate *
1917c478bdstevel@tonic-gate * After calling SERVE_INTR, the caller must check if os3 is set. If
1927c478bdstevel@tonic-gate * so, there is another interrupt to process. The caller must call
1937c478bdstevel@tonic-gate * SERVE_INTR_NEXT, immediately followed by SERVE_INTR.
1947c478bdstevel@tonic-gate *
1957c478bdstevel@tonic-gate * Before calling SERVE_INTR_NEXT, the caller may perform accounting
1967c478bdstevel@tonic-gate * and other actions which need to occur after invocation of an interrupt
1977c478bdstevel@tonic-gate * handler. However, the values of ls1 and os3 *must* be preserved and
1987c478bdstevel@tonic-gate * passed unmodified into SERVE_INTR_NEXT.
1997c478bdstevel@tonic-gate *
2007c478bdstevel@tonic-gate * Registers on return from SERVE_INTR:
2017c478bdstevel@tonic-gate *
2027c478bdstevel@tonic-gate * ls1 - the pil just processed
203b0fc0e7govinda * ls2 - the pointer to intr_vec_t (iv) just processed
2047c478bdstevel@tonic-gate * os3 - if set, another interrupt needs to be processed
2057c478bdstevel@tonic-gate * cpu, ls1, os3 - must be preserved if os3 is set
2067c478bdstevel@tonic-gate */
2077c478bdstevel@tonic-gate
2087c478bdstevel@tonic-gate#define	SERVE_INTR(os5, cpu, ls1, ls2, os1, os2, os3, os4)		\
2097c478bdstevel@tonic-gate	ldn	[ls1 + IV_HANDLER], os2;				\
210b0fc0e7govinda	ldn	[ls1 + IV_ARG1], %o0;					\
211b0fc0e7govinda	ldn	[ls1 + IV_ARG2], %o1;					\
2127c478bdstevel@tonic-gate	call	os2;							\
2137c478bdstevel@tonic-gate	lduh	[ls1 + IV_PIL], ls1;					\
2147c478bdstevel@tonic-gate	brnz,pt	%o0, 2f;						\
2157c478bdstevel@tonic-gate	mov	CE_WARN, %o0;						\
2167c478bdstevel@tonic-gate	set	_spurious, %o1;						\
2177c478bdstevel@tonic-gate	mov	ls2, %o2;						\
2187c478bdstevel@tonic-gate	call	cmn_err;						\
2197c478bdstevel@tonic-gate	rdpr	%pil, %o3;						\
2207c478bdstevel@tonic-gate2:	ldn	[THREAD_REG + T_CPU], cpu;				\
2217c478bdstevel@tonic-gate	sll	ls1, 3, os1;						\
2227c478bdstevel@tonic-gate	add	os1, CPU_STATS_SYS_INTR - 8, os2;			\
2237c478bdstevel@tonic-gate	ldx	[cpu + os2], os3;					\
2247c478bdstevel@tonic-gate	inc	os3;							\
2257c478bdstevel@tonic-gate	stx	os3, [cpu + os2];					\
2267c478bdstevel@tonic-gate	sll	ls1, CPTRSHIFT, os2;					\
2277c478bdstevel@tonic-gate	add	cpu,  INTR_HEAD, os1;					\
2287c478bdstevel@tonic-gate	add	os1, os2, os1;						\
2297c478bdstevel@tonic-gate	ldn	[os1], os3;
2307c478bdstevel@tonic-gate
2317c478bdstevel@tonic-gate/*
2327c478bdstevel@tonic-gate * Registers on entry:
2337c478bdstevel@tonic-gate *
2347c478bdstevel@tonic-gate * cpu			- cpu pointer (clobbered, set to cpu upon completion)
2357c478bdstevel@tonic-gate * ls1, os3		- preserved from prior call to SERVE_INTR
2367c478bdstevel@tonic-gate * ls2			- local scratch reg (not preserved)
2377c478bdstevel@tonic-gate * os1, os2, os4, os5	- scratch reg, can be out (not preserved)
2387c478bdstevel@tonic-gate */
2397c478bdstevel@tonic-gate#define SERVE_INTR_NEXT(os5, cpu, ls1, ls2, os1, os2, os3, os4)		\
2407c478bdstevel@tonic-gate	sll	ls1, CPTRSHIFT, os4;					\
2417c478bdstevel@tonic-gate	add	cpu, INTR_HEAD, os1;					\
2427c478bdstevel@tonic-gate	rdpr	%pstate, ls2;						\
2437c478bdstevel@tonic-gate	wrpr	ls2, PSTATE_IE, %pstate;				\
244b0fc0e7govinda	lduh	[os3 + IV_FLAGS], os2;					\
245b0fc0e7govinda	and	os2, IV_SOFTINT_MT, os2;				\
246b0fc0e7govinda	brz,pt	os2, 4f;						\
247b0fc0e7govinda	add	os3, IV_PIL_NEXT, os2;					\
248b0fc0e7govinda	ld	[cpu + CPU_ID], os5;					\
249b0fc0e7govinda	sll	os5, CPTRSHIFT, os5;					\
250b0fc0e7govinda	add	os2, os5, os2;						\
251b0fc0e7govinda4:	ldn	[os2], os5;						\
252b0fc0e7govinda	brnz,pn	os5, 5f;						\
253b0fc0e7govinda	stn	os5, [os1 + os4];					\
2547c478bdstevel@tonic-gate	add	cpu, INTR_TAIL, os1;					\
2557c478bdstevel@tonic-gate	stn	%g0, [os1 + os4];					\
2567c478bdstevel@tonic-gate	mov	1, os1;							\
2577c478bdstevel@tonic-gate	sll	os1, ls1, os1;						\
2587c478bdstevel@tonic-gate	wr	os1, CLEAR_SOFTINT;					\
259b0fc0e7govinda5:	lduh	[os3 + IV_FLAGS], ls1;                                  \
260b0fc0e7govinda	andn	ls1, IV_SOFTINT_PEND, ls1;				\
261b0fc0e7govinda	sth	ls1, [os3 + IV_FLAGS];				        \
262b0fc0e7govinda	stn	%g0, [os2];						\
2637c478bdstevel@tonic-gate	wrpr	%g0, ls2, %pstate;					\
264b0fc0e7govinda	mov	os3, ls1;						\
265b0fc0e7govinda	mov	os3, ls2;						\
266b0fc0e7govinda	SERVE_INTR_TRACE2(os5, os1, os2, os3, os4);
2677c478bdstevel@tonic-gate
2687c478bdstevel@tonic-gate#ifdef TRAPTRACE
2697c478bdstevel@tonic-gate/*
2707c478bdstevel@tonic-gate * inum - not modified, _spurious depends on it.
2717c478bdstevel@tonic-gate */
2727c478bdstevel@tonic-gate#define	SERVE_INTR_TRACE(inum, os1, os2, os3, os4)			\
2737c478bdstevel@tonic-gate	rdpr	%pstate, os3;						\
2747c478bdstevel@tonic-gate	andn	os3, PSTATE_IE | PSTATE_AM, os2;			\
2757c478bdstevel@tonic-gate	wrpr	%g0, os2, %pstate;					\
276023e71dHaik Aftandilian	TRACE_PTR(os1, os2); 						\
2777c478bdstevel@tonic-gate	ldn	[os4 + PC_OFF], os2;					\
2787c478bdstevel@tonic-gate	stna	os2, [os1 + TRAP_ENT_TPC]%asi;				\
2797c478bdstevel@tonic-gate	ldx	[os4 + TSTATE_OFF], os2;				\
2807c478bdstevel@tonic-gate	stxa	os2, [os1 + TRAP_ENT_TSTATE]%asi;			\
2817c478bdstevel@tonic-gate	mov	os3, os4;						\
282023e71dHaik Aftandilian	GET_TRACE_TICK(os2, os3);					\
2837c478bdstevel@tonic-gate	stxa	os2, [os1 + TRAP_ENT_TICK]%asi;				\
2847c478bdstevel@tonic-gate	TRACE_SAVE_TL_GL_REGS(os1, os2);				\
2857c478bdstevel@tonic-gate	set	TT_SERVE_INTR, os2;					\
2867c478bdstevel@tonic-gate	rdpr	%pil, os3;						\
2877c478bdstevel@tonic-gate	or	os2, os3, os2;						\
2887c478bdstevel@tonic-gate	stha	os2, [os1 + TRAP_ENT_TT]%asi;				\
2897c478bdstevel@tonic-gate	stna	%sp, [os1 + TRAP_ENT_SP]%asi;				\
2907c478bdstevel@tonic-gate	stna	inum, [os1 + TRAP_ENT_TR]%asi;				\
2917c478bdstevel@tonic-gate	stna	%g0, [os1 + TRAP_ENT_F1]%asi;				\
2927c478bdstevel@tonic-gate	stna	%g0, [os1 + TRAP_ENT_F2]%asi;				\
2937c478bdstevel@tonic-gate	stna	%g0, [os1 + TRAP_ENT_F3]%asi;				\
2947c478bdstevel@tonic-gate	stna	%g0, [os1 + TRAP_ENT_F4]%asi;				\
2957c478bdstevel@tonic-gate	TRACE_NEXT(os1, os2, os3);					\
2967c478bdstevel@tonic-gate	wrpr	%g0, os4, %pstate
2977c478bdstevel@tonic-gate#else	/* TRAPTRACE */
2987c478bdstevel@tonic-gate#define SERVE_INTR_TRACE(inum, os1, os2, os3, os4)
2997c478bdstevel@tonic-gate#endif	/* TRAPTRACE */
3007c478bdstevel@tonic-gate
3017c478bdstevel@tonic-gate#ifdef TRAPTRACE
3027c478bdstevel@tonic-gate/*
3037c478bdstevel@tonic-gate * inum - not modified, _spurious depends on it.
3047c478bdstevel@tonic-gate */
3057c478bdstevel@tonic-gate#define	SERVE_INTR_TRACE2(inum, os1, os2, os3, os4)			\
3067c478bdstevel@tonic-gate	rdpr	%pstate, os3;						\
3077c478bdstevel@tonic-gate	andn	os3, PSTATE_IE | PSTATE_AM, os2;			\
3087c478bdstevel@tonic-gate	wrpr	%g0, os2, %pstate;					\
309023e71dHaik Aftandilian	TRACE_PTR(os1, os2); 						\
3107c478bdstevel@tonic-gate	stna	%g0, [os1 + TRAP_ENT_TPC]%asi;				\
3117c478bdstevel@tonic-gate	stxa	%g0, [os1 + TRAP_ENT_TSTATE]%asi;			\
3127c478bdstevel@tonic-gate	mov	os3, os4;						\
313023e71dHaik Aftandilian	GET_TRACE_TICK(os2, os3);					\
3147c478bdstevel@tonic-gate	stxa	os2, [os1 + TRAP_ENT_TICK]%asi;				\
3157c478bdstevel@tonic-gate	TRACE_SAVE_TL_GL_REGS(os1, os2);				\
3167c478bdstevel@tonic-gate	set	TT_SERVE_INTR, os2;					\
3177c478bdstevel@tonic-gate	rdpr	%pil, os3;						\
3187c478bdstevel@tonic-gate	or	os2, os3, os2;						\
3197c478bdstevel@tonic-gate	stha	os2, [os1 + TRAP_ENT_TT]%asi;				\
3207c478bdstevel@tonic-gate	stna	%sp, [os1 + TRAP_ENT_SP]%asi;				\
3217c478bdstevel@tonic-gate	stna	inum, [os1 + TRAP_ENT_TR]%asi;				\
3227c478bdstevel@tonic-gate	stna	%g0, [os1 + TRAP_ENT_F1]%asi;				\
3237c478bdstevel@tonic-gate	stna	%g0, [os1 + TRAP_ENT_F2]%asi;				\
3247c478bdstevel@tonic-gate	stna	%g0, [os1 + TRAP_ENT_F3]%asi;				\
3257c478bdstevel@tonic-gate	stna	%g0, [os1 + TRAP_ENT_F4]%asi;				\
3267c478bdstevel@tonic-gate	TRACE_NEXT(os1, os2, os3);					\
3277c478bdstevel@tonic-gate	wrpr	%g0, os4, %pstate
3287c478bdstevel@tonic-gate#else	/* TRAPTRACE */
3297c478bdstevel@tonic-gate#define SERVE_INTR_TRACE2(inum, os1, os2, os3, os4)
3307c478bdstevel@tonic-gate#endif	/* TRAPTRACE */
3317c478bdstevel@tonic-gate
3327c478bdstevel@tonic-gate#endif	/* lint */
3337c478bdstevel@tonic-gate
3347c478bdstevel@tonic-gate#if defined(lint)
3357c478bdstevel@tonic-gate
3367c478bdstevel@tonic-gate/*ARGSUSED*/
3377c478bdstevel@tonic-gatevoid
338b0fc0e7govindaintr_thread(struct regs *regs, uint64_t iv_p, uint_t pil)
3397c478bdstevel@tonic-gate{}
3407c478bdstevel@tonic-gate
3417c478bdstevel@tonic-gate#else	/* lint */
3427c478bdstevel@tonic-gate
3437c478bdstevel@tonic-gate#define	INTRCNT_LIMIT 16
3447c478bdstevel@tonic-gate
3457c478bdstevel@tonic-gate/*
3467c478bdstevel@tonic-gate * Handle an interrupt in a new thread.
3477c478bdstevel@tonic-gate *	Entry:
3487c478bdstevel@tonic-gate *		%o0       = pointer to regs structure
349b0fc0e7govinda *		%o1       = pointer to current intr_vec_t (iv) to be processed
3507c478bdstevel@tonic-gate *		%o2       = pil
3517c478bdstevel@tonic-gate *		%sp       = on current thread's kernel stack
3527c478bdstevel@tonic-gate *		%o7       = return linkage to trap code
3537c478bdstevel@tonic-gate *		%g7       = current thread
3547c478bdstevel@tonic-gate *		%pstate   = normal globals, interrupts enabled,
3557c478bdstevel@tonic-gate *		            privileged, fp disabled
3567c478bdstevel@tonic-gate *		%pil      = DISP_LEVEL
3577c478bdstevel@tonic-gate *
3587c478bdstevel@tonic-gate *	Register Usage
3597c478bdstevel@tonic-gate *		%l0       = return linkage
3607c478bdstevel@tonic-gate *		%l1       = pil
3617c478bdstevel@tonic-gate *		%l2 - %l3 = scratch
3627c478bdstevel@tonic-gate *		%l4 - %l7 = reserved for sys_trap
3637c478bdstevel@tonic-gate *		%o2       = cpu
3647c478bdstevel@tonic-gate *		%o3       = intr thread
3657c478bdstevel@tonic-gate *		%o0       = scratch
3667c478bdstevel@tonic-gate *		%o4 - %o5 = scratch
3677c478bdstevel@tonic-gate */
3687c478bdstevel@tonic-gate	ENTRY_NP(intr_thread)
3697c478bdstevel@tonic-gate	mov	%o7, %l0
3707c478bdstevel@tonic-gate	mov	%o2, %l1
3717c478bdstevel@tonic-gate	!
3727c478bdstevel@tonic-gate	! See if we are interrupting another interrupt thread.
3737c478bdstevel@tonic-gate	!
3747c478bdstevel@tonic-gate	lduh	[THREAD_REG + T_FLAGS], %o3
3757c478bdstevel@tonic-gate	andcc	%o3, T_INTR_THREAD, %g0
3767c478bdstevel@tonic-gate	bz,pt	%xcc, 1f
3777c478bdstevel@tonic-gate	ldn	[THREAD_REG + T_CPU], %o2	! delay - load CPU pointer
3787c478bdstevel@tonic-gate
3797c478bdstevel@tonic-gate	! We have interrupted an interrupt thread. Take a timestamp,
3807c478bdstevel@tonic-gate	! compute its interval, and update its cumulative counter.
3817c478bdstevel@tonic-gate	add	THREAD_REG, T_INTR_START, %o5
3827c478bdstevel@tonic-gate0:
3837c478bdstevel@tonic-gate	ldx	[%o5], %o3
3847c478bdstevel@tonic-gate	brz,pn	%o3, 1f
3857c478bdstevel@tonic-gate	! We came in on top of an interrupt thread that had no timestamp.
3867c478bdstevel@tonic-gate	! This could happen if, for instance, an interrupt thread which had
3877c478bdstevel@tonic-gate	! previously blocked is being set up to run again in resume(), but
3887c478bdstevel@tonic-gate	! resume() hasn't yet stored a timestamp for it. Or, it could be in
3897c478bdstevel@tonic-gate	! swtch() after its slice has been accounted for.
3907c478bdstevel@tonic-gate	! Only account for the time slice if the starting timestamp is non-zero.
391bd28a47Prashanth Sreenivasa	RD_CLOCK_TICK(%o4,%l2,%l3,__LINE__)
3927c478bdstevel@tonic-gate	sub	%o4, %o3, %o4			! o4 has interval
3937c478bdstevel@tonic-gate
3947c478bdstevel@tonic-gate	! A high-level interrupt in current_thread() interrupting here
3957c478bdstevel@tonic-gate	! will account for the interrupted thread's time slice, but
3967c478bdstevel@tonic-gate	! only if t_intr_start is non-zero. Since this code is going to account
3977c478bdstevel@tonic-gate	! for the time slice, we want to "atomically" load the thread's
3987c478bdstevel@tonic-gate	! starting timestamp, calculate the interval with %tick, and zero
3997c478bdstevel@tonic-gate	! its starting timestamp.
4007c478bdstevel@tonic-gate	! To do this, we do a casx on the t_intr_start field, and store 0 to it.
4017c478bdstevel@tonic-gate	! If it has changed since we loaded it above, we need to re-compute the
4027c478bdstevel@tonic-gate	! interval, since a changed t_intr_start implies current_thread placed
4037c478bdstevel@tonic-gate	! a new, later timestamp there after running a high-level interrupt,
4047c478bdstevel@tonic-gate	! and the %tick val in %o4 had become stale.
4057c478bdstevel@tonic-gate	mov	%g0, %l2
4067c478bdstevel@tonic-gate	casx	[%o5], %o3, %l2
4077c478bdstevel@tonic-gate
4087c478bdstevel@tonic-gate	! If %l2 == %o3, our casx was successful. If not, the starting timestamp
4097c478bdstevel@tonic-gate	! changed between loading it (after label 0b) and computing the
4107c478bdstevel@tonic-gate	! interval above.
4117c478bdstevel@tonic-gate	cmp	%l2, %o3
4127c478bdstevel@tonic-gate	bne,pn	%xcc, 0b
4137c478bdstevel@tonic-gate
4147c478bdstevel@tonic-gate	! Check for Energy Star mode
4157c478bdstevel@tonic-gate	lduh	[%o2 + CPU_DIVISOR], %l2	! delay -- %l2 = clock divisor
4167c478bdstevel@tonic-gate	cmp	%l2, 1
4177c478bdstevel@tonic-gate	bg,a,pn	%xcc, 2f
4187c478bdstevel@tonic-gate	mulx	%o4, %l2, %o4	! multiply interval by clock divisor iff > 1
4197c478bdstevel@tonic-gate2:
4207c478bdstevel@tonic-gate	! We now know that a valid interval for the interrupted interrupt
4217c478bdstevel@tonic-gate	! thread is in %o4. Update its cumulative counter.
4227c478bdstevel@tonic-gate	ldub	[THREAD_REG + T_PIL], %l3	! load PIL
4237c478bdstevel@tonic-gate	sllx	%l3, 4, %l3		! convert PIL index to byte offset
4247c478bdstevel@tonic-gate	add	%l3, CPU_MCPU, %l3	! CPU_INTRSTAT is too big for use
4257c478bdstevel@tonic-gate	add	%l3, MCPU_INTRSTAT, %l3	! as const, add offsets separately
4267c478bdstevel@tonic-gate	ldx	[%o2 + %l3], %o5	! old counter in o5
4277c478bdstevel@tonic-gate	add	%o5, %o4, %o5		! new counter in o5
4287c478bdstevel@tonic-gate	stx	%o5, [%o2 + %l3]	! store new counter
4297c478bdstevel@tonic-gate
430eda8946esolom	! Also update intracct[]
431eda8946esolom	lduh	[%o2 + CPU_MSTATE], %l3
432eda8946esolom	sllx	%l3, 3, %l3
433eda8946esolom	add	%l3, CPU_INTRACCT, %l3
434eda8946esolom	add	%l3, %o2, %l3
435eda8946esolom0:
436eda8946esolom	ldx	[%l3], %o5
437eda8946esolom	add	%o5, %o4, %o3
438eda8946esolom	casx	[%l3], %o5, %o3
439eda8946esolom	cmp	%o5, %o3
440eda8946esolom	bne,pn	%xcc, 0b
441eda8946esolom	nop
442eda8946esolom
4437c478bdstevel@tonic-gate1:
4447c478bdstevel@tonic-gate	!
4457c478bdstevel@tonic-gate	! Get set to run interrupt thread.
4467c478bdstevel@tonic-gate	! There should always be an interrupt thread since we allocate one
4477c478bdstevel@tonic-gate	! for each level on the CPU.
4487c478bdstevel@tonic-gate	!
4497c478bdstevel@tonic-gate	! Note that the code in kcpc_overflow_intr -relies- on the ordering
4507c478bdstevel@tonic-gate	! of events here -- in particular that t->t_lwp of the interrupt thread
4517c478bdstevel@tonic-gate	! is set to the pinned thread *before* curthread is changed.
4527c478bdstevel@tonic-gate	!
4537c478bdstevel@tonic-gate	ldn	[%o2 + CPU_INTR_THREAD], %o3	! interrupt thread pool
4547c478bdstevel@tonic-gate	ldn	[%o3 + T_LINK], %o4		! unlink thread from CPU's list
4557c478bdstevel@tonic-gate	stn	%o4, [%o2 + CPU_INTR_THREAD]
4567c478bdstevel@tonic-gate	!
4577c478bdstevel@tonic-gate	! Set bit for this level in CPU's active interrupt bitmask.
4587c478bdstevel@tonic-gate	!
4597c478bdstevel@tonic-gate	ld	[%o2 + CPU_INTR_ACTV], %o5
4607c478bdstevel@tonic-gate	mov	1, %o4
4617c478bdstevel@tonic-gate	sll	%o4, %l1, %o4
4627c478bdstevel@tonic-gate#ifdef DEBUG
4637c478bdstevel@tonic-gate	!
4647c478bdstevel@tonic-gate	! ASSERT(!(CPU->cpu_intr_actv & (1 << PIL)))
4657c478bdstevel@tonic-gate	!
4667c478bdstevel@tonic-gate	andcc	%o5, %o4, %g0
4677c478bdstevel@tonic-gate	bz,pt	%xcc, 0f
4687c478bdstevel@tonic-gate	nop
4697c478bdstevel@tonic-gate	! Do not call panic if a panic is already in progress.
4707c478bdstevel@tonic-gate	sethi	%hi(panic_quiesce), %l2
4717c478bdstevel@tonic-gate	ld	[%l2 + %lo(panic_quiesce)], %l2
4727c478bdstevel@tonic-gate	brnz,pn	%l2, 0f
4737c478bdstevel@tonic-gate	nop
4747c478bdstevel@tonic-gate	sethi	%hi(intr_thread_actv_bit_set), %o0
4757c478bdstevel@tonic-gate	call	panic
4767c478bdstevel@tonic-gate	or	%o0, %lo(intr_thread_actv_bit_set), %o0
4777c478bdstevel@tonic-gate0:
4787c478bdstevel@tonic-gate#endif /* DEBUG */
4797c478bdstevel@tonic-gate	or	%o5, %o4, %o5
4807c478bdstevel@tonic-gate	st	%o5, [%o2 + CPU_INTR_ACTV]
4817c478bdstevel@tonic-gate	!
4827c478bdstevel@tonic-gate	! Consider the new thread part of the same LWP so that
4837c478bdstevel@tonic-gate	! window overflow code can find the PCB.
4847c478bdstevel@tonic-gate	!
4857c478bdstevel@tonic-gate	ldn	[THREAD_REG + T_LWP], %o4
4867c478bdstevel@tonic-gate	stn	%o4, [%o3 + T_LWP]
4877c478bdstevel@tonic-gate	!
4887c478bdstevel@tonic-gate	! Threads on the interrupt thread free list could have state already
4897c478bdstevel@tonic-gate	! set to TS_ONPROC, but it helps in debugging if they're TS_FREE
4907c478bdstevel@tonic-gate	! Could eliminate the next two instructions with a little work.
4917c478bdstevel@tonic-gate	!
4927c478bdstevel@tonic-gate	mov	TS_ONPROC, %o4
4937c478bdstevel@tonic-gate	st	%o4, [%o3 + T_STATE]
4947c478bdstevel@tonic-gate	!
4957c478bdstevel@tonic-gate	! Push interrupted thread onto list from new thread.
4967c478bdstevel@tonic-gate	! Set the new thread as the current one.
4977c478bdstevel@tonic-gate	! Set interrupted thread's T_SP because if it is the idle thread,
4987c478bdstevel@tonic-gate	! resume may use that stack between threads.
4997c478bdstevel@tonic-gate	!
5007c478bdstevel@tonic-gate	stn	%o7, [THREAD_REG + T_PC]	! mark pc for resume
5017c478bdstevel@tonic-gate	stn	%sp, [THREAD_REG + T_SP]	! mark stack for resume
5027c478bdstevel@tonic-gate	stn	THREAD_REG, [%o3 + T_INTR]	! push old thread
5037c478bdstevel@tonic-gate	stn	%o3, [%o2 + CPU_THREAD]		! set new thread
5047c478bdstevel@tonic-gate	mov	%o3, THREAD_REG			! set global curthread register
5057c478bdstevel@tonic-gate	ldn	[%o3 + T_STACK], %o4		! interrupt stack pointer
5067c478bdstevel@tonic-gate	sub	%o4, STACK_BIAS, %sp
5077c478bdstevel@tonic-gate	!
5087c478bdstevel@tonic-gate	! Initialize thread priority level from intr_pri
5097c478bdstevel@tonic-gate	!
5107c478bdstevel@tonic-gate	sethi	%hi(intr_pri), %o4
5117c478bdstevel@tonic-gate	ldsh	[%o4 + %lo(intr_pri)], %o4	! grab base interrupt priority
5127c478bdstevel@tonic-gate	add	%l1, %o4, %o4		! convert level to dispatch priority
5137c478bdstevel@tonic-gate	sth	%o4, [THREAD_REG + T_PRI]
5147c478bdstevel@tonic-gate	stub	%l1, [THREAD_REG + T_PIL]	! save pil for intr_passivate
5157c478bdstevel@tonic-gate
5167c478bdstevel@tonic-gate	! Store starting timestamp in thread structure.
5177c478bdstevel@tonic-gate	add	THREAD_REG, T_INTR_START, %o3
5187c478bdstevel@tonic-gate1:
5197c478bdstevel@tonic-gate	ldx	[%o3], %o5
520bd28a47Prashanth Sreenivasa	RD_CLOCK_TICK(%o4,%l2,%l3,__LINE__)
5217c478bdstevel@tonic-gate	casx	[%o3], %o5, %o4
5227c478bdstevel@tonic-gate	cmp	%o4, %o5
5237c478bdstevel@tonic-gate	! If a high-level interrupt occurred while we were attempting to store
5247c478bdstevel@tonic-gate	! the timestamp, try again.
5257c478bdstevel@tonic-gate	bne,pn	%xcc, 1b
5267c478bdstevel@tonic-gate	nop
5277c478bdstevel@tonic-gate
5287c478bdstevel@tonic-gate	wrpr	%g0, %l1, %pil			! lower %pil to new level
5297c478bdstevel@tonic-gate	!
5307c478bdstevel@tonic-gate	! Fast event tracing.
5317c478bdstevel@tonic-gate	!
5327c478bdstevel@tonic-gate	ld	[%o2 + CPU_FTRACE_STATE], %o4	! %o2 = curthread->t_cpu
5337c478bdstevel@tonic-gate	btst	FTRACE_ENABLED, %o4
5347c478bdstevel@tonic-gate	be,pt	%icc, 1f			! skip if ftrace disabled
5357c478bdstevel@tonic-gate	  mov	%l1, %o5
5367c478bdstevel@tonic-gate	!
5377c478bdstevel@tonic-gate	! Tracing is enabled - write the trace entry.
5387c478bdstevel@tonic-gate	!
5397c478bdstevel@tonic-gate	save	%sp, -SA(MINFRAME), %sp
5407c478bdstevel@tonic-gate	set	ftrace_intr_thread_format_str, %o0
5417c478bdstevel@tonic-gate	mov	%i0, %o1
5427c478bdstevel@tonic-gate	mov	%i1, %o2
5437c478bdstevel@tonic-gate	mov	%i5, %o3
5444df4bd6bs	call	ftrace_3
5454df4bd6bs	ldn	[%i0 + PC_OFF], %o4
5467c478bdstevel@tonic-gate	restore
5477c478bdstevel@tonic-gate1:
5487c478bdstevel@tonic-gate	!
5497c478bdstevel@tonic-gate	! call the handler
5507c478bdstevel@tonic-gate	!
5517c478bdstevel@tonic-gate	SERVE_INTR_PRE(%o1, %o2, %l1, %l3, %o4, %o5, %o3, %o0)
5527c478bdstevel@tonic-gate	!
5537c478bdstevel@tonic-gate	! %o0 and %o1 are now available as scratch registers.
5547c478bdstevel@tonic-gate	!
5557c478bdstevel@tonic-gate0:
5567c478bdstevel@tonic-gate	SERVE_INTR(%o1, %o2, %l1, %l3, %o4, %o5, %o3, %o0)
5577c478bdstevel@tonic-gate	!
558b0fc0e7govinda	! If %o3 is set, we must call serve_intr_next, and both %l1 and %o3
5597c478bdstevel@tonic-gate	! must be preserved. %l1 holds our pil, %l3 holds our inum.
5607c478bdstevel@tonic-gate	!
5617c478bdstevel@tonic-gate	! Note: %l1 is the pil level we're processing, but we may have a
5627c478bdstevel@tonic-gate	! higher effective pil because a higher-level interrupt may have
5637c478bdstevel@tonic-gate	! blocked.
5647c478bdstevel@tonic-gate	!
5657c478bdstevel@tonic-gate	wrpr	%g0, DISP_LEVEL, %pil
5667c478bdstevel@tonic-gate	!
5677c478bdstevel@tonic-gate	! Take timestamp, compute interval, update cumulative counter.
5687c478bdstevel@tonic-gate	!
5697c478bdstevel@tonic-gate	add	THREAD_REG, T_INTR_START, %o5
5707c478bdstevel@tonic-gate1:
5717c478bdstevel@tonic-gate	ldx	[%o5], %o0
5727c478bdstevel@tonic-gate#ifdef DEBUG
5737c478bdstevel@tonic-gate	brnz	%o0, 9f
5747c478bdstevel@tonic-gate	nop
5757c478bdstevel@tonic-gate	! Do not call panic if a panic is already in progress.
5767c478bdstevel@tonic-gate	sethi	%hi(panic_quiesce), %o1
5777c478bdstevel@tonic-gate	ld	[%o1 + %lo(panic_quiesce)], %o1
5787c478bdstevel@tonic-gate	brnz,pn	%o1, 9f
5797c478bdstevel@tonic-gate	nop
5807c478bdstevel@tonic-gate	sethi	%hi(intr_thread_t_intr_start_zero), %o0
5817c478bdstevel@tonic-gate	call	panic
5827c478bdstevel@tonic-gate	or	%o0, %lo(intr_thread_t_intr_start_zero), %o0
5837c478bdstevel@tonic-gate9:
5847c478bdstevel@tonic-gate#endif /* DEBUG */
585bd28a47Prashanth Sreenivasa	RD_CLOCK_TICK(%o1,%l2,%l3,__LINE__)
5867c478bdstevel@tonic-gate	sub	%o1, %o0, %l2			! l2 has interval
5877c478bdstevel@tonic-gate	!
5887c478bdstevel@tonic-gate	! The general outline of what the code here does is:
5897c478bdstevel@tonic-gate	! 1. load t_intr_start, %tick, and calculate the delta
5907c478bdstevel@tonic-gate	! 2. replace t_intr_start with %tick (if %o3 is set) or 0.
5917c478bdstevel@tonic-gate	!
5927c478bdstevel@tonic-gate	! The problem is that a high-level interrupt could arrive at any time.
5937c478bdstevel@tonic-gate	! It will account for (%tick - t_intr_start) for us when it starts,
5947c478bdstevel@tonic-gate	! unless we have set t_intr_start to zero, and then set t_intr_start
5957c478bdstevel@tonic-gate	! to a new %tick when it finishes. To account for this, our first step
5967c478bdstevel@tonic-gate	! is to load t_intr_start and the last is to use casx to store the new
5977c478bdstevel@tonic-gate	! t_intr_start. This guarantees atomicity in reading t_intr_start,
5987c478bdstevel@tonic-gate	! reading %tick, and updating t_intr_start.
5997c478bdstevel@tonic-gate	!
6007c478bdstevel@tonic-gate	movrz	%o3, %g0, %o1
6017c478bdstevel@tonic-gate	casx	[%o5], %o0, %o1
6027c478bdstevel@tonic-gate	cmp	%o0, %o1
6037c478bdstevel@tonic-gate	bne,pn	%xcc, 1b
6047c478bdstevel@tonic-gate	!
6057c478bdstevel@tonic-gate	! Check for Energy Star mode
6067c478bdstevel@tonic-gate	!
6077c478bdstevel@tonic-gate	lduh	[%o2 + CPU_DIVISOR], %o0	! delay -- %o0 = clock divisor
6087c478bdstevel@tonic-gate	cmp	%o0, 1
6097c478bdstevel@tonic-gate	bg,a,pn	%xcc, 2f
6107c478bdstevel@tonic-gate	mulx	%l2, %o0, %l2	! multiply interval by clock divisor iff > 1
6117c478bdstevel@tonic-gate2:
6127c478bdstevel@tonic-gate	!
6137c478bdstevel@tonic-gate	! Update cpu_intrstat. If o3 is set then we will be processing another
6147c478bdstevel@tonic-gate	! interrupt. Above we have set t_intr_start to %tick, not 0. This
6157c478bdstevel@tonic-gate	! means a high-level interrupt can arrive and update the same stats
6167c478bdstevel@tonic-gate	! we're updating. Need to use casx.
6177c478bdstevel@tonic-gate	!
6187c478bdstevel@tonic-gate	sllx	%l1, 4, %o1			! delay - PIL as byte offset
6197c478bdstevel@tonic-gate	add	%o1, CPU_MCPU, %o1		! CPU_INTRSTAT const too big
6207c478bdstevel@tonic-gate	add	%o1, MCPU_INTRSTAT, %o1		! add parts separately
6217c478bdstevel@tonic-gate	add	%o1, %o2, %o1
6227c478bdstevel@tonic-gate1:
6237c478bdstevel@tonic-gate	ldx	[%o1], %o5			! old counter in o5
6247c478bdstevel@tonic-gate	add	%o5, %l2, %o0			! new counter in o0
6257c478bdstevel@tonic-gate 	stx	%o0, [%o1 + 8]			! store into intrstat[pil][1]
6267c478bdstevel@tonic-gate	casx	[%o1], %o5, %o0			! and into intrstat[pil][0]
6277c478bdstevel@tonic-gate	cmp	%o5, %o0
6287c478bdstevel@tonic-gate	bne,pn	%xcc, 1b
6297c478bdstevel@tonic-gate	nop
630eda8946esolom
631eda8946esolom	! Also update intracct[]
632eda8946esolom	lduh	[%o2 + CPU_MSTATE], %o1
633eda8946esolom	sllx	%o1, 3, %o1
634eda8946esolom	add	%o1, CPU_INTRACCT, %o1
635eda8946esolom	add	%o1, %o2, %o1
636eda8946esolom1:
637eda8946esolom	ldx	[%o1], %o5
638eda8946esolom	add	%o5, %l2, %o0
639eda8946esolom	casx	[%o1], %o5, %o0
640eda8946esolom	cmp	%o5, %o0
641eda8946esolom	bne,pn	%xcc, 1b
642eda8946esolom	nop
643eda8946esolom
6447c478bdstevel@tonic-gate	!
6457c478bdstevel@tonic-gate	! Don't keep a pinned process pinned indefinitely. Bump cpu_intrcnt
6467c478bdstevel@tonic-gate	! for each interrupt handler we invoke. If we hit INTRCNT_LIMIT, then
6477c478bdstevel@tonic-gate	! we've crossed the threshold and we should unpin the pinned threads
6487c478bdstevel@tonic-gate	! by preempt()ing ourselves, which will bubble up the t_intr chain
6497c478bdstevel@tonic-gate	! until hitting the non-interrupt thread, which will then in turn
6507c478bdstevel@tonic-gate	! preempt itself allowing the interrupt processing to resume. Finally,
6517c478bdstevel@tonic-gate	! the scheduler takes over and picks the next thread to run.
6527c478bdstevel@tonic-gate	!
6537c478bdstevel@tonic-gate	! If our CPU is quiesced, we cannot preempt because the idle thread
6547c478bdstevel@tonic-gate	! won't ever re-enter the scheduler, and the interrupt will be forever
6557c478bdstevel@tonic-gate	! blocked.
6567c478bdstevel@tonic-gate	!
6577c478bdstevel@tonic-gate	! If t_intr is NULL, we're not pinning anyone, so we use a simpler
6587c478bdstevel@tonic-gate	! algorithm. Just check for cpu_kprunrun, and if set then preempt.
6597c478bdstevel@tonic-gate	! This insures we enter the scheduler if a higher-priority thread
6607c478bdstevel@tonic-gate	! has become runnable.
6617c478bdstevel@tonic-gate	!
6627c478bdstevel@tonic-gate	lduh	[%o2 + CPU_FLAGS], %o5		! don't preempt if quiesced
6637c478bdstevel@tonic-gate	andcc	%o5, CPU_QUIESCED, %g0
6647c478bdstevel@tonic-gate	bnz,pn	%xcc, 1f
6657c478bdstevel@tonic-gate
6667c478bdstevel@tonic-gate	ldn     [THREAD_REG + T_INTR], %o5      ! pinning anything?
6677c478bdstevel@tonic-gate	brz,pn  %o5, 3f				! if not, don't inc intrcnt
6687c478bdstevel@tonic-gate
6697c478bdstevel@tonic-gate	ldub	[%o2 + CPU_INTRCNT], %o5	! delay - %o5 = cpu_intrcnt
6707c478bdstevel@tonic-gate	inc	%o5
6717c478bdstevel@tonic-gate	cmp	%o5, INTRCNT_LIMIT		! have we hit the limit?
6727c478bdstevel@tonic-gate	bl,a,pt	%xcc, 1f			! no preempt if < INTRCNT_LIMIT
6737c478bdstevel@tonic-gate	stub	%o5, [%o2 + CPU_INTRCNT]	! delay annul - inc CPU_INTRCNT
6747c478bdstevel@tonic-gate	bg,pn	%xcc, 2f			! don't inc stats again
6757c478bdstevel@tonic-gate	!
6767c478bdstevel@tonic-gate	! We've reached the limit. Set cpu_intrcnt and cpu_kprunrun, and do
6777c478bdstevel@tonic-gate	! CPU_STATS_ADDQ(cp, sys, intrunpin, 1). Then call preempt.
6787c478bdstevel@tonic-gate	!
6797c478bdstevel@tonic-gate	mov	1, %o4				! delay
6807c478bdstevel@tonic-gate	stub	%o4, [%o2 + CPU_KPRUNRUN]
6817c478bdstevel@tonic-gate	ldx	[%o2 + CPU_STATS_SYS_INTRUNPIN], %o4
6827c478bdstevel@tonic-gate	inc	%o4
6837c478bdstevel@tonic-gate	stx	%o4, [%o2 + CPU_STATS_SYS_INTRUNPIN]
6847c478bdstevel@tonic-gate	ba	2f
6857c478bdstevel@tonic-gate	stub	%o5, [%o2 + CPU_INTRCNT]	! delay
6867c478bdstevel@tonic-gate3:
6877c478bdstevel@tonic-gate	! Code for t_intr == NULL
6887c478bdstevel@tonic-gate	ldub	[%o2 + CPU_KPRUNRUN], %o5
6897c478bdstevel@tonic-gate	brz,pt	%o5, 1f				! don't preempt unless kprunrun
6907c478bdstevel@tonic-gate2:
6917c478bdstevel@tonic-gate	! Time to call preempt
6927c478bdstevel@tonic-gate	mov	%o2, %l3			! delay - save %o2
6937c478bdstevel@tonic-gate	call	preempt
6947c478bdstevel@tonic-gate	mov	%o3, %l2			! delay - save %o3.
6957c478bdstevel@tonic-gate	mov	%l3, %o2			! restore %o2
6967c478bdstevel@tonic-gate	mov	%l2, %o3			! restore %o3
6977c478bdstevel@tonic-gate	wrpr	%g0, DISP_LEVEL, %pil		! up from cpu_base_spl
6987c478bdstevel@tonic-gate1:
6997c478bdstevel@tonic-gate	!
700b0fc0e7govinda	! Do we need to call serve_intr_next and do this again?
7017c478bdstevel@tonic-gate	!
7027c478bdstevel@tonic-gate	brz,a,pt %o3, 0f
7037c478bdstevel@tonic-gate	ld	[%o2 + CPU_INTR_ACTV], %o5	! delay annulled
7047c478bdstevel@tonic-gate	!
7057c478bdstevel@tonic-gate	! Restore %pil before calling serve_intr() again. We must check
7067c478bdstevel@tonic-gate	! CPU_BASE_SPL and set %pil to max(our-pil, CPU_BASE_SPL)
7077c478bdstevel@tonic-gate	!
7087c478bdstevel@tonic-gate	ld	[%o2 + CPU_BASE_SPL], %o4
7097c478bdstevel@tonic-gate	cmp	%o4, %l1
7107c478bdstevel@tonic-gate	movl	%xcc, %l1, %o4
7117c478bdstevel@tonic-gate	wrpr	%g0, %o4, %pil
7127c478bdstevel@tonic-gate	SERVE_INTR_NEXT(%o1, %o2, %l1, %l3, %o4, %o5, %o3, %o0)
7137c478bdstevel@tonic-gate	ba	0b				! compute new stats
7147c478bdstevel@tonic-gate	nop
7157c478bdstevel@tonic-gate0:
7167c478bdstevel@tonic-gate	!
7177c478bdstevel@tonic-gate	! Clear bit for this level in CPU's interrupt active bitmask.
7187c478bdstevel@tonic-gate	!
7197c478bdstevel@tonic-gate	mov	1, %o4
7207c478bdstevel@tonic-gate	sll	%o4, %l1, %o4
7217c478bdstevel@tonic-gate#ifdef DEBUG
7227c478bdstevel@tonic-gate	!
7237c478bdstevel@tonic-gate	! ASSERT(CPU->cpu_intr_actv & (1 << PIL))
7247c478bdstevel@tonic-gate	!
7257c478bdstevel@tonic-gate	andcc	%o4, %o5, %g0
7267c478bdstevel@tonic-gate	bnz,pt	%xcc, 0f
7277c478bdstevel@tonic-gate	nop
7287c478bdstevel@tonic-gate	! Do not call panic if a panic is already in progress.
7297c478bdstevel@tonic-gate	sethi	%hi(panic_quiesce), %l2
7307c478bdstevel@tonic-gate	ld	[%l2 + %lo(panic_quiesce)], %l2
7317c478bdstevel@tonic-gate	brnz,pn	%l2, 0f
7327c478bdstevel@tonic-gate	nop
7337c478bdstevel@tonic-gate	sethi	%hi(intr_thread_actv_bit_not_set), %o0
7347c478bdstevel@tonic-gate	call	panic
7357c478bdstevel@tonic-gate	or	%o0, %lo(intr_thread_actv_bit_not_set), %o0
7367c478bdstevel@tonic-gate0:
7377c478bdstevel@tonic-gate#endif /* DEBUG */
7387c478bdstevel@tonic-gate	andn	%o5, %o4, %o5
7397c478bdstevel@tonic-gate	st	%o5, [%o2 + CPU_INTR_ACTV]
7407c478bdstevel@tonic-gate	!
7417c478bdstevel@tonic-gate	! If there is still an interrupted thread underneath this one,
7427c478bdstevel@tonic-gate	! then the interrupt was never blocked and the return is fairly
7437c478bdstevel@tonic-gate	! simple.  Otherwise jump to intr_thread_exit.
7447c478bdstevel@tonic-gate	!
7457c478bdstevel@tonic-gate	ldn	[THREAD_REG + T_INTR], %o4	! pinned thread
7467c478bdstevel@tonic-gate	brz,pn	%o4, intr_thread_exit		! branch if none
7477c478bdstevel@tonic-gate	nop
7487c478bdstevel@tonic-gate	!
7497c478bdstevel@tonic-gate	! link the thread back onto the interrupt thread pool
7507c478bdstevel@tonic-gate	!
7517c478bdstevel@tonic-gate	ldn	[%o2 + CPU_INTR_THREAD], %o3
7527c478bdstevel@tonic-gate	stn	%o3, [THREAD_REG + T_LINK]
7537c478bdstevel@tonic-gate	stn	THREAD_REG, [%o2 + CPU_INTR_THREAD]
7547c478bdstevel@tonic-gate	!
7557c478bdstevel@tonic-gate	! set the thread state to free so kernel debuggers don't see it
7567c478bdstevel@tonic-gate	!
7577c478bdstevel@tonic-gate	mov	TS_FREE, %o5
7587c478bdstevel@tonic-gate	st	%o5, [THREAD_REG + T_STATE]
7597c478bdstevel@tonic-gate	!
7607c478bdstevel@tonic-gate	! Switch back to the interrupted thread and return
7617c478bdstevel@tonic-gate	!
7627c478bdstevel@tonic-gate	stn	%o4, [%o2 + CPU_THREAD]
76325cf1a3jl	membar	#StoreLoad			! sync with mutex_exit()
7647c478bdstevel@tonic-gate	mov	%o4, THREAD_REG
7657c478bdstevel@tonic-gate
7667c478bdstevel@tonic-gate	! If we pinned an interrupt thread, store its starting timestamp.
7677c478bdstevel@tonic-gate	lduh	[THREAD_REG + T_FLAGS], %o5
7687c478bdstevel@tonic-gate	andcc	%o5, T_INTR_THREAD, %g0
7697c478bdstevel@tonic-gate	bz,pt	%xcc, 1f
7707c478bdstevel@tonic-gate	ldn	[THREAD_REG + T_SP], %sp	! delay - restore %sp
7717c478bdstevel@tonic-gate
7727c478bdstevel@tonic-gate	add	THREAD_REG, T_INTR_START, %o3	! o3 has &curthread->t_intr_star
7737c478bdstevel@tonic-gate0:
7747c478bdstevel@tonic-gate	ldx	[%o3], %o4			! o4 = t_intr_start before
775bd28a47Prashanth Sreenivasa	RD_CLOCK_TICK(%o5,%l2,%l3,__LINE__)
7767c478bdstevel@tonic-gate	casx	[%o3], %o4, %o5			! put o5 in ts if o4 == ts after
7777c478bdstevel@tonic-gate	cmp	%o4, %o5
7787c478bdstevel@tonic-gate	! If a high-level interrupt occurred while we were attempting to store
7797c478bdstevel@tonic-gate	! the timestamp, try again.
7807c478bdstevel@tonic-gate	bne,pn	%xcc, 0b
7817c478bdstevel@tonic-gate	ldn	[THREAD_REG + T_SP], %sp	! delay - restore %sp
7827c478bdstevel@tonic-gate1:
7837c478bdstevel@tonic-gate	! If the thread being restarted isn't pinning anyone, and no interrupts
7847c478bdstevel@tonic-gate	! are pending, zero out cpu_intrcnt
7857c478bdstevel@tonic-gate	ldn	[THREAD_REG + T_INTR], %o4
7867c478bdstevel@tonic-gate	brnz,pn	%o4, 2f
7877c478bdstevel@tonic-gate	rd	SOFTINT, %o4			! delay
7887c478bdstevel@tonic-gate	set	SOFTINT_MASK, %o5
7897c478bdstevel@tonic-gate	andcc	%o4, %o5, %g0
7907c478bdstevel@tonic-gate	bz,a,pt	%xcc, 2f
7917c478bdstevel@tonic-gate	stub	%g0, [%o2 + CPU_INTRCNT]	! delay annul
7927c478bdstevel@tonic-gate2:
7937c478bdstevel@tonic-gate	jmp	%l0 + 8
7947c478bdstevel@tonic-gate	nop
7957c478bdstevel@tonic-gate	SET_SIZE(intr_thread)
7967c478bdstevel@tonic-gate	/* Not Reached */
7977c478bdstevel@tonic-gate
7987c478bdstevel@tonic-gate	!
7997c478bdstevel@tonic-gate	! An interrupt returned on what was once (and still might be)
8007c478bdstevel@tonic-gate	! an interrupt thread stack, but the interrupted process is no longer
8017c478bdstevel@tonic-gate	! there.  This means the interrupt must have blocked.
8027c478bdstevel@tonic-gate	!
8037c478bdstevel@tonic-gate	! There is no longer a thread under this one, so put this thread back
8047c478bdstevel@tonic-gate	! on the CPU's free list and resume the idle thread which will dispatch
8057c478bdstevel@tonic-gate	! the next thread to run.
8067c478bdstevel@tonic-gate	!
8077c478bdstevel@tonic-gate	! All traps below DISP_LEVEL are disabled here, but the mondo interrupt
8087c478bdstevel@tonic-gate	! is enabled.
8097c478bdstevel@tonic-gate	!
8107c478bdstevel@tonic-gate	ENTRY_NP(intr_thread_exit)
8117c478bdstevel@tonic-gate#ifdef TRAPTRACE
8127c478bdstevel@tonic-gate	rdpr	%pstate, %l2
8137c478bdstevel@tonic-gate	andn	%l2, PSTATE_IE | PSTATE_AM, %o4
8147c478bdstevel@tonic-gate	wrpr	%g0, %o4, %pstate			! cpu to known state
8157c478bdstevel@tonic-gate	TRACE_PTR(%o4, %o5)
816023e71dHaik Aftandilian	GET_TRACE_TICK(%o5, %o0)
8177c478bdstevel@tonic-gate	stxa	%o5, [%o4 + TRAP_ENT_TICK]%asi
8187c478bdstevel@tonic-gate	TRACE_SAVE_TL_GL_REGS(%o4, %o5)
8197c478bdstevel@tonic-gate	set	TT_INTR_EXIT, %o5
8207c478bdstevel@tonic-gate	stha	%o5, [%o4 + TRAP_ENT_TT]%asi
8217c478bdstevel@tonic-gate	stna	%g0, [%o4 + TRAP_ENT_TPC]%asi
8227c478bdstevel@tonic-gate	stxa	%g0, [%o4 + TRAP_ENT_TSTATE]%asi
8237c478bdstevel@tonic-gate	stna	%sp, [%o4 + TRAP_ENT_SP]%asi
8247c478bdstevel@tonic-gate	stna	THREAD_REG, [%o4 + TRAP_ENT_TR]%asi
8257c478bdstevel@tonic-gate	ld	[%o2 + CPU_BASE_SPL], %o5
8267c478bdstevel@tonic-gate	stna	%o5, [%o4 + TRAP_ENT_F1]%asi
8277c478bdstevel@tonic-gate	stna	%g0, [%o4 + TRAP_ENT_F2]%asi
8287c478bdstevel@tonic-gate	stna	%g0, [%o4 + TRAP_ENT_F3]%asi
8297c478bdstevel@tonic-gate	stna	%g0, [%o4 + TRAP_ENT_F4]%asi
8307c478bdstevel@tonic-gate	TRACE_NEXT(%o4, %o5, %o0)
8317c478bdstevel@tonic-gate	wrpr	%g0, %l2, %pstate
8327c478bdstevel@tonic-gate#endif /* TRAPTRACE */
8337c478bdstevel@tonic-gate	! cpu_stats.sys.intrblk++
8347c478bdstevel@tonic-gate        ldx	[%o2 + CPU_STATS_SYS_INTRBLK], %o4
8357c478bdstevel@tonic-gate        inc     %o4
8367c478bdstevel@tonic-gate        stx	%o4, [%o2 + CPU_STATS_SYS_INTRBLK]
8377c478bdstevel@tonic-gate	!
8387c478bdstevel@tonic-gate	! Put thread back on the interrupt thread list.
8397c478bdstevel@tonic-gate	!
8407c478bdstevel@tonic-gate
8417c478bdstevel@tonic-gate	!
8427c478bdstevel@tonic-gate	! Set the CPU's base SPL level.
8437c478bdstevel@tonic-gate	!
8447c478bdstevel@tonic-gate#ifdef DEBUG
8457c478bdstevel@tonic-gate	!
8467c478bdstevel@tonic-gate	! ASSERT(!(CPU->cpu_intr_actv & (1 << PIL)))
8477c478bdstevel@tonic-gate	!
8487c478bdstevel@tonic-gate	ld	[%o2 + CPU_INTR_ACTV], %o5
8497c478bdstevel@tonic-gate	mov	1, %o4
8507c478bdstevel@tonic-gate	sll	%o4, %l1, %o4
8517c478bdstevel@tonic-gate	and	%o5, %o4, %o4
8527c478bdstevel@tonic-gate	brz,pt	%o4, 0f
8537c478bdstevel@tonic-gate	nop
8547c478bdstevel@tonic-gate	! Do not call panic if a panic is already in progress.
8557c478bdstevel@tonic-gate	sethi	%hi(panic_quiesce), %l2
8567c478bdstevel@tonic-gate	ld	[%l2 + %lo(panic_quiesce)], %l2
8577c478bdstevel@tonic-gate	brnz,pn	%l2, 0f
8587c478bdstevel@tonic-gate	nop
8597c478bdstevel@tonic-gate	sethi	%hi(intr_thread_exit_actv_bit_set), %o0
8607c478bdstevel@tonic-gate	call	panic
8617c478bdstevel@tonic-gate	or	%o0, %lo(intr_thread_exit_actv_bit_set), %o0
8627c478bdstevel@tonic-gate0:
8637c478bdstevel@tonic-gate#endif /* DEBUG */
8647c478bdstevel@tonic-gate	call	_intr_set_spl			! set CPU's base SPL level
8657c478bdstevel@tonic-gate	ld	[%o2 + CPU_INTR_ACTV], %o5	! delay - load active mask
8667c478bdstevel@tonic-gate	!
8677c478bdstevel@tonic-gate	! set the thread state to free so kernel debuggers don't see it
8687c478bdstevel@tonic-gate	!
8697c478bdstevel@tonic-gate	mov	TS_FREE, %o4
8707c478bdstevel@tonic-gate	st	%o4, [THREAD_REG + T_STATE]
8717c478bdstevel@tonic-gate	!
8727c478bdstevel@tonic-gate	! Put thread on either the interrupt pool or the free pool and
8737c478bdstevel@tonic-gate	! call swtch() to resume another thread.
8747c478bdstevel@tonic-gate	!
8757c478bdstevel@tonic-gate	ldn	[%o2 + CPU_INTR_THREAD], %o5	! get list pointer
8767c478bdstevel@tonic-gate	stn	%o5, [THREAD_REG + T_LINK]
8777c478bdstevel@tonic-gate	call	swtch				! switch to best thread
8787c478bdstevel@tonic-gate	stn	THREAD_REG, [%o2 + CPU_INTR_THREAD] ! delay - put thread on list
8797c478bdstevel@tonic-gate	ba,a,pt	%xcc, .				! swtch() shouldn't return
8807c478bdstevel@tonic-gate	SET_SIZE(intr_thread_exit)
8817c478bdstevel@tonic-gate
8827c478bdstevel@tonic-gate	.global ftrace_intr_thread_format_str
8837c478bdstevel@tonic-gateftrace_intr_thread_format_str:
8847c478bdstevel@tonic-gate	.asciz	"intr_thread(): regs=0x%lx, int=0x%lx, pil=0x%lx"
8857c478bdstevel@tonic-gate#ifdef DEBUG
8867c478bdstevel@tonic-gateintr_thread_actv_bit_set:
8877c478bdstevel@tonic-gate	.asciz	"intr_thread():	cpu_intr_actv bit already set for PIL"
8887c478bdstevel@tonic-gateintr_thread_actv_bit_not_set:
8897c478bdstevel@tonic-gate	.asciz	"intr_thread():	cpu_intr_actv bit not set for PIL"
8907c478bdstevel@tonic-gateintr_thread_exit_actv_bit_set:
8917c478bdstevel@tonic-gate	.asciz	"intr_thread_exit(): cpu_intr_actv bit erroneously set for PIL"
8927c478bdstevel@tonic-gateintr_thread_t_intr_start_zero:
8937c478bdstevel@tonic-gate	.asciz	"intr_thread():	t_intr_start zero upon handler return"
8947c478bdstevel@tonic-gate#endif /* DEBUG */
8957c478bdstevel@tonic-gate#endif	/* lint */
8967c478bdstevel@tonic-gate
8977c478bdstevel@tonic-gate#if defined(lint)
8987c478bdstevel@tonic-gate
8997c478bdstevel@tonic-gate/*
9007c478bdstevel@tonic-gate * Handle an interrupt in the current thread
9017c478bdstevel@tonic-gate *	Entry:
9027c478bdstevel@tonic-gate *		%o0       = pointer to regs structure
903b0fc0e7govinda *		%o1       = pointer to current intr_vec_t (iv) to be processed
9047c478bdstevel@tonic-gate *		%o2       = pil
9057c478bdstevel@tonic-gate *		%sp       = on current thread's kernel stack
9067c478bdstevel@tonic-gate *		%o7       = return linkage to trap code
9077c478bdstevel@tonic-gate *		%g7       = current thread
9087c478bdstevel@tonic-gate *		%pstate   = normal globals, interrupts enabled,
9097c478bdstevel@tonic-gate *		            privileged, fp disabled
9107c478bdstevel@tonic-gate *		%pil      = PIL_MAX
9117c478bdstevel@tonic-gate *
9127c478bdstevel@tonic-gate *	Register Usage
9137c478bdstevel@tonic-gate *		%l0       = return linkage
9147c478bdstevel@tonic-gate *		%l1       = old stack
9157c478bdstevel@tonic-gate *		%l2 - %l3 = scratch
9167c478bdstevel@tonic-gate *		%l4 - %l7 = reserved for sys_trap
9177c478bdstevel@tonic-gate *		%o3       = cpu
9187c478bdstevel@tonic-gate *		%o0       = scratch
9197c478bdstevel@tonic-gate *		%o4 - %o5 = scratch
9207c478bdstevel@tonic-gate */
9217c478bdstevel@tonic-gate/* ARGSUSED */
9227c478bdstevel@tonic-gatevoid
923b0fc0e7govindacurrent_thread(struct regs *regs, uint64_t iv_p, uint_t pil)
9247c478bdstevel@tonic-gate{}
9257c478bdstevel@tonic-gate
9267c478bdstevel@tonic-gate#else	/* lint */
9277c478bdstevel@tonic-gate
9287c478bdstevel@tonic-gate	ENTRY_NP(current_thread)
9297c478bdstevel@tonic-gate
9307c478bdstevel@tonic-gate	mov	%o7, %l0
9317c478bdstevel@tonic-gate	ldn	[THREAD_REG + T_CPU], %o3
9322604657vb
9332604657vb	ldn	[THREAD_REG + T_ONFAULT], %l2
9342604657vb	brz,pt	%l2, no_onfault		! branch if no onfault label set
9352604657vb	nop
9362604657vb	stn	%g0, [THREAD_REG + T_ONFAULT]! clear onfault label
9372604657vb	ldn	[THREAD_REG + T_LOFAULT], %l3
9382604657vb	stn	%g0, [THREAD_REG + T_LOFAULT]! clear lofault data
9392604657vb
9402604657vb	sub	%o2, LOCK_LEVEL + 1, %o5
9412604657vb	sll	%o5, CPTRSHIFT, %o5
9422604657vb	add	%o5, CPU_OFD, %o4	! %o4 has on_fault data offset
9432604657vb	stn	%l2, [%o3 + %o4]	! save onfault label for pil %o2
9442604657vb	add	%o5, CPU_LFD, %o4	! %o4 has lofault data offset
9452604657vb	stn	%l3, [%o3 + %o4]	! save lofault data for pil %o2
9462604657vb
9472604657vbno_onfault:
9482604657vb	ldn	[THREAD_REG + T_ONTRAP], %l2
9492604657vb	brz,pt	%l2, 6f			! branch if no on_trap protection
9502604657vb	nop
9512604657vb	stn	%g0, [THREAD_REG + T_ONTRAP]! clear on_trap protection
9522604657vb	sub	%o2, LOCK_LEVEL + 1, %o5
9532604657vb	sll	%o5, CPTRSHIFT, %o5
9542604657vb	add	%o5, CPU_OTD, %o4	! %o4 has on_trap data offset
9552604657vb	stn	%l2, [%o3 + %o4]	! save on_trap label for pil %o2
9562604657vb
9577c478bdstevel@tonic-gate	!
9587c478bdstevel@tonic-gate	! Set bit for this level in CPU's active interrupt bitmask.
9597c478bdstevel@tonic-gate	!
9602604657vb6:	ld	[%o3 + CPU_INTR_ACTV], %o5	! o5 has cpu_intr_actv b4 chng
9617c478bdstevel@tonic-gate	mov	1, %o4
9627c478bdstevel@tonic-gate	sll	%o4, %o2, %o4			! construct mask for level
9637c478bdstevel@tonic-gate#ifdef DEBUG
9647c478bdstevel@tonic-gate	!
9657c478bdstevel@tonic-gate	! ASSERT(!(CPU->cpu_intr_actv & (1 << PIL)))
9667c478bdstevel@tonic-gate	!
9677c478bdstevel@tonic-gate	andcc	%o5, %o4, %g0
9687c478bdstevel@tonic-gate	bz,pt	%xcc, 0f
9697c478bdstevel@tonic-gate	nop
9707c478bdstevel@tonic-gate	! Do not call panic if a panic is already in progress.
9717c478bdstevel@tonic-gate	sethi	%hi(panic_quiesce), %l2
9727c478bdstevel@tonic-gate	ld	[%l2 + %lo(panic_quiesce)], %l2
9737c478bdstevel@tonic-gate	brnz,pn	%l2, 0f
9747c478bdstevel@tonic-gate	nop
9757c478bdstevel@tonic-gate	sethi	%hi(current_thread_actv_bit_set), %o0
9767c478bdstevel@tonic-gate	call	panic
9777c478bdstevel@tonic-gate	or	%o0, %lo(current_thread_actv_bit_set), %o0
9787c478bdstevel@tonic-gate0:
9797c478bdstevel@tonic-gate#endif /* DEBUG */
9807c478bdstevel@tonic-gate	or	%o5, %o4, %o4
9817c478bdstevel@tonic-gate	!
9827c478bdstevel@tonic-gate	! See if we are interrupting another high-level interrupt.
9837c478bdstevel@tonic-gate	!
9847c478bdstevel@tonic-gate	srl	%o5, LOCK_LEVEL + 1, %o5	! only look at high-level bits
9857c478bdstevel@tonic-gate	brz,pt	%o5, 1f
9867c478bdstevel@tonic-gate	st	%o4, [%o3 + CPU_INTR_ACTV]	! delay - store active mask
9877c478bdstevel@tonic-gate	!
9887c478bdstevel@tonic-gate	! We have interrupted another high-level interrupt. Find its PIL,
9897c478bdstevel@tonic-gate	! compute the interval it ran for, and update its cumulative counter.
9907c478bdstevel@tonic-gate	!
9917c478bdstevel@tonic-gate	! Register usage:
9927c478bdstevel@tonic-gate
9937c478bdstevel@tonic-gate	! o2 = PIL of this interrupt
9947c478bdstevel@tonic-gate	! o5 = high PIL bits of INTR_ACTV (not including this PIL)
9957c478bdstevel@tonic-gate	! l1 = bitmask used to find other active high-level PIL
9967c478bdstevel@tonic-gate	! o4 = index of bit set in l1
9977c478bdstevel@tonic-gate	! Use cpu_intr_actv to find the cpu_pil_high_start[] offset for the
9987c478bdstevel@tonic-gate	! interrupted high-level interrupt.
9997c478bdstevel@tonic-gate	! Create mask for cpu_intr_actv. Begin by looking for bits set
10007c478bdstevel@tonic-gate	! at one level below the current PIL. Since %o5 contains the active
10017c478bdstevel@tonic-gate	! mask already shifted right by (LOCK_LEVEL + 1), we start by looking
10027c478bdstevel@tonic-gate	! at bit (current_pil - (LOCK_LEVEL + 2)).
10037c478bdstevel@tonic-gate	sub	%o2, LOCK_LEVEL + 2, %o4
10047c478bdstevel@tonic-gate	mov	1, %l1
10057c478bdstevel@tonic-gate	sll	%l1, %o4, %l1
10067c478bdstevel@tonic-gate2:
10077c478bdstevel@tonic-gate#ifdef DEBUG
10087c478bdstevel@tonic-gate	! ASSERT(%l1 != 0) (we didn't shift the bit off the right edge)
10097c478bdstevel@tonic-gate	brnz,pt	%l1, 9f
10107c478bdstevel@tonic-gate	nop
10117c478bdstevel@tonic-gate
10127c478bdstevel@tonic-gate	! Don't panic if a panic is already in progress.
10137c478bdstevel@tonic-gate	sethi	%hi(panic_quiesce), %l3
10147c478bdstevel@tonic-gate	ld	[%l3 + %lo(panic_quiesce)], %l3
10157c478bdstevel@tonic-gate	brnz,pn	%l3, 9f
10167c478bdstevel@tonic-gate	nop
10177c478bdstevel@tonic-gate	sethi	%hi(current_thread_nested_PIL_not_found), %o0
10187c478bdstevel@tonic-gate	call	panic
10197c478bdstevel@tonic-gate	or	%o0, %lo(current_thread_nested_PIL_not_found), %o0
10207c478bdstevel@tonic-gate9:
10217c478bdstevel@tonic-gate#endif /* DEBUG */
10227c478bdstevel@tonic-gate	andcc	%l1, %o5, %g0		! test mask against high-level bits of
10237c478bdstevel@tonic-gate	bnz	%xcc, 3f		! cpu_intr_actv
10247c478bdstevel@tonic-gate	nop
10257c478bdstevel@tonic-gate	srl	%l1, 1, %l1		! No match. Try next lower PIL.
10267c478bdstevel@tonic-gate	ba,pt	%xcc, 2b
10277c478bdstevel@tonic-gate	sub	%o4, 1, %o4		! delay - decrement PIL
10287c478bdstevel@tonic-gate3:
10297c478bdstevel@tonic-gate	sll	%o4, 3, %o4			! index to byte offset
10307c478bdstevel@tonic-gate	add	%o4, CPU_MCPU, %l1	! CPU_PIL_HIGH_START is too large
10317c478bdstevel@tonic-gate	add	%l1, MCPU_PIL_HIGH_START, %l1
10327c478bdstevel@tonic-gate	ldx	[%o3 + %l1], %l3		! load starting timestamp
10337c478bdstevel@tonic-gate#ifdef DEBUG
10347c478bdstevel@tonic-gate	brnz,pt	%l3, 9f
10357c478bdstevel@tonic-gate	nop
10367c478bdstevel@tonic-gate	! Don't panic if a panic is already in progress.
10377c478bdstevel@tonic-gate	sethi	%hi(panic_quiesce), %l1
10387c478bdstevel@tonic-gate	ld	[%l1 + %lo(panic_quiesce)], %l1
10397c478bdstevel@tonic-gate	brnz,pn	%l1, 9f
10407c478bdstevel@tonic-gate	nop
10417c478bdstevel@tonic-gate	srl	%o4, 3, %o1			! Find interrupted PIL for panic
10427c478bdstevel@tonic-gate	add	%o1, LOCK_LEVEL + 1, %o1
10437c478bdstevel@tonic-gate	sethi	%hi(current_thread_nested_pil_zero), %o0
10447c478bdstevel@tonic-gate	call	panic
10457c478bdstevel@tonic-gate	or	%o0, %lo(current_thread_nested_pil_zero), %o0
10467c478bdstevel@tonic-gate9:
10477c478bdstevel@tonic-gate#endif /* DEBUG */
1048bd28a47Prashanth Sreenivasa	RD_CLOCK_TICK_NO_SUSPEND_CHECK(%l1, %l2)
10497c478bdstevel@tonic-gate	sub	%l1, %l3, %l3			! interval in %l3
10507c478bdstevel@tonic-gate	!
10517c478bdstevel@tonic-gate	! Check for Energy Star mode
10527c478bdstevel@tonic-gate	!
10537c478bdstevel@tonic-gate	lduh	[%o3 + CPU_DIVISOR], %l1	! %l1 = clock divisor
10547c478bdstevel@tonic-gate	cmp	%l1, 1
10557c478bdstevel@tonic-gate	bg,a,pn	%xcc, 2f
10567c478bdstevel@tonic-gate	mulx	%l3, %l1, %l3	! multiply interval by clock divisor iff > 1
10577c478bdstevel@tonic-gate2:
10587c478bdstevel@tonic-gate	!
10597c478bdstevel@tonic-gate	! We need to find the CPU offset of the cumulative counter. We start
10607c478bdstevel@tonic-gate	! with %o4, which has (PIL - (LOCK_LEVEL + 1)) * 8. We need PIL * 16,
10617c478bdstevel@tonic-gate	! so we shift left 1, then add (LOCK_LEVEL + 1) * 16, which is
10627c478bdstevel@tonic-gate	! CPU_INTRSTAT_LOW_PIL_OFFSET.
10637c478bdstevel@tonic-gate	!
10647c478bdstevel@tonic-gate	sll	%o4, 1, %o4
10657c478bdstevel@tonic-gate	add	%o4, CPU_MCPU, %o4		! CPU_INTRSTAT const too large
10667c478bdstevel@tonic-gate	add	%o4, MCPU_INTRSTAT, %o4		! add parts separately
10677c478bdstevel@tonic-gate	add	%o4, CPU_INTRSTAT_LOW_PIL_OFFSET, %o4
10687c478bdstevel@tonic-gate	ldx	[%o3 + %o4], %l1		! old counter in l1
10697c478bdstevel@tonic-gate	add	%l1, %l3, %l1			! new counter in l1
1070eda8946esolom	stx	%l1, [%o3 + %o4]		! store new counter
1071eda8946esolom
1072eda8946esolom	! Also update intracct[]
1073eda8946esolom	lduh	[%o3 + CPU_MSTATE], %o4
1074eda8946esolom	sllx	%o4, 3, %o4
1075eda8946esolom	add	%o4, CPU_INTRACCT, %o4
1076eda8946esolom	ldx	[%o3 + %o4], %l1
1077eda8946esolom	add	%l1, %l3, %l1
10787c478bdstevel@tonic-gate	! Another high-level interrupt is active below this one, so
10797c478bdstevel@tonic-gate	! there is no need to check for an interrupt thread. That will be
10807c478bdstevel@tonic-gate	! done by the lowest priority high-level interrupt active.
10817c478bdstevel@tonic-gate	ba,pt	%xcc, 5f
10827c478bdstevel@tonic-gate	stx	%l1, [%o3 + %o4]		! delay - store new counter
10837c478bdstevel@tonic-gate1:
10847c478bdstevel@tonic-gate	! If we haven't interrupted another high-level interrupt, we may be
10857c478bdstevel@tonic-gate	! interrupting a low level interrupt thread. If so, compute its interval
10867c478bdstevel@tonic-gate	! and update its cumulative counter.
10877c478bdstevel@tonic-gate	lduh	[THREAD_REG + T_FLAGS], %o4
10887c478bdstevel@tonic-gate	andcc	%o4, T_INTR_THREAD, %g0
10897c478bdstevel@tonic-gate	bz,pt	%xcc, 4f
10907c478bdstevel@tonic-gate	nop
10917c478bdstevel@tonic-gate
10927c478bdstevel@tonic-gate	! We have interrupted an interrupt thread. Take timestamp, compute
10937c478bdstevel@tonic-gate	! interval, update cumulative counter.
10947c478bdstevel@tonic-gate
10957c478bdstevel@tonic-gate	! Check t_intr_start. If it is zero, either intr_thread() or
10967c478bdstevel@tonic-gate	! current_thread() (at a lower PIL, of course) already did
10977c478bdstevel@tonic-gate	! the accounting for the underlying interrupt thread.
10987c478bdstevel@tonic-gate	ldx	[THREAD_REG + T_INTR_START], %o5
10997c478bdstevel@tonic-gate	brz,pn	%o5, 4f
11007c478bdstevel@tonic-gate	nop
11017c478bdstevel@tonic-gate
11027c478bdstevel@tonic-gate	stx	%g0, [THREAD_REG + T_INTR_START]
1103bd28a47Prashanth Sreenivasa	RD_CLOCK_TICK_NO_SUSPEND_CHECK(%o4, %l2)
11047c478bdstevel@tonic-gate	sub	%o4, %o5, %o5			! o5 has the interval
11057c478bdstevel@tonic-gate
11067c478bdstevel@tonic-gate	! Check for Energy Star mode
11077c478bdstevel@tonic-gate	lduh	[%o3 + CPU_DIVISOR], %o4	! %o4 = clock divisor
11087c478bdstevel@tonic-gate	cmp	%o4, 1
11097c478bdstevel@tonic-gate	bg,a,pn	%xcc, 2f
11107c478bdstevel@tonic-gate	mulx	%o5, %o4, %o5	! multiply interval by clock divisor iff > 1
11117c478bdstevel@tonic-gate2:
11127c478bdstevel@tonic-gate	ldub	[THREAD_REG + T_PIL], %o4
11137c478bdstevel@tonic-gate	sllx	%o4, 4, %o4			! PIL index to byte offset
11147c478bdstevel@tonic-gate	add	%o4, CPU_MCPU, %o4		! CPU_INTRSTAT const too large
11157c478bdstevel@tonic-gate	add	%o4, MCPU_INTRSTAT, %o4		! add parts separately
11167c478bdstevel@tonic-gate	ldx	[%o3 + %o4], %l2		! old counter in l2
11177c478bdstevel@tonic-gate	add	%l2, %o5, %l2			! new counter in l2
11187c478bdstevel@tonic-gate	stx	%l2, [%o3 + %o4]		! store new counter
1119eda8946esolom
1120eda8946esolom	! Also update intracct[]
1121eda8946esolom	lduh	[%o3 + CPU_MSTATE], %o4
1122eda8946esolom	sllx	%o4, 3, %o4
1123eda8946esolom	add	%o4, CPU_INTRACCT, %o4
1124eda8946esolom	ldx	[%o3 + %o4], %l2
1125eda8946esolom	add	%l2, %o5, %l2
1126eda8946esolom	stx	%l2, [%o3 + %o4]
11277c478bdstevel@tonic-gate4:
11287c478bdstevel@tonic-gate	!
11297c478bdstevel@tonic-gate	! Handle high-level interrupts on separate interrupt stack.
11307c478bdstevel@tonic-gate	! No other high-level interrupts are active, so switch to int stack.
11317c478bdstevel@tonic-gate	!
11327c478bdstevel@tonic-gate	mov	%sp, %l1
11337c478bdstevel@tonic-gate	ldn	[%o3 + CPU_INTR_STACK], %l3
11347c478bdstevel@tonic-gate	sub	%l3, STACK_BIAS, %sp
11357c478bdstevel@tonic-gate
11367c478bdstevel@tonic-gate5:
11377c478bdstevel@tonic-gate#ifdef DEBUG
11387c478bdstevel@tonic-gate	!
11397c478bdstevel@tonic-gate	! ASSERT(%o2 > LOCK_LEVEL)
11407c478bdstevel@tonic-gate	!
11417c478bdstevel@tonic-gate	cmp	%o2, LOCK_LEVEL
11427c478bdstevel@tonic-gate	bg,pt	%xcc, 3f
11437c478bdstevel@tonic-gate	nop
11447c478bdstevel@tonic-gate	mov	CE_PANIC, %o0
11457c478bdstevel@tonic-gate	sethi	%hi(current_thread_wrong_pil), %o1
11467c478bdstevel@tonic-gate	call	cmn_err				! %o2 has the %pil already
11477c478bdstevel@tonic-gate	or	%o1, %lo(current_thread_wrong_pil), %o1
11487c478bdstevel@tonic-gate#endif
11497c478bdstevel@tonic-gate3:
11507c478bdstevel@tonic-gate	! Store starting timestamp for this PIL in CPU structure at
11517c478bdstevel@tonic-gate	! cpu.cpu_m.pil_high_start[PIL - (LOCK_LEVEL + 1)]
11527c478bdstevel@tonic-gate        sub     %o2, LOCK_LEVEL + 1, %o4	! convert PIL to array index
11537c478bdstevel@tonic-gate	sllx    %o4, 3, %o4			! index to byte offset
11547c478bdstevel@tonic-gate	add	%o4, CPU_MCPU, %o4	! CPU_PIL_HIGH_START is too large
11557c478bdstevel@tonic-gate	add	%o4, MCPU_PIL_HIGH_START, %o4
1156bd28a47Prashanth Sreenivasa	RD_CLOCK_TICK_NO_SUSPEND_CHECK(%o5, %l2)
11577c478bdstevel@tonic-gate        stx     %o5, [%o3 + %o4]
11587c478bdstevel@tonic-gate
11597c478bdstevel@tonic-gate	wrpr	%g0, %o2, %pil			! enable interrupts
11607c478bdstevel@tonic-gate
11617c478bdstevel@tonic-gate	!
11627c478bdstevel@tonic-gate	! call the handler
11637c478bdstevel@tonic-gate	!
11647c478bdstevel@tonic-gate	SERVE_INTR_PRE(%o1, %o3, %l2, %l3, %o4, %o5, %o2, %o0)
11657c478bdstevel@tonic-gate1:
11667c478bdstevel@tonic-gate	SERVE_INTR(%o1, %o3, %l2, %l3, %o4, %o5, %o2, %o0)
11677c478bdstevel@tonic-gate
11687c478bdstevel@tonic-gate	brz,a,pt %o2, 0f			! if %o2, more intrs await
11697c478bdstevel@tonic-gate	rdpr	%pil, %o2			! delay annulled
11707c478bdstevel@tonic-gate	SERVE_INTR_NEXT(%o1, %o3, %l2, %l3, %o4, %o5, %o2, %o0)
11717c478bdstevel@tonic-gate	ba	1b
11727c478bdstevel@tonic-gate	nop
11737c478bdstevel@tonic-gate0:
11747c478bdstevel@tonic-gate	wrpr	%g0, PIL_MAX, %pil		! disable interrupts (1-15)
11757c478bdstevel@tonic-gate
11767c478bdstevel@tonic-gate	cmp	%o2, PIL_15
11777c478bdstevel@tonic-gate	bne,pt	%xcc, 3f
11787c478bdstevel@tonic-gate	nop
11797c478bdstevel@tonic-gate
11807c478bdstevel@tonic-gate	sethi	%hi(cpc_level15_inum), %o1
1181b0fc0e7govinda	ldx	[%o1 + %lo(cpc_level15_inum)], %o1 ! arg for intr_enqueue_req
11827c478bdstevel@tonic-gate	brz	%o1, 3f
11837c478bdstevel@tonic-gate	nop
11847c478bdstevel@tonic-gate
11857c478bdstevel@tonic-gate	rdpr 	%pstate, %g5
11867c478bdstevel@tonic-gate	andn	%g5, PSTATE_IE, %g1
11877c478bdstevel@tonic-gate	wrpr	%g0, %g1, %pstate		! Disable vec interrupts
11887c478bdstevel@tonic-gate
11897c478bdstevel@tonic-gate	call	intr_enqueue_req		! preserves %g5
11907c478bdstevel@tonic-gate	mov	PIL_15, %o0
11917c478bdstevel@tonic-gate
11927c478bdstevel@tonic-gate	! clear perfcntr overflow
11937c478bdstevel@tonic-gate	mov	1, %o0
11947c478bdstevel@tonic-gate	sllx	%o0, PIL_15, %o0
11957c478bdstevel@tonic-gate	wr	%o0, CLEAR_SOFTINT
11967c478bdstevel@tonic-gate
11977c478bdstevel@tonic-gate	wrpr	%g0, %g5, %pstate		! Enable vec interrupts
11987c478bdstevel@tonic-gate
11997c478bdstevel@tonic-gate3:
12007c478bdstevel@tonic-gate	cmp	%o2, PIL_14
12017c478bdstevel@tonic-gate	be	tick_rtt			!  cpu-specific tick processing
12027c478bdstevel@tonic-gate	nop
12037c478bdstevel@tonic-gate	.global	current_thread_complete
12047c478bdstevel@tonic-gatecurrent_thread_complete:
12057c478bdstevel@tonic-gate	!
12067c478bdstevel@tonic-gate	! Register usage:
12077c478bdstevel@tonic-gate	!
12087c478bdstevel@tonic-gate	! %l1 = stack pointer
12097c478bdstevel@tonic-gate	! %l2 = CPU_INTR_ACTV >> (LOCK_LEVEL + 1)
12107c478bdstevel@tonic-gate	! %o2 = PIL
12117c478bdstevel@tonic-gate	! %o3 = CPU pointer
12127c478bdstevel@tonic-gate	! %o4, %o5, %l3, %l4, %l5 = scratch
12137c478bdstevel@tonic-gate	!
12147c478bdstevel@tonic-gate	ldn	[THREAD_REG + T_CPU], %o3
12157c478bdstevel@tonic-gate	!
12167c478bdstevel@tonic-gate	! Clear bit for this level in CPU's interrupt active bitmask.
12177c478bdstevel@tonic-gate	!
12187c478bdstevel@tonic-gate	ld	[%o3 + CPU_INTR_ACTV], %l2
12197c478bdstevel@tonic-gate	mov	1, %o5
12207c478bdstevel@tonic-gate	sll	%o5, %o2, %o5
12217c478bdstevel@tonic-gate#ifdef DEBUG
12227c478bdstevel@tonic-gate	!
12237c478bdstevel@tonic-gate	! ASSERT(CPU->cpu_intr_actv & (1 << PIL))
12247c478bdstevel@tonic-gate	!
12257c478bdstevel@tonic-gate	andcc	%l2, %o5, %g0
12267c478bdstevel@tonic-gate	bnz,pt	%xcc, 0f
12277c478bdstevel@tonic-gate	nop
12287c478bdstevel@tonic-gate	! Do not call panic if a panic is already in progress.
12297c478bdstevel@tonic-gate	sethi	%hi(panic_quiesce), %l2
12307c478bdstevel@tonic-gate	ld	[%l2 + %lo(panic_quiesce)], %l2
12317c478bdstevel@tonic-gate	brnz,pn	%l2, 0f
12327c478bdstevel@tonic-gate	nop
12337c478bdstevel@tonic-gate	sethi	%hi(current_thread_actv_bit_not_set), %o0
12347c478bdstevel@tonic-gate	call	panic
12357c478bdstevel@tonic-gate	or	%o0, %lo(current_thread_actv_bit_not_set), %o0
12367c478bdstevel@tonic-gate0:
12377c478bdstevel@tonic-gate#endif /* DEBUG */
12387c478bdstevel@tonic-gate	andn	%l2, %o5, %l2
12397c478bdstevel@tonic-gate	st	%l2, [%o3 + CPU_INTR_ACTV]
12407c478bdstevel@tonic-gate
12417c478bdstevel@tonic-gate	! Take timestamp, compute interval, update cumulative counter.
12427c478bdstevel@tonic-gate        sub     %o2, LOCK_LEVEL + 1, %o4	! PIL to array index
12437c478bdstevel@tonic-gate	sllx    %o4, 3, %o4			! index to byte offset
12447c478bdstevel@tonic-gate	add	%o4, CPU_MCPU, %o4	! CPU_PIL_HIGH_START is too large
12457c478bdstevel@tonic-gate	add	%o4, MCPU_PIL_HIGH_START, %o4
1246bd28a47Prashanth Sreenivasa	RD_CLOCK_TICK_NO_SUSPEND_CHECK(%o5, %o0)
12477c478bdstevel@tonic-gate	ldx     [%o3 + %o4], %o0
12487c478bdstevel@tonic-gate#ifdef DEBUG
12497c478bdstevel@tonic-gate	! ASSERT(cpu.cpu_m.pil_high_start[pil - (LOCK_LEVEL + 1)] != 0)
12507c478bdstevel@tonic-gate	brnz,pt	%o0, 9f
12517c478bdstevel@tonic-gate	nop
12527c478bdstevel@tonic-gate	! Don't panic if a panic is already in progress.
12537c478bdstevel@tonic-gate	sethi	%hi(panic_quiesce), %l2
12547c478bdstevel@tonic-gate	ld	[%l2 + %lo(panic_quiesce)], %l2
12557c478bdstevel@tonic-gate	brnz,pn	%l2, 9f
12567c478bdstevel@tonic-gate	nop
12577c478bdstevel@tonic-gate	sethi	%hi(current_thread_timestamp_zero), %o0
12587c478bdstevel@tonic-gate	call	panic
12597c478bdstevel@tonic-gate	or	%o0, %lo(current_thread_timestamp_zero), %o0
12607c478bdstevel@tonic-gate9:
12617c478bdstevel@tonic-gate#endif /* DEBUG */
12627c478bdstevel@tonic-gate	stx	%g0, [%o3 + %o4]
12637c478bdstevel@tonic-gate	sub	%o5, %o0, %o5			! interval in o5
12647c478bdstevel@tonic-gate
12657c478bdstevel@tonic-gate	! Check for Energy Star mode
12667c478bdstevel@tonic-gate	lduh	[%o3 + CPU_DIVISOR], %o4	! %o4 = clock divisor
12677c478bdstevel@tonic-gate	cmp	%o4, 1
12687c478bdstevel@tonic-gate	bg,a,pn	%xcc, 2f
12697c478bdstevel@tonic-gate	mulx	%o5, %o4, %o5	! multiply interval by clock divisor iff > 1
12707c478bdstevel@tonic-gate2:
12717c478bdstevel@tonic-gate	sllx	%o2, 4, %o4			! PIL index to byte offset
12727c478bdstevel@tonic-gate	add	%o4, CPU_MCPU, %o4		! CPU_INTRSTAT too large
12737c478bdstevel@tonic-gate	add	%o4, MCPU_INTRSTAT, %o4		! add parts separately
12747c478bdstevel@tonic-gate	ldx	[%o3 + %o4], %o0		! old counter in o0
12757c478bdstevel@tonic-gate	add	%o0, %o5, %o0			! new counter in o0
12767c478bdstevel@tonic-gate	stx	%o0, [%o3 + %o4]		! store new counter
1277eda8946esolom
1278eda8946esolom	! Also update intracct[]
1279eda8946esolom	lduh	[%o3 + CPU_MSTATE], %o4
1280eda8946esolom	sllx	%o4, 3, %o4
1281eda8946esolom	add	%o4, CPU_INTRACCT, %o4
1282eda8946esolom	ldx	[%o3 + %o4], %o0
1283eda8946esolom	add	%o0, %o5, %o0
1284eda8946esolom	stx	%o0, [%o3 + %o4]
12857c478bdstevel@tonic-gate
12867c478bdstevel@tonic-gate	!
12877c478bdstevel@tonic-gate	! get back on current thread's stack
12887c478bdstevel@tonic-gate	!
12897c478bdstevel@tonic-gate	srl	%l2, LOCK_LEVEL + 1, %l2
12907c478bdstevel@tonic-gate	tst	%l2				! any more high-level ints?
12917c478bdstevel@tonic-gate	movz	%xcc, %l1, %sp
12927c478bdstevel@tonic-gate	!
12937c478bdstevel@tonic-gate	! Current register usage:
12947c478bdstevel@tonic-gate	! o2 = PIL
12957c478bdstevel@tonic-gate	! o3 = CPU pointer
12967c478bdstevel@tonic-gate	! l0 = return address
12977c478bdstevel@tonic-gate	! l2 = intr_actv shifted right
12987c478bdstevel@tonic-gate	!
12997c478bdstevel@tonic-gate	bz,pt	%xcc, 3f			! if l2 was zero, no more ints
13007c478bdstevel@tonic-gate	nop
13017c478bdstevel@tonic-gate	!
13027c478bdstevel@tonic-gate	! We found another high-level interrupt active below the one that just
13037c478bdstevel@tonic-gate	! returned. Store a starting timestamp for it in the CPU structure.
13047c478bdstevel@tonic-gate	!
13057c478bdstevel@tonic-gate	! Use cpu_intr_actv to find the cpu_pil_high_start[] offset for the
13067c478bdstevel@tonic-gate	! interrupted high-level interrupt.
13077c478bdstevel@tonic-gate	! Create mask for cpu_intr_actv. Begin by looking for bits set
13087c478bdstevel@tonic-gate	! at one level below the current PIL. Since %l2 contains the active
13097c478bdstevel@tonic-gate	! mask already shifted right by (LOCK_LEVEL + 1), we start by looking
13107c478bdstevel@tonic-gate	! at bit (current_pil - (LOCK_LEVEL + 2)).
13117c478bdstevel@tonic-gate	! %l1 = mask, %o5 = index of bit set in mask
13127c478bdstevel@tonic-gate	!
13137c478bdstevel@tonic-gate	mov	1, %l1
13147c478bdstevel@tonic-gate	sub	%o2, LOCK_LEVEL + 2, %o5
13157c478bdstevel@tonic-gate	sll	%l1, %o5, %l1			! l1 = mask for level
13167c478bdstevel@tonic-gate1:
13177c478bdstevel@tonic-gate#ifdef DEBUG
13187c478bdstevel@tonic-gate	! ASSERT(%l1 != 0) (we didn't shift the bit off the right edge)
13197c478bdstevel@tonic-gate	brnz,pt	%l1, 9f
13207c478bdstevel@tonic-gate	nop
13217c478bdstevel@tonic-gate	sethi	%hi(current_thread_nested_PIL_not_found), %o0
13227c478bdstevel@tonic-gate	call	panic
13237c478bdstevel@tonic-gate	or	%o0, %lo(current_thread_nested_PIL_not_found), %o0
13247c478bdstevel@tonic-gate9:
13257c478bdstevel@tonic-gate#endif /* DEBUG */
13267c478bdstevel@tonic-gate	andcc	%l1, %l2, %g0		! test mask against high-level bits of
13277c478bdstevel@tonic-gate	bnz	%xcc, 2f		! cpu_intr_actv
13287c478bdstevel@tonic-gate	nop
13297c478bdstevel@tonic-gate	srl	%l1, 1, %l1		! No match. Try next lower PIL.
13307c478bdstevel@tonic-gate	ba,pt	%xcc, 1b
13317c478bdstevel@tonic-gate	sub	%o5, 1, %o5		! delay - decrement PIL
13327c478bdstevel@tonic-gate2:
13337c478bdstevel@tonic-gate	sll	%o5, 3, %o5		! convert array index to byte offset
13347c478bdstevel@tonic-gate	add	%o5, CPU_MCPU, %o5	! CPU_PIL_HIGH_START is too large
13357c478bdstevel@tonic-gate	add	%o5, MCPU_PIL_HIGH_START, %o5
1336bd28a47Prashanth Sreenivasa	RD_CLOCK_TICK_NO_SUSPEND_CHECK(%o4, %l2)
13377c478bdstevel@tonic-gate	! Another high-level interrupt is active below this one, so
13387c478bdstevel@tonic-gate	! there is no need to check for an interrupt thread. That will be
13397c478bdstevel@tonic-gate	! done by the lowest priority high-level interrupt active.
13402604657vb	ba,pt	%xcc, 7f
13417c478bdstevel@tonic-gate	stx	%o4, [%o3 + %o5]	! delay - store timestamp
13427c478bdstevel@tonic-gate3:
13437c478bdstevel@tonic-gate	! If we haven't interrupted another high-level interrupt, we may have
13447c478bdstevel@tonic-gate	! interrupted a low level interrupt thread. If so, store a starting
13457c478bdstevel@tonic-gate	! timestamp in its thread structure.
13467c478bdstevel@tonic-gate	lduh	[THREAD_REG + T_FLAGS], %o4
13477c478bdstevel@tonic-gate	andcc	%o4, T_INTR_THREAD, %g0
13482604657vb	bz,pt	%xcc, 7f
13497c478bdstevel@tonic-gate	nop
13507c478bdstevel@tonic-gate
1351bd28a47Prashanth Sreenivasa	RD_CLOCK_TICK_NO_SUSPEND_CHECK(%o4, %l2)
13527c478bdstevel@tonic-gate	stx	%o4, [THREAD_REG + T_INTR_START]
13532604657vb
13542604657vb7:
13552604657vb	sub	%o2, LOCK_LEVEL + 1, %o4
13562604657vb	sll	%o4, CPTRSHIFT, %o5
13572604657vb
13582604657vb	! Check on_trap saved area and restore as needed
13592604657vb	add	%o5, CPU_OTD, %o4
13602604657vb	ldn	[%o3 + %o4], %l2
13612604657vb	brz,pt %l2, no_ontrp_restore
13622604657vb	nop
13632604657vb	stn	%l2, [THREAD_REG + T_ONTRAP] ! restore
13642604657vb	stn	%g0, [%o3 + %o4]	! clear
13652604657vb
13662604657vbno_ontrp_restore:
13672604657vb	! Check on_fault saved area and restore as needed
13682604657vb	add	%o5, CPU_OFD, %o4
13692604657vb	ldn	[%o3 + %o4], %l2
13702604657vb	brz,pt %l2, 8f
13712604657vb	nop
13722604657vb	stn	%l2, [THREAD_REG + T_ONFAULT] ! restore
13732604657vb	stn	%g0, [%o3 + %o4]	! clear
13742604657vb	add	%o5, CPU_LFD, %o4
13752604657vb	ldn	[%o3 + %o4], %l2
13762604657vb	stn	%l2, [THREAD_REG + T_LOFAULT] ! restore
13772604657vb	stn	%g0, [%o3 + %o4]	! clear
13782604657vb
13792604657vb
13802604657vb8:
13817c478bdstevel@tonic-gate	! Enable interrupts and return
13827c478bdstevel@tonic-gate	jmp	%l0 + 8
13837c478bdstevel@tonic-gate	wrpr	%g0, %o2, %pil			! enable interrupts
13847c478bdstevel@tonic-gate	SET_SIZE(current_thread)
13857c478bdstevel@tonic-gate
13867c478bdstevel@tonic-gate
13877c478bdstevel@tonic-gate#ifdef DEBUG
13887c478bdstevel@tonic-gatecurrent_thread_wrong_pil:
13897c478bdstevel@tonic-gate	.asciz	"current_thread: unexpected pil level: %d"
13907c478bdstevel@tonic-gatecurrent_thread_actv_bit_set:
13917c478bdstevel@tonic-gate	.asciz	"current_thread(): cpu_intr_actv bit already set for PIL"
13927c478bdstevel@tonic-gatecurrent_thread_actv_bit_not_set:
13937c478bdstevel@tonic-gate	.asciz	"current_thread(): cpu_intr_actv bit not set for PIL"
13947c478bdstevel@tonic-gatecurrent_thread_nested_pil_zero:
13957c478bdstevel@tonic-gate	.asciz	"current_thread(): timestamp zero for nested PIL %d"
13967c478bdstevel@tonic-gatecurrent_thread_timestamp_zero:
13977c478bdstevel@tonic-gate	.asciz	"current_thread(): timestamp zero upon handler return"
13987c478bdstevel@tonic-gatecurrent_thread_nested_PIL_not_found:
13997c478bdstevel@tonic-gate	.asciz	"current_thread: couldn't find nested high-level PIL"
14007c478bdstevel@tonic-gate#endif /* DEBUG */
14017c478bdstevel@tonic-gate#endif /* lint */
14027c478bdstevel@tonic-gate
14037c478bdstevel@tonic-gate/*
14047c478bdstevel@tonic-gate * Return a thread's interrupt level.
14057c478bdstevel@tonic-gate * Since this isn't saved anywhere but in %l4 on interrupt entry, we
14067c478bdstevel@tonic-gate * must dig it out of the save area.
14077c478bdstevel@tonic-gate *
14087c478bdstevel@tonic-gate * Caller 'swears' that this really is an interrupt thread.
14097c478bdstevel@tonic-gate *
14107c478bdstevel@tonic-gate * int
14117c478bdstevel@tonic-gate * intr_level(t)
14127c478bdstevel@tonic-gate *	kthread_id_t	t;
14137c478bdstevel@tonic-gate */
14147c478bdstevel@tonic-gate
14157c478bdstevel@tonic-gate#if defined(lint)
14167c478bdstevel@tonic-gate
14177c478bdstevel@tonic-gate/* ARGSUSED */
14187c478bdstevel@tonic-gateint
14197c478bdstevel@tonic-gateintr_level(kthread_id_t t)
14207c478bdstevel@tonic-gate{ return (0); }
14217c478bdstevel@tonic-gate
14227c478bdstevel@tonic-gate#else	/* lint */
14237c478bdstevel@tonic-gate
14247c478bdstevel@tonic-gate	ENTRY_NP(intr_level)
14257c478bdstevel@tonic-gate	retl
14267c478bdstevel@tonic-gate	ldub	[%o0 + T_PIL], %o0		! return saved pil
14277c478bdstevel@tonic-gate	SET_SIZE(intr_level)
14287c478bdstevel@tonic-gate
14297c478bdstevel@tonic-gate#endif	/* lint */
14307c478bdstevel@tonic-gate
14317c478bdstevel@tonic-gate#if defined(lint)
14327c478bdstevel@tonic-gate
14337c478bdstevel@tonic-gate/* ARGSUSED */
14347c478bdstevel@tonic-gateint
14357c478bdstevel@tonic-gatedisable_pil_intr()
14367c478bdstevel@tonic-gate{ return (0); }
14377c478bdstevel@tonic-gate
14387c478bdstevel@tonic-gate#else	/* lint */
14397c478bdstevel@tonic-gate
14407c478bdstevel@tonic-gate	ENTRY_NP(disable_pil_intr)
14417c478bdstevel@tonic-gate	rdpr	%pil, %o0
14427c478bdstevel@tonic-gate	retl
14437c478bdstevel@tonic-gate	wrpr	%g0, PIL_MAX, %pil		! disable interrupts (1-15)
14447c478bdstevel@tonic-gate	SET_SIZE(disable_pil_intr)
14457c478bdstevel@tonic-gate
14467c478bdstevel@tonic-gate#endif	/* lint */
14477c478bdstevel@tonic-gate
14487c478bdstevel@tonic-gate#if defined(lint)
14497c478bdstevel@tonic-gate
14507c478bdstevel@tonic-gate/* ARGSUSED */
14517c478bdstevel@tonic-gatevoid
14527c478bdstevel@tonic-gateenable_pil_intr(int pil_save)
14537c478bdstevel@tonic-gate{}
14547c478bdstevel@tonic-gate
14557c478bdstevel@tonic-gate#else	/* lint */
14567c478bdstevel@tonic-gate
14577c478bdstevel@tonic-gate	ENTRY_NP(enable_pil_intr)
14587c478bdstevel@tonic-gate	retl
14597c478bdstevel@tonic-gate	wrpr	%o0, %pil
14607c478bdstevel@tonic-gate	SET_SIZE(enable_pil_intr)
14617c478bdstevel@tonic-gate
14627c478bdstevel@tonic-gate#endif	/* lint */
14637c478bdstevel@tonic-gate
14647c478bdstevel@tonic-gate#if defined(lint)
14657c478bdstevel@tonic-gate
14667c478bdstevel@tonic-gate/* ARGSUSED */
14677c478bdstevel@tonic-gateuint_t
14687c478bdstevel@tonic-gatedisable_vec_intr(void)
14697c478bdstevel@tonic-gate{ return (0); }
14707c478bdstevel@tonic-gate
14717c478bdstevel@tonic-gate#else	/* lint */
14727c478bdstevel@tonic-gate
14737c478bdstevel@tonic-gate	ENTRY_NP(disable_vec_intr)
14747c478bdstevel@tonic-gate	rdpr	%pstate, %o0
14757c478bdstevel@tonic-gate	andn	%o0, PSTATE_IE, %g1
14767c478bdstevel@tonic-gate	retl
14777c478bdstevel@tonic-gate	wrpr	%g0, %g1, %pstate		! disable interrupt
14787c478bdstevel@tonic-gate	SET_SIZE(disable_vec_intr)
14797c478bdstevel@tonic-gate
14807c478bdstevel@tonic-gate#endif	/* lint */
14817c478bdstevel@tonic-gate
14827c478bdstevel@tonic-gate#if defined(lint)
14837c478bdstevel@tonic-gate
14847c478bdstevel@tonic-gate/* ARGSUSED */
14857c478bdstevel@tonic-gatevoid
14867c478bdstevel@tonic-gateenable_vec_intr(uint_t pstate_save)
14877c478bdstevel@tonic-gate{}
14887c478bdstevel@tonic-gate
14897c478bdstevel@tonic-gate#else	/* lint */
14907c478bdstevel@tonic-gate
14917c478bdstevel@tonic-gate	ENTRY_NP(enable_vec_intr)
14927c478bdstevel@tonic-gate	retl
14937c478bdstevel@tonic-gate	wrpr	%g0, %o0, %pstate
14947c478bdstevel@tonic-gate	SET_SIZE(enable_vec_intr)
14957c478bdstevel@tonic-gate
14967c478bdstevel@tonic-gate#endif	/* lint */
14977c478bdstevel@tonic-gate
14987c478bdstevel@tonic-gate#if defined(lint)
14997c478bdstevel@tonic-gate
15007c478bdstevel@tonic-gatevoid
15017c478bdstevel@tonic-gatecbe_level14(void)
15027c478bdstevel@tonic-gate{}
15037c478bdstevel@tonic-gate
15047c478bdstevel@tonic-gate#else   /* lint */
15057c478bdstevel@tonic-gate
15067c478bdstevel@tonic-gate	ENTRY_NP(cbe_level14)
15077c478bdstevel@tonic-gate	save    %sp, -SA(MINFRAME), %sp ! get a new window
15087c478bdstevel@tonic-gate	!
15097c478bdstevel@tonic-gate	! Make sure that this is from TICK_COMPARE; if not just return
15107c478bdstevel@tonic-gate	!
15117c478bdstevel@tonic-gate	rd	SOFTINT, %l1
15127c478bdstevel@tonic-gate	set	(TICK_INT_MASK | STICK_INT_MASK), %o2
15137c478bdstevel@tonic-gate	andcc	%l1, %o2, %g0
15147c478bdstevel@tonic-gate	bz,pn	%icc, 2f
15157c478bdstevel@tonic-gate	nop
15167c478bdstevel@tonic-gate
15177c478bdstevel@tonic-gate	CPU_ADDR(%o1, %o2)
15187c478bdstevel@tonic-gate	call	cyclic_fire
15197c478bdstevel@tonic-gate	mov	%o1, %o0
15207c478bdstevel@tonic-gate2:
15217c478bdstevel@tonic-gate	ret
15227c478bdstevel@tonic-gate	restore	%g0, 1, %o0
15237c478bdstevel@tonic-gate	SET_SIZE(cbe_level14)
15247c478bdstevel@tonic-gate
15257c478bdstevel@tonic-gate#endif  /* lint */
15267c478bdstevel@tonic-gate
15277c478bdstevel@tonic-gate
15287c478bdstevel@tonic-gate#if defined(lint)
15297c478bdstevel@tonic-gate
15307c478bdstevel@tonic-gate/* ARGSUSED */
15317c478bdstevel@tonic-gatevoid
1532a1af7bacwbkdi_setsoftint(uint64_t iv_p)
15337c478bdstevel@tonic-gate{}
15347c478bdstevel@tonic-gate
15357c478bdstevel@tonic-gate#else	/* lint */
15367c478bdstevel@tonic-gate
1537a1af7bacwb	ENTRY_NP(kdi_setsoftint)
15387c478bdstevel@tonic-gate	save	%sp, -SA(MINFRAME), %sp	! get a new window
15397c478bdstevel@tonic-gate	rdpr	%pstate, %l5
15407c478bdstevel@tonic-gate	andn	%l5, PSTATE_IE, %l1
15417c478bdstevel@tonic-gate	wrpr	%l1, %pstate		! disable interrupt
15427c478bdstevel@tonic-gate	!
1543b0fc0e7govinda	! We have a pointer to an interrupt vector data structure.
1544b0fc0e7govinda	! Put the request on the cpu's softint priority list and
1545b0fc0e7govinda	! set %set_softint.
15467c478bdstevel@tonic-gate	!
15477c478bdstevel@tonic-gate	! Register usage
1548b0fc0e7govinda	! 	%i0 - pointer to intr_vec_t (iv)
15497c478bdstevel@tonic-gate	!	%l2 - requested pil
1550b0fc0e7govinda	!	%l4 - cpu
1551b0fc0e7govinda	!	%l5 - pstate
1552b0fc0e7govinda	!	%l1, %l3, %l6 - temps
15537c478bdstevel@tonic-gate	!
1554b0fc0e7govinda	! check if a softint is pending for this softint,
1555b0fc0e7govinda	! if one is pending, don't bother queuing another.
15567c478bdstevel@tonic-gate	!
1557b0fc0e7govinda	lduh	[%i0 + IV_FLAGS], %l1	! %l1 = iv->iv_flags
1558b0fc0e7govinda	and	%l1, IV_SOFTINT_PEND, %l6 ! %l6 = iv->iv_flags & IV_SOFTINT_PEND
1559b0fc0e7govinda	brnz,pn	%l6, 4f			! branch if softint is already pending
1560b0fc0e7govinda	or	%l1, IV_SOFTINT_PEND, %l2
1561b0fc0e7govinda	sth	%l2, [%i0 + IV_FLAGS]	! Set IV_SOFTINT_PEND flag
1562b0fc0e7govinda
1563b0fc0e7govinda	CPU_ADDR(%l4, %l2)		! %l4 = cpu
1564b0fc0e7govinda	lduh	[%i0 + IV_PIL], %l2	! %l2 = iv->iv_pil
1565b0fc0e7govinda
15667c478bdstevel@tonic-gate	!
1567b0fc0e7govinda	! Insert intr_vec_t (iv) to appropriate cpu's softint priority list
15687c478bdstevel@tonic-gate	!
1569b0fc0e7govinda	sll	%l2, CPTRSHIFT, %l0	! %l0 = offset to pil entry
1570b0fc0e7govinda	add	%l4, INTR_TAIL, %l6	! %l6 = &cpu->m_cpu.intr_tail
1571b0fc0e7govinda	ldn	[%l6 + %l0], %l1	! %l1 = cpu->m_cpu.intr_tail[pil]
1572b0fc0e7govinda					!       current tail (ct)
1573b0fc0e7govinda	brz,pt	%l1, 2f			! branch if current tail is NULL
1574b0fc0e7govinda	stn	%i0, [%l6 + %l0]	! make intr_vec_t (iv) as new tail
15757c478bdstevel@tonic-gate	!
1576b0fc0e7govinda	! there's pending intr_vec_t already
15777c478bdstevel@tonic-gate	!
1578b0fc0e7govinda	lduh	[%l1 + IV_FLAGS], %l6	! %l6 = ct->iv_flags
1579b0fc0e7govinda	and	%l6, IV_SOFTINT_MT, %l6	! %l6 = ct->iv_flags & IV_SOFTINT_MT
1580b0fc0e7govinda	brz,pt	%l6, 1f			! check for Multi target softint flag
1581b0fc0e7govinda	add	%l1, IV_PIL_NEXT, %l3	! %l3 = &ct->iv_pil_next
1582b0fc0e7govinda	ld	[%l4 + CPU_ID], %l6	! for multi target softint, use cpuid
1583b0fc0e7govinda	sll	%l6, CPTRSHIFT, %l6	! calculate offset address from cpuid
1584b0fc0e7govinda	add	%l3, %l6, %l3		! %l3 =  &ct->iv_xpil_next[cpuid]
1585b0fc0e7govinda1:
15867c478bdstevel@tonic-gate	!
1587b0fc0e7govinda	! update old tail
15887c478bdstevel@tonic-gate	!
15897c478bdstevel@tonic-gate	ba,pt	%xcc, 3f
1590b0fc0e7govinda	stn	%i0, [%l3]		! [%l3] = iv, set pil_next field
15917c478bdstevel@tonic-gate2:
15927c478bdstevel@tonic-gate	!
1593b0fc0e7govinda	! no pending intr_vec_t; make intr_vec_t as new head
15947c478bdstevel@tonic-gate	!
1595b0fc0e7govinda	add	%l4, INTR_HEAD, %l6	! %l6 = &cpu->m_cpu.intr_head[pil]
1596b0fc0e7govinda	stn	%i0, [%l6 + %l0]	! cpu->m_cpu.intr_head[pil] = iv
15977c478bdstevel@tonic-gate3:
15987c478bdstevel@tonic-gate	!
15997c478bdstevel@tonic-gate	! Write %set_softint with (1<<pil) to cause a "pil" level trap
16007c478bdstevel@tonic-gate	!
1601b0fc0e7govinda	mov	1, %l1			! %l1 = 1
1602b0fc0e7govinda	sll	%l1, %l2, %l1		! %l1 = 1 << pil
1603b0fc0e7govinda	wr	%l1, SET_SOFTINT	! trigger required pil softint
16047c478bdstevel@tonic-gate4:
1605b0fc0e7govinda	wrpr	%g0, %l5, %pstate	! %pstate = saved %pstate (in %l5)
16067c478bdstevel@tonic-gate	ret
16077c478bdstevel@tonic-gate	restore
1608a1af7bacwb	SET_SIZE(kdi_setsoftint)
1609b0fc0e7govinda
16107c478bdstevel@tonic-gate#endif	/* lint */
16117c478bdstevel@tonic-gate
16127c478bdstevel@tonic-gate#if defined(lint)
16137c478bdstevel@tonic-gate
16147c478bdstevel@tonic-gate/*ARGSUSED*/
16157c478bdstevel@tonic-gatevoid
1616b0fc0e7govindasetsoftint_tl1(uint64_t iv_p, uint64_t dummy)
16177c478bdstevel@tonic-gate{}
16187c478bdstevel@tonic-gate
16197c478bdstevel@tonic-gate#else	/* lint */
16207c478bdstevel@tonic-gate
16217c478bdstevel@tonic-gate	!
16227c478bdstevel@tonic-gate	! Register usage
1623b0fc0e7govinda	!	Arguments:
1624b0fc0e7govinda	! 	%g1 - Pointer to intr_vec_t (iv)
16257c478bdstevel@tonic-gate	!
1626b0fc0e7govinda	!	Internal:
1627b0fc0e7govinda	!	%g2 - pil
1628b0fc0e7govinda	!	%g4 - cpu
1629b0fc0e7govinda	!	%g3,%g5-g7 - temps
16307c478bdstevel@tonic-gate	!
16317c478bdstevel@tonic-gate	ENTRY_NP(setsoftint_tl1)
16327c478bdstevel@tonic-gate	!
1633b0fc0e7govinda	! We have a pointer to an interrupt vector data structure.
1634b0fc0e7govinda	! Put the request on the cpu's softint priority list and
1635b0fc0e7govinda	! set %set_softint.
16367c478bdstevel@tonic-gate	!
1637b0fc0e7govinda	CPU_ADDR(%g4, %g2)		! %g4 = cpu
1638b0fc0e7govinda	lduh	[%g1 + IV_PIL], %g2	! %g2 = iv->iv_pil
1639b0fc0e7govinda
16407c478bdstevel@tonic-gate	!
1641b0fc0e7govinda	! Insert intr_vec_t (iv) to appropriate cpu's softint priority list
16427c478bdstevel@tonic-gate	!
1643b0fc0e7govinda	sll	%g2, CPTRSHIFT, %g7	! %g7 = offset to pil entry
1644b0fc0e7govinda	add	%g4, INTR_TAIL, %g6	! %g6 = &cpu->m_cpu.intr_tail
1645b0fc0e7govinda	ldn	[%g6 + %g7], %g5	! %g5 = cpu->m_cpu.intr_tail[pil]
1646b0fc0e7govinda					!       current tail (ct)
1647b0fc0e7govinda	brz,pt	%g5, 1f			! branch if current tail is NULL
1648b0fc0e7govinda	stn	%g1, [%g6 + %g7]	! make intr_rec_t (iv) as new tail
16497c478bdstevel@tonic-gate	!
1650b0fc0e7govinda	! there's pending intr_vec_t already
1651b0fc0e7govinda	!
1652b0fc0e7govinda	lduh	[%g5 + IV_FLAGS], %g6	! %g6 = ct->iv_flags
1653b0fc0e7govinda	and	%g6, IV_SOFTINT_MT, %g6	! %g6 = ct->iv_flags & IV_SOFTINT_MT
1654b0fc0e7govinda	brz,pt	%g6, 0f			! check for Multi target softint flag
1655b0fc0e7govinda	add	%g5, IV_PIL_NEXT, %g3	! %g3 = &ct->iv_pil_next
1656b0fc0e7govinda	ld	[%g4 + CPU_ID], %g6	! for multi target softint, use cpuid
1657b0fc0e7govinda	sll	%g6, CPTRSHIFT, %g6	! calculate offset address from cpuid
1658b0fc0e7govinda	add	%g3, %g6, %g3		! %g3 = &ct->iv_xpil_next[cpuid]
1659b0fc0e7govinda0:
1660b0fc0e7govinda	!
1661b0fc0e7govinda	! update old tail
1662b0fc0e7govinda	!
1663b0fc0e7govinda	ba,pt	%xcc, 2f
1664b0fc0e7govinda	stn	%g1, [%g3]		! [%g3] = iv, set pil_next field
1665b0fc0e7govinda1:
1666b0fc0e7govinda	!
1667b0fc0e7govinda	! no pending intr_vec_t; make intr_vec_t as new head
16687c478bdstevel@tonic-gate	!
1669b0fc0e7govinda	add	%g4, INTR_HEAD, %g6	! %g6 = &cpu->m_cpu.intr_head[pil]
1670b0fc0e7govinda	stn	%g1, [%g6 + %g7]	! cpu->m_cpu.intr_head[pil] = iv
1671b0fc0e7govinda2:
1672b0fc0e7govinda#ifdef TRAPTRACE
1673b0fc0e7govinda	TRACE_PTR(%g5, %g6)
1674023e71dHaik Aftandilian	GET_TRACE_TICK(%g6, %g3)
1675b0fc0e7govinda	stxa	%g6, [%g5 + TRAP_ENT_TICK]%asi	! trap_tick = %tick
1676b0fc0e7govinda	TRACE_SAVE_TL_GL_REGS(%g5, %g6)
1677b0fc0e7govinda	rdpr	%tt, %g6
1678b0fc0e7govinda	stha	%g6, [%g5 + TRAP_ENT_TT]%asi	! trap_type = %tt
1679b0fc0e7govinda	rdpr	%tpc, %g6
1680b0fc0e7govinda	stna	%g6, [%g5 + TRAP_ENT_TPC]%asi	! trap_pc = %tpc
1681b0fc0e7govinda	rdpr	%tstate, %g6
1682b0fc0e7govinda	stxa	%g6, [%g5 + TRAP_ENT_TSTATE]%asi ! trap_tstate = %tstate
1683b0fc0e7govinda	stna	%sp, [%g5 + TRAP_ENT_SP]%asi	! trap_sp = %sp
1684b0fc0e7govinda	stna	%g1, [%g5 + TRAP_ENT_TR]%asi	! trap_tr = iv
1685b0fc0e7govinda	ldn	[%g1 + IV_PIL_NEXT], %g6	!
1686b0fc0e7govinda	stna	%g6, [%g5 + TRAP_ENT_F1]%asi	! trap_f1 = iv->iv_pil_next
1687b0fc0e7govinda	add	%g4, INTR_HEAD, %g6
1688b0fc0e7govinda	ldn	[%g6 + %g7], %g6		! %g6=cpu->m_cpu.intr_head[pil]
1689b0fc0e7govinda	stna	%g6, [%g5 + TRAP_ENT_F2]%asi	! trap_f2 = intr_head[pil]
1690b0fc0e7govinda	add	%g4, INTR_TAIL, %g6
1691b0fc0e7govinda	ldn	[%g6 + %g7], %g6		! %g6=cpu->m_cpu.intr_tail[pil]
1692b0fc0e7govinda	stna	%g6, [%g5 + TRAP_ENT_F3]%asi	! trap_f3 = intr_tail[pil]
1693b0fc0e7govinda	stna	%g2, [%g5 + TRAP_ENT_F4]%asi	! trap_f4 = pil
1694b0fc0e7govinda	TRACE_NEXT(%g5, %g6, %g3)
1695b0fc0e7govinda#endif /* TRAPTRACE */
16967c478bdstevel@tonic-gate	!
1697b0fc0e7govinda	! Write %set_softint with (1<<pil) to cause a "pil" level trap
1698b0fc0e7govinda	!
1699b0fc0e7govinda	mov	1, %g5			! %g5 = 1
1700b0fc0e7govinda	sll	%g5, %g2, %g5		! %g5 = 1 << pil
1701b0fc0e7govinda	wr	%g5, SET_SOFTINT	! trigger required pil softint
1702b0fc0e7govinda	retry
1703b0fc0e7govinda	SET_SIZE(setsoftint_tl1)
17047c478bdstevel@tonic-gate
1705b0fc0e7govinda#endif	/* lint */
17067c478bdstevel@tonic-gate
1707b0fc0e7govinda#if defined(lint)
17087c478bdstevel@tonic-gate
1709b0fc0e7govinda/*ARGSUSED*/
1710b0fc0e7govindavoid
1711b0fc0e7govindasetvecint_tl1(uint64_t inum, uint64_t dummy)
1712b0fc0e7govinda{}
1713b0fc0e7govinda
1714b0fc0e7govinda#else	/* lint */
1715b0fc0e7govinda
1716b0fc0e7govinda	!
1717b0fc0e7govinda	! Register usage
1718b0fc0e7govinda	!	Arguments:
1719b0fc0e7govinda	! 	%g1 - inumber
1720b0fc0e7govinda	!
1721b0fc0e7govinda	!	Internal:
1722b0fc0e7govinda	! 	%g1 - softint pil mask
1723b0fc0e7govinda	!	%g2 - pil of intr_vec_t
1724b0fc0e7govinda	!	%g3 - pointer to current intr_vec_t (iv)
1725b0fc0e7govinda	!	%g4 - cpu
1726b0fc0e7govinda	!	%g5, %g6,%g7 - temps
1727b0fc0e7govinda	!
1728b0fc0e7govinda	ENTRY_NP(setvecint_tl1)
1729b0fc0e7govinda	!
1730b0fc0e7govinda	! Verify the inumber received (should be inum < MAXIVNUM).
1731b0fc0e7govinda	!
1732b0fc0e7govinda	set	MAXIVNUM, %g2
1733b0fc0e7govinda	cmp	%g1, %g2
1734b0fc0e7govinda	bgeu,pn	%xcc, .no_ivintr
1735b0fc0e7govinda	clr	%g2			! expected in .no_ivintr
17367c478bdstevel@tonic-gate
17377c478bdstevel@tonic-gate	!
1738b0fc0e7govinda	! Fetch data from intr_vec_table according to the inum.
1739b0fc0e7govinda	!
1740b0fc0e7govinda	! We have an interrupt number. Fetch the interrupt vector requests
1741b0fc0e7govinda	! from the interrupt vector table for a given interrupt number and
1742b0fc0e7govinda	! insert them into cpu's softint priority lists and set %set_softint.
17437c478bdstevel@tonic-gate	!
1744b0fc0e7govinda	set	intr_vec_table, %g5	! %g5 = intr_vec_table
1745b0fc0e7govinda	sll	%g1, CPTRSHIFT, %g6	! %g6 = offset to inum entry in table
1746b0fc0e7govinda	add	%g5, %g6, %g5		! %g5 = &intr_vec_table[inum]
1747b0fc0e7govinda	ldn	[%g5], %g3		! %g3 = pointer to first entry of
1748b0fc0e7govinda					!       intr_vec_t list
1749b0fc0e7govinda
1750b0fc0e7govinda	! Verify the first intr_vec_t pointer for a given inum and it should
1751b0fc0e7govinda	! not be NULL. This used to be guarded by DEBUG but broken drivers can
1752b0fc0e7govinda	! cause spurious tick interrupts when the softint register is programmed
1753b0fc0e7govinda	! with 1 << 0 at the end of this routine. Now we always check for a
1754b0fc0e7govinda	! valid intr_vec_t pointer.
1755b0fc0e7govinda	brz,pn	%g3, .no_ivintr
1756b0fc0e7govinda	nop
17577c478bdstevel@tonic-gate
17587c478bdstevel@tonic-gate	!
1759b0fc0e7govinda	! Traverse the intr_vec_t link list, put each item on to corresponding
1760b0fc0e7govinda	! CPU softint priority queue, and compose the final softint pil mask.
17617c478bdstevel@tonic-gate	!
1762b0fc0e7govinda	! At this point:
1763b0fc0e7govinda	!	%g3 = intr_vec_table[inum]
17647c478bdstevel@tonic-gate	!
1765b0fc0e7govinda	CPU_ADDR(%g4, %g2)		! %g4 = cpu
1766b0fc0e7govinda	mov	%g0, %g1		! %g1 = 0, initialize pil mask to 0
1767b0fc0e7govinda0:
17687c478bdstevel@tonic-gate	!
1769b0fc0e7govinda	! Insert next intr_vec_t (iv) to appropriate cpu's softint priority list
1770b0fc0e7govinda	!
1771b0fc0e7govinda	! At this point:
1772b0fc0e7govinda	!	%g1 = softint pil mask
1773b0fc0e7govinda	!	%g3 = pointer to next intr_vec_t (iv)
1774b0fc0e7govinda	!	%g4 = cpu
1775b0fc0e7govinda	!
1776b0fc0e7govinda	lduh	[%g3 + IV_PIL], %g2	! %g2 = iv->iv_pil
1777b0fc0e7govinda	sll	%g2, CPTRSHIFT, %g7	! %g7 = offset to pil entry
1778b0fc0e7govinda	add	%g4, INTR_TAIL, %g6	! %g6 = &cpu->m_cpu.intr_tail
1779b0fc0e7govinda	ldn	[%g6 + %g7], %g5	! %g5 = cpu->m_cpu.intr_tail[pil]
1780b0fc0e7govinda					! 	current tail (ct)
1781b0fc0e7govinda	brz,pt	%g5, 2f			! branch if current tail is NULL
1782b0fc0e7govinda	stn	%g3, [%g6 + %g7]	! make intr_vec_t (iv) as new tail
1783b0fc0e7govinda					! cpu->m_cpu.intr_tail[pil] = iv
1784b0fc0e7govinda	!
1785b0fc0e7govinda	! there's pending intr_vec_t already
1786b0fc0e7govinda	!
1787b0fc0e7govinda	lduh	[%g5 + IV_FLAGS], %g6	! %g6 = ct->iv_flags
1788b0fc0e7govinda	and	%g6, IV_SOFTINT_MT, %g6	! %g6 = ct->iv_flags & IV_SOFTINT_MT
1789b0fc0e7govinda	brz,pt	%g6, 1f			! check for Multi target softint flag
1790b0fc0e7govinda	add	%g5, IV_PIL_NEXT, %g5	! %g5 = &ct->iv_pil_next
1791b0fc0e7govinda	ld	[%g4 + CPU_ID], %g6	! for multi target softint, use cpuid
1792b0fc0e7govinda	sll	%g6, CPTRSHIFT, %g6	! calculate offset address from cpuid
1793b0fc0e7govinda	add	%g5, %g6, %g5		! %g5 = &ct->iv_xpil_next[cpuid]
1794b0fc0e7govinda1:
17957c478bdstevel@tonic-gate	!
1796b0fc0e7govinda	! update old tail
17977c478bdstevel@tonic-gate	!
17987c478bdstevel@tonic-gate	ba,pt	%xcc, 3f
1799b0fc0e7govinda	stn	%g3, [%g5]		! [%g5] = iv, set pil_next field
18007c478bdstevel@tonic-gate2:
18017c478bdstevel@tonic-gate	!
1802b0fc0e7govinda	! no pending intr_vec_t; make intr_vec_t as new head
18037c478bdstevel@tonic-gate	!
1804b0fc0e7govinda	add	%g4, INTR_HEAD, %g6	!  %g6 = &cpu->m_cpu.intr_head[pil]
1805b0fc0e7govinda	stn	%g3, [%g6 + %g7]	!  cpu->m_cpu.intr_head[pil] = iv
18067c478bdstevel@tonic-gate3:
18077c478bdstevel@tonic-gate#ifdef TRAPTRACE
1808b0fc0e7govinda	TRACE_PTR(%g5, %g6)
1809b0fc0e7govinda	TRACE_SAVE_TL_GL_REGS(%g5, %g6)
18107c478bdstevel@tonic-gate	rdpr	%tt, %g6
1811b0fc0e7govinda	stha	%g6, [%g5 + TRAP_ENT_TT]%asi	! trap_type = %tt`
18127c478bdstevel@tonic-gate	rdpr	%tpc, %g6
1813b0fc0e7govinda	stna	%g6, [%g5 + TRAP_ENT_TPC]%asi	! trap_pc = %tpc
18147c478bdstevel@tonic-gate	rdpr	%tstate, %g6
1815b0fc0e7govinda	stxa	%g6, [%g5 + TRAP_ENT_TSTATE]%asi ! trap_tstate = %tstate
1816b0fc0e7govinda	stna	%sp, [%g5 + TRAP_ENT_SP]%asi	! trap_sp = %sp
1817b0fc0e7govinda	stna	%g3, [%g5 + TRAP_ENT_TR]%asi	! trap_tr = iv
1818b0fc0e7govinda	stna	%g1, [%g5 + TRAP_ENT_F1]%asi	! trap_f1 = pil mask
18197c478bdstevel@tonic-gate	add	%g4, INTR_HEAD, %g6
1820b0fc0e7govinda	ldn	[%g6 + %g7], %g6		! %g6=cpu->m_cpu.intr_head[pil]
1821b0fc0e7govinda	stna	%g6, [%g5 + TRAP_ENT_F2]%asi	! trap_f2 = intr_head[pil]
18227c478bdstevel@tonic-gate	add	%g4, INTR_TAIL, %g6
1823b0fc0e7govinda	ldn	[%g6 + %g7], %g6		! %g6=cpu->m_cpu.intr_tail[pil]
1824b0fc0e7govinda	stna	%g6, [%g5 + TRAP_ENT_F3]%asi	! trap_f3 = intr_tail[pil]
1825b0fc0e7govinda	stna	%g2, [%g5 + TRAP_ENT_F4]%asi	! trap_f4 = pil
1826023e71dHaik Aftandilian	GET_TRACE_TICK(%g6, %g7)
1827023e71dHaik Aftandilian	stxa	%g6, [%g5 + TRAP_ENT_TICK]%asi	! trap_tick = %tick
1828b0fc0e7govinda	TRACE_NEXT(%g5, %g6, %g7)
18297c478bdstevel@tonic-gate#endif /* TRAPTRACE */
1830b0fc0e7govinda	mov	1, %g6			! %g6 = 1
1831b0fc0e7govinda	sll	%g6, %g2, %g6		! %g6 = 1 << pil
1832b0fc0e7govinda	or	%g1, %g6, %g1		! %g1 |= (1 << pil), pil mask
1833b0fc0e7govinda	ldn	[%g3 + IV_VEC_NEXT], %g3 ! %g3 = pointer to next intr_vec_t (iv)
1834b0fc0e7govinda	brnz,pn	%g3, 0b			! iv->iv_vec_next is non NULL, goto 0b
1835b0fc0e7govinda	nop
1836b0fc0e7govinda	wr	%g1, SET_SOFTINT	! triggered one or more pil softints
18377c478bdstevel@tonic-gate	retry
18387c478bdstevel@tonic-gate
18397c478bdstevel@tonic-gate.no_ivintr:
18407c478bdstevel@tonic-gate	! no_ivintr: arguments: rp, inum (%g1), pil (%g2 == 0)
18417c478bdstevel@tonic-gate	mov	%g2, %g3
18427c478bdstevel@tonic-gate	mov	%g1, %g2
18437c478bdstevel@tonic-gate	set	no_ivintr, %g1
18447c478bdstevel@tonic-gate	ba,pt	%xcc, sys_trap
18457c478bdstevel@tonic-gate	mov	PIL_15, %g4
1846b0fc0e7govinda	SET_SIZE(setvecint_tl1)
18477c478bdstevel@tonic-gate
18487c478bdstevel@tonic-gate#endif	/* lint */
18497c478bdstevel@tonic-gate
18507c478bdstevel@tonic-gate#if defined(lint)
18517c478bdstevel@tonic-gate
18527c478bdstevel@tonic-gate/*ARGSUSED*/
18537c478bdstevel@tonic-gatevoid
18547c478bdstevel@tonic-gatewr_clr_softint(uint_t value)
18557c478bdstevel@tonic-gate{}
18567c478bdstevel@tonic-gate
18577c478bdstevel@tonic-gate#else
18587c478bdstevel@tonic-gate
18597c478bdstevel@tonic-gate	ENTRY_NP(wr_clr_softint)
18607c478bdstevel@tonic-gate	retl
18617c478bdstevel@tonic-gate	wr	%o0, CLEAR_SOFTINT
18627c478bdstevel@tonic-gate	SET_SIZE(wr_clr_softint)
18637c478bdstevel@tonic-gate
18647c478bdstevel@tonic-gate#endif /* lint */
18657c478bdstevel@tonic-gate
18667c478bdstevel@tonic-gate#if defined(lint)
18677c478bdstevel@tonic-gate
18687c478bdstevel@tonic-gate/*ARGSUSED*/
18697c478bdstevel@tonic-gatevoid
1870b0fc0e7govindaintr_enqueue_req(uint_t pil, uint64_t inum)
18717c478bdstevel@tonic-gate{}
18727c478bdstevel@tonic-gate
18737c478bdstevel@tonic-gate#else   /* lint */
18747c478bdstevel@tonic-gate
18757c478bdstevel@tonic-gate/*
18767c478bdstevel@tonic-gate * intr_enqueue_req
18777c478bdstevel@tonic-gate *
18787c478bdstevel@tonic-gate * %o0 - pil
1879b0fc0e7govinda * %o1 - pointer to intr_vec_t (iv)
18807c478bdstevel@tonic-gate * %o5 - preserved
18817c478bdstevel@tonic-gate * %g5 - preserved
18827c478bdstevel@tonic-gate */
18837c478bdstevel@tonic-gate	ENTRY_NP(intr_enqueue_req)
1884b0fc0e7govinda	!
1885b0fc0e7govinda	CPU_ADDR(%g4, %g1)		! %g4 = cpu
18867c478bdstevel@tonic-gate
1887b0fc0e7govinda	!
1888b0fc0e7govinda	! Insert intr_vec_t (iv) to appropriate cpu's softint priority list
1889b0fc0e7govinda	!
1890b0fc0e7govinda	sll	%o0, CPTRSHIFT, %o0	! %o0 = offset to pil entry
1891b0fc0e7govinda	add	%g4, INTR_TAIL, %g6	! %g6 = &cpu->m_cpu.intr_tail
1892b0fc0e7govinda	ldn	[%o0 + %g6], %g1	! %g1 = cpu->m_cpu.intr_tail[pil]
1893b0fc0e7govinda					!       current tail (ct)
1894b0fc0e7govinda	brz,pt	%g1, 2f			! branch if current tail is NULL
1895b0fc0e7govinda	stn	%o1, [%g6 + %o0]	! make intr_vec_t (iv) as new tail
18967c478bdstevel@tonic-gate
1897b0fc0e7govinda	!
1898b0fc0e7govinda	! there's pending intr_vec_t already
1899b0fc0e7govinda	!
1900b0fc0e7govinda	lduh	[%g1 + IV_FLAGS], %g6	! %g6 = ct->iv_flags
1901b0fc0e7govinda	and	%g6, IV_SOFTINT_MT, %g6	! %g6 = ct->iv_flags & IV_SOFTINT_MT
1902b0fc0e7govinda	brz,pt	%g6, 1f			! check for Multi target softint flag
1903b0fc0e7govinda	add	%g1, IV_PIL_NEXT, %g3	! %g3 = &ct->iv_pil_next
1904b0fc0e7govinda	ld	[%g4 + CPU_ID], %g6	! for multi target softint, use cpuid
1905b0fc0e7govinda	sll	%g6, CPTRSHIFT, %g6	! calculate offset address from cpuid
1906b0fc0e7govinda	add	%g3, %g6, %g3		! %g3 = &ct->iv_xpil_next[cpuid]
1907b0fc0e7govinda1:
1908b0fc0e7govinda	!
1909b0fc0e7govinda	! update old tail
1910b0fc0e7govinda	!
19117c478bdstevel@tonic-gate	ba,pt	%xcc, 3f
1912b0fc0e7govinda	stn	%o1, [%g3]		! {%g5] = iv, set pil_next field
19137c478bdstevel@tonic-gate2:
1914b0fc0e7govinda	!
1915b0fc0e7govinda	! no intr_vec_t's queued so make intr_vec_t as new head
1916b0fc0e7govinda	!
1917b0fc0e7govinda	add	%g4, INTR_HEAD, %g6	! %g6 = &cpu->m_cpu.intr_head[pil]
1918b0fc0e7govinda	stn	%o1, [%g6 + %o0]	! cpu->m_cpu.intr_head[pil] = iv
19197c478bdstevel@tonic-gate3:
19207c478bdstevel@tonic-gate	retl
19217c478bdstevel@tonic-gate	nop
19227c478bdstevel@tonic-gate	SET_SIZE(intr_enqueue_req)
19237c478bdstevel@tonic-gate
19247c478bdstevel@tonic-gate#endif  /* lint */
19257c478bdstevel@tonic-gate
19267c478bdstevel@tonic-gate/*
19277c478bdstevel@tonic-gate * Set CPU's base SPL level, based on which interrupt levels are active.
19287c478bdstevel@tonic-gate * 	Called at spl7 or above.
19297c478bdstevel@tonic-gate */
19307c478bdstevel@tonic-gate
19317c478bdstevel@tonic-gate#if defined(lint)
19327c478bdstevel@tonic-gate
19337c478bdstevel@tonic-gatevoid
19347c478bdstevel@tonic-gateset_base_spl(void)
19357c478bdstevel@tonic-gate{}
19367c478bdstevel@tonic-gate
19377c478bdstevel@tonic-gate#else	/* lint */
19387c478bdstevel@tonic-gate
19397c478bdstevel@tonic-gate	ENTRY_NP(set_base_spl)
19407c478bdstevel@tonic-gate	ldn	[THREAD_REG + T_CPU], %o2	! load CPU pointer
19417c478bdstevel@tonic-gate	ld	[%o2 + CPU_INTR_ACTV], %o5	! load active interrupts mask
19427c478bdstevel@tonic-gate
19437c478bdstevel@tonic-gate/*
19447c478bdstevel@tonic-gate * WARNING: non-standard callinq sequence; do not call from C
19457c478bdstevel@tonic-gate *	%o2 = pointer to CPU
19467c478bdstevel@tonic-gate *	%o5 = updated CPU_INTR_ACTV
19477c478bdstevel@tonic-gate */
19487c478bdstevel@tonic-gate_intr_set_spl:					! intr_thread_exit enters here
19497c478bdstevel@tonic-gate	!
19507c478bdstevel@tonic-gate	! Determine highest interrupt level active.  Several could be blocked
19517c478bdstevel@tonic-gate	! at higher levels than this one, so must convert flags to a PIL
19527c478bdstevel@tonic-gate	! Normally nothing will be blocked, so test this first.
19537c478bdstevel@tonic-gate	!
19547c478bdstevel@tonic-gate	brz,pt	%o5, 1f				! nothing active
19557c478bdstevel@tonic-gate	sra	%o5, 11, %o3			! delay - set %o3 to bits 15-11
19567c478bdstevel@tonic-gate	set	_intr_flag_table, %o1
19577c478bdstevel@tonic-gate	tst	%o3				! see if any of the bits set
19587c478bdstevel@tonic-gate	ldub	[%o1 + %o3], %o3		! load bit number
19597c478bdstevel@tonic-gate	bnz,a,pn %xcc, 1f			! yes, add 10 and we're done
19607c478bdstevel@tonic-gate	add	%o3, 11-1, %o3			! delay - add bit number - 1
19617c478bdstevel@tonic-gate
19627c478bdstevel@tonic-gate	sra	%o5, 6, %o3			! test bits 10-6
19637c478bdstevel@tonic-gate	tst	%o3
19647c478bdstevel@tonic-gate	ldub	[%o1 + %o3], %o3
19657c478bdstevel@tonic-gate	bnz,a,pn %xcc, 1f
19667c478bdstevel@tonic-gate	add	%o3, 6-1, %o3
19677c478bdstevel@tonic-gate
19687c478bdstevel@tonic-gate	sra	%o5, 1, %o3			! test bits 5-1
19697c478bdstevel@tonic-gate	ldub	[%o1 + %o3], %o3
19707c478bdstevel@tonic-gate
19717c478bdstevel@tonic-gate	!
19727c478bdstevel@tonic-gate	! highest interrupt level number active is in %l6
19737c478bdstevel@tonic-gate	!
19747c478bdstevel@tonic-gate1:
19757c478bdstevel@tonic-gate	retl
19767c478bdstevel@tonic-gate	st	%o3, [%o2 + CPU_BASE_SPL]	! delay - store base priority
19777c478bdstevel@tonic-gate	SET_SIZE(set_base_spl)
19787c478bdstevel@tonic-gate
19797c478bdstevel@tonic-gate/*
19807c478bdstevel@tonic-gate * Table that finds the most significant bit set in a five bit field.
19817c478bdstevel@tonic-gate * Each entry is the high-order bit number + 1 of it's index in the table.
19827c478bdstevel@tonic-gate * This read-only data is in the text segment.
19837c478bdstevel@tonic-gate */
19847c478bdstevel@tonic-gate_intr_flag_table:
19857c478bdstevel@tonic-gate	.byte	0, 1, 2, 2,	3, 3, 3, 3,	4, 4, 4, 4,	4, 4, 4, 4
19867c478bdstevel@tonic-gate	.byte	5, 5, 5, 5,	5, 5, 5, 5,	5, 5, 5, 5,	5, 5, 5, 5
19877c478bdstevel@tonic-gate	.align	4
19887c478bdstevel@tonic-gate
19897c478bdstevel@tonic-gate#endif	/* lint */
19907c478bdstevel@tonic-gate
19917c478bdstevel@tonic-gate/*
19927c478bdstevel@tonic-gate * int
19937c478bdstevel@tonic-gate * intr_passivate(from, to)
19947c478bdstevel@tonic-gate *	kthread_id_t	from;		interrupt thread
19957c478bdstevel@tonic-gate *	kthread_id_t	to;		interrupted thread
19967c478bdstevel@tonic-gate */
19977c478bdstevel@tonic-gate
19987c478bdstevel@tonic-gate#if defined(lint)
19997c478bdstevel@tonic-gate
20007c478bdstevel@tonic-gate/* ARGSUSED */
20017c478bdstevel@tonic-gateint
20027c478bdstevel@tonic-gateintr_passivate(kthread_id_t from, kthread_id_t to)
20037c478bdstevel@tonic-gate{ return (0); }
20047c478bdstevel@tonic-gate
20057c478bdstevel@tonic-gate#else	/* lint */
20067c478bdstevel@tonic-gate
20077c478bdstevel@tonic-gate	ENTRY_NP(intr_passivate)
20087c478bdstevel@tonic-gate	save	%sp, -SA(MINFRAME), %sp	! get a new window
20097c478bdstevel@tonic-gate
20107c478bdstevel@tonic-gate	flushw				! force register windows to stack
20117c478bdstevel@tonic-gate	!
20127c478bdstevel@tonic-gate	! restore registers from the base of the stack of the interrupt thread.
20137c478bdstevel@tonic-gate	!
20147c478bdstevel@tonic-gate	ldn	[%i0 + T_STACK], %i2	! get stack save area pointer
20157c478bdstevel@tonic-gate	ldn	[%i2 + (0*GREGSIZE)], %l0	! load locals
20167c478bdstevel@tonic-gate	ldn	[%i2 + (1*GREGSIZE)], %l1
20177c478bdstevel@tonic-gate	ldn	[%i2 + (2*GREGSIZE)], %l2
20187c478bdstevel@tonic-gate	ldn	[%i2 + (3*GREGSIZE)], %l3
20197c478bdstevel@tonic-gate	ldn	[%i2 + (4*GREGSIZE)], %l4
20207c478bdstevel@tonic-gate	ldn	[%i2 + (5*GREGSIZE)], %l5
20217c478bdstevel@tonic-gate	ldn	[%i2 + (6*GREGSIZE)], %l6
20227c478bdstevel@tonic-gate	ldn	[%i2 + (7*GREGSIZE)], %l7
20237c478bdstevel@tonic-gate	ldn	[%i2 + (8*GREGSIZE)], %o0	! put ins from stack in outs
20247c478bdstevel@tonic-gate	ldn	[%i2 + (9*GREGSIZE)], %o1
20257c478bdstevel@tonic-gate	ldn	[%i2 + (10*GREGSIZE)], %o2
20267c478bdstevel@tonic-gate	ldn	[%i2 + (11*GREGSIZE)], %o3
20277c478bdstevel@tonic-gate	ldn	[%i2 + (12*GREGSIZE)], %o4
20287c478bdstevel@tonic-gate	ldn	[%i2 + (13*GREGSIZE)], %o5
20297c478bdstevel@tonic-gate	ldn	[%i2 + (14*GREGSIZE)], %i4
20307c478bdstevel@tonic-gate					! copy stack/pointer without using %sp
20317c478bdstevel@tonic-gate	ldn	[%i2 + (15*GREGSIZE)], %i5
20327c478bdstevel@tonic-gate	!
20337c478bdstevel@tonic-gate	! put registers into the save area at the top of the interrupted
20347c478bdstevel@tonic-gate	! thread's stack, pointed to by %l7 in the save area just loaded.
20357c478bdstevel@tonic-gate	!
20367c478bdstevel@tonic-gate	ldn	[%i1 + T_SP], %i3	! get stack save area pointer
20377c478bdstevel@tonic-gate	stn	%l0, [%i3 + STACK_BIAS + (0*GREGSIZE)]	! save locals
20387c478bdstevel@tonic-gate	stn	%l1, [%i3 + STACK_BIAS + (1*GREGSIZE)]
20397c478bdstevel@tonic-gate	stn	%l2, [%i3 + STACK_BIAS + (2*GREGSIZE)]
20407c478bdstevel@tonic-gate	stn	%l3, [%i3 + STACK_BIAS + (3*GREGSIZE)]
20417c478bdstevel@tonic-gate	stn	%l4, [%i3 + STACK_BIAS + (4*GREGSIZE)]
20427c478bdstevel@tonic-gate	stn	%l5, [%i3 + STACK_BIAS + (5*GREGSIZE)]
20437c478bdstevel@tonic-gate	stn	%l6, [%i3 + STACK_BIAS + (6*GREGSIZE)]
20447c478bdstevel@tonic-gate	stn	%l7, [%i3 + STACK_BIAS + (7*GREGSIZE)]
20457c478bdstevel@tonic-gate	stn	%o0, [%i3 + STACK_BIAS + (8*GREGSIZE)]	! save ins using outs
20467c478bdstevel@tonic-gate	stn	%o1, [%i3 + STACK_BIAS + (9*GREGSIZE)]
20477c478bdstevel@tonic-gate	stn	%o2, [%i3 + STACK_BIAS + (10*GREGSIZE)]
20487c478bdstevel@tonic-gate	stn	%o3, [%i3 + STACK_BIAS + (11*GREGSIZE)]
20497c478bdstevel@tonic-gate	stn	%o4, [%i3 + STACK_BIAS + (12*GREGSIZE)]
20507c478bdstevel@tonic-gate	stn	%o5, [%i3 + STACK_BIAS + (13*GREGSIZE)]
20517c478bdstevel@tonic-gate	stn	%i4, [%i3 + STACK_BIAS + (14*GREGSIZE)]
20527c478bdstevel@tonic-gate						! fp, %i7 copied using %i4
20537c478bdstevel@tonic-gate	stn	%i5, [%i3 + STACK_BIAS + (15*GREGSIZE)]
20547c478bdstevel@tonic-gate	stn	%g0, [%i2 + ((8+6)*GREGSIZE)]
20557c478bdstevel@tonic-gate						! clear fp in save area
20567c478bdstevel@tonic-gate
20577c478bdstevel@tonic-gate	! load saved pil for return
20587c478bdstevel@tonic-gate	ldub	[%i0 + T_PIL], %i0
20597c478bdstevel@tonic-gate	ret
20607c478bdstevel@tonic-gate	restore
20617c478bdstevel@tonic-gate	SET_SIZE(intr_passivate)
20627c478bdstevel@tonic-gate
20637c478bdstevel@tonic-gate#endif	/* lint */
20647c478bdstevel@tonic-gate
20657c478bdstevel@tonic-gate#if defined(lint)
20667c478bdstevel@tonic-gate
20677c478bdstevel@tonic-gate/*
20687c478bdstevel@tonic-gate * intr_get_time() is a resource for interrupt handlers to determine how
20697c478bdstevel@tonic-gate * much time has been spent handling the current interrupt. Such a function
20707c478bdstevel@tonic-gate * is needed because higher level interrupts can arrive during the
20717c478bdstevel@tonic-gate * processing of an interrupt, thus making direct comparisons of %tick by
20727c478bdstevel@tonic-gate * the handler inaccurate. intr_get_time() only returns time spent in the
20737c478bdstevel@tonic-gate * current interrupt handler.
20747c478bdstevel@tonic-gate *
20757c478bdstevel@tonic-gate * The caller must be calling from an interrupt handler running at a pil
20767c478bdstevel@tonic-gate * below or at lock level. Timings are not provided for high-level
20777c478bdstevel@tonic-gate * interrupts.
20787c478bdstevel@tonic-gate *
20797c478bdstevel@tonic-gate * The first time intr_get_time() is called while handling an interrupt,
20807c478bdstevel@tonic-gate * it returns the time since the interrupt handler was invoked. Subsequent
20817c478bdstevel@tonic-gate * calls will return the time since the prior call to intr_get_time(). Time
20827c478bdstevel@tonic-gate * is returned as ticks, adjusted for any clock divisor due to power
20837c478bdstevel@tonic-gate * management. Use tick2ns() to convert ticks to nsec. Warning: ticks may
20847c478bdstevel@tonic-gate * not be the same across CPUs.
20857c478bdstevel@tonic-gate *
20867c478bdstevel@tonic-gate * Theory Of Intrstat[][]:
20877c478bdstevel@tonic-gate *
20887c478bdstevel@tonic-gate * uint64_t intrstat[pil][0..1] is an array indexed by pil level, with two
20897c478bdstevel@tonic-gate * uint64_ts per pil.
20907c478bdstevel@tonic-gate *
20917c478bdstevel@tonic-gate * intrstat[pil][0] is a cumulative count of the number of ticks spent
20927c478bdstevel@tonic-gate * handling all interrupts at the specified pil on this CPU. It is
20937c478bdstevel@tonic-gate * exported via kstats to the user.
20947c478bdstevel@tonic-gate *
20957c478bdstevel@tonic-gate * intrstat[pil][1] is always a count of ticks less than or equal to the
20967c478bdstevel@tonic-gate * value in [0]. The difference between [1] and [0] is the value returned
20977c478bdstevel@tonic-gate * by a call to intr_get_time(). At the start of interrupt processing,
20987c478bdstevel@tonic-gate * [0] and [1] will be equal (or nearly so). As the interrupt consumes
20997c478bdstevel@tonic-gate * time, [0] will increase, but [1] will remain the same. A call to