xref: /illumos-gate/usr/src/uts/sun4/os/intr.c (revision c3377ee9)
17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate  * CDDL HEADER START
37c478bd9Sstevel@tonic-gate  *
47c478bd9Sstevel@tonic-gate  * The contents of this file are subject to the terms of the
5100b72f4Sandrei  * Common Development and Distribution License (the "License").
6100b72f4Sandrei  * You may not use this file except in compliance with the License.
77c478bd9Sstevel@tonic-gate  *
87c478bd9Sstevel@tonic-gate  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bd9Sstevel@tonic-gate  * or http://www.opensolaris.org/os/licensing.
107c478bd9Sstevel@tonic-gate  * See the License for the specific language governing permissions
117c478bd9Sstevel@tonic-gate  * and limitations under the License.
127c478bd9Sstevel@tonic-gate  *
137c478bd9Sstevel@tonic-gate  * When distributing Covered Code, include this CDDL HEADER in each
147c478bd9Sstevel@tonic-gate  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bd9Sstevel@tonic-gate  * If applicable, add the following below this CDDL HEADER, with the
167c478bd9Sstevel@tonic-gate  * fields enclosed by brackets "[]" replaced with your own identifying
177c478bd9Sstevel@tonic-gate  * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bd9Sstevel@tonic-gate  *
197c478bd9Sstevel@tonic-gate  * CDDL HEADER END
207c478bd9Sstevel@tonic-gate  */
217c478bd9Sstevel@tonic-gate /*
2207247649SMadhavan Venkataraman  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
237c478bd9Sstevel@tonic-gate  * Use is subject to license terms.
247c478bd9Sstevel@tonic-gate  */
25a288e5a9SJoshua M. Clulow /*
26*c3377ee9SJohn Levon  * Copyright 2019 Joyent, Inc.
27a288e5a9SJoshua M. Clulow  */
282a1fd0ffSPeter Tribble /*
292a1fd0ffSPeter Tribble  * Copyright 2019 Peter Tribble.
302a1fd0ffSPeter Tribble  */
327c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h>
337c478bd9Sstevel@tonic-gate #include <sys/stack.h>
347c478bd9Sstevel@tonic-gate #include <sys/cpuvar.h>
357c478bd9Sstevel@tonic-gate #include <sys/ivintr.h>
367c478bd9Sstevel@tonic-gate #include <sys/intreg.h>
377c478bd9Sstevel@tonic-gate #include <sys/membar.h>
387c478bd9Sstevel@tonic-gate #include <sys/kmem.h>
397c478bd9Sstevel@tonic-gate #include <sys/intr.h>
40dd4eeefdSeota #include <sys/sunddi.h>
417c478bd9Sstevel@tonic-gate #include <sys/sunndi.h>
427c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h>
437c478bd9Sstevel@tonic-gate #include <sys/privregs.h>
447c478bd9Sstevel@tonic-gate #include <sys/systm.h>
457c478bd9Sstevel@tonic-gate #include <sys/archsystm.h>
467c478bd9Sstevel@tonic-gate #include <sys/machsystm.h>
477c478bd9Sstevel@tonic-gate #include <sys/x_call.h>
487c478bd9Sstevel@tonic-gate #include <vm/seg_kp.h>
497c478bd9Sstevel@tonic-gate #include <sys/debug.h>
507c478bd9Sstevel@tonic-gate #include <sys/cyclic.h>
51a1af7ba0Scwb #include <sys/kdi_impl.h>
52a288e5a9SJoshua M. Clulow #include <sys/ddi_periodic.h>
547c478bd9Sstevel@tonic-gate #include <sys/cpu_sgnblk_defs.h>
567c478bd9Sstevel@tonic-gate /* Global locks which protect the interrupt distribution lists */
577c478bd9Sstevel@tonic-gate static kmutex_t intr_dist_lock;
587c478bd9Sstevel@tonic-gate static kmutex_t intr_dist_cpu_lock;
607c478bd9Sstevel@tonic-gate /* Head of the interrupt distribution lists */
617c478bd9Sstevel@tonic-gate static struct intr_dist *intr_dist_head = NULL;
627c478bd9Sstevel@tonic-gate static struct intr_dist *intr_dist_whead = NULL;
64dd4eeefdSeota static uint64_t siron_inum[DDI_IPL_10]; /* software interrupt numbers */
653aedfe0bSmishra uint64_t *siron_cpu_inum = NULL;
663aedfe0bSmishra uint64_t siron_poke_cpu_inum;
673aedfe0bSmishra static int siron_cpu_setup(cpu_setup_t, int, void *);
683aedfe0bSmishra extern uint_t softlevel1();
70dd4eeefdSeota static uint64_t siron1_inum; /* backward compatibility */
71b0fc0e77Sgovinda uint64_t poke_cpu_inum;
72b0fc0e77Sgovinda uint_t poke_cpu_intr(caddr_t arg1, caddr_t arg2);
733aedfe0bSmishra uint_t siron_poke_cpu_intr(caddr_t arg1, caddr_t arg2);
75492887eeSChristopher Baumbauer - Sun Microsystems - San Diego United States /*
76492887eeSChristopher Baumbauer - Sun Microsystems - San Diego United States  * Variable to enable/disable printing a message when an invalid vecintr
77492887eeSChristopher Baumbauer - Sun Microsystems - San Diego United States  * is received.
78492887eeSChristopher Baumbauer - Sun Microsystems - San Diego United States  */
79492887eeSChristopher Baumbauer - Sun Microsystems - San Diego United States uint_t ignore_invalid_vecintr = 0;
80492887eeSChristopher Baumbauer - Sun Microsystems - San Diego United States 
81f8047eabSsudheer /*
82f8047eabSsudheer  * Note:-
83f8047eabSsudheer  * siron_pending was originally created to prevent a resource over consumption
84f8047eabSsudheer  * bug in setsoftint(exhaustion of interrupt pool free list).
85f8047eabSsudheer  * It's original intention is obsolete with the use of iv_pending in
86f8047eabSsudheer  * setsoftint. However, siron_pending stayed around, acting as a second
87f8047eabSsudheer  * gatekeeper preventing soft interrupts from being queued. In this capacity,
88f8047eabSsudheer  * it can lead to hangs on MP systems, where due to global visibility issues
89f8047eabSsudheer  * it can end up set while iv_pending is reset, preventing soft interrupts from
90f8047eabSsudheer  * ever being processed. In addition to its gatekeeper role, init_intr also
91f8047eabSsudheer  * uses it to flag the situation where siron() was called before siron_inum has
92f8047eabSsudheer  * been defined.
93f8047eabSsudheer  *
94f8047eabSsudheer  * siron() does not need an extra gatekeeper; any cpu that wishes should be
95f8047eabSsudheer  * allowed to queue a soft interrupt. It is softint()'s job to ensure
96f8047eabSsudheer  * correct handling of the queues. Therefore, siron_pending has been
97f8047eabSsudheer  * stripped of its gatekeeper task, retaining only its intr_init job, where
98f8047eabSsudheer  * it indicates that there is a pending need to call siron().
99f8047eabSsudheer  */
100dd4eeefdSeota static int siron_pending[DDI_IPL_10]; /* software interrupt pending flags */
101dd4eeefdSeota static int siron1_pending; /* backward compatibility */
1037c478bd9Sstevel@tonic-gate int intr_policy = INTR_WEIGHTED_DIST;	/* interrupt distribution policy */
1047c478bd9Sstevel@tonic-gate int intr_dist_debug = 0;
1057c478bd9Sstevel@tonic-gate int32_t intr_dist_weight_max = 1;
1067c478bd9Sstevel@tonic-gate int32_t intr_dist_weight_maxmax = 1000;
1077c478bd9Sstevel@tonic-gate int intr_dist_weight_maxfactor = 2;
1087c478bd9Sstevel@tonic-gate #define	INTR_DEBUG(args) if (intr_dist_debug) cmn_err args
1107c478bd9Sstevel@tonic-gate /*
111b0fc0e77Sgovinda  * intr_init() - Interrupt initialization
112b0fc0e77Sgovinda  *	Initialize the system's interrupt vector table.
1137c478bd9Sstevel@tonic-gate  */
1147c478bd9Sstevel@tonic-gate void
intr_init(cpu_t * cp)1157c478bd9Sstevel@tonic-gate intr_init(cpu_t *cp)
1167c478bd9Sstevel@tonic-gate {
117dd4eeefdSeota 	int i;
118b0fc0e77Sgovinda 	extern uint_t softlevel1();
1207c478bd9Sstevel@tonic-gate 	init_ivintr();
122dd4eeefdSeota 	/*
123dd4eeefdSeota 	 * Register these software interrupts for ddi timer.
124dd4eeefdSeota 	 * Software interrupts up to the level 10 are supported.
125dd4eeefdSeota 	 */
126dd4eeefdSeota 	for (i = DDI_IPL_1; i <= DDI_IPL_10; i++) {
127a288e5a9SJoshua M. Clulow 		siron_inum[i - 1] = add_softintr(i,
128a288e5a9SJoshua M. Clulow 		    (softintrfunc)ddi_periodic_softintr,
129dd4eeefdSeota 		    (caddr_t)(uintptr_t)(i), SOFTINT_ST);
130dd4eeefdSeota 	}
132dd4eeefdSeota 	siron1_inum = add_softintr(PIL_1, softlevel1, 0, SOFTINT_ST);
133b0fc0e77Sgovinda 	poke_cpu_inum = add_softintr(PIL_13, poke_cpu_intr, 0, SOFTINT_MT);
1343aedfe0bSmishra 	siron_poke_cpu_inum = add_softintr(PIL_13,
1353aedfe0bSmishra 	    siron_poke_cpu_intr, 0, SOFTINT_MT);
136b0fc0e77Sgovinda 	cp->cpu_m.poke_cpu_outstanding = B_FALSE;
1387c478bd9Sstevel@tonic-gate 	mutex_init(&intr_dist_lock, NULL, MUTEX_DEFAULT, NULL);
1397c478bd9Sstevel@tonic-gate 	mutex_init(&intr_dist_cpu_lock, NULL, MUTEX_DEFAULT, NULL);
1417c478bd9Sstevel@tonic-gate 	/*
1427c478bd9Sstevel@tonic-gate 	 * A soft interrupt may have been requested prior to the initialization
1437c478bd9Sstevel@tonic-gate 	 * of soft interrupts.  Soft interrupts can't be dispatched until after
144b0fc0e77Sgovinda 	 * init_intr(), so we have to wait until now before we can dispatch the
145b0fc0e77Sgovinda 	 * pending soft interrupt (if any).
1467c478bd9Sstevel@tonic-gate 	 */
147dd4eeefdSeota 	for (i = DDI_IPL_1; i <= DDI_IPL_10; i++) {
148dd4eeefdSeota 		if (siron_pending[i-1]) {
149dd4eeefdSeota 			siron_pending[i-1] = 0;
150dd4eeefdSeota 			sir_on(i);
151dd4eeefdSeota 		}
152dd4eeefdSeota 	}
153dd4eeefdSeota 	if (siron1_pending) {
154dd4eeefdSeota 		siron1_pending = 0;
155f8047eabSsudheer 		siron();
156f8047eabSsudheer 	}
1577c478bd9Sstevel@tonic-gate }
1597c478bd9Sstevel@tonic-gate /*
1607c478bd9Sstevel@tonic-gate  * poke_cpu_intr - fall through when poke_cpu calls
1617c478bd9Sstevel@tonic-gate  */
1627c478bd9Sstevel@tonic-gate /* ARGSUSED */
1637c478bd9Sstevel@tonic-gate uint_t
poke_cpu_intr(caddr_t arg1,caddr_t arg2)1647c478bd9Sstevel@tonic-gate poke_cpu_intr(caddr_t arg1, caddr_t arg2)
1657c478bd9Sstevel@tonic-gate {
1667c478bd9Sstevel@tonic-gate 	CPU->cpu_m.poke_cpu_outstanding = B_FALSE;
1677c478bd9Sstevel@tonic-gate 	membar_stld_stst();
1687c478bd9Sstevel@tonic-gate 	return (1);
1697c478bd9Sstevel@tonic-gate }
171dd4eeefdSeota /*
172dd4eeefdSeota  * Trigger software interrupts dedicated to ddi timer.
173dd4eeefdSeota  */
174dd4eeefdSeota void
sir_on(int level)175dd4eeefdSeota sir_on(int level)
176dd4eeefdSeota {
177dd4eeefdSeota 	ASSERT(level >= DDI_IPL_1 && level <= DDI_IPL_10);
178dd4eeefdSeota 	if (siron_inum[level-1])
179dd4eeefdSeota 		setsoftint(siron_inum[level-1]);
180dd4eeefdSeota 	else
181dd4eeefdSeota 		siron_pending[level-1] = 1;
182dd4eeefdSeota }
1847c478bd9Sstevel@tonic-gate /*
185a1af7ba0Scwb  * kmdb uses siron (and thus setsoftint) while the world is stopped in order to
186a1af7ba0Scwb  * inform its driver component that there's work to be done.  We need to keep
187a1af7ba0Scwb  * DTrace from instrumenting kmdb's siron and setsoftint.  We duplicate siron,
188a1af7ba0Scwb  * giving kmdb's version a kdi_ prefix to keep DTrace at bay.  The
189a1af7ba0Scwb  * implementation of setsoftint is complicated enough that we don't want to
190a1af7ba0Scwb  * duplicate it, but at the same time we don't want to preclude tracing either.
191a1af7ba0Scwb  * The meat of setsoftint() therefore goes into kdi_setsoftint, with
192a1af7ba0Scwb  * setsoftint() implemented as a wrapper.  This allows tracing, while still
193a1af7ba0Scwb  * providing a way for kmdb to sneak in unmolested.
1947c478bd9Sstevel@tonic-gate  */
195a1af7ba0Scwb void
kdi_siron(void)196a1af7ba0Scwb kdi_siron(void)
197a1af7ba0Scwb {
198dd4eeefdSeota 	if (siron1_inum != 0)
199dd4eeefdSeota 		kdi_setsoftint(siron1_inum);
200a1af7ba0Scwb 	else
201dd4eeefdSeota 		siron1_pending = 1;
202a1af7ba0Scwb }
204a1af7ba0Scwb void
setsoftint(uint64_t inum)205a1af7ba0Scwb setsoftint(uint64_t inum)
206a1af7ba0Scwb {
207a1af7ba0Scwb 	kdi_setsoftint(inum);
208a1af7ba0Scwb }
2103aedfe0bSmishra /*
2113aedfe0bSmishra  * Generates softlevel1 interrupt on current CPU if it
2123aedfe0bSmishra  * is not pending already.
2133aedfe0bSmishra  */
2147c478bd9Sstevel@tonic-gate void
siron(void)2157c478bd9Sstevel@tonic-gate siron(void)
2167c478bd9Sstevel@tonic-gate {
2173aedfe0bSmishra 	uint64_t inum;
219dd4eeefdSeota 	if (siron1_inum != 0) {
22012ceefb6Smishra 		/*
22112ceefb6Smishra 		 * Once siron_cpu_inum has been allocated, we can
22212ceefb6Smishra 		 * use per-CPU siron inum.
22312ceefb6Smishra 		 */
22412ceefb6Smishra 		if (siron_cpu_inum && siron_cpu_inum[CPU->cpu_id] != 0)
2253aedfe0bSmishra 			inum = siron_cpu_inum[CPU->cpu_id];
2263aedfe0bSmishra 		else
227dd4eeefdSeota 			inum = siron1_inum;
2293aedfe0bSmishra 		setsoftint(inum);
2303aedfe0bSmishra 	} else
231dd4eeefdSeota 		siron1_pending = 1;
2327c478bd9Sstevel@tonic-gate }
23512ceefb6Smishra static void
siron_init(void)23612ceefb6Smishra siron_init(void)
23712ceefb6Smishra {
23812ceefb6Smishra 	/*
23912ceefb6Smishra 	 * We just allocate memory for per-cpu siron right now. Rest of
24012ceefb6Smishra 	 * the work is done when CPU is configured.
24112ceefb6Smishra 	 */
24212ceefb6Smishra 	siron_cpu_inum = kmem_zalloc(sizeof (uint64_t) * NCPU, KM_SLEEP);
24312ceefb6Smishra }
2453aedfe0bSmishra /*
2463aedfe0bSmishra  * This routine creates per-CPU siron inum for CPUs which are
2473aedfe0bSmishra  * configured during boot.
2483aedfe0bSmishra  */
2493aedfe0bSmishra void
siron_mp_init()2503aedfe0bSmishra siron_mp_init()
2513aedfe0bSmishra {
2523aedfe0bSmishra 	cpu_t *c;
25412ceefb6Smishra 	/*
25512ceefb6Smishra 	 * Get the memory for per-CPU siron inums
25612ceefb6Smishra 	 */
25712ceefb6Smishra 	siron_init();
2593aedfe0bSmishra 	mutex_enter(&cpu_lock);
2603aedfe0bSmishra 	c = cpu_list;
2613aedfe0bSmishra 	do {
2623aedfe0bSmishra 		(void) siron_cpu_setup(CPU_CONFIG, c->cpu_id, NULL);
2633aedfe0bSmishra 	} while ((c = c->cpu_next) != cpu_list);
2653aedfe0bSmishra 	register_cpu_setup_func(siron_cpu_setup, NULL);
2663aedfe0bSmishra 	mutex_exit(&cpu_lock);
2673aedfe0bSmishra }
2693aedfe0bSmishra /*
2703aedfe0bSmishra  * siron_poke_cpu_intr - cross-call handler.
2713aedfe0bSmishra  */
2723aedfe0bSmishra /* ARGSUSED */
2733aedfe0bSmishra uint_t
siron_poke_cpu_intr(caddr_t arg1,caddr_t arg2)2743aedfe0bSmishra siron_poke_cpu_intr(caddr_t arg1, caddr_t arg2)
2753aedfe0bSmishra {
2763aedfe0bSmishra 	/* generate level1 softint */
2773aedfe0bSmishra 	siron();
2783aedfe0bSmishra 	return (1);
2793aedfe0bSmishra }
2813aedfe0bSmishra /*
2823aedfe0bSmishra  * This routine generates a cross-call on target CPU(s).
2833aedfe0bSmishra  */
2843aedfe0bSmishra void
siron_poke_cpu(cpuset_t poke)2853aedfe0bSmishra siron_poke_cpu(cpuset_t poke)
2863aedfe0bSmishra {
2873aedfe0bSmishra 	int cpuid = CPU->cpu_id;
2893aedfe0bSmishra 	if (CPU_IN_SET(poke, cpuid)) {
2903aedfe0bSmishra 		siron();
2913aedfe0bSmishra 		CPUSET_DEL(poke, cpuid);
2923aedfe0bSmishra 		if (CPUSET_ISNULL(poke))
2933aedfe0bSmishra 			return;
2943aedfe0bSmishra 	}
2963aedfe0bSmishra 	xt_some(poke, setsoftint_tl1, siron_poke_cpu_inum, 0);
2973aedfe0bSmishra }
2993aedfe0bSmishra /*
3003aedfe0bSmishra  * This callback function allows us to create per-CPU siron inum.
3013aedfe0bSmishra  */
3023aedfe0bSmishra /* ARGSUSED */
3033aedfe0bSmishra static int
siron_cpu_setup(cpu_setup_t what,int id,void * arg)3043aedfe0bSmishra siron_cpu_setup(cpu_setup_t what, int id, void *arg)
3053aedfe0bSmishra {
3063aedfe0bSmishra 	cpu_t *cp = cpu[id];
3083aedfe0bSmishra 	ASSERT(MUTEX_HELD(&cpu_lock));
3093aedfe0bSmishra 	ASSERT(cp != NULL);
3113aedfe0bSmishra 	switch (what) {
3123aedfe0bSmishra 	case CPU_CONFIG:
3133aedfe0bSmishra 		siron_cpu_inum[cp->cpu_id] = add_softintr(PIL_1,
3143aedfe0bSmishra 		    (softintrfunc)softlevel1, 0, SOFTINT_ST);
3153aedfe0bSmishra 		break;
3163aedfe0bSmishra 	case CPU_UNCONFIG:
3173aedfe0bSmishra 		(void) rem_softintr(siron_cpu_inum[cp->cpu_id]);
3183aedfe0bSmishra 		siron_cpu_inum[cp->cpu_id] = 0;
3193aedfe0bSmishra 		break;
3203aedfe0bSmishra 	default:
3213aedfe0bSmishra 		break;
3223aedfe0bSmishra 	}
3243aedfe0bSmishra 	return (0);
3253aedfe0bSmishra }
3277c478bd9Sstevel@tonic-gate /*
3287c478bd9Sstevel@tonic-gate  * no_ivintr()
329*c3377ee9SJohn Levon  *	called by setvecint_tl1() through sys_trap()
3307c478bd9Sstevel@tonic-gate  *	vector interrupt received but not valid or not
331b0fc0e77Sgovinda  *	registered in intr_vec_table
3327c478bd9Sstevel@tonic-gate  *	considered as a spurious mondo interrupt
3337c478bd9Sstevel@tonic-gate  */
3347c478bd9Sstevel@tonic-gate /* ARGSUSED */
3357c478bd9Sstevel@tonic-gate void
no_ivintr(struct regs * rp,int inum,int pil)3367c478bd9Sstevel@tonic-gate no_ivintr(struct regs *rp, int inum, int pil)
3377c478bd9Sstevel@tonic-gate {
338492887eeSChristopher Baumbauer - Sun Microsystems - San Diego United States 	if (!ignore_invalid_vecintr)
339492887eeSChristopher Baumbauer - Sun Microsystems - San Diego United States 		cmn_err(CE_WARN, "invalid vector intr: number 0x%x, pil 0x%x",
340492887eeSChristopher Baumbauer - Sun Microsystems - San Diego United States 		    inum, pil);
3427c478bd9Sstevel@tonic-gate #ifdef DEBUG_VEC_INTR
3437c478bd9Sstevel@tonic-gate 	prom_enter_mon();
3447c478bd9Sstevel@tonic-gate #endif /* DEBUG_VEC_INTR */
3457c478bd9Sstevel@tonic-gate }
3477c478bd9Sstevel@tonic-gate void
intr_dequeue_req(uint_t pil,uint64_t inum)348b0fc0e77Sgovinda intr_dequeue_req(uint_t pil, uint64_t inum)
3497c478bd9Sstevel@tonic-gate {
350b0fc0e77Sgovinda 	intr_vec_t	*iv, *next, *prev;
351b0fc0e77Sgovinda 	struct machcpu	*mcpu;
352b0fc0e77Sgovinda 	uint32_t	clr;
353b0fc0e77Sgovinda 	processorid_t	cpu_id;
354b0fc0e77Sgovinda 	extern uint_t	getpstate(void);
3567c478bd9Sstevel@tonic-gate 	ASSERT((getpstate() & PSTATE_IE) == 0);
3587c478bd9Sstevel@tonic-gate 	mcpu = &CPU->cpu_m;
359b0fc0e77Sgovinda 	cpu_id = CPU->cpu_id;
361b0fc0e77Sgovinda 	iv = (intr_vec_t *)inum;
3627c478bd9Sstevel@tonic-gate 	prev = NULL;
363b0fc0e77Sgovinda 	next = mcpu->intr_head[pil];
365b0fc0e77Sgovinda 	/* Find a matching entry in the list */
366b0fc0e77Sgovinda 	while (next != NULL) {
367b0fc0e77Sgovinda 		if (next == iv)
3687c478bd9Sstevel@tonic-gate 			break;
369b0fc0e77Sgovinda 		prev = next;
370b0fc0e77Sgovinda 		next = IV_GET_PIL_NEXT(next, cpu_id);
3717c478bd9Sstevel@tonic-gate 	}
373b0fc0e77Sgovinda 	if (next != NULL) {
374b0fc0e77Sgovinda 		intr_vec_t	*next_iv = IV_GET_PIL_NEXT(next, cpu_id);
376b0fc0e77Sgovinda 		/* Remove entry from list */
3777c478bd9Sstevel@tonic-gate 		if (prev != NULL)
378b0fc0e77Sgovinda 			IV_SET_PIL_NEXT(prev, cpu_id, next_iv); /* non-head */
3797c478bd9Sstevel@tonic-gate 		else
380b0fc0e77Sgovinda 			mcpu->intr_head[pil] = next_iv; /* head */
382b0fc0e77Sgovinda 		if (next_iv == NULL)
383b0fc0e77Sgovinda 			mcpu->intr_tail[pil] = prev; /* tail */
3847c478bd9Sstevel@tonic-gate 	}
386b0fc0e77Sgovinda 	/* Clear pending interrupts at this level if the list is empty */
3877c478bd9Sstevel@tonic-gate 	if (mcpu->intr_head[pil] == NULL) {
3887c478bd9Sstevel@tonic-gate 		clr = 1 << pil;
3897c478bd9Sstevel@tonic-gate 		if (pil == PIL_14)
3907c478bd9Sstevel@tonic-gate 			clr |= (TICK_INT_MASK | STICK_INT_MASK);
3917c478bd9Sstevel@tonic-gate 		wr_clr_softint(clr);
3927c478bd9Sstevel@tonic-gate 	}
3937c478bd9Sstevel@tonic-gate }
3967c478bd9Sstevel@tonic-gate /*
3977c478bd9Sstevel@tonic-gate  * Send a directed interrupt of specified interrupt number id to a cpu.
3987c478bd9Sstevel@tonic-gate  */
3997c478bd9Sstevel@tonic-gate void
send_dirint(int cpuix,int intr_id)4007c478bd9Sstevel@tonic-gate send_dirint(
4017c478bd9Sstevel@tonic-gate 	int cpuix,		/* cpu to be interrupted */
4027c478bd9Sstevel@tonic-gate 	int intr_id)		/* interrupt number id */
4037c478bd9Sstevel@tonic-gate {
4047c478bd9Sstevel@tonic-gate 	xt_one(cpuix, setsoftint_tl1, intr_id, 0);
4057c478bd9Sstevel@tonic-gate }
4077c478bd9Sstevel@tonic-gate /*
4087c478bd9Sstevel@tonic-gate  * Take the specified CPU out of participation in interrupts.
4097c478bd9Sstevel@tonic-gate  *	Called by p_online(2) when a processor is being taken off-line.
4107c478bd9Sstevel@tonic-gate  *	This allows interrupt threads being handled on the processor to
4117c478bd9Sstevel@tonic-gate  *	complete before the processor is idled.
4127c478bd9Sstevel@tonic-gate  */
4137c478bd9Sstevel@tonic-gate int
cpu_disable_intr(struct cpu * cp)4147c478bd9Sstevel@tonic-gate cpu_disable_intr(struct cpu *cp)
4157c478bd9Sstevel@tonic-gate {
4167c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&cpu_lock));
4187c478bd9Sstevel@tonic-gate 	/*
4197c478bd9Sstevel@tonic-gate 	 * Turn off the CPU_ENABLE flag before calling the redistribution
4207c478bd9Sstevel@tonic-gate 	 * function, since it checks for this in the cpu flags.
4217c478bd9Sstevel@tonic-gate 	 */
4227c478bd9Sstevel@tonic-gate 	cp->cpu_flags &= ~CPU_ENABLE;
423*c3377ee9SJohn Levon 	ncpus_intr_enabled--;
4257c478bd9Sstevel@tonic-gate 	intr_redist_all_cpus();
4277c478bd9Sstevel@tonic-gate 	return (0);
4287c478bd9Sstevel@tonic-gate }
4307c478bd9Sstevel@tonic-gate /*
4317c478bd9Sstevel@tonic-gate  * Allow the specified CPU to participate in interrupts.
4327c478bd9Sstevel@tonic-gate  *	Called by p_online(2) if a processor could not be taken off-line
4337c478bd9Sstevel@tonic-gate  *	because of bound threads, in order to resume processing interrupts.
4347c478bd9Sstevel@tonic-gate  *	Also called after starting a processor.
4357c478bd9Sstevel@tonic-gate  */
4367c478bd9Sstevel@tonic-gate void
cpu_enable_intr(struct cpu * cp)4377c478bd9Sstevel@tonic-gate cpu_enable_intr(struct cpu *cp)
4387c478bd9Sstevel@tonic-gate {
4397c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&cpu_lock));
4417c478bd9Sstevel@tonic-gate 	cp->cpu_flags |= CPU_ENABLE;
442*c3377ee9SJohn Levon 	ncpus_intr_enabled++;
4447c478bd9Sstevel@tonic-gate 	intr_redist_all_cpus();
4457c478bd9Sstevel@tonic-gate }
4477c478bd9Sstevel@tonic-gate /*
4487c478bd9Sstevel@tonic-gate  * Add function to callback list for intr_redist_all_cpus.  We keep two lists,
4497c478bd9Sstevel@tonic-gate  * one for weighted callbacks and one for normal callbacks. Weighted callbacks
4507c478bd9Sstevel@tonic-gate  * are issued to redirect interrupts of a specified weight, from heavy to
4517c478bd9Sstevel@tonic-gate  * light.  This allows all the interrupts of a given weight to be redistributed
4527c478bd9Sstevel@tonic-gate  * for all weighted nexus drivers prior to those of less weight.
4537c478bd9Sstevel@tonic-gate  */
4547c478bd9Sstevel@tonic-gate static void
intr_dist_add_list(struct intr_dist ** phead,void (* func)(void *),void * arg)4557c478bd9Sstevel@tonic-gate intr_dist_add_list(struct intr_dist **phead, void (*func)(void *), void *arg)
4567c478bd9Sstevel@tonic-gate {
4577c478bd9Sstevel@tonic-gate 	struct intr_dist *new = kmem_alloc(sizeof (*new), KM_SLEEP);
4587c478bd9Sstevel@tonic-gate 	struct intr_dist *iptr;
4597c478bd9Sstevel@tonic-gate 	struct intr_dist **pptr;
4617c478bd9Sstevel@tonic-gate 	ASSERT(func);
4627c478bd9Sstevel@tonic-gate 	new->func = func;
4637c478bd9Sstevel@tonic-gate 	new->arg = arg;
4647c478bd9Sstevel@tonic-gate 	new->next = NULL;
4667c478bd9Sstevel@tonic-gate 	/* Add to tail so that redistribution occurs in original order. */
4677c478bd9Sstevel@tonic-gate 	mutex_enter(&intr_dist_lock);
4687c478bd9Sstevel@tonic-gate 	for (iptr = *phead, pptr = phead; iptr != NULL;
4697c478bd9Sstevel@tonic-gate 	    pptr = &iptr->next, iptr = iptr->next) {
4707c478bd9Sstevel@tonic-gate 		/* check for problems as we locate the tail */
4717c478bd9Sstevel@tonic-gate 		if ((iptr->func == func) && (iptr->arg == arg)) {
4727c478bd9Sstevel@tonic-gate 			cmn_err(CE_PANIC, "intr_dist_add_list(): duplicate");
4737c478bd9Sstevel@tonic-gate 			/*NOTREACHED*/
4747c478bd9Sstevel@tonic-gate 		}
4757c478bd9Sstevel@tonic-gate 	}
4767c478bd9Sstevel@tonic-gate 	*pptr = new;
4787c478bd9Sstevel@tonic-gate 	mutex_exit(&intr_dist_lock);
4797c478bd9Sstevel@tonic-gate }
4817c478bd9Sstevel@tonic-gate void
intr_dist_add(void (* func)(void *),void * arg)4827c478bd9Sstevel@tonic-gate intr_dist_add(void (*func)(void *), void *arg)
4837c478bd9Sstevel@tonic-gate {
4847c478bd9Sstevel@tonic-gate 	intr_dist_add_list(&intr_dist_head, (void (*)(void *))func, arg);
4857c478bd9Sstevel@tonic-gate }
4877c478bd9Sstevel@tonic-gate void
intr_dist_add_weighted(void (* func)(void *,int32_t,int32_t),void * arg)4887c478bd9Sstevel@tonic-gate intr_dist_add_weighted(void (*func)(void *, int32_t, int32_t), void *arg)
4897c478bd9Sstevel@tonic-gate {
4907c478bd9Sstevel@tonic-gate 	intr_dist_add_list(&intr_dist_whead, (void (*)(void *))func, arg);
4917c478bd9Sstevel@tonic-gate }
4937c478bd9Sstevel@tonic-gate /*
4947c478bd9Sstevel@tonic-gate  * Search for the interrupt distribution structure with the specified
4957c478bd9Sstevel@tonic-gate  * mondo vec reg in the interrupt distribution list. If a match is found,
4967c478bd9Sstevel@tonic-gate  * then delete the entry from the list. The caller is responsible for
4977c478bd9Sstevel@tonic-gate  * modifying the mondo vector registers.
4987c478bd9Sstevel@tonic-gate  */
4997c478bd9Sstevel@tonic-gate static void
intr_dist_rem_list(struct intr_dist ** headp,void (* func)(void *),void * arg)5007c478bd9Sstevel@tonic-gate intr_dist_rem_list(struct intr_dist **headp, void (*func)(void *), void *arg)
5017c478bd9Sstevel@tonic-gate {
5027c478bd9Sstevel@tonic-gate 	struct intr_dist *iptr;
5037c478bd9Sstevel@tonic-gate 	struct intr_dist **vect;
5057c478bd9Sstevel@tonic-gate 	mutex_enter(&intr_dist_lock);
5067c478bd9Sstevel@tonic-gate 	for (iptr = *headp, vect = headp;
5077c478bd9Sstevel@tonic-gate 	    iptr != NULL; vect = &iptr->next, iptr = iptr->next) {
5087c478bd9Sstevel@tonic-gate 		if ((iptr->func == func) && (iptr->arg == arg)) {
5097c478bd9Sstevel@tonic-gate 			*vect = iptr->next;
5107c478bd9Sstevel@tonic-gate 			kmem_free(iptr, sizeof (struct intr_dist));
5117c478bd9Sstevel@tonic-gate 			mutex_exit(&intr_dist_lock);
5127c478bd9Sstevel@tonic-gate 			return;
5137c478bd9Sstevel@tonic-gate 		}
5147c478bd9Sstevel@tonic-gate 	}
5167c478bd9Sstevel@tonic-gate 	if (!panicstr)
5177c478bd9Sstevel@tonic-gate 		cmn_err(CE_PANIC, "intr_dist_rem_list: not found");
5187c478bd9Sstevel@tonic-gate 	mutex_exit(&intr_dist_lock);
5197c478bd9Sstevel@tonic-gate }
5217c478bd9Sstevel@tonic-gate void
intr_dist_rem(void (* func)(void *),void * arg)5227c478bd9Sstevel@tonic-gate intr_dist_rem(void (*func)(void *), void *arg)
5237c478bd9Sstevel@tonic-gate {
5247c478bd9Sstevel@tonic-gate 	intr_dist_rem_list(&intr_dist_head, (void (*)(void *))func, arg);
5257c478bd9Sstevel@tonic-gate }
5277c478bd9Sstevel@tonic-gate void
intr_dist_rem_weighted(void (* func)(void *,int32_t,int32_t),void * arg)5287c478bd9Sstevel@tonic-gate intr_dist_rem_weighted(void (*func)(void *, int32_t, int32_t), void *arg)
5297c478bd9Sstevel@tonic-gate {
5307c478bd9Sstevel@tonic-gate 	intr_dist_rem_list(&intr_dist_whead, (void (*)(void *))func, arg);
5317c478bd9Sstevel@tonic-gate }
5337c478bd9Sstevel@tonic-gate /*
5347c478bd9Sstevel@tonic-gate  * Initiate interrupt redistribution.  Redistribution improves the isolation
5357c478bd9Sstevel@tonic-gate  * associated with interrupt weights by ordering operations from heavy weight
5367c478bd9Sstevel@tonic-gate  * to light weight.  When a CPUs orientation changes relative to interrupts,
5377c478bd9Sstevel@tonic-gate  * there is *always* a redistribution to accommodate this change (call to
5387c478bd9Sstevel@tonic-gate  * intr_redist_all_cpus()).  As devices (not CPUs) attach/detach it is possible
5397c478bd9Sstevel@tonic-gate  * that a redistribution could improve the quality of an initialization. For
5407c478bd9Sstevel@tonic-gate  * example, if you are not using a NIC it may not be attached with s10 (devfs).
5417c478bd9Sstevel@tonic-gate  * If you then configure the NIC (ifconfig), this may cause the NIC to attach
5427c478bd9Sstevel@tonic-gate  * and plumb interrupts.  The CPU assignment for the NIC's interrupts is
5437c478bd9Sstevel@tonic-gate  * occurring late, so optimal "isolation" relative to weight is not occurring.
5447c478bd9Sstevel@tonic-gate  * The same applies to detach, although in this case doing the redistribution
5457c478bd9Sstevel@tonic-gate  * might improve "spread" for medium weight devices since the "isolation" of
5467c478bd9Sstevel@tonic-gate  * a higher weight device may no longer be present.
5477c478bd9Sstevel@tonic-gate  *
5487c478bd9Sstevel@tonic-gate  * NB: We should provide a utility to trigger redistribution (ala "intradm -r").
5497c478bd9Sstevel@tonic-gate  *
5507c478bd9Sstevel@tonic-gate  * NB: There is risk associated with automatically triggering execution of the
5517c478bd9Sstevel@tonic-gate  * redistribution code at arbitrary times. The risk comes from the fact that
5527c478bd9Sstevel@tonic-gate  * there is a lot of low-level hardware interaction associated with a
5537c478bd9Sstevel@tonic-gate  * redistribution.  At some point we may want this code to perform automatic
5547c478bd9Sstevel@tonic-gate  * redistribution (redistribution thread; trigger timeout when add/remove
5557c478bd9Sstevel@tonic-gate  * weight delta is large enough, and call cv_signal from timeout - causing
5567c478bd9Sstevel@tonic-gate  * thead to call i_ddi_intr_redist_all_cpus()) but this is considered too
5577c478bd9Sstevel@tonic-gate  * risky at this time.
5587c478bd9Sstevel@tonic-gate  */
5597c478bd9Sstevel@tonic-gate void
i_ddi_intr_redist_all_cpus()5607c478bd9Sstevel@tonic-gate i_ddi_intr_redist_all_cpus()
5617c478bd9Sstevel@tonic-gate {
5627c478bd9Sstevel@tonic-gate 	mutex_enter(&cpu_lock);
5637c478bd9Sstevel@tonic-gate 	INTR_DEBUG((CE_CONT, "intr_dist: i_ddi_intr_redist_all_cpus\n"));
5647c478bd9Sstevel@tonic-gate 	intr_redist_all_cpus();
5657c478bd9Sstevel@tonic-gate 	mutex_exit(&cpu_lock);
5667c478bd9Sstevel@tonic-gate }
5687c478bd9Sstevel@tonic-gate /*
5697c478bd9Sstevel@tonic-gate  * Redistribute all interrupts
5707c478bd9Sstevel@tonic-gate  *
5717c478bd9Sstevel@tonic-gate  * This function redistributes all interrupting devices, running the
5727c478bd9Sstevel@tonic-gate  * parent callback functions for each node.
5737c478bd9Sstevel@tonic-gate  */
5747c478bd9Sstevel@tonic-gate void
intr_redist_all_cpus(void)5757c478bd9Sstevel@tonic-gate intr_redist_all_cpus(void)
5767c478bd9Sstevel@tonic-gate {
5777c478bd9Sstevel@tonic-gate 	struct cpu *cp;
5787c478bd9Sstevel@tonic-gate 	struct intr_dist *iptr;
5797c478bd9Sstevel@tonic-gate 	int32_t weight, max_weight;
5817c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&cpu_lock));
5827c478bd9Sstevel@tonic-gate 	mutex_enter(&intr_dist_lock);
5847c478bd9Sstevel@tonic-gate 	/*
5857c478bd9Sstevel@tonic-gate 	 * zero cpu_intr_weight on all cpus - it is safe to traverse
5867c478bd9Sstevel@tonic-gate 	 * cpu_list since we hold cpu_lock.
5877c478bd9Sstevel@tonic-gate 	 */
5887c478bd9Sstevel@tonic-gate 	cp = cpu_list;
5897c478bd9Sstevel@tonic-gate 	do {
5907c478bd9Sstevel@tonic-gate 		cp->cpu_intr_weight = 0;
5917c478bd9Sstevel@tonic-gate 	} while ((cp = cp->cpu_next) != cpu_list);
5937c478bd9Sstevel@tonic-gate 	/*
5947c478bd9Sstevel@tonic-gate 	 * Assume that this redistribution may encounter a device weight
5957c478bd9Sstevel@tonic-gate 	 * via driver.conf tuning of "ddi-intr-weight" that is at most
5967c478bd9Sstevel@tonic-gate 	 * intr_dist_weight_maxfactor times larger.
5977c478bd9Sstevel@tonic-gate 	 */
5987c478bd9Sstevel@tonic-gate 	max_weight = intr_dist_weight_max * intr_dist_weight_maxfactor;
5997c478bd9Sstevel@tonic-gate 	if (max_weight > intr_dist_weight_maxmax)
6007c478bd9Sstevel@tonic-gate 		max_weight = intr_dist_weight_maxmax;
6017c478bd9Sstevel@tonic-gate 	intr_dist_weight_max = 1;
6037c478bd9Sstevel@tonic-gate 	INTR_DEBUG((CE_CONT, "intr_dist: "
6047c478bd9Sstevel@tonic-gate 	    "intr_redist_all_cpus: %d-0\n", max_weight));
6067c478bd9Sstevel@tonic-gate 	/*
6077c478bd9Sstevel@tonic-gate 	 * Redistribute weighted, from heavy to light.  The callback that
6087c478bd9Sstevel@tonic-gate 	 * specifies a weight equal to weight_max should redirect all
6097c478bd9Sstevel@tonic-gate 	 * interrupts of weight weight_max or greater [weight_max, inf.).
6107c478bd9Sstevel@tonic-gate 	 * Interrupts of lesser weight should be processed on the call with
6117c478bd9Sstevel@tonic-gate 	 * the matching weight. This allows all the heaver weight interrupts
6127c478bd9Sstevel@tonic-gate 	 * on all weighted busses (multiple pci busses) to be redirected prior
6137c478bd9Sstevel@tonic-gate 	 * to any lesser weight interrupts.
6147c478bd9Sstevel@tonic-gate 	 */
6157c478bd9Sstevel@tonic-gate 	for (weight = max_weight; weight >= 0; weight--)
6167c478bd9Sstevel@tonic-gate 		for (iptr = intr_dist_whead; iptr != NULL; iptr = iptr->next)
6177c478bd9Sstevel@tonic-gate 			((void (*)(void *, int32_t, int32_t))iptr->func)
6187c478bd9Sstevel@tonic-gate 			    (iptr->arg, max_weight, weight);
6207c478bd9Sstevel@tonic-gate 	/* redistribute normal (non-weighted) interrupts */
6217c478bd9Sstevel@tonic-gate 	for (iptr = intr_dist_head; iptr != NULL; iptr = iptr->next)
6227c478bd9Sstevel@tonic-gate 		((void (*)(void *))iptr->func)(iptr->arg);
6237c478bd9Sstevel@tonic-gate 	mutex_exit(&intr_dist_lock);
6247c478bd9Sstevel@tonic-gate }
6267c478bd9Sstevel@tonic-gate void
intr_redist_all_cpus_shutdown(void)6277c478bd9Sstevel@tonic-gate intr_redist_all_cpus_shutdown(void)
6287c478bd9Sstevel@tonic-gate {
6297c478bd9Sstevel@tonic-gate 	intr_policy = INTR_CURRENT_CPU;
6307c478bd9Sstevel@tonic-gate 	intr_redist_all_cpus();
6317c478bd9Sstevel@tonic-gate }
6337c478bd9Sstevel@tonic-gate /*
6347c478bd9Sstevel@tonic-gate  * Determine what CPU to target, based on interrupt policy.
6357c478bd9Sstevel@tonic-gate  *
6367c478bd9Sstevel@tonic-gate  * INTR_FLAT_DIST: hold a current CPU pointer in a static variable and
6377c478bd9Sstevel@tonic-gate  *	advance through interrupt enabled cpus (round-robin).
6387c478bd9Sstevel@tonic-gate  *
6397c478bd9Sstevel@tonic-gate  * INTR_WEIGHTED_DIST: search for an enabled CPU with the lowest
6407c478bd9Sstevel@tonic-gate  *	cpu_intr_weight, round robin when all equal.
6417c478bd9Sstevel@tonic-gate  *
6427c478bd9Sstevel@tonic-gate  *	Weighted interrupt distribution provides two things: "spread" of weight
6437c478bd9Sstevel@tonic-gate  *	(associated with algorithm itself) and "isolation" (associated with a
6447c478bd9Sstevel@tonic-gate  *	particular device weight). A redistribution is what provides optimal
6457c478bd9Sstevel@tonic-gate  *	"isolation" of heavy weight interrupts, optimal "spread" of weight
6467c478bd9Sstevel@tonic-gate  *	(relative to what came before) is always occurring.
6477c478bd9Sstevel@tonic-gate  *
6487c478bd9Sstevel@tonic-gate  *	An interrupt weight is a subjective number that represents the
6497c478bd9Sstevel@tonic-gate  *	percentage of a CPU required to service a device's interrupts: the
6507c478bd9Sstevel@tonic-gate  *	default weight is 0% (however the algorithm still maintains
6517c478bd9Sstevel@tonic-gate  *	round-robin), a network interface controller (NIC) may have a large
6527c478bd9Sstevel@tonic-gate  *	weight (35%). Interrupt weight only has meaning relative to the
6537c478bd9Sstevel@tonic-gate  *	interrupt weight of other devices: a CPU can be weighted more than
6547c478bd9Sstevel@tonic-gate  *	100%, and a single device might consume more than 100% of a CPU.
6557c478bd9Sstevel@tonic-gate  *
6567c478bd9Sstevel@tonic-gate  *	A coarse interrupt weight can be defined by the parent nexus driver
6577c478bd9Sstevel@tonic-gate  *	based on bus specific information, like pci class codes. A nexus
6587c478bd9Sstevel@tonic-gate  *	driver that supports device interrupt weighting for its children
6597c478bd9Sstevel@tonic-gate  *	should call intr_dist_cpuid_add/rem_device_weight(), which adds
6607c478bd9Sstevel@tonic-gate  *	and removes the weight of a device from the CPU that an interrupt
6617c478bd9Sstevel@tonic-gate  *	is directed at.  The quality of initialization improves when the
6627c478bd9Sstevel@tonic-gate  *	device interrupt weights more accuracy reflect actual run-time weights,
6637c478bd9Sstevel@tonic-gate  *	and as the assignments are ordered from is heavy to light.
6647c478bd9Sstevel@tonic-gate  *
6657c478bd9Sstevel@tonic-gate  *	The implementation also supports interrupt weight being specified in
6667c478bd9Sstevel@tonic-gate  *	driver.conf files via the property "ddi-intr-weight", which takes
6677c478bd9Sstevel@tonic-gate  *	precedence over the nexus supplied weight.  This support is added to
6687c478bd9Sstevel@tonic-gate  *	permit possible tweaking in the product in response to customer
6697c478bd9Sstevel@tonic-gate  *	problems. This is not a formal or committed interface.
6707c478bd9Sstevel@tonic-gate  *
6717c478bd9Sstevel@tonic-gate  *	While a weighted approach chooses the CPU providing the best spread
6727c478bd9Sstevel@tonic-gate  *	given past weights, less than optimal isolation can result in cases
6737c478bd9Sstevel@tonic-gate  *	where heavy weight devices show up last. The nexus driver's interrupt
6747c478bd9Sstevel@tonic-gate  *	redistribution logic should use intr_dist_add/rem_weighted so that
6757c478bd9Sstevel@tonic-gate  *	interrupts can be redistributed heavy first for optimal isolation.
6767c478bd9Sstevel@tonic-gate  */
6777c478bd9Sstevel@tonic-gate uint32_t
intr_dist_cpuid(void)6787c478bd9Sstevel@tonic-gate intr_dist_cpuid(void)
6797c478bd9Sstevel@tonic-gate {
6807c478bd9Sstevel@tonic-gate 	static struct cpu	*curr_cpu;
6817c478bd9Sstevel@tonic-gate 	struct cpu		*start_cpu;
6827c478bd9Sstevel@tonic-gate 	struct cpu		*new_cpu;
6837c478bd9Sstevel@tonic-gate 	struct cpu		*cp;
6847c478bd9Sstevel@tonic-gate 	int			cpuid = -1;
6867c478bd9Sstevel@tonic-gate 	/* Establish exclusion for curr_cpu and cpu_intr_weight manipulation */
6877c478bd9Sstevel@tonic-gate 	mutex_enter(&intr_dist_cpu_lock);
6897c478bd9Sstevel@tonic-gate 	switch (intr_policy) {
6907c478bd9Sstevel@tonic-gate 	case INTR_CURRENT_CPU:
6917c478bd9Sstevel@tonic-gate 		cpuid = CPU->cpu_id;
6927c478bd9Sstevel@tonic-gate 		break;
6947c478bd9Sstevel@tonic-gate 	case INTR_BOOT_CPU:
6957c478bd9Sstevel@tonic-gate 		panic("INTR_BOOT_CPU no longer supported.");
6967c478bd9Sstevel@tonic-gate 		/*NOTREACHED*/
6987c478bd9Sstevel@tonic-gate 	case INTR_FLAT_DIST:
6997c478bd9Sstevel@tonic-gate 	case INTR_WEIGHTED_DIST:
7007c478bd9Sstevel@tonic-gate 	default:
7017c478bd9Sstevel@tonic-gate 		/*
7027c478bd9Sstevel@tonic-gate 		 * Ensure that curr_cpu is valid - cpu_next will be NULL if
7037c478bd9Sstevel@tonic-gate 		 * the cpu has been deleted (cpu structs are never freed).
7047c478bd9Sstevel@tonic-gate 		 */
7057c478bd9Sstevel@tonic-gate 		if (curr_cpu == NULL || curr_cpu->cpu_next == NULL)
7067c478bd9Sstevel@tonic-gate 			curr_cpu = CPU;
7087c478bd9Sstevel@tonic-gate 		/*
7097c478bd9Sstevel@tonic-gate 		 * Advance to online CPU after curr_cpu (round-robin). For
7107c478bd9Sstevel@tonic-gate 		 * INTR_WEIGHTED_DIST we choose the cpu with the lightest
7117c478bd9Sstevel@tonic-gate 		 * weight.  For a nexus that does not support weight the
7127c478bd9Sstevel@tonic-gate 		 * default weight of zero is used. We degrade to round-robin
7137c478bd9Sstevel@tonic-gate 		 * behavior among equal weightes.  The default weight is zero
7147c478bd9Sstevel@tonic-gate 		 * and round-robin behavior continues.
7157c478bd9Sstevel@tonic-gate 		 *
7167c478bd9Sstevel@tonic-gate 		 * Disable preemption while traversing cpu_next_onln to
7177c478bd9Sstevel@tonic-gate 		 * ensure the list does not change.  This works because
7187c478bd9Sstevel@tonic-gate 		 * modifiers of this list and other lists in a struct cpu
7197c478bd9Sstevel@tonic-gate 		 * call pause_cpus() before making changes.
7207c478bd9Sstevel@tonic-gate 		 */
7217c478bd9Sstevel@tonic-gate 		kpreempt_disable();
7227c478bd9Sstevel@tonic-gate 		cp = start_cpu = curr_cpu->cpu_next_onln;
7237c478bd9Sstevel@tonic-gate 		new_cpu = NULL;
7247c478bd9Sstevel@tonic-gate 		do {
7257c478bd9Sstevel@tonic-gate 			/* Skip CPUs with interrupts disabled */
7267c478bd9Sstevel@tonic-gate 			if ((cp->cpu_flags & CPU_ENABLE) == 0)
7277c478bd9Sstevel@tonic-gate 				continue;
7297c478bd9Sstevel@tonic-gate 			if (intr_policy == INTR_FLAT_DIST) {
7307c478bd9Sstevel@tonic-gate 				/* select CPU */
7317c478bd9Sstevel@tonic-gate 				new_cpu = cp;
7327c478bd9Sstevel@tonic-gate 				break;
7337c478bd9Sstevel@tonic-gate 			} else if ((new_cpu == NULL) ||
7347c478bd9Sstevel@tonic-gate 			    (cp->cpu_intr_weight < new_cpu->cpu_intr_weight)) {
7357c478bd9Sstevel@tonic-gate 				/* Choose if lighter weight */
7367c478bd9Sstevel@tonic-gate 				new_cpu = cp;
7377c478bd9Sstevel@tonic-gate 			}
7387c478bd9Sstevel@tonic-gate 		} while ((cp = cp->cpu_next_onln) != start_cpu);
7397c478bd9Sstevel@tonic-gate 		ASSERT(new_cpu);
7407c478bd9Sstevel@tonic-gate 		cpuid = new_cpu->cpu_id;
7427c478bd9Sstevel@tonic-gate 		INTR_DEBUG((CE_CONT, "intr_dist: cpu %2d weight %3d: "
7437c478bd9Sstevel@tonic-gate 		    "targeted\n", cpuid, new_cpu->cpu_intr_weight));
7457c478bd9Sstevel@tonic-gate 		/* update static pointer for next round-robin */
7467c478bd9Sstevel@tonic-gate 		curr_cpu = new_cpu;
7477c478bd9Sstevel@tonic-gate 		kpreempt_enable();
7487c478bd9Sstevel@tonic-gate 		break;
7497c478bd9Sstevel@tonic-gate 	}
7507c478bd9Sstevel@tonic-gate 	mutex_exit(&intr_dist_cpu_lock);
7517c478bd9Sstevel@tonic-gate 	return (cpuid);
7527c478bd9Sstevel@tonic-gate }
7547c478bd9Sstevel@tonic-gate /*
7557c478bd9Sstevel@tonic-gate  * Add or remove the the weight of a device from a CPUs interrupt weight.
7567c478bd9Sstevel@tonic-gate  *
7577c478bd9Sstevel@tonic-gate  * We expect nexus drivers to call intr_dist_cpuid_add/rem_device_weight for
7587c478bd9Sstevel@tonic-gate  * their children to improve the overall quality of interrupt initialization.
7597c478bd9Sstevel@tonic-gate  *
7607c478bd9Sstevel@tonic-gate  * If a nexues shares the CPU returned by a single intr_dist_cpuid() call
7617c478bd9Sstevel@tonic-gate  * among multiple devices (sharing ino) then the nexus should call
7627c478bd9Sstevel@tonic-gate  * intr_dist_cpuid_add/rem_device_weight for each device separately. Devices
7637c478bd9Sstevel@tonic-gate  * that share must specify the same cpuid.
7647c478bd9Sstevel@tonic-gate  *
7657c478bd9Sstevel@tonic-gate  * If a nexus driver is unable to determine the cpu at remove_intr time
7667c478bd9Sstevel@tonic-gate  * for some of its interrupts, then it should not call add_device_weight -
7677c478bd9Sstevel@tonic-gate  * intr_dist_cpuid will still provide round-robin.
7687c478bd9Sstevel@tonic-gate  *
7697c478bd9Sstevel@tonic-gate  * An established device weight (from dev_info node) takes precedence over
7707c478bd9Sstevel@tonic-gate  * the weight passed in.  If a device weight is not already established
7717c478bd9Sstevel@tonic-gate  * then the passed in nexus weight is established.
7727c478bd9Sstevel@tonic-gate  */
7737c478bd9Sstevel@tonic-gate void
intr_dist_cpuid_add_device_weight(uint32_t cpuid,dev_info_t * dip,int32_t nweight)7747c478bd9Sstevel@tonic-gate intr_dist_cpuid_add_device_weight(uint32_t cpuid,
7757c478bd9Sstevel@tonic-gate     dev_info_t *dip, int32_t nweight)
7767c478bd9Sstevel@tonic-gate {
7777c478bd9Sstevel@tonic-gate 	int32_t		eweight;
7797c478bd9Sstevel@tonic-gate 	/*
7807c478bd9Sstevel@tonic-gate 	 * For non-weighted policy everything has weight of zero (and we get
7817c478bd9Sstevel@tonic-gate 	 * round-robin distribution from intr_dist_cpuid).
7827c478bd9Sstevel@tonic-gate 	 * NB: intr_policy is limited to this file. A weighted nexus driver is
7837c478bd9Sstevel@tonic-gate 	 * calls this rouitne even if intr_policy has been patched to
7847c478bd9Sstevel@tonic-gate 	 * INTR_FLAG_DIST.
7857c478bd9Sstevel@tonic-gate 	 */
7867c478bd9Sstevel@tonic-gate 	ASSERT(dip);
7877c478bd9Sstevel@tonic-gate 	if (intr_policy != INTR_WEIGHTED_DIST)
7887c478bd9Sstevel@tonic-gate 		return;
7907c478bd9Sstevel@tonic-gate 	eweight = i_ddi_get_intr_weight(dip);
7917c478bd9Sstevel@tonic-gate 	INTR_DEBUG((CE_CONT, "intr_dist: cpu %2d weight %3d: +%2d/%2d for "
7927c478bd9Sstevel@tonic-gate 	    "%s#%d/%s#%d\n", cpuid, cpu[cpuid]->cpu_intr_weight,
7937c478bd9Sstevel@tonic-gate 	    nweight, eweight, ddi_driver_name(ddi_get_parent(dip)),
7947c478bd9Sstevel@tonic-gate 	    ddi_get_instance(ddi_get_parent(dip)),
7957c478bd9Sstevel@tonic-gate 	    ddi_driver_name(dip), ddi_get_instance(dip)));
7977c478bd9Sstevel@tonic-gate 	/* if no establish weight, establish nexus weight */
7987c478bd9Sstevel@tonic-gate 	if (eweight < 0) {
7997c478bd9Sstevel@tonic-gate 		if (nweight > 0)
8007c478bd9Sstevel@tonic-gate 			(void) i_ddi_set_intr_weight(dip, nweight);
8017c478bd9Sstevel@tonic-gate 		else
8027c478bd9Sstevel@tonic-gate 			nweight = 0;
8037c478bd9Sstevel@tonic-gate 	} else
8047c478bd9Sstevel@tonic-gate 		nweight = eweight;	/* use established weight */
8067c478bd9Sstevel@tonic-gate 	/* Establish exclusion for cpu_intr_weight manipulation */
8077c478bd9Sstevel@tonic-gate 	mutex_enter(&intr_dist_cpu_lock);
8087c478bd9Sstevel@tonic-gate 	cpu[cpuid]->cpu_intr_weight += nweight;
8107c478bd9Sstevel@tonic-gate 	/* update intr_dist_weight_max */
8117c478bd9Sstevel@tonic-gate 	if (nweight > intr_dist_weight_max)
8127c478bd9Sstevel@tonic-gate 		intr_dist_weight_max = nweight;
8137c478bd9Sstevel@tonic-gate 	mutex_exit(&intr_dist_cpu_lock);
8147c478bd9Sstevel@tonic-gate }
8167c478bd9Sstevel@tonic-gate void
intr_dist_cpuid_rem_device_weight(uint32_t cpuid,dev_info_t * dip)8177c478bd9Sstevel@tonic-gate intr_dist_cpuid_rem_device_weight(uint32_t cpuid, dev_info_t *dip)
8187c478bd9Sstevel@tonic-gate {
8197c478bd9Sstevel@tonic-gate 	struct cpu	*cp;
8207c478bd9Sstevel@tonic-gate 	int32_t		weight;
8227c478bd9Sstevel@tonic-gate 	ASSERT(dip);
8237c478bd9Sstevel@tonic-gate 	if (intr_policy != INTR_WEIGHTED_DIST)
8247c478bd9Sstevel@tonic-gate 		return;
8267c478bd9Sstevel@tonic-gate 	/* remove weight of device from cpu */
8277c478bd9Sstevel@tonic-gate 	weight = i_ddi_get_intr_weight(dip);
8287c478bd9Sstevel@tonic-gate 	if (weight < 0)
8297c478bd9Sstevel@tonic-gate 		weight = 0;
8307c478bd9Sstevel@tonic-gate 	INTR_DEBUG((CE_CONT, "intr_dist: cpu %2d weight %3d: -%2d    for "
8317c478bd9Sstevel@tonic-gate 	    "%s#%d/%s#%d\n", cpuid, cpu[cpuid]->cpu_intr_weight, weight,
8327c478bd9Sstevel@tonic-gate 	    ddi_driver_name(ddi_get_parent(dip)),
8337c478bd9Sstevel@tonic-gate 	    ddi_get_instance(ddi_get_parent(dip)),
8347c478bd9Sstevel@tonic-gate 	    ddi_driver_name(dip), ddi_get_instance(dip)));
8367c478bd9Sstevel@tonic-gate 	/* Establish exclusion for cpu_intr_weight manipulation */
8377c478bd9Sstevel@tonic-gate 	mutex_enter(&intr_dist_cpu_lock);
8387c478bd9Sstevel@tonic-gate 	cp = cpu[cpuid];
8397c478bd9Sstevel@tonic-gate 	cp->cpu_intr_weight -= weight;
8407c478bd9Sstevel@tonic-gate 	if (cp->cpu_intr_weight < 0)
8417c478bd9Sstevel@tonic-gate 		cp->cpu_intr_weight = 0;	/* sanity */
8427c478bd9Sstevel@tonic-gate 	mutex_exit(&intr_dist_cpu_lock);
8437c478bd9Sstevel@tonic-gate }
8452850d85bSmv ulong_t
create_softint(uint_t pil,uint_t (* func)(caddr_t,caddr_t),caddr_t arg1)8462850d85bSmv create_softint(uint_t pil, uint_t (*func)(caddr_t, caddr_t), caddr_t arg1)
8472850d85bSmv {
8482850d85bSmv 	uint64_t inum;
85007247649SMadhavan Venkataraman 	inum = add_softintr(pil, func, arg1, SOFTINT_MT);
8512850d85bSmv 	return ((ulong_t)inum);
8522850d85bSmv }
8542850d85bSmv void
invoke_softint(processorid_t cpuid,ulong_t hdl)8552850d85bSmv invoke_softint(processorid_t cpuid, ulong_t hdl)
8562850d85bSmv {
8572850d85bSmv 	uint64_t inum = hdl;
8592850d85bSmv 	if (cpuid == CPU->cpu_id)
8602850d85bSmv 		setsoftint(inum);
8612850d85bSmv 	else
8622850d85bSmv 		xt_one(cpuid, setsoftint_tl1, inum, 0);
8632850d85bSmv }
8652850d85bSmv void
remove_softint(ulong_t hdl)8662850d85bSmv remove_softint(ulong_t hdl)
8672850d85bSmv {
8682850d85bSmv 	uint64_t inum = hdl;
8702850d85bSmv 	(void) rem_softintr(inum);
8712850d85bSmv }
8732850d85bSmv void
sync_softint(cpuset_t set)8742850d85bSmv sync_softint(cpuset_t set)
8752850d85bSmv {
8762850d85bSmv 	xt_sync(set);
8772850d85bSmv }