xref: /illumos-gate/usr/src/uts/sun4/os/intr.c (revision 12ceefb6)
17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate  * CDDL HEADER START
37c478bd9Sstevel@tonic-gate  *
47c478bd9Sstevel@tonic-gate  * The contents of this file are subject to the terms of the
5100b72f4Sandrei  * Common Development and Distribution License (the "License").
6100b72f4Sandrei  * You may not use this file except in compliance with the License.
77c478bd9Sstevel@tonic-gate  *
87c478bd9Sstevel@tonic-gate  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bd9Sstevel@tonic-gate  * or http://www.opensolaris.org/os/licensing.
107c478bd9Sstevel@tonic-gate  * See the License for the specific language governing permissions
117c478bd9Sstevel@tonic-gate  * and limitations under the License.
127c478bd9Sstevel@tonic-gate  *
137c478bd9Sstevel@tonic-gate  * When distributing Covered Code, include this CDDL HEADER in each
147c478bd9Sstevel@tonic-gate  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bd9Sstevel@tonic-gate  * If applicable, add the following below this CDDL HEADER, with the
167c478bd9Sstevel@tonic-gate  * fields enclosed by brackets "[]" replaced with your own identifying
177c478bd9Sstevel@tonic-gate  * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bd9Sstevel@tonic-gate  *
197c478bd9Sstevel@tonic-gate  * CDDL HEADER END
207c478bd9Sstevel@tonic-gate  */
217c478bd9Sstevel@tonic-gate /*
222850d85bSmv  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
237c478bd9Sstevel@tonic-gate  * Use is subject to license terms.
247c478bd9Sstevel@tonic-gate  */
257c478bd9Sstevel@tonic-gate 
267c478bd9Sstevel@tonic-gate #pragma ident	"%Z%%M%	%I%	%E% SMI"
277c478bd9Sstevel@tonic-gate 
287c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h>
297c478bd9Sstevel@tonic-gate #include <sys/stack.h>
307c478bd9Sstevel@tonic-gate #include <sys/cpuvar.h>
317c478bd9Sstevel@tonic-gate #include <sys/ivintr.h>
327c478bd9Sstevel@tonic-gate #include <sys/intreg.h>
337c478bd9Sstevel@tonic-gate #include <sys/membar.h>
347c478bd9Sstevel@tonic-gate #include <sys/kmem.h>
357c478bd9Sstevel@tonic-gate #include <sys/intr.h>
36dd4eeefdSeota #include <sys/sunddi.h>
377c478bd9Sstevel@tonic-gate #include <sys/sunndi.h>
387c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h>
397c478bd9Sstevel@tonic-gate #include <sys/privregs.h>
407c478bd9Sstevel@tonic-gate #include <sys/systm.h>
417c478bd9Sstevel@tonic-gate #include <sys/archsystm.h>
427c478bd9Sstevel@tonic-gate #include <sys/machsystm.h>
437c478bd9Sstevel@tonic-gate #include <sys/x_call.h>
447c478bd9Sstevel@tonic-gate #include <vm/seg_kp.h>
457c478bd9Sstevel@tonic-gate #include <sys/debug.h>
467c478bd9Sstevel@tonic-gate #include <sys/cyclic.h>
47a1af7ba0Scwb #include <sys/kdi_impl.h>
48dd4eeefdSeota #include <sys/ddi_timer.h>
497c478bd9Sstevel@tonic-gate 
507c478bd9Sstevel@tonic-gate #include <sys/cpu_sgnblk_defs.h>
517c478bd9Sstevel@tonic-gate 
527c478bd9Sstevel@tonic-gate /* Global locks which protect the interrupt distribution lists */
537c478bd9Sstevel@tonic-gate static kmutex_t intr_dist_lock;
547c478bd9Sstevel@tonic-gate static kmutex_t intr_dist_cpu_lock;
557c478bd9Sstevel@tonic-gate 
567c478bd9Sstevel@tonic-gate /* Head of the interrupt distribution lists */
577c478bd9Sstevel@tonic-gate static struct intr_dist *intr_dist_head = NULL;
587c478bd9Sstevel@tonic-gate static struct intr_dist *intr_dist_whead = NULL;
597c478bd9Sstevel@tonic-gate 
60dd4eeefdSeota static uint64_t siron_inum[DDI_IPL_10]; /* software interrupt numbers */
613aedfe0bSmishra uint64_t *siron_cpu_inum = NULL;
623aedfe0bSmishra uint64_t siron_poke_cpu_inum;
633aedfe0bSmishra static int siron_cpu_setup(cpu_setup_t, int, void *);
643aedfe0bSmishra extern uint_t softlevel1();
653aedfe0bSmishra 
66dd4eeefdSeota static uint64_t siron1_inum; /* backward compatibility */
67b0fc0e77Sgovinda uint64_t poke_cpu_inum;
68b0fc0e77Sgovinda uint_t poke_cpu_intr(caddr_t arg1, caddr_t arg2);
693aedfe0bSmishra uint_t siron_poke_cpu_intr(caddr_t arg1, caddr_t arg2);
70b0fc0e77Sgovinda 
71f8047eabSsudheer /*
72f8047eabSsudheer  * Note:-
73f8047eabSsudheer  * siron_pending was originally created to prevent a resource over consumption
74f8047eabSsudheer  * bug in setsoftint(exhaustion of interrupt pool free list).
75f8047eabSsudheer  * It's original intention is obsolete with the use of iv_pending in
76f8047eabSsudheer  * setsoftint. However, siron_pending stayed around, acting as a second
77f8047eabSsudheer  * gatekeeper preventing soft interrupts from being queued. In this capacity,
78f8047eabSsudheer  * it can lead to hangs on MP systems, where due to global visibility issues
79f8047eabSsudheer  * it can end up set while iv_pending is reset, preventing soft interrupts from
80f8047eabSsudheer  * ever being processed. In addition to its gatekeeper role, init_intr also
81f8047eabSsudheer  * uses it to flag the situation where siron() was called before siron_inum has
82f8047eabSsudheer  * been defined.
83f8047eabSsudheer  *
84f8047eabSsudheer  * siron() does not need an extra gatekeeper; any cpu that wishes should be
85f8047eabSsudheer  * allowed to queue a soft interrupt. It is softint()'s job to ensure
86f8047eabSsudheer  * correct handling of the queues. Therefore, siron_pending has been
87f8047eabSsudheer  * stripped of its gatekeeper task, retaining only its intr_init job, where
88f8047eabSsudheer  * it indicates that there is a pending need to call siron().
89f8047eabSsudheer  */
90dd4eeefdSeota static int siron_pending[DDI_IPL_10]; /* software interrupt pending flags */
91dd4eeefdSeota static int siron1_pending; /* backward compatibility */
927c478bd9Sstevel@tonic-gate 
937c478bd9Sstevel@tonic-gate int intr_policy = INTR_WEIGHTED_DIST;	/* interrupt distribution policy */
947c478bd9Sstevel@tonic-gate int intr_dist_debug = 0;
957c478bd9Sstevel@tonic-gate int32_t intr_dist_weight_max = 1;
967c478bd9Sstevel@tonic-gate int32_t intr_dist_weight_maxmax = 1000;
977c478bd9Sstevel@tonic-gate int intr_dist_weight_maxfactor = 2;
987c478bd9Sstevel@tonic-gate #define	INTR_DEBUG(args) if (intr_dist_debug) cmn_err args
997c478bd9Sstevel@tonic-gate 
1007c478bd9Sstevel@tonic-gate /*
101b0fc0e77Sgovinda  * intr_init() - Interrupt initialization
102b0fc0e77Sgovinda  *	Initialize the system's interrupt vector table.
1037c478bd9Sstevel@tonic-gate  */
1047c478bd9Sstevel@tonic-gate void
1057c478bd9Sstevel@tonic-gate intr_init(cpu_t *cp)
1067c478bd9Sstevel@tonic-gate {
107dd4eeefdSeota 	int i;
108b0fc0e77Sgovinda 	extern uint_t softlevel1();
109b0fc0e77Sgovinda 
1107c478bd9Sstevel@tonic-gate 	init_ivintr();
111b0fc0e77Sgovinda 	REGISTER_BBUS_INTR();
112b0fc0e77Sgovinda 
113dd4eeefdSeota 	/*
114dd4eeefdSeota 	 * Register these software interrupts for ddi timer.
115dd4eeefdSeota 	 * Software interrupts up to the level 10 are supported.
116dd4eeefdSeota 	 */
117dd4eeefdSeota 	for (i = DDI_IPL_1; i <= DDI_IPL_10; i++) {
118dd4eeefdSeota 		siron_inum[i-1] = add_softintr(i, (softintrfunc)timer_softintr,
119dd4eeefdSeota 		    (caddr_t)(uintptr_t)(i), SOFTINT_ST);
120dd4eeefdSeota 	}
121dd4eeefdSeota 
122dd4eeefdSeota 	siron1_inum = add_softintr(PIL_1, softlevel1, 0, SOFTINT_ST);
123b0fc0e77Sgovinda 	poke_cpu_inum = add_softintr(PIL_13, poke_cpu_intr, 0, SOFTINT_MT);
1243aedfe0bSmishra 	siron_poke_cpu_inum = add_softintr(PIL_13,
1253aedfe0bSmishra 	    siron_poke_cpu_intr, 0, SOFTINT_MT);
126b0fc0e77Sgovinda 	cp->cpu_m.poke_cpu_outstanding = B_FALSE;
1277c478bd9Sstevel@tonic-gate 
1287c478bd9Sstevel@tonic-gate 	mutex_init(&intr_dist_lock, NULL, MUTEX_DEFAULT, NULL);
1297c478bd9Sstevel@tonic-gate 	mutex_init(&intr_dist_cpu_lock, NULL, MUTEX_DEFAULT, NULL);
1307c478bd9Sstevel@tonic-gate 
1317c478bd9Sstevel@tonic-gate 	/*
1327c478bd9Sstevel@tonic-gate 	 * A soft interrupt may have been requested prior to the initialization
1337c478bd9Sstevel@tonic-gate 	 * of soft interrupts.  Soft interrupts can't be dispatched until after
134b0fc0e77Sgovinda 	 * init_intr(), so we have to wait until now before we can dispatch the
135b0fc0e77Sgovinda 	 * pending soft interrupt (if any).
1367c478bd9Sstevel@tonic-gate 	 */
137dd4eeefdSeota 	for (i = DDI_IPL_1; i <= DDI_IPL_10; i++) {
138dd4eeefdSeota 		if (siron_pending[i-1]) {
139dd4eeefdSeota 			siron_pending[i-1] = 0;
140dd4eeefdSeota 			sir_on(i);
141dd4eeefdSeota 		}
142dd4eeefdSeota 	}
143dd4eeefdSeota 	if (siron1_pending) {
144dd4eeefdSeota 		siron1_pending = 0;
145f8047eabSsudheer 		siron();
146f8047eabSsudheer 	}
1477c478bd9Sstevel@tonic-gate }
1487c478bd9Sstevel@tonic-gate 
1497c478bd9Sstevel@tonic-gate /*
1507c478bd9Sstevel@tonic-gate  * poke_cpu_intr - fall through when poke_cpu calls
1517c478bd9Sstevel@tonic-gate  */
1527c478bd9Sstevel@tonic-gate /* ARGSUSED */
1537c478bd9Sstevel@tonic-gate uint_t
1547c478bd9Sstevel@tonic-gate poke_cpu_intr(caddr_t arg1, caddr_t arg2)
1557c478bd9Sstevel@tonic-gate {
1567c478bd9Sstevel@tonic-gate 	CPU->cpu_m.poke_cpu_outstanding = B_FALSE;
1577c478bd9Sstevel@tonic-gate 	membar_stld_stst();
1587c478bd9Sstevel@tonic-gate 	return (1);
1597c478bd9Sstevel@tonic-gate }
1607c478bd9Sstevel@tonic-gate 
161dd4eeefdSeota /*
162dd4eeefdSeota  * Trigger software interrupts dedicated to ddi timer.
163dd4eeefdSeota  */
164dd4eeefdSeota void
165dd4eeefdSeota sir_on(int level)
166dd4eeefdSeota {
167dd4eeefdSeota 	ASSERT(level >= DDI_IPL_1 && level <= DDI_IPL_10);
168dd4eeefdSeota 	if (siron_inum[level-1])
169dd4eeefdSeota 		setsoftint(siron_inum[level-1]);
170dd4eeefdSeota 	else
171dd4eeefdSeota 		siron_pending[level-1] = 1;
172dd4eeefdSeota }
173dd4eeefdSeota 
1747c478bd9Sstevel@tonic-gate /*
175a1af7ba0Scwb  * kmdb uses siron (and thus setsoftint) while the world is stopped in order to
176a1af7ba0Scwb  * inform its driver component that there's work to be done.  We need to keep
177a1af7ba0Scwb  * DTrace from instrumenting kmdb's siron and setsoftint.  We duplicate siron,
178a1af7ba0Scwb  * giving kmdb's version a kdi_ prefix to keep DTrace at bay.  The
179a1af7ba0Scwb  * implementation of setsoftint is complicated enough that we don't want to
180a1af7ba0Scwb  * duplicate it, but at the same time we don't want to preclude tracing either.
181a1af7ba0Scwb  * The meat of setsoftint() therefore goes into kdi_setsoftint, with
182a1af7ba0Scwb  * setsoftint() implemented as a wrapper.  This allows tracing, while still
183a1af7ba0Scwb  * providing a way for kmdb to sneak in unmolested.
1847c478bd9Sstevel@tonic-gate  */
185a1af7ba0Scwb void
186a1af7ba0Scwb kdi_siron(void)
187a1af7ba0Scwb {
188dd4eeefdSeota 	if (siron1_inum != 0)
189dd4eeefdSeota 		kdi_setsoftint(siron1_inum);
190a1af7ba0Scwb 	else
191dd4eeefdSeota 		siron1_pending = 1;
192a1af7ba0Scwb }
193a1af7ba0Scwb 
194a1af7ba0Scwb void
195a1af7ba0Scwb setsoftint(uint64_t inum)
196a1af7ba0Scwb {
197a1af7ba0Scwb 	kdi_setsoftint(inum);
198a1af7ba0Scwb }
199a1af7ba0Scwb 
2003aedfe0bSmishra /*
2013aedfe0bSmishra  * Generates softlevel1 interrupt on current CPU if it
2023aedfe0bSmishra  * is not pending already.
2033aedfe0bSmishra  */
2047c478bd9Sstevel@tonic-gate void
2057c478bd9Sstevel@tonic-gate siron(void)
2067c478bd9Sstevel@tonic-gate {
2073aedfe0bSmishra 	uint64_t inum;
2083aedfe0bSmishra 
209dd4eeefdSeota 	if (siron1_inum != 0) {
210*12ceefb6Smishra 		/*
211*12ceefb6Smishra 		 * Once siron_cpu_inum has been allocated, we can
212*12ceefb6Smishra 		 * use per-CPU siron inum.
213*12ceefb6Smishra 		 */
214*12ceefb6Smishra 		if (siron_cpu_inum && siron_cpu_inum[CPU->cpu_id] != 0)
2153aedfe0bSmishra 			inum = siron_cpu_inum[CPU->cpu_id];
2163aedfe0bSmishra 		else
217dd4eeefdSeota 			inum = siron1_inum;
2183aedfe0bSmishra 
2193aedfe0bSmishra 		setsoftint(inum);
2203aedfe0bSmishra 	} else
221dd4eeefdSeota 		siron1_pending = 1;
2227c478bd9Sstevel@tonic-gate }
2237c478bd9Sstevel@tonic-gate 
224*12ceefb6Smishra 
225*12ceefb6Smishra static void
226*12ceefb6Smishra siron_init(void)
227*12ceefb6Smishra {
228*12ceefb6Smishra 	/*
229*12ceefb6Smishra 	 * We just allocate memory for per-cpu siron right now. Rest of
230*12ceefb6Smishra 	 * the work is done when CPU is configured.
231*12ceefb6Smishra 	 */
232*12ceefb6Smishra 	siron_cpu_inum = kmem_zalloc(sizeof (uint64_t) * NCPU, KM_SLEEP);
233*12ceefb6Smishra }
234*12ceefb6Smishra 
2353aedfe0bSmishra /*
2363aedfe0bSmishra  * This routine creates per-CPU siron inum for CPUs which are
2373aedfe0bSmishra  * configured during boot.
2383aedfe0bSmishra  */
2393aedfe0bSmishra void
2403aedfe0bSmishra siron_mp_init()
2413aedfe0bSmishra {
2423aedfe0bSmishra 	cpu_t *c;
2433aedfe0bSmishra 
244*12ceefb6Smishra 	/*
245*12ceefb6Smishra 	 * Get the memory for per-CPU siron inums
246*12ceefb6Smishra 	 */
247*12ceefb6Smishra 	siron_init();
248*12ceefb6Smishra 
2493aedfe0bSmishra 	mutex_enter(&cpu_lock);
2503aedfe0bSmishra 	c = cpu_list;
2513aedfe0bSmishra 	do {
2523aedfe0bSmishra 		(void) siron_cpu_setup(CPU_CONFIG, c->cpu_id, NULL);
2533aedfe0bSmishra 	} while ((c = c->cpu_next) != cpu_list);
2543aedfe0bSmishra 
2553aedfe0bSmishra 	register_cpu_setup_func(siron_cpu_setup, NULL);
2563aedfe0bSmishra 	mutex_exit(&cpu_lock);
2573aedfe0bSmishra }
2583aedfe0bSmishra 
2593aedfe0bSmishra /*
2603aedfe0bSmishra  * siron_poke_cpu_intr - cross-call handler.
2613aedfe0bSmishra  */
2623aedfe0bSmishra /* ARGSUSED */
2633aedfe0bSmishra uint_t
2643aedfe0bSmishra siron_poke_cpu_intr(caddr_t arg1, caddr_t arg2)
2653aedfe0bSmishra {
2663aedfe0bSmishra 	/* generate level1 softint */
2673aedfe0bSmishra 	siron();
2683aedfe0bSmishra 	return (1);
2693aedfe0bSmishra }
2703aedfe0bSmishra 
2713aedfe0bSmishra /*
2723aedfe0bSmishra  * This routine generates a cross-call on target CPU(s).
2733aedfe0bSmishra  */
2743aedfe0bSmishra void
2753aedfe0bSmishra siron_poke_cpu(cpuset_t poke)
2763aedfe0bSmishra {
2773aedfe0bSmishra 	int cpuid = CPU->cpu_id;
2783aedfe0bSmishra 
2793aedfe0bSmishra 	if (CPU_IN_SET(poke, cpuid)) {
2803aedfe0bSmishra 		siron();
2813aedfe0bSmishra 		CPUSET_DEL(poke, cpuid);
2823aedfe0bSmishra 		if (CPUSET_ISNULL(poke))
2833aedfe0bSmishra 			return;
2843aedfe0bSmishra 	}
2853aedfe0bSmishra 
2863aedfe0bSmishra 	xt_some(poke, setsoftint_tl1, siron_poke_cpu_inum, 0);
2873aedfe0bSmishra }
2883aedfe0bSmishra 
2893aedfe0bSmishra /*
2903aedfe0bSmishra  * This callback function allows us to create per-CPU siron inum.
2913aedfe0bSmishra  */
2923aedfe0bSmishra /* ARGSUSED */
2933aedfe0bSmishra static int
2943aedfe0bSmishra siron_cpu_setup(cpu_setup_t what, int id, void *arg)
2953aedfe0bSmishra {
2963aedfe0bSmishra 	cpu_t *cp = cpu[id];
2973aedfe0bSmishra 
2983aedfe0bSmishra 	ASSERT(MUTEX_HELD(&cpu_lock));
2993aedfe0bSmishra 	ASSERT(cp != NULL);
3003aedfe0bSmishra 
3013aedfe0bSmishra 	switch (what) {
3023aedfe0bSmishra 	case CPU_CONFIG:
3033aedfe0bSmishra 		siron_cpu_inum[cp->cpu_id] = add_softintr(PIL_1,
3043aedfe0bSmishra 		    (softintrfunc)softlevel1, 0, SOFTINT_ST);
3053aedfe0bSmishra 		break;
3063aedfe0bSmishra 	case CPU_UNCONFIG:
3073aedfe0bSmishra 		(void) rem_softintr(siron_cpu_inum[cp->cpu_id]);
3083aedfe0bSmishra 		siron_cpu_inum[cp->cpu_id] = 0;
3093aedfe0bSmishra 		break;
3103aedfe0bSmishra 	default:
3113aedfe0bSmishra 		break;
3123aedfe0bSmishra 	}
3133aedfe0bSmishra 
3143aedfe0bSmishra 	return (0);
3153aedfe0bSmishra }
3163aedfe0bSmishra 
3177c478bd9Sstevel@tonic-gate /*
3187c478bd9Sstevel@tonic-gate  * no_ivintr()
319b0fc0e77Sgovinda  * 	called by setvecint_tl1() through sys_trap()
3207c478bd9Sstevel@tonic-gate  *	vector interrupt received but not valid or not
321b0fc0e77Sgovinda  *	registered in intr_vec_table
3227c478bd9Sstevel@tonic-gate  *	considered as a spurious mondo interrupt
3237c478bd9Sstevel@tonic-gate  */
3247c478bd9Sstevel@tonic-gate /* ARGSUSED */
3257c478bd9Sstevel@tonic-gate void
3267c478bd9Sstevel@tonic-gate no_ivintr(struct regs *rp, int inum, int pil)
3277c478bd9Sstevel@tonic-gate {
3287c478bd9Sstevel@tonic-gate 	cmn_err(CE_WARN, "invalid vector intr: number 0x%x, pil 0x%x",
3297c478bd9Sstevel@tonic-gate 	    inum, pil);
3307c478bd9Sstevel@tonic-gate 
3317c478bd9Sstevel@tonic-gate #ifdef DEBUG_VEC_INTR
3327c478bd9Sstevel@tonic-gate 	prom_enter_mon();
3337c478bd9Sstevel@tonic-gate #endif /* DEBUG_VEC_INTR */
3347c478bd9Sstevel@tonic-gate }
3357c478bd9Sstevel@tonic-gate 
3367c478bd9Sstevel@tonic-gate void
337b0fc0e77Sgovinda intr_dequeue_req(uint_t pil, uint64_t inum)
3387c478bd9Sstevel@tonic-gate {
339b0fc0e77Sgovinda 	intr_vec_t	*iv, *next, *prev;
340b0fc0e77Sgovinda 	struct machcpu	*mcpu;
341b0fc0e77Sgovinda 	uint32_t	clr;
342b0fc0e77Sgovinda 	processorid_t	cpu_id;
343b0fc0e77Sgovinda 	extern uint_t	getpstate(void);
3447c478bd9Sstevel@tonic-gate 
3457c478bd9Sstevel@tonic-gate 	ASSERT((getpstate() & PSTATE_IE) == 0);
3467c478bd9Sstevel@tonic-gate 
3477c478bd9Sstevel@tonic-gate 	mcpu = &CPU->cpu_m;
348b0fc0e77Sgovinda 	cpu_id = CPU->cpu_id;
3497c478bd9Sstevel@tonic-gate 
350b0fc0e77Sgovinda 	iv = (intr_vec_t *)inum;
3517c478bd9Sstevel@tonic-gate 	prev = NULL;
352b0fc0e77Sgovinda 	next = mcpu->intr_head[pil];
353b0fc0e77Sgovinda 
354b0fc0e77Sgovinda 	/* Find a matching entry in the list */
355b0fc0e77Sgovinda 	while (next != NULL) {
356b0fc0e77Sgovinda 		if (next == iv)
3577c478bd9Sstevel@tonic-gate 			break;
358b0fc0e77Sgovinda 		prev = next;
359b0fc0e77Sgovinda 		next = IV_GET_PIL_NEXT(next, cpu_id);
3607c478bd9Sstevel@tonic-gate 	}
361b0fc0e77Sgovinda 
362b0fc0e77Sgovinda 	if (next != NULL) {
363b0fc0e77Sgovinda 		intr_vec_t	*next_iv = IV_GET_PIL_NEXT(next, cpu_id);
364b0fc0e77Sgovinda 
365b0fc0e77Sgovinda 		/* Remove entry from list */
3667c478bd9Sstevel@tonic-gate 		if (prev != NULL)
367b0fc0e77Sgovinda 			IV_SET_PIL_NEXT(prev, cpu_id, next_iv); /* non-head */
3687c478bd9Sstevel@tonic-gate 		else
369b0fc0e77Sgovinda 			mcpu->intr_head[pil] = next_iv; /* head */
3707c478bd9Sstevel@tonic-gate 
371b0fc0e77Sgovinda 		if (next_iv == NULL)
372b0fc0e77Sgovinda 			mcpu->intr_tail[pil] = prev; /* tail */
3737c478bd9Sstevel@tonic-gate 	}
3747c478bd9Sstevel@tonic-gate 
375b0fc0e77Sgovinda 	/* Clear pending interrupts at this level if the list is empty */
3767c478bd9Sstevel@tonic-gate 	if (mcpu->intr_head[pil] == NULL) {
3777c478bd9Sstevel@tonic-gate 		clr = 1 << pil;
3787c478bd9Sstevel@tonic-gate 		if (pil == PIL_14)
3797c478bd9Sstevel@tonic-gate 			clr |= (TICK_INT_MASK | STICK_INT_MASK);
3807c478bd9Sstevel@tonic-gate 		wr_clr_softint(clr);
3817c478bd9Sstevel@tonic-gate 	}
3827c478bd9Sstevel@tonic-gate }
3837c478bd9Sstevel@tonic-gate 
3847c478bd9Sstevel@tonic-gate 
3857c478bd9Sstevel@tonic-gate /*
3867c478bd9Sstevel@tonic-gate  * Send a directed interrupt of specified interrupt number id to a cpu.
3877c478bd9Sstevel@tonic-gate  */
3887c478bd9Sstevel@tonic-gate void
3897c478bd9Sstevel@tonic-gate send_dirint(
3907c478bd9Sstevel@tonic-gate 	int cpuix,		/* cpu to be interrupted */
3917c478bd9Sstevel@tonic-gate 	int intr_id)		/* interrupt number id */
3927c478bd9Sstevel@tonic-gate {
3937c478bd9Sstevel@tonic-gate 	xt_one(cpuix, setsoftint_tl1, intr_id, 0);
3947c478bd9Sstevel@tonic-gate }
3957c478bd9Sstevel@tonic-gate 
3967c478bd9Sstevel@tonic-gate /*
3977c478bd9Sstevel@tonic-gate  * Take the specified CPU out of participation in interrupts.
3987c478bd9Sstevel@tonic-gate  *	Called by p_online(2) when a processor is being taken off-line.
3997c478bd9Sstevel@tonic-gate  *	This allows interrupt threads being handled on the processor to
4007c478bd9Sstevel@tonic-gate  *	complete before the processor is idled.
4017c478bd9Sstevel@tonic-gate  */
4027c478bd9Sstevel@tonic-gate int
4037c478bd9Sstevel@tonic-gate cpu_disable_intr(struct cpu *cp)
4047c478bd9Sstevel@tonic-gate {
4057c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&cpu_lock));
4067c478bd9Sstevel@tonic-gate 
4077c478bd9Sstevel@tonic-gate 	/*
4087c478bd9Sstevel@tonic-gate 	 * Turn off the CPU_ENABLE flag before calling the redistribution
4097c478bd9Sstevel@tonic-gate 	 * function, since it checks for this in the cpu flags.
4107c478bd9Sstevel@tonic-gate 	 */
4117c478bd9Sstevel@tonic-gate 	cp->cpu_flags &= ~CPU_ENABLE;
4127c478bd9Sstevel@tonic-gate 
4137c478bd9Sstevel@tonic-gate 	intr_redist_all_cpus();
4147c478bd9Sstevel@tonic-gate 
4157c478bd9Sstevel@tonic-gate 	return (0);
4167c478bd9Sstevel@tonic-gate }
4177c478bd9Sstevel@tonic-gate 
4187c478bd9Sstevel@tonic-gate /*
4197c478bd9Sstevel@tonic-gate  * Allow the specified CPU to participate in interrupts.
4207c478bd9Sstevel@tonic-gate  *	Called by p_online(2) if a processor could not be taken off-line
4217c478bd9Sstevel@tonic-gate  *	because of bound threads, in order to resume processing interrupts.
4227c478bd9Sstevel@tonic-gate  *	Also called after starting a processor.
4237c478bd9Sstevel@tonic-gate  */
4247c478bd9Sstevel@tonic-gate void
4257c478bd9Sstevel@tonic-gate cpu_enable_intr(struct cpu *cp)
4267c478bd9Sstevel@tonic-gate {
4277c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&cpu_lock));
4287c478bd9Sstevel@tonic-gate 
4297c478bd9Sstevel@tonic-gate 	cp->cpu_flags |= CPU_ENABLE;
4307c478bd9Sstevel@tonic-gate 
4317c478bd9Sstevel@tonic-gate 	intr_redist_all_cpus();
4327c478bd9Sstevel@tonic-gate }
4337c478bd9Sstevel@tonic-gate 
4347c478bd9Sstevel@tonic-gate /*
4357c478bd9Sstevel@tonic-gate  * Add function to callback list for intr_redist_all_cpus.  We keep two lists,
4367c478bd9Sstevel@tonic-gate  * one for weighted callbacks and one for normal callbacks. Weighted callbacks
4377c478bd9Sstevel@tonic-gate  * are issued to redirect interrupts of a specified weight, from heavy to
4387c478bd9Sstevel@tonic-gate  * light.  This allows all the interrupts of a given weight to be redistributed
4397c478bd9Sstevel@tonic-gate  * for all weighted nexus drivers prior to those of less weight.
4407c478bd9Sstevel@tonic-gate  */
4417c478bd9Sstevel@tonic-gate static void
4427c478bd9Sstevel@tonic-gate intr_dist_add_list(struct intr_dist **phead, void (*func)(void *), void *arg)
4437c478bd9Sstevel@tonic-gate {
4447c478bd9Sstevel@tonic-gate 	struct intr_dist *new = kmem_alloc(sizeof (*new), KM_SLEEP);
4457c478bd9Sstevel@tonic-gate 	struct intr_dist *iptr;
4467c478bd9Sstevel@tonic-gate 	struct intr_dist **pptr;
4477c478bd9Sstevel@tonic-gate 
4487c478bd9Sstevel@tonic-gate 	ASSERT(func);
4497c478bd9Sstevel@tonic-gate 	new->func = func;
4507c478bd9Sstevel@tonic-gate 	new->arg = arg;
4517c478bd9Sstevel@tonic-gate 	new->next = NULL;
4527c478bd9Sstevel@tonic-gate 
4537c478bd9Sstevel@tonic-gate 	/* Add to tail so that redistribution occurs in original order. */
4547c478bd9Sstevel@tonic-gate 	mutex_enter(&intr_dist_lock);
4557c478bd9Sstevel@tonic-gate 	for (iptr = *phead, pptr = phead; iptr != NULL;
4567c478bd9Sstevel@tonic-gate 	    pptr = &iptr->next, iptr = iptr->next) {
4577c478bd9Sstevel@tonic-gate 		/* check for problems as we locate the tail */
4587c478bd9Sstevel@tonic-gate 		if ((iptr->func == func) && (iptr->arg == arg)) {
4597c478bd9Sstevel@tonic-gate 			cmn_err(CE_PANIC, "intr_dist_add_list(): duplicate");
4607c478bd9Sstevel@tonic-gate 			/*NOTREACHED*/
4617c478bd9Sstevel@tonic-gate 		}
4627c478bd9Sstevel@tonic-gate 	}
4637c478bd9Sstevel@tonic-gate 	*pptr = new;
4647c478bd9Sstevel@tonic-gate 
4657c478bd9Sstevel@tonic-gate 	mutex_exit(&intr_dist_lock);
4667c478bd9Sstevel@tonic-gate }
4677c478bd9Sstevel@tonic-gate 
4687c478bd9Sstevel@tonic-gate void
4697c478bd9Sstevel@tonic-gate intr_dist_add(void (*func)(void *), void *arg)
4707c478bd9Sstevel@tonic-gate {
4717c478bd9Sstevel@tonic-gate 	intr_dist_add_list(&intr_dist_head, (void (*)(void *))func, arg);
4727c478bd9Sstevel@tonic-gate }
4737c478bd9Sstevel@tonic-gate 
4747c478bd9Sstevel@tonic-gate void
4757c478bd9Sstevel@tonic-gate intr_dist_add_weighted(void (*func)(void *, int32_t, int32_t), void *arg)
4767c478bd9Sstevel@tonic-gate {
4777c478bd9Sstevel@tonic-gate 	intr_dist_add_list(&intr_dist_whead, (void (*)(void *))func, arg);
4787c478bd9Sstevel@tonic-gate }
4797c478bd9Sstevel@tonic-gate 
4807c478bd9Sstevel@tonic-gate /*
4817c478bd9Sstevel@tonic-gate  * Search for the interrupt distribution structure with the specified
4827c478bd9Sstevel@tonic-gate  * mondo vec reg in the interrupt distribution list. If a match is found,
4837c478bd9Sstevel@tonic-gate  * then delete the entry from the list. The caller is responsible for
4847c478bd9Sstevel@tonic-gate  * modifying the mondo vector registers.
4857c478bd9Sstevel@tonic-gate  */
4867c478bd9Sstevel@tonic-gate static void
4877c478bd9Sstevel@tonic-gate intr_dist_rem_list(struct intr_dist **headp, void (*func)(void *), void *arg)
4887c478bd9Sstevel@tonic-gate {
4897c478bd9Sstevel@tonic-gate 	struct intr_dist *iptr;
4907c478bd9Sstevel@tonic-gate 	struct intr_dist **vect;
4917c478bd9Sstevel@tonic-gate 
4927c478bd9Sstevel@tonic-gate 	mutex_enter(&intr_dist_lock);
4937c478bd9Sstevel@tonic-gate 	for (iptr = *headp, vect = headp;
4947c478bd9Sstevel@tonic-gate 	    iptr != NULL; vect = &iptr->next, iptr = iptr->next) {
4957c478bd9Sstevel@tonic-gate 		if ((iptr->func == func) && (iptr->arg == arg)) {
4967c478bd9Sstevel@tonic-gate 			*vect = iptr->next;
4977c478bd9Sstevel@tonic-gate 			kmem_free(iptr, sizeof (struct intr_dist));
4987c478bd9Sstevel@tonic-gate 			mutex_exit(&intr_dist_lock);
4997c478bd9Sstevel@tonic-gate 			return;
5007c478bd9Sstevel@tonic-gate 		}
5017c478bd9Sstevel@tonic-gate 	}
5027c478bd9Sstevel@tonic-gate 
5037c478bd9Sstevel@tonic-gate 	if (!panicstr)
5047c478bd9Sstevel@tonic-gate 		cmn_err(CE_PANIC, "intr_dist_rem_list: not found");
5057c478bd9Sstevel@tonic-gate 	mutex_exit(&intr_dist_lock);
5067c478bd9Sstevel@tonic-gate }
5077c478bd9Sstevel@tonic-gate 
5087c478bd9Sstevel@tonic-gate void
5097c478bd9Sstevel@tonic-gate intr_dist_rem(void (*func)(void *), void *arg)
5107c478bd9Sstevel@tonic-gate {
5117c478bd9Sstevel@tonic-gate 	intr_dist_rem_list(&intr_dist_head, (void (*)(void *))func, arg);
5127c478bd9Sstevel@tonic-gate }
5137c478bd9Sstevel@tonic-gate 
5147c478bd9Sstevel@tonic-gate void
5157c478bd9Sstevel@tonic-gate intr_dist_rem_weighted(void (*func)(void *, int32_t, int32_t), void *arg)
5167c478bd9Sstevel@tonic-gate {
5177c478bd9Sstevel@tonic-gate 	intr_dist_rem_list(&intr_dist_whead, (void (*)(void *))func, arg);
5187c478bd9Sstevel@tonic-gate }
5197c478bd9Sstevel@tonic-gate 
5207c478bd9Sstevel@tonic-gate /*
5217c478bd9Sstevel@tonic-gate  * Initiate interrupt redistribution.  Redistribution improves the isolation
5227c478bd9Sstevel@tonic-gate  * associated with interrupt weights by ordering operations from heavy weight
5237c478bd9Sstevel@tonic-gate  * to light weight.  When a CPUs orientation changes relative to interrupts,
5247c478bd9Sstevel@tonic-gate  * there is *always* a redistribution to accommodate this change (call to
5257c478bd9Sstevel@tonic-gate  * intr_redist_all_cpus()).  As devices (not CPUs) attach/detach it is possible
5267c478bd9Sstevel@tonic-gate  * that a redistribution could improve the quality of an initialization. For
5277c478bd9Sstevel@tonic-gate  * example, if you are not using a NIC it may not be attached with s10 (devfs).
5287c478bd9Sstevel@tonic-gate  * If you then configure the NIC (ifconfig), this may cause the NIC to attach
5297c478bd9Sstevel@tonic-gate  * and plumb interrupts.  The CPU assignment for the NIC's interrupts is
5307c478bd9Sstevel@tonic-gate  * occurring late, so optimal "isolation" relative to weight is not occurring.
5317c478bd9Sstevel@tonic-gate  * The same applies to detach, although in this case doing the redistribution
5327c478bd9Sstevel@tonic-gate  * might improve "spread" for medium weight devices since the "isolation" of
5337c478bd9Sstevel@tonic-gate  * a higher weight device may no longer be present.
5347c478bd9Sstevel@tonic-gate  *
5357c478bd9Sstevel@tonic-gate  * NB: We should provide a utility to trigger redistribution (ala "intradm -r").
5367c478bd9Sstevel@tonic-gate  *
5377c478bd9Sstevel@tonic-gate  * NB: There is risk associated with automatically triggering execution of the
5387c478bd9Sstevel@tonic-gate  * redistribution code at arbitrary times. The risk comes from the fact that
5397c478bd9Sstevel@tonic-gate  * there is a lot of low-level hardware interaction associated with a
5407c478bd9Sstevel@tonic-gate  * redistribution.  At some point we may want this code to perform automatic
5417c478bd9Sstevel@tonic-gate  * redistribution (redistribution thread; trigger timeout when add/remove
5427c478bd9Sstevel@tonic-gate  * weight delta is large enough, and call cv_signal from timeout - causing
5437c478bd9Sstevel@tonic-gate  * thead to call i_ddi_intr_redist_all_cpus()) but this is considered too
5447c478bd9Sstevel@tonic-gate  * risky at this time.
5457c478bd9Sstevel@tonic-gate  */
5467c478bd9Sstevel@tonic-gate void
5477c478bd9Sstevel@tonic-gate i_ddi_intr_redist_all_cpus()
5487c478bd9Sstevel@tonic-gate {
5497c478bd9Sstevel@tonic-gate 	mutex_enter(&cpu_lock);
5507c478bd9Sstevel@tonic-gate 	INTR_DEBUG((CE_CONT, "intr_dist: i_ddi_intr_redist_all_cpus\n"));
5517c478bd9Sstevel@tonic-gate 	intr_redist_all_cpus();
5527c478bd9Sstevel@tonic-gate 	mutex_exit(&cpu_lock);
5537c478bd9Sstevel@tonic-gate }
5547c478bd9Sstevel@tonic-gate 
5557c478bd9Sstevel@tonic-gate /*
5567c478bd9Sstevel@tonic-gate  * Redistribute all interrupts
5577c478bd9Sstevel@tonic-gate  *
5587c478bd9Sstevel@tonic-gate  * This function redistributes all interrupting devices, running the
5597c478bd9Sstevel@tonic-gate  * parent callback functions for each node.
5607c478bd9Sstevel@tonic-gate  */
5617c478bd9Sstevel@tonic-gate void
5627c478bd9Sstevel@tonic-gate intr_redist_all_cpus(void)
5637c478bd9Sstevel@tonic-gate {
5647c478bd9Sstevel@tonic-gate 	struct cpu *cp;
5657c478bd9Sstevel@tonic-gate 	struct intr_dist *iptr;
5667c478bd9Sstevel@tonic-gate 	int32_t weight, max_weight;
5677c478bd9Sstevel@tonic-gate 
5687c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&cpu_lock));
5697c478bd9Sstevel@tonic-gate 	mutex_enter(&intr_dist_lock);
5707c478bd9Sstevel@tonic-gate 
5717c478bd9Sstevel@tonic-gate 	/*
5727c478bd9Sstevel@tonic-gate 	 * zero cpu_intr_weight on all cpus - it is safe to traverse
5737c478bd9Sstevel@tonic-gate 	 * cpu_list since we hold cpu_lock.
5747c478bd9Sstevel@tonic-gate 	 */
5757c478bd9Sstevel@tonic-gate 	cp = cpu_list;
5767c478bd9Sstevel@tonic-gate 	do {
5777c478bd9Sstevel@tonic-gate 		cp->cpu_intr_weight = 0;
5787c478bd9Sstevel@tonic-gate 	} while ((cp = cp->cpu_next) != cpu_list);
5797c478bd9Sstevel@tonic-gate 
5807c478bd9Sstevel@tonic-gate 	/*
5817c478bd9Sstevel@tonic-gate 	 * Assume that this redistribution may encounter a device weight
5827c478bd9Sstevel@tonic-gate 	 * via driver.conf tuning of "ddi-intr-weight" that is at most
5837c478bd9Sstevel@tonic-gate 	 * intr_dist_weight_maxfactor times larger.
5847c478bd9Sstevel@tonic-gate 	 */
5857c478bd9Sstevel@tonic-gate 	max_weight = intr_dist_weight_max * intr_dist_weight_maxfactor;
5867c478bd9Sstevel@tonic-gate 	if (max_weight > intr_dist_weight_maxmax)
5877c478bd9Sstevel@tonic-gate 		max_weight = intr_dist_weight_maxmax;
5887c478bd9Sstevel@tonic-gate 	intr_dist_weight_max = 1;
5897c478bd9Sstevel@tonic-gate 
5907c478bd9Sstevel@tonic-gate 	INTR_DEBUG((CE_CONT, "intr_dist: "
5917c478bd9Sstevel@tonic-gate 	    "intr_redist_all_cpus: %d-0\n", max_weight));
5927c478bd9Sstevel@tonic-gate 
5937c478bd9Sstevel@tonic-gate 	/*
5947c478bd9Sstevel@tonic-gate 	 * Redistribute weighted, from heavy to light.  The callback that
5957c478bd9Sstevel@tonic-gate 	 * specifies a weight equal to weight_max should redirect all
5967c478bd9Sstevel@tonic-gate 	 * interrupts of weight weight_max or greater [weight_max, inf.).
5977c478bd9Sstevel@tonic-gate 	 * Interrupts of lesser weight should be processed on the call with
5987c478bd9Sstevel@tonic-gate 	 * the matching weight. This allows all the heaver weight interrupts
5997c478bd9Sstevel@tonic-gate 	 * on all weighted busses (multiple pci busses) to be redirected prior
6007c478bd9Sstevel@tonic-gate 	 * to any lesser weight interrupts.
6017c478bd9Sstevel@tonic-gate 	 */
6027c478bd9Sstevel@tonic-gate 	for (weight = max_weight; weight >= 0; weight--)
6037c478bd9Sstevel@tonic-gate 		for (iptr = intr_dist_whead; iptr != NULL; iptr = iptr->next)
6047c478bd9Sstevel@tonic-gate 			((void (*)(void *, int32_t, int32_t))iptr->func)
6057c478bd9Sstevel@tonic-gate 			    (iptr->arg, max_weight, weight);
6067c478bd9Sstevel@tonic-gate 
6077c478bd9Sstevel@tonic-gate 	/* redistribute normal (non-weighted) interrupts */
6087c478bd9Sstevel@tonic-gate 	for (iptr = intr_dist_head; iptr != NULL; iptr = iptr->next)
6097c478bd9Sstevel@tonic-gate 		((void (*)(void *))iptr->func)(iptr->arg);
6107c478bd9Sstevel@tonic-gate 	mutex_exit(&intr_dist_lock);
6117c478bd9Sstevel@tonic-gate }
6127c478bd9Sstevel@tonic-gate 
6137c478bd9Sstevel@tonic-gate void
6147c478bd9Sstevel@tonic-gate intr_redist_all_cpus_shutdown(void)
6157c478bd9Sstevel@tonic-gate {
6167c478bd9Sstevel@tonic-gate 	intr_policy = INTR_CURRENT_CPU;
6177c478bd9Sstevel@tonic-gate 	intr_redist_all_cpus();
6187c478bd9Sstevel@tonic-gate }
6197c478bd9Sstevel@tonic-gate 
6207c478bd9Sstevel@tonic-gate /*
6217c478bd9Sstevel@tonic-gate  * Determine what CPU to target, based on interrupt policy.
6227c478bd9Sstevel@tonic-gate  *
6237c478bd9Sstevel@tonic-gate  * INTR_FLAT_DIST: hold a current CPU pointer in a static variable and
6247c478bd9Sstevel@tonic-gate  *	advance through interrupt enabled cpus (round-robin).
6257c478bd9Sstevel@tonic-gate  *
6267c478bd9Sstevel@tonic-gate  * INTR_WEIGHTED_DIST: search for an enabled CPU with the lowest
6277c478bd9Sstevel@tonic-gate  *	cpu_intr_weight, round robin when all equal.
6287c478bd9Sstevel@tonic-gate  *
6297c478bd9Sstevel@tonic-gate  *	Weighted interrupt distribution provides two things: "spread" of weight
6307c478bd9Sstevel@tonic-gate  *	(associated with algorithm itself) and "isolation" (associated with a
6317c478bd9Sstevel@tonic-gate  *	particular device weight). A redistribution is what provides optimal
6327c478bd9Sstevel@tonic-gate  *	"isolation" of heavy weight interrupts, optimal "spread" of weight
6337c478bd9Sstevel@tonic-gate  *	(relative to what came before) is always occurring.
6347c478bd9Sstevel@tonic-gate  *
6357c478bd9Sstevel@tonic-gate  *	An interrupt weight is a subjective number that represents the
6367c478bd9Sstevel@tonic-gate  *	percentage of a CPU required to service a device's interrupts: the
6377c478bd9Sstevel@tonic-gate  *	default weight is 0% (however the algorithm still maintains
6387c478bd9Sstevel@tonic-gate  *	round-robin), a network interface controller (NIC) may have a large
6397c478bd9Sstevel@tonic-gate  *	weight (35%). Interrupt weight only has meaning relative to the
6407c478bd9Sstevel@tonic-gate  *	interrupt weight of other devices: a CPU can be weighted more than
6417c478bd9Sstevel@tonic-gate  *	100%, and a single device might consume more than 100% of a CPU.
6427c478bd9Sstevel@tonic-gate  *
6437c478bd9Sstevel@tonic-gate  *	A coarse interrupt weight can be defined by the parent nexus driver
6447c478bd9Sstevel@tonic-gate  *	based on bus specific information, like pci class codes. A nexus
6457c478bd9Sstevel@tonic-gate  *	driver that supports device interrupt weighting for its children
6467c478bd9Sstevel@tonic-gate  *	should call intr_dist_cpuid_add/rem_device_weight(), which adds
6477c478bd9Sstevel@tonic-gate  *	and removes the weight of a device from the CPU that an interrupt
6487c478bd9Sstevel@tonic-gate  *	is directed at.  The quality of initialization improves when the
6497c478bd9Sstevel@tonic-gate  *	device interrupt weights more accuracy reflect actual run-time weights,
6507c478bd9Sstevel@tonic-gate  *	and as the assignments are ordered from is heavy to light.
6517c478bd9Sstevel@tonic-gate  *
6527c478bd9Sstevel@tonic-gate  *	The implementation also supports interrupt weight being specified in
6537c478bd9Sstevel@tonic-gate  *	driver.conf files via the property "ddi-intr-weight", which takes
6547c478bd9Sstevel@tonic-gate  *	precedence over the nexus supplied weight.  This support is added to
6557c478bd9Sstevel@tonic-gate  *	permit possible tweaking in the product in response to customer
6567c478bd9Sstevel@tonic-gate  *	problems. This is not a formal or committed interface.
6577c478bd9Sstevel@tonic-gate  *
6587c478bd9Sstevel@tonic-gate  *	While a weighted approach chooses the CPU providing the best spread
6597c478bd9Sstevel@tonic-gate  *	given past weights, less than optimal isolation can result in cases
6607c478bd9Sstevel@tonic-gate  *	where heavy weight devices show up last. The nexus driver's interrupt
6617c478bd9Sstevel@tonic-gate  *	redistribution logic should use intr_dist_add/rem_weighted so that
6627c478bd9Sstevel@tonic-gate  *	interrupts can be redistributed heavy first for optimal isolation.
6637c478bd9Sstevel@tonic-gate  */
6647c478bd9Sstevel@tonic-gate uint32_t
6657c478bd9Sstevel@tonic-gate intr_dist_cpuid(void)
6667c478bd9Sstevel@tonic-gate {
6677c478bd9Sstevel@tonic-gate 	static struct cpu	*curr_cpu;
6687c478bd9Sstevel@tonic-gate 	struct cpu		*start_cpu;
6697c478bd9Sstevel@tonic-gate 	struct cpu		*new_cpu;
6707c478bd9Sstevel@tonic-gate 	struct cpu		*cp;
6717c478bd9Sstevel@tonic-gate 	int			cpuid = -1;
6727c478bd9Sstevel@tonic-gate 
6737c478bd9Sstevel@tonic-gate 	/* Establish exclusion for curr_cpu and cpu_intr_weight manipulation */
6747c478bd9Sstevel@tonic-gate 	mutex_enter(&intr_dist_cpu_lock);
6757c478bd9Sstevel@tonic-gate 
6767c478bd9Sstevel@tonic-gate 	switch (intr_policy) {
6777c478bd9Sstevel@tonic-gate 	case INTR_CURRENT_CPU:
6787c478bd9Sstevel@tonic-gate 		cpuid = CPU->cpu_id;
6797c478bd9Sstevel@tonic-gate 		break;
6807c478bd9Sstevel@tonic-gate 
6817c478bd9Sstevel@tonic-gate 	case INTR_BOOT_CPU:
6827c478bd9Sstevel@tonic-gate 		panic("INTR_BOOT_CPU no longer supported.");
6837c478bd9Sstevel@tonic-gate 		/*NOTREACHED*/
6847c478bd9Sstevel@tonic-gate 
6857c478bd9Sstevel@tonic-gate 	case INTR_FLAT_DIST:
6867c478bd9Sstevel@tonic-gate 	case INTR_WEIGHTED_DIST:
6877c478bd9Sstevel@tonic-gate 	default:
6887c478bd9Sstevel@tonic-gate 		/*
6897c478bd9Sstevel@tonic-gate 		 * Ensure that curr_cpu is valid - cpu_next will be NULL if
6907c478bd9Sstevel@tonic-gate 		 * the cpu has been deleted (cpu structs are never freed).
6917c478bd9Sstevel@tonic-gate 		 */
6927c478bd9Sstevel@tonic-gate 		if (curr_cpu == NULL || curr_cpu->cpu_next == NULL)
6937c478bd9Sstevel@tonic-gate 			curr_cpu = CPU;
6947c478bd9Sstevel@tonic-gate 
6957c478bd9Sstevel@tonic-gate 		/*
6967c478bd9Sstevel@tonic-gate 		 * Advance to online CPU after curr_cpu (round-robin). For
6977c478bd9Sstevel@tonic-gate 		 * INTR_WEIGHTED_DIST we choose the cpu with the lightest
6987c478bd9Sstevel@tonic-gate 		 * weight.  For a nexus that does not support weight the
6997c478bd9Sstevel@tonic-gate 		 * default weight of zero is used. We degrade to round-robin
7007c478bd9Sstevel@tonic-gate 		 * behavior among equal weightes.  The default weight is zero
7017c478bd9Sstevel@tonic-gate 		 * and round-robin behavior continues.
7027c478bd9Sstevel@tonic-gate 		 *
7037c478bd9Sstevel@tonic-gate 		 * Disable preemption while traversing cpu_next_onln to
7047c478bd9Sstevel@tonic-gate 		 * ensure the list does not change.  This works because
7057c478bd9Sstevel@tonic-gate 		 * modifiers of this list and other lists in a struct cpu
7067c478bd9Sstevel@tonic-gate 		 * call pause_cpus() before making changes.
7077c478bd9Sstevel@tonic-gate 		 */
7087c478bd9Sstevel@tonic-gate 		kpreempt_disable();
7097c478bd9Sstevel@tonic-gate 		cp = start_cpu = curr_cpu->cpu_next_onln;
7107c478bd9Sstevel@tonic-gate 		new_cpu = NULL;
7117c478bd9Sstevel@tonic-gate 		do {
7127c478bd9Sstevel@tonic-gate 			/* Skip CPUs with interrupts disabled */
7137c478bd9Sstevel@tonic-gate 			if ((cp->cpu_flags & CPU_ENABLE) == 0)
7147c478bd9Sstevel@tonic-gate 				continue;
7157c478bd9Sstevel@tonic-gate 
7167c478bd9Sstevel@tonic-gate 			if (intr_policy == INTR_FLAT_DIST) {
7177c478bd9Sstevel@tonic-gate 				/* select CPU */
7187c478bd9Sstevel@tonic-gate 				new_cpu = cp;
7197c478bd9Sstevel@tonic-gate 				break;
7207c478bd9Sstevel@tonic-gate 			} else if ((new_cpu == NULL) ||
7217c478bd9Sstevel@tonic-gate 			    (cp->cpu_intr_weight < new_cpu->cpu_intr_weight)) {
7227c478bd9Sstevel@tonic-gate 				/* Choose if lighter weight */
7237c478bd9Sstevel@tonic-gate 				new_cpu = cp;
7247c478bd9Sstevel@tonic-gate 			}
7257c478bd9Sstevel@tonic-gate 		} while ((cp = cp->cpu_next_onln) != start_cpu);
7267c478bd9Sstevel@tonic-gate 		ASSERT(new_cpu);
7277c478bd9Sstevel@tonic-gate 		cpuid = new_cpu->cpu_id;
7287c478bd9Sstevel@tonic-gate 
7297c478bd9Sstevel@tonic-gate 		INTR_DEBUG((CE_CONT, "intr_dist: cpu %2d weight %3d: "
7307c478bd9Sstevel@tonic-gate 		    "targeted\n", cpuid, new_cpu->cpu_intr_weight));
7317c478bd9Sstevel@tonic-gate 
7327c478bd9Sstevel@tonic-gate 		/* update static pointer for next round-robin */
7337c478bd9Sstevel@tonic-gate 		curr_cpu = new_cpu;
7347c478bd9Sstevel@tonic-gate 		kpreempt_enable();
7357c478bd9Sstevel@tonic-gate 		break;
7367c478bd9Sstevel@tonic-gate 	}
7377c478bd9Sstevel@tonic-gate 	mutex_exit(&intr_dist_cpu_lock);
7387c478bd9Sstevel@tonic-gate 	return (cpuid);
7397c478bd9Sstevel@tonic-gate }
7407c478bd9Sstevel@tonic-gate 
7417c478bd9Sstevel@tonic-gate /*
7427c478bd9Sstevel@tonic-gate  * Add or remove the the weight of a device from a CPUs interrupt weight.
7437c478bd9Sstevel@tonic-gate  *
7447c478bd9Sstevel@tonic-gate  * We expect nexus drivers to call intr_dist_cpuid_add/rem_device_weight for
7457c478bd9Sstevel@tonic-gate  * their children to improve the overall quality of interrupt initialization.
7467c478bd9Sstevel@tonic-gate  *
7477c478bd9Sstevel@tonic-gate  * If a nexues shares the CPU returned by a single intr_dist_cpuid() call
7487c478bd9Sstevel@tonic-gate  * among multiple devices (sharing ino) then the nexus should call
7497c478bd9Sstevel@tonic-gate  * intr_dist_cpuid_add/rem_device_weight for each device separately. Devices
7507c478bd9Sstevel@tonic-gate  * that share must specify the same cpuid.
7517c478bd9Sstevel@tonic-gate  *
7527c478bd9Sstevel@tonic-gate  * If a nexus driver is unable to determine the cpu at remove_intr time
7537c478bd9Sstevel@tonic-gate  * for some of its interrupts, then it should not call add_device_weight -
7547c478bd9Sstevel@tonic-gate  * intr_dist_cpuid will still provide round-robin.
7557c478bd9Sstevel@tonic-gate  *
7567c478bd9Sstevel@tonic-gate  * An established device weight (from dev_info node) takes precedence over
7577c478bd9Sstevel@tonic-gate  * the weight passed in.  If a device weight is not already established
7587c478bd9Sstevel@tonic-gate  * then the passed in nexus weight is established.
7597c478bd9Sstevel@tonic-gate  */
7607c478bd9Sstevel@tonic-gate void
7617c478bd9Sstevel@tonic-gate intr_dist_cpuid_add_device_weight(uint32_t cpuid,
7627c478bd9Sstevel@tonic-gate     dev_info_t *dip, int32_t nweight)
7637c478bd9Sstevel@tonic-gate {
7647c478bd9Sstevel@tonic-gate 	int32_t		eweight;
7657c478bd9Sstevel@tonic-gate 
7667c478bd9Sstevel@tonic-gate 	/*
7677c478bd9Sstevel@tonic-gate 	 * For non-weighted policy everything has weight of zero (and we get
7687c478bd9Sstevel@tonic-gate 	 * round-robin distribution from intr_dist_cpuid).
7697c478bd9Sstevel@tonic-gate 	 * NB: intr_policy is limited to this file. A weighted nexus driver is
7707c478bd9Sstevel@tonic-gate 	 * calls this rouitne even if intr_policy has been patched to
7717c478bd9Sstevel@tonic-gate 	 * INTR_FLAG_DIST.
7727c478bd9Sstevel@tonic-gate 	 */
7737c478bd9Sstevel@tonic-gate 	ASSERT(dip);
7747c478bd9Sstevel@tonic-gate 	if (intr_policy != INTR_WEIGHTED_DIST)
7757c478bd9Sstevel@tonic-gate 		return;
7767c478bd9Sstevel@tonic-gate 
7777c478bd9Sstevel@tonic-gate 	eweight = i_ddi_get_intr_weight(dip);
7787c478bd9Sstevel@tonic-gate 	INTR_DEBUG((CE_CONT, "intr_dist: cpu %2d weight %3d: +%2d/%2d for "
7797c478bd9Sstevel@tonic-gate 	    "%s#%d/%s#%d\n", cpuid, cpu[cpuid]->cpu_intr_weight,
7807c478bd9Sstevel@tonic-gate 	    nweight, eweight, ddi_driver_name(ddi_get_parent(dip)),
7817c478bd9Sstevel@tonic-gate 	    ddi_get_instance(ddi_get_parent(dip)),
7827c478bd9Sstevel@tonic-gate 	    ddi_driver_name(dip), ddi_get_instance(dip)));
7837c478bd9Sstevel@tonic-gate 
7847c478bd9Sstevel@tonic-gate 	/* if no establish weight, establish nexus weight */
7857c478bd9Sstevel@tonic-gate 	if (eweight < 0) {
7867c478bd9Sstevel@tonic-gate 		if (nweight > 0)
7877c478bd9Sstevel@tonic-gate 			(void) i_ddi_set_intr_weight(dip, nweight);
7887c478bd9Sstevel@tonic-gate 		else
7897c478bd9Sstevel@tonic-gate 			nweight = 0;
7907c478bd9Sstevel@tonic-gate 	} else
7917c478bd9Sstevel@tonic-gate 		nweight = eweight;	/* use established weight */
7927c478bd9Sstevel@tonic-gate 
7937c478bd9Sstevel@tonic-gate 	/* Establish exclusion for cpu_intr_weight manipulation */
7947c478bd9Sstevel@tonic-gate 	mutex_enter(&intr_dist_cpu_lock);
7957c478bd9Sstevel@tonic-gate 	cpu[cpuid]->cpu_intr_weight += nweight;
7967c478bd9Sstevel@tonic-gate 
7977c478bd9Sstevel@tonic-gate 	/* update intr_dist_weight_max */
7987c478bd9Sstevel@tonic-gate 	if (nweight > intr_dist_weight_max)
7997c478bd9Sstevel@tonic-gate 		intr_dist_weight_max = nweight;
8007c478bd9Sstevel@tonic-gate 	mutex_exit(&intr_dist_cpu_lock);
8017c478bd9Sstevel@tonic-gate }
8027c478bd9Sstevel@tonic-gate 
8037c478bd9Sstevel@tonic-gate void
8047c478bd9Sstevel@tonic-gate intr_dist_cpuid_rem_device_weight(uint32_t cpuid, dev_info_t *dip)
8057c478bd9Sstevel@tonic-gate {
8067c478bd9Sstevel@tonic-gate 	struct cpu	*cp;
8077c478bd9Sstevel@tonic-gate 	int32_t		weight;
8087c478bd9Sstevel@tonic-gate 
8097c478bd9Sstevel@tonic-gate 	ASSERT(dip);
8107c478bd9Sstevel@tonic-gate 	if (intr_policy != INTR_WEIGHTED_DIST)
8117c478bd9Sstevel@tonic-gate 		return;
8127c478bd9Sstevel@tonic-gate 
8137c478bd9Sstevel@tonic-gate 	/* remove weight of device from cpu */
8147c478bd9Sstevel@tonic-gate 	weight = i_ddi_get_intr_weight(dip);
8157c478bd9Sstevel@tonic-gate 	if (weight < 0)
8167c478bd9Sstevel@tonic-gate 		weight = 0;
8177c478bd9Sstevel@tonic-gate 	INTR_DEBUG((CE_CONT, "intr_dist: cpu %2d weight %3d: -%2d    for "
8187c478bd9Sstevel@tonic-gate 	    "%s#%d/%s#%d\n", cpuid, cpu[cpuid]->cpu_intr_weight, weight,
8197c478bd9Sstevel@tonic-gate 	    ddi_driver_name(ddi_get_parent(dip)),
8207c478bd9Sstevel@tonic-gate 	    ddi_get_instance(ddi_get_parent(dip)),
8217c478bd9Sstevel@tonic-gate 	    ddi_driver_name(dip), ddi_get_instance(dip)));
8227c478bd9Sstevel@tonic-gate 
8237c478bd9Sstevel@tonic-gate 	/* Establish exclusion for cpu_intr_weight manipulation */
8247c478bd9Sstevel@tonic-gate 	mutex_enter(&intr_dist_cpu_lock);
8257c478bd9Sstevel@tonic-gate 	cp = cpu[cpuid];
8267c478bd9Sstevel@tonic-gate 	cp->cpu_intr_weight -= weight;
8277c478bd9Sstevel@tonic-gate 	if (cp->cpu_intr_weight < 0)
8287c478bd9Sstevel@tonic-gate 		cp->cpu_intr_weight = 0;	/* sanity */
8297c478bd9Sstevel@tonic-gate 	mutex_exit(&intr_dist_cpu_lock);
8307c478bd9Sstevel@tonic-gate }
8312850d85bSmv 
8322850d85bSmv ulong_t
8332850d85bSmv create_softint(uint_t pil, uint_t (*func)(caddr_t, caddr_t), caddr_t arg1)
8342850d85bSmv {
8352850d85bSmv 	uint64_t inum;
8362850d85bSmv 
8372850d85bSmv 	inum = add_softintr(pil, func, arg1, SOFTINT_ST);
8382850d85bSmv 	return ((ulong_t)inum);
8392850d85bSmv }
8402850d85bSmv 
8412850d85bSmv void
8422850d85bSmv invoke_softint(processorid_t cpuid, ulong_t hdl)
8432850d85bSmv {
8442850d85bSmv 	uint64_t inum = hdl;
8452850d85bSmv 
8462850d85bSmv 	if (cpuid == CPU->cpu_id)
8472850d85bSmv 		setsoftint(inum);
8482850d85bSmv 	else
8492850d85bSmv 		xt_one(cpuid, setsoftint_tl1, inum, 0);
8502850d85bSmv }
8512850d85bSmv 
8522850d85bSmv void
8532850d85bSmv remove_softint(ulong_t hdl)
8542850d85bSmv {
8552850d85bSmv 	uint64_t inum = hdl;
8562850d85bSmv 
8572850d85bSmv 	(void) rem_softintr(inum);
8582850d85bSmv }
8592850d85bSmv 
8602850d85bSmv void
8612850d85bSmv sync_softint(cpuset_t set)
8622850d85bSmv {
8632850d85bSmv 	xt_sync(set);
8642850d85bSmv }
865