17c478bd9Sstevel@tonic-gate /* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 5100b72f4Sandrei * Common Development and Distribution License (the "License"). 6100b72f4Sandrei * You may not use this file except in compliance with the License. 77c478bd9Sstevel@tonic-gate * 87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 117c478bd9Sstevel@tonic-gate * and limitations under the License. 127c478bd9Sstevel@tonic-gate * 137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 187c478bd9Sstevel@tonic-gate * 197c478bd9Sstevel@tonic-gate * CDDL HEADER END 207c478bd9Sstevel@tonic-gate */ 217c478bd9Sstevel@tonic-gate /* 222850d85bSmv * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 237c478bd9Sstevel@tonic-gate * Use is subject to license terms. 247c478bd9Sstevel@tonic-gate */ 257c478bd9Sstevel@tonic-gate 267c478bd9Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI" 277c478bd9Sstevel@tonic-gate 287c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h> 297c478bd9Sstevel@tonic-gate #include <sys/stack.h> 307c478bd9Sstevel@tonic-gate #include <sys/cpuvar.h> 317c478bd9Sstevel@tonic-gate #include <sys/ivintr.h> 327c478bd9Sstevel@tonic-gate #include <sys/intreg.h> 337c478bd9Sstevel@tonic-gate #include <sys/membar.h> 347c478bd9Sstevel@tonic-gate #include <sys/kmem.h> 357c478bd9Sstevel@tonic-gate #include <sys/intr.h> 36dd4eeefdSeota #include <sys/sunddi.h> 377c478bd9Sstevel@tonic-gate #include <sys/sunndi.h> 387c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h> 397c478bd9Sstevel@tonic-gate #include <sys/privregs.h> 407c478bd9Sstevel@tonic-gate #include <sys/systm.h> 417c478bd9Sstevel@tonic-gate #include <sys/archsystm.h> 427c478bd9Sstevel@tonic-gate #include <sys/machsystm.h> 437c478bd9Sstevel@tonic-gate #include <sys/x_call.h> 447c478bd9Sstevel@tonic-gate #include <vm/seg_kp.h> 457c478bd9Sstevel@tonic-gate #include <sys/debug.h> 467c478bd9Sstevel@tonic-gate #include <sys/cyclic.h> 47a1af7ba0Scwb #include <sys/kdi_impl.h> 48dd4eeefdSeota #include <sys/ddi_timer.h> 497c478bd9Sstevel@tonic-gate 507c478bd9Sstevel@tonic-gate #include <sys/cpu_sgnblk_defs.h> 517c478bd9Sstevel@tonic-gate 527c478bd9Sstevel@tonic-gate /* Global locks which protect the interrupt distribution lists */ 537c478bd9Sstevel@tonic-gate static kmutex_t intr_dist_lock; 547c478bd9Sstevel@tonic-gate static kmutex_t intr_dist_cpu_lock; 557c478bd9Sstevel@tonic-gate 567c478bd9Sstevel@tonic-gate /* Head of the interrupt distribution lists */ 577c478bd9Sstevel@tonic-gate static struct intr_dist *intr_dist_head = NULL; 587c478bd9Sstevel@tonic-gate static struct intr_dist *intr_dist_whead = NULL; 597c478bd9Sstevel@tonic-gate 60dd4eeefdSeota static uint64_t siron_inum[DDI_IPL_10]; /* software interrupt numbers */ 613aedfe0bSmishra uint64_t *siron_cpu_inum = NULL; 623aedfe0bSmishra uint64_t siron_poke_cpu_inum; 633aedfe0bSmishra static int siron_cpu_setup(cpu_setup_t, int, void *); 643aedfe0bSmishra extern uint_t softlevel1(); 653aedfe0bSmishra 66dd4eeefdSeota static uint64_t siron1_inum; /* backward compatibility */ 67b0fc0e77Sgovinda uint64_t poke_cpu_inum; 68b0fc0e77Sgovinda uint_t poke_cpu_intr(caddr_t arg1, caddr_t arg2); 693aedfe0bSmishra uint_t siron_poke_cpu_intr(caddr_t arg1, caddr_t arg2); 70b0fc0e77Sgovinda 71f8047eabSsudheer /* 72f8047eabSsudheer * Note:- 73f8047eabSsudheer * siron_pending was originally created to prevent a resource over consumption 74f8047eabSsudheer * bug in setsoftint(exhaustion of interrupt pool free list). 75f8047eabSsudheer * It's original intention is obsolete with the use of iv_pending in 76f8047eabSsudheer * setsoftint. However, siron_pending stayed around, acting as a second 77f8047eabSsudheer * gatekeeper preventing soft interrupts from being queued. In this capacity, 78f8047eabSsudheer * it can lead to hangs on MP systems, where due to global visibility issues 79f8047eabSsudheer * it can end up set while iv_pending is reset, preventing soft interrupts from 80f8047eabSsudheer * ever being processed. In addition to its gatekeeper role, init_intr also 81f8047eabSsudheer * uses it to flag the situation where siron() was called before siron_inum has 82f8047eabSsudheer * been defined. 83f8047eabSsudheer * 84f8047eabSsudheer * siron() does not need an extra gatekeeper; any cpu that wishes should be 85f8047eabSsudheer * allowed to queue a soft interrupt. It is softint()'s job to ensure 86f8047eabSsudheer * correct handling of the queues. Therefore, siron_pending has been 87f8047eabSsudheer * stripped of its gatekeeper task, retaining only its intr_init job, where 88f8047eabSsudheer * it indicates that there is a pending need to call siron(). 89f8047eabSsudheer */ 90dd4eeefdSeota static int siron_pending[DDI_IPL_10]; /* software interrupt pending flags */ 91dd4eeefdSeota static int siron1_pending; /* backward compatibility */ 927c478bd9Sstevel@tonic-gate 937c478bd9Sstevel@tonic-gate int intr_policy = INTR_WEIGHTED_DIST; /* interrupt distribution policy */ 947c478bd9Sstevel@tonic-gate int intr_dist_debug = 0; 957c478bd9Sstevel@tonic-gate int32_t intr_dist_weight_max = 1; 967c478bd9Sstevel@tonic-gate int32_t intr_dist_weight_maxmax = 1000; 977c478bd9Sstevel@tonic-gate int intr_dist_weight_maxfactor = 2; 987c478bd9Sstevel@tonic-gate #define INTR_DEBUG(args) if (intr_dist_debug) cmn_err args 997c478bd9Sstevel@tonic-gate 1007c478bd9Sstevel@tonic-gate /* 101b0fc0e77Sgovinda * intr_init() - Interrupt initialization 102b0fc0e77Sgovinda * Initialize the system's interrupt vector table. 1037c478bd9Sstevel@tonic-gate */ 1047c478bd9Sstevel@tonic-gate void 1057c478bd9Sstevel@tonic-gate intr_init(cpu_t *cp) 1067c478bd9Sstevel@tonic-gate { 107dd4eeefdSeota int i; 108b0fc0e77Sgovinda extern uint_t softlevel1(); 109b0fc0e77Sgovinda 1107c478bd9Sstevel@tonic-gate init_ivintr(); 111b0fc0e77Sgovinda REGISTER_BBUS_INTR(); 112b0fc0e77Sgovinda 113dd4eeefdSeota /* 114dd4eeefdSeota * Register these software interrupts for ddi timer. 115dd4eeefdSeota * Software interrupts up to the level 10 are supported. 116dd4eeefdSeota */ 117dd4eeefdSeota for (i = DDI_IPL_1; i <= DDI_IPL_10; i++) { 118dd4eeefdSeota siron_inum[i-1] = add_softintr(i, (softintrfunc)timer_softintr, 119dd4eeefdSeota (caddr_t)(uintptr_t)(i), SOFTINT_ST); 120dd4eeefdSeota } 121dd4eeefdSeota 122dd4eeefdSeota siron1_inum = add_softintr(PIL_1, softlevel1, 0, SOFTINT_ST); 123b0fc0e77Sgovinda poke_cpu_inum = add_softintr(PIL_13, poke_cpu_intr, 0, SOFTINT_MT); 1243aedfe0bSmishra siron_poke_cpu_inum = add_softintr(PIL_13, 1253aedfe0bSmishra siron_poke_cpu_intr, 0, SOFTINT_MT); 126b0fc0e77Sgovinda cp->cpu_m.poke_cpu_outstanding = B_FALSE; 1277c478bd9Sstevel@tonic-gate 1287c478bd9Sstevel@tonic-gate mutex_init(&intr_dist_lock, NULL, MUTEX_DEFAULT, NULL); 1297c478bd9Sstevel@tonic-gate mutex_init(&intr_dist_cpu_lock, NULL, MUTEX_DEFAULT, NULL); 1307c478bd9Sstevel@tonic-gate 1317c478bd9Sstevel@tonic-gate /* 1327c478bd9Sstevel@tonic-gate * A soft interrupt may have been requested prior to the initialization 1337c478bd9Sstevel@tonic-gate * of soft interrupts. Soft interrupts can't be dispatched until after 134b0fc0e77Sgovinda * init_intr(), so we have to wait until now before we can dispatch the 135b0fc0e77Sgovinda * pending soft interrupt (if any). 1367c478bd9Sstevel@tonic-gate */ 137dd4eeefdSeota for (i = DDI_IPL_1; i <= DDI_IPL_10; i++) { 138dd4eeefdSeota if (siron_pending[i-1]) { 139dd4eeefdSeota siron_pending[i-1] = 0; 140dd4eeefdSeota sir_on(i); 141dd4eeefdSeota } 142dd4eeefdSeota } 143dd4eeefdSeota if (siron1_pending) { 144dd4eeefdSeota siron1_pending = 0; 145f8047eabSsudheer siron(); 146f8047eabSsudheer } 1477c478bd9Sstevel@tonic-gate } 1487c478bd9Sstevel@tonic-gate 1497c478bd9Sstevel@tonic-gate /* 1507c478bd9Sstevel@tonic-gate * poke_cpu_intr - fall through when poke_cpu calls 1517c478bd9Sstevel@tonic-gate */ 1527c478bd9Sstevel@tonic-gate /* ARGSUSED */ 1537c478bd9Sstevel@tonic-gate uint_t 1547c478bd9Sstevel@tonic-gate poke_cpu_intr(caddr_t arg1, caddr_t arg2) 1557c478bd9Sstevel@tonic-gate { 1567c478bd9Sstevel@tonic-gate CPU->cpu_m.poke_cpu_outstanding = B_FALSE; 1577c478bd9Sstevel@tonic-gate membar_stld_stst(); 1587c478bd9Sstevel@tonic-gate return (1); 1597c478bd9Sstevel@tonic-gate } 1607c478bd9Sstevel@tonic-gate 161dd4eeefdSeota /* 162dd4eeefdSeota * Trigger software interrupts dedicated to ddi timer. 163dd4eeefdSeota */ 164dd4eeefdSeota void 165dd4eeefdSeota sir_on(int level) 166dd4eeefdSeota { 167dd4eeefdSeota ASSERT(level >= DDI_IPL_1 && level <= DDI_IPL_10); 168dd4eeefdSeota if (siron_inum[level-1]) 169dd4eeefdSeota setsoftint(siron_inum[level-1]); 170dd4eeefdSeota else 171dd4eeefdSeota siron_pending[level-1] = 1; 172dd4eeefdSeota } 173dd4eeefdSeota 1747c478bd9Sstevel@tonic-gate /* 175a1af7ba0Scwb * kmdb uses siron (and thus setsoftint) while the world is stopped in order to 176a1af7ba0Scwb * inform its driver component that there's work to be done. We need to keep 177a1af7ba0Scwb * DTrace from instrumenting kmdb's siron and setsoftint. We duplicate siron, 178a1af7ba0Scwb * giving kmdb's version a kdi_ prefix to keep DTrace at bay. The 179a1af7ba0Scwb * implementation of setsoftint is complicated enough that we don't want to 180a1af7ba0Scwb * duplicate it, but at the same time we don't want to preclude tracing either. 181a1af7ba0Scwb * The meat of setsoftint() therefore goes into kdi_setsoftint, with 182a1af7ba0Scwb * setsoftint() implemented as a wrapper. This allows tracing, while still 183a1af7ba0Scwb * providing a way for kmdb to sneak in unmolested. 1847c478bd9Sstevel@tonic-gate */ 185a1af7ba0Scwb void 186a1af7ba0Scwb kdi_siron(void) 187a1af7ba0Scwb { 188dd4eeefdSeota if (siron1_inum != 0) 189dd4eeefdSeota kdi_setsoftint(siron1_inum); 190a1af7ba0Scwb else 191dd4eeefdSeota siron1_pending = 1; 192a1af7ba0Scwb } 193a1af7ba0Scwb 194a1af7ba0Scwb void 195a1af7ba0Scwb setsoftint(uint64_t inum) 196a1af7ba0Scwb { 197a1af7ba0Scwb kdi_setsoftint(inum); 198a1af7ba0Scwb } 199a1af7ba0Scwb 2003aedfe0bSmishra /* 2013aedfe0bSmishra * Generates softlevel1 interrupt on current CPU if it 2023aedfe0bSmishra * is not pending already. 2033aedfe0bSmishra */ 2047c478bd9Sstevel@tonic-gate void 2057c478bd9Sstevel@tonic-gate siron(void) 2067c478bd9Sstevel@tonic-gate { 2073aedfe0bSmishra uint64_t inum; 2083aedfe0bSmishra 209dd4eeefdSeota if (siron1_inum != 0) { 210*12ceefb6Smishra /* 211*12ceefb6Smishra * Once siron_cpu_inum has been allocated, we can 212*12ceefb6Smishra * use per-CPU siron inum. 213*12ceefb6Smishra */ 214*12ceefb6Smishra if (siron_cpu_inum && siron_cpu_inum[CPU->cpu_id] != 0) 2153aedfe0bSmishra inum = siron_cpu_inum[CPU->cpu_id]; 2163aedfe0bSmishra else 217dd4eeefdSeota inum = siron1_inum; 2183aedfe0bSmishra 2193aedfe0bSmishra setsoftint(inum); 2203aedfe0bSmishra } else 221dd4eeefdSeota siron1_pending = 1; 2227c478bd9Sstevel@tonic-gate } 2237c478bd9Sstevel@tonic-gate 224*12ceefb6Smishra 225*12ceefb6Smishra static void 226*12ceefb6Smishra siron_init(void) 227*12ceefb6Smishra { 228*12ceefb6Smishra /* 229*12ceefb6Smishra * We just allocate memory for per-cpu siron right now. Rest of 230*12ceefb6Smishra * the work is done when CPU is configured. 231*12ceefb6Smishra */ 232*12ceefb6Smishra siron_cpu_inum = kmem_zalloc(sizeof (uint64_t) * NCPU, KM_SLEEP); 233*12ceefb6Smishra } 234*12ceefb6Smishra 2353aedfe0bSmishra /* 2363aedfe0bSmishra * This routine creates per-CPU siron inum for CPUs which are 2373aedfe0bSmishra * configured during boot. 2383aedfe0bSmishra */ 2393aedfe0bSmishra void 2403aedfe0bSmishra siron_mp_init() 2413aedfe0bSmishra { 2423aedfe0bSmishra cpu_t *c; 2433aedfe0bSmishra 244*12ceefb6Smishra /* 245*12ceefb6Smishra * Get the memory for per-CPU siron inums 246*12ceefb6Smishra */ 247*12ceefb6Smishra siron_init(); 248*12ceefb6Smishra 2493aedfe0bSmishra mutex_enter(&cpu_lock); 2503aedfe0bSmishra c = cpu_list; 2513aedfe0bSmishra do { 2523aedfe0bSmishra (void) siron_cpu_setup(CPU_CONFIG, c->cpu_id, NULL); 2533aedfe0bSmishra } while ((c = c->cpu_next) != cpu_list); 2543aedfe0bSmishra 2553aedfe0bSmishra register_cpu_setup_func(siron_cpu_setup, NULL); 2563aedfe0bSmishra mutex_exit(&cpu_lock); 2573aedfe0bSmishra } 2583aedfe0bSmishra 2593aedfe0bSmishra /* 2603aedfe0bSmishra * siron_poke_cpu_intr - cross-call handler. 2613aedfe0bSmishra */ 2623aedfe0bSmishra /* ARGSUSED */ 2633aedfe0bSmishra uint_t 2643aedfe0bSmishra siron_poke_cpu_intr(caddr_t arg1, caddr_t arg2) 2653aedfe0bSmishra { 2663aedfe0bSmishra /* generate level1 softint */ 2673aedfe0bSmishra siron(); 2683aedfe0bSmishra return (1); 2693aedfe0bSmishra } 2703aedfe0bSmishra 2713aedfe0bSmishra /* 2723aedfe0bSmishra * This routine generates a cross-call on target CPU(s). 2733aedfe0bSmishra */ 2743aedfe0bSmishra void 2753aedfe0bSmishra siron_poke_cpu(cpuset_t poke) 2763aedfe0bSmishra { 2773aedfe0bSmishra int cpuid = CPU->cpu_id; 2783aedfe0bSmishra 2793aedfe0bSmishra if (CPU_IN_SET(poke, cpuid)) { 2803aedfe0bSmishra siron(); 2813aedfe0bSmishra CPUSET_DEL(poke, cpuid); 2823aedfe0bSmishra if (CPUSET_ISNULL(poke)) 2833aedfe0bSmishra return; 2843aedfe0bSmishra } 2853aedfe0bSmishra 2863aedfe0bSmishra xt_some(poke, setsoftint_tl1, siron_poke_cpu_inum, 0); 2873aedfe0bSmishra } 2883aedfe0bSmishra 2893aedfe0bSmishra /* 2903aedfe0bSmishra * This callback function allows us to create per-CPU siron inum. 2913aedfe0bSmishra */ 2923aedfe0bSmishra /* ARGSUSED */ 2933aedfe0bSmishra static int 2943aedfe0bSmishra siron_cpu_setup(cpu_setup_t what, int id, void *arg) 2953aedfe0bSmishra { 2963aedfe0bSmishra cpu_t *cp = cpu[id]; 2973aedfe0bSmishra 2983aedfe0bSmishra ASSERT(MUTEX_HELD(&cpu_lock)); 2993aedfe0bSmishra ASSERT(cp != NULL); 3003aedfe0bSmishra 3013aedfe0bSmishra switch (what) { 3023aedfe0bSmishra case CPU_CONFIG: 3033aedfe0bSmishra siron_cpu_inum[cp->cpu_id] = add_softintr(PIL_1, 3043aedfe0bSmishra (softintrfunc)softlevel1, 0, SOFTINT_ST); 3053aedfe0bSmishra break; 3063aedfe0bSmishra case CPU_UNCONFIG: 3073aedfe0bSmishra (void) rem_softintr(siron_cpu_inum[cp->cpu_id]); 3083aedfe0bSmishra siron_cpu_inum[cp->cpu_id] = 0; 3093aedfe0bSmishra break; 3103aedfe0bSmishra default: 3113aedfe0bSmishra break; 3123aedfe0bSmishra } 3133aedfe0bSmishra 3143aedfe0bSmishra return (0); 3153aedfe0bSmishra } 3163aedfe0bSmishra 3177c478bd9Sstevel@tonic-gate /* 3187c478bd9Sstevel@tonic-gate * no_ivintr() 319b0fc0e77Sgovinda * called by setvecint_tl1() through sys_trap() 3207c478bd9Sstevel@tonic-gate * vector interrupt received but not valid or not 321b0fc0e77Sgovinda * registered in intr_vec_table 3227c478bd9Sstevel@tonic-gate * considered as a spurious mondo interrupt 3237c478bd9Sstevel@tonic-gate */ 3247c478bd9Sstevel@tonic-gate /* ARGSUSED */ 3257c478bd9Sstevel@tonic-gate void 3267c478bd9Sstevel@tonic-gate no_ivintr(struct regs *rp, int inum, int pil) 3277c478bd9Sstevel@tonic-gate { 3287c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, "invalid vector intr: number 0x%x, pil 0x%x", 3297c478bd9Sstevel@tonic-gate inum, pil); 3307c478bd9Sstevel@tonic-gate 3317c478bd9Sstevel@tonic-gate #ifdef DEBUG_VEC_INTR 3327c478bd9Sstevel@tonic-gate prom_enter_mon(); 3337c478bd9Sstevel@tonic-gate #endif /* DEBUG_VEC_INTR */ 3347c478bd9Sstevel@tonic-gate } 3357c478bd9Sstevel@tonic-gate 3367c478bd9Sstevel@tonic-gate void 337b0fc0e77Sgovinda intr_dequeue_req(uint_t pil, uint64_t inum) 3387c478bd9Sstevel@tonic-gate { 339b0fc0e77Sgovinda intr_vec_t *iv, *next, *prev; 340b0fc0e77Sgovinda struct machcpu *mcpu; 341b0fc0e77Sgovinda uint32_t clr; 342b0fc0e77Sgovinda processorid_t cpu_id; 343b0fc0e77Sgovinda extern uint_t getpstate(void); 3447c478bd9Sstevel@tonic-gate 3457c478bd9Sstevel@tonic-gate ASSERT((getpstate() & PSTATE_IE) == 0); 3467c478bd9Sstevel@tonic-gate 3477c478bd9Sstevel@tonic-gate mcpu = &CPU->cpu_m; 348b0fc0e77Sgovinda cpu_id = CPU->cpu_id; 3497c478bd9Sstevel@tonic-gate 350b0fc0e77Sgovinda iv = (intr_vec_t *)inum; 3517c478bd9Sstevel@tonic-gate prev = NULL; 352b0fc0e77Sgovinda next = mcpu->intr_head[pil]; 353b0fc0e77Sgovinda 354b0fc0e77Sgovinda /* Find a matching entry in the list */ 355b0fc0e77Sgovinda while (next != NULL) { 356b0fc0e77Sgovinda if (next == iv) 3577c478bd9Sstevel@tonic-gate break; 358b0fc0e77Sgovinda prev = next; 359b0fc0e77Sgovinda next = IV_GET_PIL_NEXT(next, cpu_id); 3607c478bd9Sstevel@tonic-gate } 361b0fc0e77Sgovinda 362b0fc0e77Sgovinda if (next != NULL) { 363b0fc0e77Sgovinda intr_vec_t *next_iv = IV_GET_PIL_NEXT(next, cpu_id); 364b0fc0e77Sgovinda 365b0fc0e77Sgovinda /* Remove entry from list */ 3667c478bd9Sstevel@tonic-gate if (prev != NULL) 367b0fc0e77Sgovinda IV_SET_PIL_NEXT(prev, cpu_id, next_iv); /* non-head */ 3687c478bd9Sstevel@tonic-gate else 369b0fc0e77Sgovinda mcpu->intr_head[pil] = next_iv; /* head */ 3707c478bd9Sstevel@tonic-gate 371b0fc0e77Sgovinda if (next_iv == NULL) 372b0fc0e77Sgovinda mcpu->intr_tail[pil] = prev; /* tail */ 3737c478bd9Sstevel@tonic-gate } 3747c478bd9Sstevel@tonic-gate 375b0fc0e77Sgovinda /* Clear pending interrupts at this level if the list is empty */ 3767c478bd9Sstevel@tonic-gate if (mcpu->intr_head[pil] == NULL) { 3777c478bd9Sstevel@tonic-gate clr = 1 << pil; 3787c478bd9Sstevel@tonic-gate if (pil == PIL_14) 3797c478bd9Sstevel@tonic-gate clr |= (TICK_INT_MASK | STICK_INT_MASK); 3807c478bd9Sstevel@tonic-gate wr_clr_softint(clr); 3817c478bd9Sstevel@tonic-gate } 3827c478bd9Sstevel@tonic-gate } 3837c478bd9Sstevel@tonic-gate 3847c478bd9Sstevel@tonic-gate 3857c478bd9Sstevel@tonic-gate /* 3867c478bd9Sstevel@tonic-gate * Send a directed interrupt of specified interrupt number id to a cpu. 3877c478bd9Sstevel@tonic-gate */ 3887c478bd9Sstevel@tonic-gate void 3897c478bd9Sstevel@tonic-gate send_dirint( 3907c478bd9Sstevel@tonic-gate int cpuix, /* cpu to be interrupted */ 3917c478bd9Sstevel@tonic-gate int intr_id) /* interrupt number id */ 3927c478bd9Sstevel@tonic-gate { 3937c478bd9Sstevel@tonic-gate xt_one(cpuix, setsoftint_tl1, intr_id, 0); 3947c478bd9Sstevel@tonic-gate } 3957c478bd9Sstevel@tonic-gate 3967c478bd9Sstevel@tonic-gate /* 3977c478bd9Sstevel@tonic-gate * Take the specified CPU out of participation in interrupts. 3987c478bd9Sstevel@tonic-gate * Called by p_online(2) when a processor is being taken off-line. 3997c478bd9Sstevel@tonic-gate * This allows interrupt threads being handled on the processor to 4007c478bd9Sstevel@tonic-gate * complete before the processor is idled. 4017c478bd9Sstevel@tonic-gate */ 4027c478bd9Sstevel@tonic-gate int 4037c478bd9Sstevel@tonic-gate cpu_disable_intr(struct cpu *cp) 4047c478bd9Sstevel@tonic-gate { 4057c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 4067c478bd9Sstevel@tonic-gate 4077c478bd9Sstevel@tonic-gate /* 4087c478bd9Sstevel@tonic-gate * Turn off the CPU_ENABLE flag before calling the redistribution 4097c478bd9Sstevel@tonic-gate * function, since it checks for this in the cpu flags. 4107c478bd9Sstevel@tonic-gate */ 4117c478bd9Sstevel@tonic-gate cp->cpu_flags &= ~CPU_ENABLE; 4127c478bd9Sstevel@tonic-gate 4137c478bd9Sstevel@tonic-gate intr_redist_all_cpus(); 4147c478bd9Sstevel@tonic-gate 4157c478bd9Sstevel@tonic-gate return (0); 4167c478bd9Sstevel@tonic-gate } 4177c478bd9Sstevel@tonic-gate 4187c478bd9Sstevel@tonic-gate /* 4197c478bd9Sstevel@tonic-gate * Allow the specified CPU to participate in interrupts. 4207c478bd9Sstevel@tonic-gate * Called by p_online(2) if a processor could not be taken off-line 4217c478bd9Sstevel@tonic-gate * because of bound threads, in order to resume processing interrupts. 4227c478bd9Sstevel@tonic-gate * Also called after starting a processor. 4237c478bd9Sstevel@tonic-gate */ 4247c478bd9Sstevel@tonic-gate void 4257c478bd9Sstevel@tonic-gate cpu_enable_intr(struct cpu *cp) 4267c478bd9Sstevel@tonic-gate { 4277c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 4287c478bd9Sstevel@tonic-gate 4297c478bd9Sstevel@tonic-gate cp->cpu_flags |= CPU_ENABLE; 4307c478bd9Sstevel@tonic-gate 4317c478bd9Sstevel@tonic-gate intr_redist_all_cpus(); 4327c478bd9Sstevel@tonic-gate } 4337c478bd9Sstevel@tonic-gate 4347c478bd9Sstevel@tonic-gate /* 4357c478bd9Sstevel@tonic-gate * Add function to callback list for intr_redist_all_cpus. We keep two lists, 4367c478bd9Sstevel@tonic-gate * one for weighted callbacks and one for normal callbacks. Weighted callbacks 4377c478bd9Sstevel@tonic-gate * are issued to redirect interrupts of a specified weight, from heavy to 4387c478bd9Sstevel@tonic-gate * light. This allows all the interrupts of a given weight to be redistributed 4397c478bd9Sstevel@tonic-gate * for all weighted nexus drivers prior to those of less weight. 4407c478bd9Sstevel@tonic-gate */ 4417c478bd9Sstevel@tonic-gate static void 4427c478bd9Sstevel@tonic-gate intr_dist_add_list(struct intr_dist **phead, void (*func)(void *), void *arg) 4437c478bd9Sstevel@tonic-gate { 4447c478bd9Sstevel@tonic-gate struct intr_dist *new = kmem_alloc(sizeof (*new), KM_SLEEP); 4457c478bd9Sstevel@tonic-gate struct intr_dist *iptr; 4467c478bd9Sstevel@tonic-gate struct intr_dist **pptr; 4477c478bd9Sstevel@tonic-gate 4487c478bd9Sstevel@tonic-gate ASSERT(func); 4497c478bd9Sstevel@tonic-gate new->func = func; 4507c478bd9Sstevel@tonic-gate new->arg = arg; 4517c478bd9Sstevel@tonic-gate new->next = NULL; 4527c478bd9Sstevel@tonic-gate 4537c478bd9Sstevel@tonic-gate /* Add to tail so that redistribution occurs in original order. */ 4547c478bd9Sstevel@tonic-gate mutex_enter(&intr_dist_lock); 4557c478bd9Sstevel@tonic-gate for (iptr = *phead, pptr = phead; iptr != NULL; 4567c478bd9Sstevel@tonic-gate pptr = &iptr->next, iptr = iptr->next) { 4577c478bd9Sstevel@tonic-gate /* check for problems as we locate the tail */ 4587c478bd9Sstevel@tonic-gate if ((iptr->func == func) && (iptr->arg == arg)) { 4597c478bd9Sstevel@tonic-gate cmn_err(CE_PANIC, "intr_dist_add_list(): duplicate"); 4607c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 4617c478bd9Sstevel@tonic-gate } 4627c478bd9Sstevel@tonic-gate } 4637c478bd9Sstevel@tonic-gate *pptr = new; 4647c478bd9Sstevel@tonic-gate 4657c478bd9Sstevel@tonic-gate mutex_exit(&intr_dist_lock); 4667c478bd9Sstevel@tonic-gate } 4677c478bd9Sstevel@tonic-gate 4687c478bd9Sstevel@tonic-gate void 4697c478bd9Sstevel@tonic-gate intr_dist_add(void (*func)(void *), void *arg) 4707c478bd9Sstevel@tonic-gate { 4717c478bd9Sstevel@tonic-gate intr_dist_add_list(&intr_dist_head, (void (*)(void *))func, arg); 4727c478bd9Sstevel@tonic-gate } 4737c478bd9Sstevel@tonic-gate 4747c478bd9Sstevel@tonic-gate void 4757c478bd9Sstevel@tonic-gate intr_dist_add_weighted(void (*func)(void *, int32_t, int32_t), void *arg) 4767c478bd9Sstevel@tonic-gate { 4777c478bd9Sstevel@tonic-gate intr_dist_add_list(&intr_dist_whead, (void (*)(void *))func, arg); 4787c478bd9Sstevel@tonic-gate } 4797c478bd9Sstevel@tonic-gate 4807c478bd9Sstevel@tonic-gate /* 4817c478bd9Sstevel@tonic-gate * Search for the interrupt distribution structure with the specified 4827c478bd9Sstevel@tonic-gate * mondo vec reg in the interrupt distribution list. If a match is found, 4837c478bd9Sstevel@tonic-gate * then delete the entry from the list. The caller is responsible for 4847c478bd9Sstevel@tonic-gate * modifying the mondo vector registers. 4857c478bd9Sstevel@tonic-gate */ 4867c478bd9Sstevel@tonic-gate static void 4877c478bd9Sstevel@tonic-gate intr_dist_rem_list(struct intr_dist **headp, void (*func)(void *), void *arg) 4887c478bd9Sstevel@tonic-gate { 4897c478bd9Sstevel@tonic-gate struct intr_dist *iptr; 4907c478bd9Sstevel@tonic-gate struct intr_dist **vect; 4917c478bd9Sstevel@tonic-gate 4927c478bd9Sstevel@tonic-gate mutex_enter(&intr_dist_lock); 4937c478bd9Sstevel@tonic-gate for (iptr = *headp, vect = headp; 4947c478bd9Sstevel@tonic-gate iptr != NULL; vect = &iptr->next, iptr = iptr->next) { 4957c478bd9Sstevel@tonic-gate if ((iptr->func == func) && (iptr->arg == arg)) { 4967c478bd9Sstevel@tonic-gate *vect = iptr->next; 4977c478bd9Sstevel@tonic-gate kmem_free(iptr, sizeof (struct intr_dist)); 4987c478bd9Sstevel@tonic-gate mutex_exit(&intr_dist_lock); 4997c478bd9Sstevel@tonic-gate return; 5007c478bd9Sstevel@tonic-gate } 5017c478bd9Sstevel@tonic-gate } 5027c478bd9Sstevel@tonic-gate 5037c478bd9Sstevel@tonic-gate if (!panicstr) 5047c478bd9Sstevel@tonic-gate cmn_err(CE_PANIC, "intr_dist_rem_list: not found"); 5057c478bd9Sstevel@tonic-gate mutex_exit(&intr_dist_lock); 5067c478bd9Sstevel@tonic-gate } 5077c478bd9Sstevel@tonic-gate 5087c478bd9Sstevel@tonic-gate void 5097c478bd9Sstevel@tonic-gate intr_dist_rem(void (*func)(void *), void *arg) 5107c478bd9Sstevel@tonic-gate { 5117c478bd9Sstevel@tonic-gate intr_dist_rem_list(&intr_dist_head, (void (*)(void *))func, arg); 5127c478bd9Sstevel@tonic-gate } 5137c478bd9Sstevel@tonic-gate 5147c478bd9Sstevel@tonic-gate void 5157c478bd9Sstevel@tonic-gate intr_dist_rem_weighted(void (*func)(void *, int32_t, int32_t), void *arg) 5167c478bd9Sstevel@tonic-gate { 5177c478bd9Sstevel@tonic-gate intr_dist_rem_list(&intr_dist_whead, (void (*)(void *))func, arg); 5187c478bd9Sstevel@tonic-gate } 5197c478bd9Sstevel@tonic-gate 5207c478bd9Sstevel@tonic-gate /* 5217c478bd9Sstevel@tonic-gate * Initiate interrupt redistribution. Redistribution improves the isolation 5227c478bd9Sstevel@tonic-gate * associated with interrupt weights by ordering operations from heavy weight 5237c478bd9Sstevel@tonic-gate * to light weight. When a CPUs orientation changes relative to interrupts, 5247c478bd9Sstevel@tonic-gate * there is *always* a redistribution to accommodate this change (call to 5257c478bd9Sstevel@tonic-gate * intr_redist_all_cpus()). As devices (not CPUs) attach/detach it is possible 5267c478bd9Sstevel@tonic-gate * that a redistribution could improve the quality of an initialization. For 5277c478bd9Sstevel@tonic-gate * example, if you are not using a NIC it may not be attached with s10 (devfs). 5287c478bd9Sstevel@tonic-gate * If you then configure the NIC (ifconfig), this may cause the NIC to attach 5297c478bd9Sstevel@tonic-gate * and plumb interrupts. The CPU assignment for the NIC's interrupts is 5307c478bd9Sstevel@tonic-gate * occurring late, so optimal "isolation" relative to weight is not occurring. 5317c478bd9Sstevel@tonic-gate * The same applies to detach, although in this case doing the redistribution 5327c478bd9Sstevel@tonic-gate * might improve "spread" for medium weight devices since the "isolation" of 5337c478bd9Sstevel@tonic-gate * a higher weight device may no longer be present. 5347c478bd9Sstevel@tonic-gate * 5357c478bd9Sstevel@tonic-gate * NB: We should provide a utility to trigger redistribution (ala "intradm -r"). 5367c478bd9Sstevel@tonic-gate * 5377c478bd9Sstevel@tonic-gate * NB: There is risk associated with automatically triggering execution of the 5387c478bd9Sstevel@tonic-gate * redistribution code at arbitrary times. The risk comes from the fact that 5397c478bd9Sstevel@tonic-gate * there is a lot of low-level hardware interaction associated with a 5407c478bd9Sstevel@tonic-gate * redistribution. At some point we may want this code to perform automatic 5417c478bd9Sstevel@tonic-gate * redistribution (redistribution thread; trigger timeout when add/remove 5427c478bd9Sstevel@tonic-gate * weight delta is large enough, and call cv_signal from timeout - causing 5437c478bd9Sstevel@tonic-gate * thead to call i_ddi_intr_redist_all_cpus()) but this is considered too 5447c478bd9Sstevel@tonic-gate * risky at this time. 5457c478bd9Sstevel@tonic-gate */ 5467c478bd9Sstevel@tonic-gate void 5477c478bd9Sstevel@tonic-gate i_ddi_intr_redist_all_cpus() 5487c478bd9Sstevel@tonic-gate { 5497c478bd9Sstevel@tonic-gate mutex_enter(&cpu_lock); 5507c478bd9Sstevel@tonic-gate INTR_DEBUG((CE_CONT, "intr_dist: i_ddi_intr_redist_all_cpus\n")); 5517c478bd9Sstevel@tonic-gate intr_redist_all_cpus(); 5527c478bd9Sstevel@tonic-gate mutex_exit(&cpu_lock); 5537c478bd9Sstevel@tonic-gate } 5547c478bd9Sstevel@tonic-gate 5557c478bd9Sstevel@tonic-gate /* 5567c478bd9Sstevel@tonic-gate * Redistribute all interrupts 5577c478bd9Sstevel@tonic-gate * 5587c478bd9Sstevel@tonic-gate * This function redistributes all interrupting devices, running the 5597c478bd9Sstevel@tonic-gate * parent callback functions for each node. 5607c478bd9Sstevel@tonic-gate */ 5617c478bd9Sstevel@tonic-gate void 5627c478bd9Sstevel@tonic-gate intr_redist_all_cpus(void) 5637c478bd9Sstevel@tonic-gate { 5647c478bd9Sstevel@tonic-gate struct cpu *cp; 5657c478bd9Sstevel@tonic-gate struct intr_dist *iptr; 5667c478bd9Sstevel@tonic-gate int32_t weight, max_weight; 5677c478bd9Sstevel@tonic-gate 5687c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 5697c478bd9Sstevel@tonic-gate mutex_enter(&intr_dist_lock); 5707c478bd9Sstevel@tonic-gate 5717c478bd9Sstevel@tonic-gate /* 5727c478bd9Sstevel@tonic-gate * zero cpu_intr_weight on all cpus - it is safe to traverse 5737c478bd9Sstevel@tonic-gate * cpu_list since we hold cpu_lock. 5747c478bd9Sstevel@tonic-gate */ 5757c478bd9Sstevel@tonic-gate cp = cpu_list; 5767c478bd9Sstevel@tonic-gate do { 5777c478bd9Sstevel@tonic-gate cp->cpu_intr_weight = 0; 5787c478bd9Sstevel@tonic-gate } while ((cp = cp->cpu_next) != cpu_list); 5797c478bd9Sstevel@tonic-gate 5807c478bd9Sstevel@tonic-gate /* 5817c478bd9Sstevel@tonic-gate * Assume that this redistribution may encounter a device weight 5827c478bd9Sstevel@tonic-gate * via driver.conf tuning of "ddi-intr-weight" that is at most 5837c478bd9Sstevel@tonic-gate * intr_dist_weight_maxfactor times larger. 5847c478bd9Sstevel@tonic-gate */ 5857c478bd9Sstevel@tonic-gate max_weight = intr_dist_weight_max * intr_dist_weight_maxfactor; 5867c478bd9Sstevel@tonic-gate if (max_weight > intr_dist_weight_maxmax) 5877c478bd9Sstevel@tonic-gate max_weight = intr_dist_weight_maxmax; 5887c478bd9Sstevel@tonic-gate intr_dist_weight_max = 1; 5897c478bd9Sstevel@tonic-gate 5907c478bd9Sstevel@tonic-gate INTR_DEBUG((CE_CONT, "intr_dist: " 5917c478bd9Sstevel@tonic-gate "intr_redist_all_cpus: %d-0\n", max_weight)); 5927c478bd9Sstevel@tonic-gate 5937c478bd9Sstevel@tonic-gate /* 5947c478bd9Sstevel@tonic-gate * Redistribute weighted, from heavy to light. The callback that 5957c478bd9Sstevel@tonic-gate * specifies a weight equal to weight_max should redirect all 5967c478bd9Sstevel@tonic-gate * interrupts of weight weight_max or greater [weight_max, inf.). 5977c478bd9Sstevel@tonic-gate * Interrupts of lesser weight should be processed on the call with 5987c478bd9Sstevel@tonic-gate * the matching weight. This allows all the heaver weight interrupts 5997c478bd9Sstevel@tonic-gate * on all weighted busses (multiple pci busses) to be redirected prior 6007c478bd9Sstevel@tonic-gate * to any lesser weight interrupts. 6017c478bd9Sstevel@tonic-gate */ 6027c478bd9Sstevel@tonic-gate for (weight = max_weight; weight >= 0; weight--) 6037c478bd9Sstevel@tonic-gate for (iptr = intr_dist_whead; iptr != NULL; iptr = iptr->next) 6047c478bd9Sstevel@tonic-gate ((void (*)(void *, int32_t, int32_t))iptr->func) 6057c478bd9Sstevel@tonic-gate (iptr->arg, max_weight, weight); 6067c478bd9Sstevel@tonic-gate 6077c478bd9Sstevel@tonic-gate /* redistribute normal (non-weighted) interrupts */ 6087c478bd9Sstevel@tonic-gate for (iptr = intr_dist_head; iptr != NULL; iptr = iptr->next) 6097c478bd9Sstevel@tonic-gate ((void (*)(void *))iptr->func)(iptr->arg); 6107c478bd9Sstevel@tonic-gate mutex_exit(&intr_dist_lock); 6117c478bd9Sstevel@tonic-gate } 6127c478bd9Sstevel@tonic-gate 6137c478bd9Sstevel@tonic-gate void 6147c478bd9Sstevel@tonic-gate intr_redist_all_cpus_shutdown(void) 6157c478bd9Sstevel@tonic-gate { 6167c478bd9Sstevel@tonic-gate intr_policy = INTR_CURRENT_CPU; 6177c478bd9Sstevel@tonic-gate intr_redist_all_cpus(); 6187c478bd9Sstevel@tonic-gate } 6197c478bd9Sstevel@tonic-gate 6207c478bd9Sstevel@tonic-gate /* 6217c478bd9Sstevel@tonic-gate * Determine what CPU to target, based on interrupt policy. 6227c478bd9Sstevel@tonic-gate * 6237c478bd9Sstevel@tonic-gate * INTR_FLAT_DIST: hold a current CPU pointer in a static variable and 6247c478bd9Sstevel@tonic-gate * advance through interrupt enabled cpus (round-robin). 6257c478bd9Sstevel@tonic-gate * 6267c478bd9Sstevel@tonic-gate * INTR_WEIGHTED_DIST: search for an enabled CPU with the lowest 6277c478bd9Sstevel@tonic-gate * cpu_intr_weight, round robin when all equal. 6287c478bd9Sstevel@tonic-gate * 6297c478bd9Sstevel@tonic-gate * Weighted interrupt distribution provides two things: "spread" of weight 6307c478bd9Sstevel@tonic-gate * (associated with algorithm itself) and "isolation" (associated with a 6317c478bd9Sstevel@tonic-gate * particular device weight). A redistribution is what provides optimal 6327c478bd9Sstevel@tonic-gate * "isolation" of heavy weight interrupts, optimal "spread" of weight 6337c478bd9Sstevel@tonic-gate * (relative to what came before) is always occurring. 6347c478bd9Sstevel@tonic-gate * 6357c478bd9Sstevel@tonic-gate * An interrupt weight is a subjective number that represents the 6367c478bd9Sstevel@tonic-gate * percentage of a CPU required to service a device's interrupts: the 6377c478bd9Sstevel@tonic-gate * default weight is 0% (however the algorithm still maintains 6387c478bd9Sstevel@tonic-gate * round-robin), a network interface controller (NIC) may have a large 6397c478bd9Sstevel@tonic-gate * weight (35%). Interrupt weight only has meaning relative to the 6407c478bd9Sstevel@tonic-gate * interrupt weight of other devices: a CPU can be weighted more than 6417c478bd9Sstevel@tonic-gate * 100%, and a single device might consume more than 100% of a CPU. 6427c478bd9Sstevel@tonic-gate * 6437c478bd9Sstevel@tonic-gate * A coarse interrupt weight can be defined by the parent nexus driver 6447c478bd9Sstevel@tonic-gate * based on bus specific information, like pci class codes. A nexus 6457c478bd9Sstevel@tonic-gate * driver that supports device interrupt weighting for its children 6467c478bd9Sstevel@tonic-gate * should call intr_dist_cpuid_add/rem_device_weight(), which adds 6477c478bd9Sstevel@tonic-gate * and removes the weight of a device from the CPU that an interrupt 6487c478bd9Sstevel@tonic-gate * is directed at. The quality of initialization improves when the 6497c478bd9Sstevel@tonic-gate * device interrupt weights more accuracy reflect actual run-time weights, 6507c478bd9Sstevel@tonic-gate * and as the assignments are ordered from is heavy to light. 6517c478bd9Sstevel@tonic-gate * 6527c478bd9Sstevel@tonic-gate * The implementation also supports interrupt weight being specified in 6537c478bd9Sstevel@tonic-gate * driver.conf files via the property "ddi-intr-weight", which takes 6547c478bd9Sstevel@tonic-gate * precedence over the nexus supplied weight. This support is added to 6557c478bd9Sstevel@tonic-gate * permit possible tweaking in the product in response to customer 6567c478bd9Sstevel@tonic-gate * problems. This is not a formal or committed interface. 6577c478bd9Sstevel@tonic-gate * 6587c478bd9Sstevel@tonic-gate * While a weighted approach chooses the CPU providing the best spread 6597c478bd9Sstevel@tonic-gate * given past weights, less than optimal isolation can result in cases 6607c478bd9Sstevel@tonic-gate * where heavy weight devices show up last. The nexus driver's interrupt 6617c478bd9Sstevel@tonic-gate * redistribution logic should use intr_dist_add/rem_weighted so that 6627c478bd9Sstevel@tonic-gate * interrupts can be redistributed heavy first for optimal isolation. 6637c478bd9Sstevel@tonic-gate */ 6647c478bd9Sstevel@tonic-gate uint32_t 6657c478bd9Sstevel@tonic-gate intr_dist_cpuid(void) 6667c478bd9Sstevel@tonic-gate { 6677c478bd9Sstevel@tonic-gate static struct cpu *curr_cpu; 6687c478bd9Sstevel@tonic-gate struct cpu *start_cpu; 6697c478bd9Sstevel@tonic-gate struct cpu *new_cpu; 6707c478bd9Sstevel@tonic-gate struct cpu *cp; 6717c478bd9Sstevel@tonic-gate int cpuid = -1; 6727c478bd9Sstevel@tonic-gate 6737c478bd9Sstevel@tonic-gate /* Establish exclusion for curr_cpu and cpu_intr_weight manipulation */ 6747c478bd9Sstevel@tonic-gate mutex_enter(&intr_dist_cpu_lock); 6757c478bd9Sstevel@tonic-gate 6767c478bd9Sstevel@tonic-gate switch (intr_policy) { 6777c478bd9Sstevel@tonic-gate case INTR_CURRENT_CPU: 6787c478bd9Sstevel@tonic-gate cpuid = CPU->cpu_id; 6797c478bd9Sstevel@tonic-gate break; 6807c478bd9Sstevel@tonic-gate 6817c478bd9Sstevel@tonic-gate case INTR_BOOT_CPU: 6827c478bd9Sstevel@tonic-gate panic("INTR_BOOT_CPU no longer supported."); 6837c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 6847c478bd9Sstevel@tonic-gate 6857c478bd9Sstevel@tonic-gate case INTR_FLAT_DIST: 6867c478bd9Sstevel@tonic-gate case INTR_WEIGHTED_DIST: 6877c478bd9Sstevel@tonic-gate default: 6887c478bd9Sstevel@tonic-gate /* 6897c478bd9Sstevel@tonic-gate * Ensure that curr_cpu is valid - cpu_next will be NULL if 6907c478bd9Sstevel@tonic-gate * the cpu has been deleted (cpu structs are never freed). 6917c478bd9Sstevel@tonic-gate */ 6927c478bd9Sstevel@tonic-gate if (curr_cpu == NULL || curr_cpu->cpu_next == NULL) 6937c478bd9Sstevel@tonic-gate curr_cpu = CPU; 6947c478bd9Sstevel@tonic-gate 6957c478bd9Sstevel@tonic-gate /* 6967c478bd9Sstevel@tonic-gate * Advance to online CPU after curr_cpu (round-robin). For 6977c478bd9Sstevel@tonic-gate * INTR_WEIGHTED_DIST we choose the cpu with the lightest 6987c478bd9Sstevel@tonic-gate * weight. For a nexus that does not support weight the 6997c478bd9Sstevel@tonic-gate * default weight of zero is used. We degrade to round-robin 7007c478bd9Sstevel@tonic-gate * behavior among equal weightes. The default weight is zero 7017c478bd9Sstevel@tonic-gate * and round-robin behavior continues. 7027c478bd9Sstevel@tonic-gate * 7037c478bd9Sstevel@tonic-gate * Disable preemption while traversing cpu_next_onln to 7047c478bd9Sstevel@tonic-gate * ensure the list does not change. This works because 7057c478bd9Sstevel@tonic-gate * modifiers of this list and other lists in a struct cpu 7067c478bd9Sstevel@tonic-gate * call pause_cpus() before making changes. 7077c478bd9Sstevel@tonic-gate */ 7087c478bd9Sstevel@tonic-gate kpreempt_disable(); 7097c478bd9Sstevel@tonic-gate cp = start_cpu = curr_cpu->cpu_next_onln; 7107c478bd9Sstevel@tonic-gate new_cpu = NULL; 7117c478bd9Sstevel@tonic-gate do { 7127c478bd9Sstevel@tonic-gate /* Skip CPUs with interrupts disabled */ 7137c478bd9Sstevel@tonic-gate if ((cp->cpu_flags & CPU_ENABLE) == 0) 7147c478bd9Sstevel@tonic-gate continue; 7157c478bd9Sstevel@tonic-gate 7167c478bd9Sstevel@tonic-gate if (intr_policy == INTR_FLAT_DIST) { 7177c478bd9Sstevel@tonic-gate /* select CPU */ 7187c478bd9Sstevel@tonic-gate new_cpu = cp; 7197c478bd9Sstevel@tonic-gate break; 7207c478bd9Sstevel@tonic-gate } else if ((new_cpu == NULL) || 7217c478bd9Sstevel@tonic-gate (cp->cpu_intr_weight < new_cpu->cpu_intr_weight)) { 7227c478bd9Sstevel@tonic-gate /* Choose if lighter weight */ 7237c478bd9Sstevel@tonic-gate new_cpu = cp; 7247c478bd9Sstevel@tonic-gate } 7257c478bd9Sstevel@tonic-gate } while ((cp = cp->cpu_next_onln) != start_cpu); 7267c478bd9Sstevel@tonic-gate ASSERT(new_cpu); 7277c478bd9Sstevel@tonic-gate cpuid = new_cpu->cpu_id; 7287c478bd9Sstevel@tonic-gate 7297c478bd9Sstevel@tonic-gate INTR_DEBUG((CE_CONT, "intr_dist: cpu %2d weight %3d: " 7307c478bd9Sstevel@tonic-gate "targeted\n", cpuid, new_cpu->cpu_intr_weight)); 7317c478bd9Sstevel@tonic-gate 7327c478bd9Sstevel@tonic-gate /* update static pointer for next round-robin */ 7337c478bd9Sstevel@tonic-gate curr_cpu = new_cpu; 7347c478bd9Sstevel@tonic-gate kpreempt_enable(); 7357c478bd9Sstevel@tonic-gate break; 7367c478bd9Sstevel@tonic-gate } 7377c478bd9Sstevel@tonic-gate mutex_exit(&intr_dist_cpu_lock); 7387c478bd9Sstevel@tonic-gate return (cpuid); 7397c478bd9Sstevel@tonic-gate } 7407c478bd9Sstevel@tonic-gate 7417c478bd9Sstevel@tonic-gate /* 7427c478bd9Sstevel@tonic-gate * Add or remove the the weight of a device from a CPUs interrupt weight. 7437c478bd9Sstevel@tonic-gate * 7447c478bd9Sstevel@tonic-gate * We expect nexus drivers to call intr_dist_cpuid_add/rem_device_weight for 7457c478bd9Sstevel@tonic-gate * their children to improve the overall quality of interrupt initialization. 7467c478bd9Sstevel@tonic-gate * 7477c478bd9Sstevel@tonic-gate * If a nexues shares the CPU returned by a single intr_dist_cpuid() call 7487c478bd9Sstevel@tonic-gate * among multiple devices (sharing ino) then the nexus should call 7497c478bd9Sstevel@tonic-gate * intr_dist_cpuid_add/rem_device_weight for each device separately. Devices 7507c478bd9Sstevel@tonic-gate * that share must specify the same cpuid. 7517c478bd9Sstevel@tonic-gate * 7527c478bd9Sstevel@tonic-gate * If a nexus driver is unable to determine the cpu at remove_intr time 7537c478bd9Sstevel@tonic-gate * for some of its interrupts, then it should not call add_device_weight - 7547c478bd9Sstevel@tonic-gate * intr_dist_cpuid will still provide round-robin. 7557c478bd9Sstevel@tonic-gate * 7567c478bd9Sstevel@tonic-gate * An established device weight (from dev_info node) takes precedence over 7577c478bd9Sstevel@tonic-gate * the weight passed in. If a device weight is not already established 7587c478bd9Sstevel@tonic-gate * then the passed in nexus weight is established. 7597c478bd9Sstevel@tonic-gate */ 7607c478bd9Sstevel@tonic-gate void 7617c478bd9Sstevel@tonic-gate intr_dist_cpuid_add_device_weight(uint32_t cpuid, 7627c478bd9Sstevel@tonic-gate dev_info_t *dip, int32_t nweight) 7637c478bd9Sstevel@tonic-gate { 7647c478bd9Sstevel@tonic-gate int32_t eweight; 7657c478bd9Sstevel@tonic-gate 7667c478bd9Sstevel@tonic-gate /* 7677c478bd9Sstevel@tonic-gate * For non-weighted policy everything has weight of zero (and we get 7687c478bd9Sstevel@tonic-gate * round-robin distribution from intr_dist_cpuid). 7697c478bd9Sstevel@tonic-gate * NB: intr_policy is limited to this file. A weighted nexus driver is 7707c478bd9Sstevel@tonic-gate * calls this rouitne even if intr_policy has been patched to 7717c478bd9Sstevel@tonic-gate * INTR_FLAG_DIST. 7727c478bd9Sstevel@tonic-gate */ 7737c478bd9Sstevel@tonic-gate ASSERT(dip); 7747c478bd9Sstevel@tonic-gate if (intr_policy != INTR_WEIGHTED_DIST) 7757c478bd9Sstevel@tonic-gate return; 7767c478bd9Sstevel@tonic-gate 7777c478bd9Sstevel@tonic-gate eweight = i_ddi_get_intr_weight(dip); 7787c478bd9Sstevel@tonic-gate INTR_DEBUG((CE_CONT, "intr_dist: cpu %2d weight %3d: +%2d/%2d for " 7797c478bd9Sstevel@tonic-gate "%s#%d/%s#%d\n", cpuid, cpu[cpuid]->cpu_intr_weight, 7807c478bd9Sstevel@tonic-gate nweight, eweight, ddi_driver_name(ddi_get_parent(dip)), 7817c478bd9Sstevel@tonic-gate ddi_get_instance(ddi_get_parent(dip)), 7827c478bd9Sstevel@tonic-gate ddi_driver_name(dip), ddi_get_instance(dip))); 7837c478bd9Sstevel@tonic-gate 7847c478bd9Sstevel@tonic-gate /* if no establish weight, establish nexus weight */ 7857c478bd9Sstevel@tonic-gate if (eweight < 0) { 7867c478bd9Sstevel@tonic-gate if (nweight > 0) 7877c478bd9Sstevel@tonic-gate (void) i_ddi_set_intr_weight(dip, nweight); 7887c478bd9Sstevel@tonic-gate else 7897c478bd9Sstevel@tonic-gate nweight = 0; 7907c478bd9Sstevel@tonic-gate } else 7917c478bd9Sstevel@tonic-gate nweight = eweight; /* use established weight */ 7927c478bd9Sstevel@tonic-gate 7937c478bd9Sstevel@tonic-gate /* Establish exclusion for cpu_intr_weight manipulation */ 7947c478bd9Sstevel@tonic-gate mutex_enter(&intr_dist_cpu_lock); 7957c478bd9Sstevel@tonic-gate cpu[cpuid]->cpu_intr_weight += nweight; 7967c478bd9Sstevel@tonic-gate 7977c478bd9Sstevel@tonic-gate /* update intr_dist_weight_max */ 7987c478bd9Sstevel@tonic-gate if (nweight > intr_dist_weight_max) 7997c478bd9Sstevel@tonic-gate intr_dist_weight_max = nweight; 8007c478bd9Sstevel@tonic-gate mutex_exit(&intr_dist_cpu_lock); 8017c478bd9Sstevel@tonic-gate } 8027c478bd9Sstevel@tonic-gate 8037c478bd9Sstevel@tonic-gate void 8047c478bd9Sstevel@tonic-gate intr_dist_cpuid_rem_device_weight(uint32_t cpuid, dev_info_t *dip) 8057c478bd9Sstevel@tonic-gate { 8067c478bd9Sstevel@tonic-gate struct cpu *cp; 8077c478bd9Sstevel@tonic-gate int32_t weight; 8087c478bd9Sstevel@tonic-gate 8097c478bd9Sstevel@tonic-gate ASSERT(dip); 8107c478bd9Sstevel@tonic-gate if (intr_policy != INTR_WEIGHTED_DIST) 8117c478bd9Sstevel@tonic-gate return; 8127c478bd9Sstevel@tonic-gate 8137c478bd9Sstevel@tonic-gate /* remove weight of device from cpu */ 8147c478bd9Sstevel@tonic-gate weight = i_ddi_get_intr_weight(dip); 8157c478bd9Sstevel@tonic-gate if (weight < 0) 8167c478bd9Sstevel@tonic-gate weight = 0; 8177c478bd9Sstevel@tonic-gate INTR_DEBUG((CE_CONT, "intr_dist: cpu %2d weight %3d: -%2d for " 8187c478bd9Sstevel@tonic-gate "%s#%d/%s#%d\n", cpuid, cpu[cpuid]->cpu_intr_weight, weight, 8197c478bd9Sstevel@tonic-gate ddi_driver_name(ddi_get_parent(dip)), 8207c478bd9Sstevel@tonic-gate ddi_get_instance(ddi_get_parent(dip)), 8217c478bd9Sstevel@tonic-gate ddi_driver_name(dip), ddi_get_instance(dip))); 8227c478bd9Sstevel@tonic-gate 8237c478bd9Sstevel@tonic-gate /* Establish exclusion for cpu_intr_weight manipulation */ 8247c478bd9Sstevel@tonic-gate mutex_enter(&intr_dist_cpu_lock); 8257c478bd9Sstevel@tonic-gate cp = cpu[cpuid]; 8267c478bd9Sstevel@tonic-gate cp->cpu_intr_weight -= weight; 8277c478bd9Sstevel@tonic-gate if (cp->cpu_intr_weight < 0) 8287c478bd9Sstevel@tonic-gate cp->cpu_intr_weight = 0; /* sanity */ 8297c478bd9Sstevel@tonic-gate mutex_exit(&intr_dist_cpu_lock); 8307c478bd9Sstevel@tonic-gate } 8312850d85bSmv 8322850d85bSmv ulong_t 8332850d85bSmv create_softint(uint_t pil, uint_t (*func)(caddr_t, caddr_t), caddr_t arg1) 8342850d85bSmv { 8352850d85bSmv uint64_t inum; 8362850d85bSmv 8372850d85bSmv inum = add_softintr(pil, func, arg1, SOFTINT_ST); 8382850d85bSmv return ((ulong_t)inum); 8392850d85bSmv } 8402850d85bSmv 8412850d85bSmv void 8422850d85bSmv invoke_softint(processorid_t cpuid, ulong_t hdl) 8432850d85bSmv { 8442850d85bSmv uint64_t inum = hdl; 8452850d85bSmv 8462850d85bSmv if (cpuid == CPU->cpu_id) 8472850d85bSmv setsoftint(inum); 8482850d85bSmv else 8492850d85bSmv xt_one(cpuid, setsoftint_tl1, inum, 0); 8502850d85bSmv } 8512850d85bSmv 8522850d85bSmv void 8532850d85bSmv remove_softint(ulong_t hdl) 8542850d85bSmv { 8552850d85bSmv uint64_t inum = hdl; 8562850d85bSmv 8572850d85bSmv (void) rem_softintr(inum); 8582850d85bSmv } 8592850d85bSmv 8602850d85bSmv void 8612850d85bSmv sync_softint(cpuset_t set) 8622850d85bSmv { 8632850d85bSmv xt_sync(set); 8642850d85bSmv } 865