17c478bd9Sstevel@tonic-gate /* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 57c478bd9Sstevel@tonic-gate * Common Development and Distribution License, Version 1.0 only 67c478bd9Sstevel@tonic-gate * (the "License"). You may not use this file except in compliance 77c478bd9Sstevel@tonic-gate * with the License. 87c478bd9Sstevel@tonic-gate * 97c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 107c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 117c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 127c478bd9Sstevel@tonic-gate * and limitations under the License. 137c478bd9Sstevel@tonic-gate * 147c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 157c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 167c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 177c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 187c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 197c478bd9Sstevel@tonic-gate * 207c478bd9Sstevel@tonic-gate * CDDL HEADER END 217c478bd9Sstevel@tonic-gate */ 227c478bd9Sstevel@tonic-gate /* 237c478bd9Sstevel@tonic-gate * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 247c478bd9Sstevel@tonic-gate * Use is subject to license terms. 257c478bd9Sstevel@tonic-gate */ 267c478bd9Sstevel@tonic-gate 277c478bd9Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI" 287c478bd9Sstevel@tonic-gate 297c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h> 307c478bd9Sstevel@tonic-gate #include <sys/stack.h> 317c478bd9Sstevel@tonic-gate #include <sys/cpuvar.h> 327c478bd9Sstevel@tonic-gate #include <sys/ivintr.h> 337c478bd9Sstevel@tonic-gate #include <sys/intreg.h> 347c478bd9Sstevel@tonic-gate #include <sys/membar.h> 357c478bd9Sstevel@tonic-gate #include <sys/kmem.h> 367c478bd9Sstevel@tonic-gate #include <sys/intr.h> 377c478bd9Sstevel@tonic-gate #include <sys/sunndi.h> 387c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h> 397c478bd9Sstevel@tonic-gate #include <sys/privregs.h> 407c478bd9Sstevel@tonic-gate #include <sys/systm.h> 417c478bd9Sstevel@tonic-gate #include <sys/archsystm.h> 427c478bd9Sstevel@tonic-gate #include <sys/machsystm.h> 437c478bd9Sstevel@tonic-gate #include <sys/x_call.h> 447c478bd9Sstevel@tonic-gate #include <vm/seg_kp.h> 457c478bd9Sstevel@tonic-gate #include <sys/debug.h> 467c478bd9Sstevel@tonic-gate #include <sys/cyclic.h> 477c478bd9Sstevel@tonic-gate 487c478bd9Sstevel@tonic-gate #include <sys/cpu_sgnblk_defs.h> 497c478bd9Sstevel@tonic-gate 507c478bd9Sstevel@tonic-gate kmutex_t soft_iv_lock; /* protect software interrupt vector table */ 517c478bd9Sstevel@tonic-gate /* Global locks which protect the interrupt distribution lists */ 527c478bd9Sstevel@tonic-gate static kmutex_t intr_dist_lock; 537c478bd9Sstevel@tonic-gate static kmutex_t intr_dist_cpu_lock; 547c478bd9Sstevel@tonic-gate 557c478bd9Sstevel@tonic-gate /* Head of the interrupt distribution lists */ 567c478bd9Sstevel@tonic-gate static struct intr_dist *intr_dist_head = NULL; 577c478bd9Sstevel@tonic-gate static struct intr_dist *intr_dist_whead = NULL; 587c478bd9Sstevel@tonic-gate 597c478bd9Sstevel@tonic-gate uint_t swinum_base; 607c478bd9Sstevel@tonic-gate uint_t maxswinum; 617c478bd9Sstevel@tonic-gate uint_t siron_inum; 627c478bd9Sstevel@tonic-gate uint_t poke_cpu_inum; 63*f8047eabSsudheer /* 64*f8047eabSsudheer * Note:- 65*f8047eabSsudheer * siron_pending was originally created to prevent a resource over consumption 66*f8047eabSsudheer * bug in setsoftint(exhaustion of interrupt pool free list). 67*f8047eabSsudheer * It's original intention is obsolete with the use of iv_pending in 68*f8047eabSsudheer * setsoftint. However, siron_pending stayed around, acting as a second 69*f8047eabSsudheer * gatekeeper preventing soft interrupts from being queued. In this capacity, 70*f8047eabSsudheer * it can lead to hangs on MP systems, where due to global visibility issues 71*f8047eabSsudheer * it can end up set while iv_pending is reset, preventing soft interrupts from 72*f8047eabSsudheer * ever being processed. In addition to its gatekeeper role, init_intr also 73*f8047eabSsudheer * uses it to flag the situation where siron() was called before siron_inum has 74*f8047eabSsudheer * been defined. 75*f8047eabSsudheer * 76*f8047eabSsudheer * siron() does not need an extra gatekeeper; any cpu that wishes should be 77*f8047eabSsudheer * allowed to queue a soft interrupt. It is softint()'s job to ensure 78*f8047eabSsudheer * correct handling of the queues. Therefore, siron_pending has been 79*f8047eabSsudheer * stripped of its gatekeeper task, retaining only its intr_init job, where 80*f8047eabSsudheer * it indicates that there is a pending need to call siron(). 81*f8047eabSsudheer */ 827c478bd9Sstevel@tonic-gate int siron_pending; 837c478bd9Sstevel@tonic-gate 847c478bd9Sstevel@tonic-gate int intr_policy = INTR_WEIGHTED_DIST; /* interrupt distribution policy */ 857c478bd9Sstevel@tonic-gate int intr_dist_debug = 0; 867c478bd9Sstevel@tonic-gate int32_t intr_dist_weight_max = 1; 877c478bd9Sstevel@tonic-gate int32_t intr_dist_weight_maxmax = 1000; 887c478bd9Sstevel@tonic-gate int intr_dist_weight_maxfactor = 2; 897c478bd9Sstevel@tonic-gate #define INTR_DEBUG(args) if (intr_dist_debug) cmn_err args 907c478bd9Sstevel@tonic-gate 917c478bd9Sstevel@tonic-gate static void sw_ivintr_init(cpu_t *); 927c478bd9Sstevel@tonic-gate 937c478bd9Sstevel@tonic-gate /* 947c478bd9Sstevel@tonic-gate * intr_init() - interrupt initialization 957c478bd9Sstevel@tonic-gate * Initialize the system's software interrupt vector table and 967c478bd9Sstevel@tonic-gate * CPU's interrupt free list 977c478bd9Sstevel@tonic-gate */ 987c478bd9Sstevel@tonic-gate void 997c478bd9Sstevel@tonic-gate intr_init(cpu_t *cp) 1007c478bd9Sstevel@tonic-gate { 1017c478bd9Sstevel@tonic-gate init_ivintr(); 1027c478bd9Sstevel@tonic-gate sw_ivintr_init(cp); 1037c478bd9Sstevel@tonic-gate init_intr_pool(cp); 1047c478bd9Sstevel@tonic-gate 1057c478bd9Sstevel@tonic-gate mutex_init(&intr_dist_lock, NULL, MUTEX_DEFAULT, NULL); 1067c478bd9Sstevel@tonic-gate mutex_init(&intr_dist_cpu_lock, NULL, MUTEX_DEFAULT, NULL); 1077c478bd9Sstevel@tonic-gate 1087c478bd9Sstevel@tonic-gate /* 1097c478bd9Sstevel@tonic-gate * A soft interrupt may have been requested prior to the initialization 1107c478bd9Sstevel@tonic-gate * of soft interrupts. Soft interrupts can't be dispatched until after 1117c478bd9Sstevel@tonic-gate * init_intr_pool, so we have to wait until now before we can dispatch 1127c478bd9Sstevel@tonic-gate * the pending soft interrupt (if any). 1137c478bd9Sstevel@tonic-gate */ 114*f8047eabSsudheer if (siron_pending) { 115*f8047eabSsudheer siron_pending = 0; 116*f8047eabSsudheer siron(); 117*f8047eabSsudheer } 1187c478bd9Sstevel@tonic-gate } 1197c478bd9Sstevel@tonic-gate 1207c478bd9Sstevel@tonic-gate /* 1217c478bd9Sstevel@tonic-gate * poke_cpu_intr - fall through when poke_cpu calls 1227c478bd9Sstevel@tonic-gate */ 1237c478bd9Sstevel@tonic-gate 1247c478bd9Sstevel@tonic-gate /* ARGSUSED */ 1257c478bd9Sstevel@tonic-gate uint_t 1267c478bd9Sstevel@tonic-gate poke_cpu_intr(caddr_t arg1, caddr_t arg2) 1277c478bd9Sstevel@tonic-gate { 1287c478bd9Sstevel@tonic-gate CPU->cpu_m.poke_cpu_outstanding = B_FALSE; 1297c478bd9Sstevel@tonic-gate membar_stld_stst(); 1307c478bd9Sstevel@tonic-gate return (1); 1317c478bd9Sstevel@tonic-gate } 1327c478bd9Sstevel@tonic-gate 1337c478bd9Sstevel@tonic-gate /* 1347c478bd9Sstevel@tonic-gate * sw_ivintr_init() - software interrupt vector initialization 1357c478bd9Sstevel@tonic-gate * called after CPU is active 1367c478bd9Sstevel@tonic-gate * the software interrupt vector table is part of the intr_vector[] 1377c478bd9Sstevel@tonic-gate */ 1387c478bd9Sstevel@tonic-gate static void 1397c478bd9Sstevel@tonic-gate sw_ivintr_init(cpu_t *cp) 1407c478bd9Sstevel@tonic-gate { 1417c478bd9Sstevel@tonic-gate extern uint_t softlevel1(); 1427c478bd9Sstevel@tonic-gate 1437c478bd9Sstevel@tonic-gate mutex_init(&soft_iv_lock, NULL, MUTEX_DEFAULT, NULL); 1447c478bd9Sstevel@tonic-gate 1457c478bd9Sstevel@tonic-gate swinum_base = SOFTIVNUM; 1467c478bd9Sstevel@tonic-gate 1477c478bd9Sstevel@tonic-gate /* 1487c478bd9Sstevel@tonic-gate * the maximum software interrupt == MAX_SOFT_INO 1497c478bd9Sstevel@tonic-gate */ 1507c478bd9Sstevel@tonic-gate maxswinum = swinum_base + MAX_SOFT_INO; 1517c478bd9Sstevel@tonic-gate 1527c478bd9Sstevel@tonic-gate REGISTER_BBUS_INTR(); 1537c478bd9Sstevel@tonic-gate 1547c478bd9Sstevel@tonic-gate siron_inum = add_softintr(PIL_1, softlevel1, 0); 1557c478bd9Sstevel@tonic-gate poke_cpu_inum = add_softintr(PIL_13, poke_cpu_intr, 0); 1567c478bd9Sstevel@tonic-gate cp->cpu_m.poke_cpu_outstanding = B_FALSE; 1577c478bd9Sstevel@tonic-gate } 1587c478bd9Sstevel@tonic-gate 1597c478bd9Sstevel@tonic-gate cpuset_t intr_add_pools_inuse; 1607c478bd9Sstevel@tonic-gate 1617c478bd9Sstevel@tonic-gate /* 1627c478bd9Sstevel@tonic-gate * cleanup_intr_pool() 1637c478bd9Sstevel@tonic-gate * Free up the extra intr request pool for this cpu. 1647c478bd9Sstevel@tonic-gate */ 1657c478bd9Sstevel@tonic-gate void 1667c478bd9Sstevel@tonic-gate cleanup_intr_pool(cpu_t *cp) 1677c478bd9Sstevel@tonic-gate { 1687c478bd9Sstevel@tonic-gate extern struct intr_req *intr_add_head; 1697c478bd9Sstevel@tonic-gate int poolno; 1707c478bd9Sstevel@tonic-gate struct intr_req *pool; 1717c478bd9Sstevel@tonic-gate 1727c478bd9Sstevel@tonic-gate poolno = cp->cpu_m.intr_pool_added; 1737c478bd9Sstevel@tonic-gate if (poolno >= 0) { 1747c478bd9Sstevel@tonic-gate cp->cpu_m.intr_pool_added = -1; 1757c478bd9Sstevel@tonic-gate pool = (poolno * INTR_PENDING_MAX * intr_add_pools) + 1767c478bd9Sstevel@tonic-gate 1777c478bd9Sstevel@tonic-gate intr_add_head; /* not byte arithmetic */ 1787c478bd9Sstevel@tonic-gate bzero(pool, INTR_PENDING_MAX * intr_add_pools * 1797c478bd9Sstevel@tonic-gate sizeof (struct intr_req)); 1807c478bd9Sstevel@tonic-gate 1817c478bd9Sstevel@tonic-gate CPUSET_DEL(intr_add_pools_inuse, poolno); 1827c478bd9Sstevel@tonic-gate } 1837c478bd9Sstevel@tonic-gate } 1847c478bd9Sstevel@tonic-gate 1857c478bd9Sstevel@tonic-gate /* 1867c478bd9Sstevel@tonic-gate * init_intr_pool() 1877c478bd9Sstevel@tonic-gate * initialize the intr request pool for the cpu 1887c478bd9Sstevel@tonic-gate * should be called for each cpu 1897c478bd9Sstevel@tonic-gate */ 1907c478bd9Sstevel@tonic-gate void 1917c478bd9Sstevel@tonic-gate init_intr_pool(cpu_t *cp) 1927c478bd9Sstevel@tonic-gate { 1937c478bd9Sstevel@tonic-gate extern struct intr_req *intr_add_head; 1947c478bd9Sstevel@tonic-gate #ifdef DEBUG 1957c478bd9Sstevel@tonic-gate extern struct intr_req *intr_add_tail; 1967c478bd9Sstevel@tonic-gate #endif /* DEBUG */ 1977c478bd9Sstevel@tonic-gate int i, pool; 1987c478bd9Sstevel@tonic-gate 1997c478bd9Sstevel@tonic-gate cp->cpu_m.intr_pool_added = -1; 2007c478bd9Sstevel@tonic-gate 2017c478bd9Sstevel@tonic-gate for (i = 0; i < INTR_PENDING_MAX-1; i++) { 2027c478bd9Sstevel@tonic-gate cp->cpu_m.intr_pool[i].intr_next = 2037c478bd9Sstevel@tonic-gate &cp->cpu_m.intr_pool[i+1]; 2047c478bd9Sstevel@tonic-gate } 2057c478bd9Sstevel@tonic-gate cp->cpu_m.intr_pool[INTR_PENDING_MAX-1].intr_next = NULL; 2067c478bd9Sstevel@tonic-gate 2077c478bd9Sstevel@tonic-gate cp->cpu_m.intr_head[0] = &cp->cpu_m.intr_pool[0]; 2087c478bd9Sstevel@tonic-gate cp->cpu_m.intr_tail[0] = &cp->cpu_m.intr_pool[INTR_PENDING_MAX-1]; 2097c478bd9Sstevel@tonic-gate 2107c478bd9Sstevel@tonic-gate if (intr_add_pools != 0) { 2117c478bd9Sstevel@tonic-gate 2127c478bd9Sstevel@tonic-gate /* 2137c478bd9Sstevel@tonic-gate * If additional interrupt pools have been allocated, 2147c478bd9Sstevel@tonic-gate * initialize those too and add them to the free list. 2157c478bd9Sstevel@tonic-gate */ 2167c478bd9Sstevel@tonic-gate 2177c478bd9Sstevel@tonic-gate struct intr_req *trace; 2187c478bd9Sstevel@tonic-gate 2197c478bd9Sstevel@tonic-gate for (pool = 0; pool < max_ncpus; pool++) { 2207c478bd9Sstevel@tonic-gate if (!(CPU_IN_SET(intr_add_pools_inuse, pool))) 2217c478bd9Sstevel@tonic-gate break; 2227c478bd9Sstevel@tonic-gate } 2237c478bd9Sstevel@tonic-gate if (pool >= max_ncpus) { 2247c478bd9Sstevel@tonic-gate /* 2257c478bd9Sstevel@tonic-gate * XXX - intr pools are alloc'd, just not as 2267c478bd9Sstevel@tonic-gate * much as we would like. 2277c478bd9Sstevel@tonic-gate */ 2287c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, "Failed to alloc all requested intr " 2297c478bd9Sstevel@tonic-gate "pools for cpu%d", cp->cpu_id); 2307c478bd9Sstevel@tonic-gate return; 2317c478bd9Sstevel@tonic-gate } 2327c478bd9Sstevel@tonic-gate CPUSET_ADD(intr_add_pools_inuse, pool); 2337c478bd9Sstevel@tonic-gate cp->cpu_m.intr_pool_added = pool; 2347c478bd9Sstevel@tonic-gate 2357c478bd9Sstevel@tonic-gate trace = (pool * INTR_PENDING_MAX * intr_add_pools) + 2367c478bd9Sstevel@tonic-gate intr_add_head; /* not byte arithmetic */ 2377c478bd9Sstevel@tonic-gate 2387c478bd9Sstevel@tonic-gate cp->cpu_m.intr_pool[INTR_PENDING_MAX-1].intr_next = trace; 2397c478bd9Sstevel@tonic-gate 2407c478bd9Sstevel@tonic-gate for (i = 1; i < intr_add_pools * INTR_PENDING_MAX; i++, trace++) 2417c478bd9Sstevel@tonic-gate trace->intr_next = trace + 1; 2427c478bd9Sstevel@tonic-gate trace->intr_next = NULL; 2437c478bd9Sstevel@tonic-gate 2447c478bd9Sstevel@tonic-gate ASSERT(trace >= intr_add_head && trace <= intr_add_tail); 2457c478bd9Sstevel@tonic-gate 2467c478bd9Sstevel@tonic-gate cp->cpu_m.intr_tail[0] = trace; 2477c478bd9Sstevel@tonic-gate } 2487c478bd9Sstevel@tonic-gate } 2497c478bd9Sstevel@tonic-gate 2507c478bd9Sstevel@tonic-gate 2517c478bd9Sstevel@tonic-gate /* 2527c478bd9Sstevel@tonic-gate * siron - primitive for sun/os/softint.c 2537c478bd9Sstevel@tonic-gate */ 2547c478bd9Sstevel@tonic-gate void 2557c478bd9Sstevel@tonic-gate siron(void) 2567c478bd9Sstevel@tonic-gate { 257*f8047eabSsudheer if (siron_inum != 0) 258*f8047eabSsudheer setsoftint(siron_inum); 259*f8047eabSsudheer else 2607c478bd9Sstevel@tonic-gate siron_pending = 1; 2617c478bd9Sstevel@tonic-gate } 2627c478bd9Sstevel@tonic-gate 2637c478bd9Sstevel@tonic-gate /* 2647c478bd9Sstevel@tonic-gate * no_ivintr() 2657c478bd9Sstevel@tonic-gate * called by vec_interrupt() through sys_trap() 2667c478bd9Sstevel@tonic-gate * vector interrupt received but not valid or not 2677c478bd9Sstevel@tonic-gate * registered in intr_vector[] 2687c478bd9Sstevel@tonic-gate * considered as a spurious mondo interrupt 2697c478bd9Sstevel@tonic-gate */ 2707c478bd9Sstevel@tonic-gate /* ARGSUSED */ 2717c478bd9Sstevel@tonic-gate void 2727c478bd9Sstevel@tonic-gate no_ivintr(struct regs *rp, int inum, int pil) 2737c478bd9Sstevel@tonic-gate { 2747c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, "invalid vector intr: number 0x%x, pil 0x%x", 2757c478bd9Sstevel@tonic-gate inum, pil); 2767c478bd9Sstevel@tonic-gate 2777c478bd9Sstevel@tonic-gate 2787c478bd9Sstevel@tonic-gate #ifdef DEBUG_VEC_INTR 2797c478bd9Sstevel@tonic-gate prom_enter_mon(); 2807c478bd9Sstevel@tonic-gate #endif /* DEBUG_VEC_INTR */ 2817c478bd9Sstevel@tonic-gate } 2827c478bd9Sstevel@tonic-gate 2837c478bd9Sstevel@tonic-gate /* 2847c478bd9Sstevel@tonic-gate * no_intr_pool() 2857c478bd9Sstevel@tonic-gate * called by vec_interrupt() through sys_trap() 2867c478bd9Sstevel@tonic-gate * vector interrupt received but no intr_req entries 2877c478bd9Sstevel@tonic-gate */ 2887c478bd9Sstevel@tonic-gate /* ARGSUSED */ 2897c478bd9Sstevel@tonic-gate void 2907c478bd9Sstevel@tonic-gate no_intr_pool(struct regs *rp, int inum, int pil) 2917c478bd9Sstevel@tonic-gate { 2927c478bd9Sstevel@tonic-gate #ifdef DEBUG_VEC_INTR 2937c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, "intr_req pool empty: num 0x%x, pil 0x%x", 2947c478bd9Sstevel@tonic-gate inum, pil); 2957c478bd9Sstevel@tonic-gate prom_enter_mon(); 2967c478bd9Sstevel@tonic-gate #else 2977c478bd9Sstevel@tonic-gate cmn_err(CE_PANIC, "intr_req pool empty: num 0x%x, pil 0x%x", 2987c478bd9Sstevel@tonic-gate inum, pil); 2997c478bd9Sstevel@tonic-gate #endif /* DEBUG_VEC_INTR */ 3007c478bd9Sstevel@tonic-gate } 3017c478bd9Sstevel@tonic-gate 3027c478bd9Sstevel@tonic-gate void 3037c478bd9Sstevel@tonic-gate intr_dequeue_req(uint_t pil, uint32_t inum) 3047c478bd9Sstevel@tonic-gate { 3057c478bd9Sstevel@tonic-gate struct intr_req *ir, *prev; 3067c478bd9Sstevel@tonic-gate struct machcpu *mcpu; 3077c478bd9Sstevel@tonic-gate uint32_t clr; 3087c478bd9Sstevel@tonic-gate extern uint_t getpstate(void); 3097c478bd9Sstevel@tonic-gate 3107c478bd9Sstevel@tonic-gate ASSERT((getpstate() & PSTATE_IE) == 0); 3117c478bd9Sstevel@tonic-gate 3127c478bd9Sstevel@tonic-gate mcpu = &CPU->cpu_m; 3137c478bd9Sstevel@tonic-gate 3147c478bd9Sstevel@tonic-gate /* Find a matching entry in the list */ 3157c478bd9Sstevel@tonic-gate prev = NULL; 3167c478bd9Sstevel@tonic-gate ir = mcpu->intr_head[pil]; 3177c478bd9Sstevel@tonic-gate while (ir != NULL) { 3187c478bd9Sstevel@tonic-gate if (ir->intr_number == inum) 3197c478bd9Sstevel@tonic-gate break; 3207c478bd9Sstevel@tonic-gate prev = ir; 3217c478bd9Sstevel@tonic-gate ir = ir->intr_next; 3227c478bd9Sstevel@tonic-gate } 3237c478bd9Sstevel@tonic-gate if (ir != NULL) { 3247c478bd9Sstevel@tonic-gate /* 3257c478bd9Sstevel@tonic-gate * Remove entry from list 3267c478bd9Sstevel@tonic-gate */ 3277c478bd9Sstevel@tonic-gate if (prev != NULL) 3287c478bd9Sstevel@tonic-gate prev->intr_next = ir->intr_next; /* non-head */ 3297c478bd9Sstevel@tonic-gate else 3307c478bd9Sstevel@tonic-gate mcpu->intr_head[pil] = ir->intr_next; /* head */ 3317c478bd9Sstevel@tonic-gate 3327c478bd9Sstevel@tonic-gate if (ir->intr_next == NULL) 3337c478bd9Sstevel@tonic-gate mcpu->intr_tail[pil] = prev; /* tail */ 3347c478bd9Sstevel@tonic-gate 3357c478bd9Sstevel@tonic-gate /* 3367c478bd9Sstevel@tonic-gate * Place on free list 3377c478bd9Sstevel@tonic-gate */ 3387c478bd9Sstevel@tonic-gate ir->intr_next = mcpu->intr_head[0]; 3397c478bd9Sstevel@tonic-gate mcpu->intr_head[0] = ir; 3407c478bd9Sstevel@tonic-gate } 3417c478bd9Sstevel@tonic-gate 3427c478bd9Sstevel@tonic-gate /* 3437c478bd9Sstevel@tonic-gate * clear pending interrupts at this level if the list is empty 3447c478bd9Sstevel@tonic-gate */ 3457c478bd9Sstevel@tonic-gate if (mcpu->intr_head[pil] == NULL) { 3467c478bd9Sstevel@tonic-gate clr = 1 << pil; 3477c478bd9Sstevel@tonic-gate if (pil == PIL_14) 3487c478bd9Sstevel@tonic-gate clr |= (TICK_INT_MASK | STICK_INT_MASK); 3497c478bd9Sstevel@tonic-gate wr_clr_softint(clr); 3507c478bd9Sstevel@tonic-gate } 3517c478bd9Sstevel@tonic-gate } 3527c478bd9Sstevel@tonic-gate 3537c478bd9Sstevel@tonic-gate 3547c478bd9Sstevel@tonic-gate /* 3557c478bd9Sstevel@tonic-gate * Send a directed interrupt of specified interrupt number id to a cpu. 3567c478bd9Sstevel@tonic-gate */ 3577c478bd9Sstevel@tonic-gate void 3587c478bd9Sstevel@tonic-gate send_dirint( 3597c478bd9Sstevel@tonic-gate int cpuix, /* cpu to be interrupted */ 3607c478bd9Sstevel@tonic-gate int intr_id) /* interrupt number id */ 3617c478bd9Sstevel@tonic-gate { 3627c478bd9Sstevel@tonic-gate xt_one(cpuix, setsoftint_tl1, intr_id, 0); 3637c478bd9Sstevel@tonic-gate } 3647c478bd9Sstevel@tonic-gate 3657c478bd9Sstevel@tonic-gate void 3667c478bd9Sstevel@tonic-gate init_intr_threads(struct cpu *cp) 3677c478bd9Sstevel@tonic-gate { 3687c478bd9Sstevel@tonic-gate int i; 3697c478bd9Sstevel@tonic-gate 3707c478bd9Sstevel@tonic-gate for (i = 0; i < NINTR_THREADS; i++) 3717c478bd9Sstevel@tonic-gate thread_create_intr(cp); 3727c478bd9Sstevel@tonic-gate 3737c478bd9Sstevel@tonic-gate cp->cpu_intr_stack = (caddr_t)segkp_get(segkp, INTR_STACK_SIZE, 3747c478bd9Sstevel@tonic-gate KPD_HASREDZONE | KPD_NO_ANON | KPD_LOCKED) + 3757c478bd9Sstevel@tonic-gate INTR_STACK_SIZE - SA(MINFRAME); 3767c478bd9Sstevel@tonic-gate } 3777c478bd9Sstevel@tonic-gate 3787c478bd9Sstevel@tonic-gate /* 3797c478bd9Sstevel@tonic-gate * Take the specified CPU out of participation in interrupts. 3807c478bd9Sstevel@tonic-gate * Called by p_online(2) when a processor is being taken off-line. 3817c478bd9Sstevel@tonic-gate * This allows interrupt threads being handled on the processor to 3827c478bd9Sstevel@tonic-gate * complete before the processor is idled. 3837c478bd9Sstevel@tonic-gate */ 3847c478bd9Sstevel@tonic-gate int 3857c478bd9Sstevel@tonic-gate cpu_disable_intr(struct cpu *cp) 3867c478bd9Sstevel@tonic-gate { 3877c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 3887c478bd9Sstevel@tonic-gate 3897c478bd9Sstevel@tonic-gate /* 3907c478bd9Sstevel@tonic-gate * Turn off the CPU_ENABLE flag before calling the redistribution 3917c478bd9Sstevel@tonic-gate * function, since it checks for this in the cpu flags. 3927c478bd9Sstevel@tonic-gate */ 3937c478bd9Sstevel@tonic-gate cp->cpu_flags &= ~CPU_ENABLE; 3947c478bd9Sstevel@tonic-gate 3957c478bd9Sstevel@tonic-gate intr_redist_all_cpus(); 3967c478bd9Sstevel@tonic-gate 3977c478bd9Sstevel@tonic-gate return (0); 3987c478bd9Sstevel@tonic-gate } 3997c478bd9Sstevel@tonic-gate 4007c478bd9Sstevel@tonic-gate /* 4017c478bd9Sstevel@tonic-gate * Allow the specified CPU to participate in interrupts. 4027c478bd9Sstevel@tonic-gate * Called by p_online(2) if a processor could not be taken off-line 4037c478bd9Sstevel@tonic-gate * because of bound threads, in order to resume processing interrupts. 4047c478bd9Sstevel@tonic-gate * Also called after starting a processor. 4057c478bd9Sstevel@tonic-gate */ 4067c478bd9Sstevel@tonic-gate void 4077c478bd9Sstevel@tonic-gate cpu_enable_intr(struct cpu *cp) 4087c478bd9Sstevel@tonic-gate { 4097c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 4107c478bd9Sstevel@tonic-gate 4117c478bd9Sstevel@tonic-gate cp->cpu_flags |= CPU_ENABLE; 4127c478bd9Sstevel@tonic-gate 4137c478bd9Sstevel@tonic-gate intr_redist_all_cpus(); 4147c478bd9Sstevel@tonic-gate } 4157c478bd9Sstevel@tonic-gate 4167c478bd9Sstevel@tonic-gate /* 4177c478bd9Sstevel@tonic-gate * Add function to callback list for intr_redist_all_cpus. We keep two lists, 4187c478bd9Sstevel@tonic-gate * one for weighted callbacks and one for normal callbacks. Weighted callbacks 4197c478bd9Sstevel@tonic-gate * are issued to redirect interrupts of a specified weight, from heavy to 4207c478bd9Sstevel@tonic-gate * light. This allows all the interrupts of a given weight to be redistributed 4217c478bd9Sstevel@tonic-gate * for all weighted nexus drivers prior to those of less weight. 4227c478bd9Sstevel@tonic-gate */ 4237c478bd9Sstevel@tonic-gate static void 4247c478bd9Sstevel@tonic-gate intr_dist_add_list(struct intr_dist **phead, void (*func)(void *), void *arg) 4257c478bd9Sstevel@tonic-gate { 4267c478bd9Sstevel@tonic-gate struct intr_dist *new = kmem_alloc(sizeof (*new), KM_SLEEP); 4277c478bd9Sstevel@tonic-gate struct intr_dist *iptr; 4287c478bd9Sstevel@tonic-gate struct intr_dist **pptr; 4297c478bd9Sstevel@tonic-gate 4307c478bd9Sstevel@tonic-gate ASSERT(func); 4317c478bd9Sstevel@tonic-gate new->func = func; 4327c478bd9Sstevel@tonic-gate new->arg = arg; 4337c478bd9Sstevel@tonic-gate new->next = NULL; 4347c478bd9Sstevel@tonic-gate 4357c478bd9Sstevel@tonic-gate /* Add to tail so that redistribution occurs in original order. */ 4367c478bd9Sstevel@tonic-gate mutex_enter(&intr_dist_lock); 4377c478bd9Sstevel@tonic-gate for (iptr = *phead, pptr = phead; iptr != NULL; 4387c478bd9Sstevel@tonic-gate pptr = &iptr->next, iptr = iptr->next) { 4397c478bd9Sstevel@tonic-gate /* check for problems as we locate the tail */ 4407c478bd9Sstevel@tonic-gate if ((iptr->func == func) && (iptr->arg == arg)) { 4417c478bd9Sstevel@tonic-gate cmn_err(CE_PANIC, "intr_dist_add_list(): duplicate"); 4427c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 4437c478bd9Sstevel@tonic-gate } 4447c478bd9Sstevel@tonic-gate } 4457c478bd9Sstevel@tonic-gate *pptr = new; 4467c478bd9Sstevel@tonic-gate 4477c478bd9Sstevel@tonic-gate mutex_exit(&intr_dist_lock); 4487c478bd9Sstevel@tonic-gate } 4497c478bd9Sstevel@tonic-gate 4507c478bd9Sstevel@tonic-gate void 4517c478bd9Sstevel@tonic-gate intr_dist_add(void (*func)(void *), void *arg) 4527c478bd9Sstevel@tonic-gate { 4537c478bd9Sstevel@tonic-gate intr_dist_add_list(&intr_dist_head, (void (*)(void *))func, arg); 4547c478bd9Sstevel@tonic-gate } 4557c478bd9Sstevel@tonic-gate 4567c478bd9Sstevel@tonic-gate void 4577c478bd9Sstevel@tonic-gate intr_dist_add_weighted(void (*func)(void *, int32_t, int32_t), void *arg) 4587c478bd9Sstevel@tonic-gate { 4597c478bd9Sstevel@tonic-gate intr_dist_add_list(&intr_dist_whead, (void (*)(void *))func, arg); 4607c478bd9Sstevel@tonic-gate } 4617c478bd9Sstevel@tonic-gate 4627c478bd9Sstevel@tonic-gate /* 4637c478bd9Sstevel@tonic-gate * Search for the interrupt distribution structure with the specified 4647c478bd9Sstevel@tonic-gate * mondo vec reg in the interrupt distribution list. If a match is found, 4657c478bd9Sstevel@tonic-gate * then delete the entry from the list. The caller is responsible for 4667c478bd9Sstevel@tonic-gate * modifying the mondo vector registers. 4677c478bd9Sstevel@tonic-gate */ 4687c478bd9Sstevel@tonic-gate static void 4697c478bd9Sstevel@tonic-gate intr_dist_rem_list(struct intr_dist **headp, void (*func)(void *), void *arg) 4707c478bd9Sstevel@tonic-gate { 4717c478bd9Sstevel@tonic-gate struct intr_dist *iptr; 4727c478bd9Sstevel@tonic-gate struct intr_dist **vect; 4737c478bd9Sstevel@tonic-gate 4747c478bd9Sstevel@tonic-gate mutex_enter(&intr_dist_lock); 4757c478bd9Sstevel@tonic-gate for (iptr = *headp, vect = headp; 4767c478bd9Sstevel@tonic-gate iptr != NULL; vect = &iptr->next, iptr = iptr->next) { 4777c478bd9Sstevel@tonic-gate if ((iptr->func == func) && (iptr->arg == arg)) { 4787c478bd9Sstevel@tonic-gate *vect = iptr->next; 4797c478bd9Sstevel@tonic-gate kmem_free(iptr, sizeof (struct intr_dist)); 4807c478bd9Sstevel@tonic-gate mutex_exit(&intr_dist_lock); 4817c478bd9Sstevel@tonic-gate return; 4827c478bd9Sstevel@tonic-gate } 4837c478bd9Sstevel@tonic-gate } 4847c478bd9Sstevel@tonic-gate 4857c478bd9Sstevel@tonic-gate if (!panicstr) 4867c478bd9Sstevel@tonic-gate cmn_err(CE_PANIC, "intr_dist_rem_list: not found"); 4877c478bd9Sstevel@tonic-gate mutex_exit(&intr_dist_lock); 4887c478bd9Sstevel@tonic-gate } 4897c478bd9Sstevel@tonic-gate 4907c478bd9Sstevel@tonic-gate void 4917c478bd9Sstevel@tonic-gate intr_dist_rem(void (*func)(void *), void *arg) 4927c478bd9Sstevel@tonic-gate { 4937c478bd9Sstevel@tonic-gate intr_dist_rem_list(&intr_dist_head, (void (*)(void *))func, arg); 4947c478bd9Sstevel@tonic-gate } 4957c478bd9Sstevel@tonic-gate 4967c478bd9Sstevel@tonic-gate void 4977c478bd9Sstevel@tonic-gate intr_dist_rem_weighted(void (*func)(void *, int32_t, int32_t), void *arg) 4987c478bd9Sstevel@tonic-gate { 4997c478bd9Sstevel@tonic-gate intr_dist_rem_list(&intr_dist_whead, (void (*)(void *))func, arg); 5007c478bd9Sstevel@tonic-gate } 5017c478bd9Sstevel@tonic-gate 5027c478bd9Sstevel@tonic-gate /* 5037c478bd9Sstevel@tonic-gate * Initiate interrupt redistribution. Redistribution improves the isolation 5047c478bd9Sstevel@tonic-gate * associated with interrupt weights by ordering operations from heavy weight 5057c478bd9Sstevel@tonic-gate * to light weight. When a CPUs orientation changes relative to interrupts, 5067c478bd9Sstevel@tonic-gate * there is *always* a redistribution to accommodate this change (call to 5077c478bd9Sstevel@tonic-gate * intr_redist_all_cpus()). As devices (not CPUs) attach/detach it is possible 5087c478bd9Sstevel@tonic-gate * that a redistribution could improve the quality of an initialization. For 5097c478bd9Sstevel@tonic-gate * example, if you are not using a NIC it may not be attached with s10 (devfs). 5107c478bd9Sstevel@tonic-gate * If you then configure the NIC (ifconfig), this may cause the NIC to attach 5117c478bd9Sstevel@tonic-gate * and plumb interrupts. The CPU assignment for the NIC's interrupts is 5127c478bd9Sstevel@tonic-gate * occurring late, so optimal "isolation" relative to weight is not occurring. 5137c478bd9Sstevel@tonic-gate * The same applies to detach, although in this case doing the redistribution 5147c478bd9Sstevel@tonic-gate * might improve "spread" for medium weight devices since the "isolation" of 5157c478bd9Sstevel@tonic-gate * a higher weight device may no longer be present. 5167c478bd9Sstevel@tonic-gate * 5177c478bd9Sstevel@tonic-gate * NB: We should provide a utility to trigger redistribution (ala "intradm -r"). 5187c478bd9Sstevel@tonic-gate * 5197c478bd9Sstevel@tonic-gate * NB: There is risk associated with automatically triggering execution of the 5207c478bd9Sstevel@tonic-gate * redistribution code at arbitrary times. The risk comes from the fact that 5217c478bd9Sstevel@tonic-gate * there is a lot of low-level hardware interaction associated with a 5227c478bd9Sstevel@tonic-gate * redistribution. At some point we may want this code to perform automatic 5237c478bd9Sstevel@tonic-gate * redistribution (redistribution thread; trigger timeout when add/remove 5247c478bd9Sstevel@tonic-gate * weight delta is large enough, and call cv_signal from timeout - causing 5257c478bd9Sstevel@tonic-gate * thead to call i_ddi_intr_redist_all_cpus()) but this is considered too 5267c478bd9Sstevel@tonic-gate * risky at this time. 5277c478bd9Sstevel@tonic-gate */ 5287c478bd9Sstevel@tonic-gate void 5297c478bd9Sstevel@tonic-gate i_ddi_intr_redist_all_cpus() 5307c478bd9Sstevel@tonic-gate { 5317c478bd9Sstevel@tonic-gate mutex_enter(&cpu_lock); 5327c478bd9Sstevel@tonic-gate INTR_DEBUG((CE_CONT, "intr_dist: i_ddi_intr_redist_all_cpus\n")); 5337c478bd9Sstevel@tonic-gate intr_redist_all_cpus(); 5347c478bd9Sstevel@tonic-gate mutex_exit(&cpu_lock); 5357c478bd9Sstevel@tonic-gate } 5367c478bd9Sstevel@tonic-gate 5377c478bd9Sstevel@tonic-gate /* 5387c478bd9Sstevel@tonic-gate * Redistribute all interrupts 5397c478bd9Sstevel@tonic-gate * 5407c478bd9Sstevel@tonic-gate * This function redistributes all interrupting devices, running the 5417c478bd9Sstevel@tonic-gate * parent callback functions for each node. 5427c478bd9Sstevel@tonic-gate */ 5437c478bd9Sstevel@tonic-gate void 5447c478bd9Sstevel@tonic-gate intr_redist_all_cpus(void) 5457c478bd9Sstevel@tonic-gate { 5467c478bd9Sstevel@tonic-gate struct cpu *cp; 5477c478bd9Sstevel@tonic-gate struct intr_dist *iptr; 5487c478bd9Sstevel@tonic-gate int32_t weight, max_weight; 5497c478bd9Sstevel@tonic-gate 5507c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 5517c478bd9Sstevel@tonic-gate mutex_enter(&intr_dist_lock); 5527c478bd9Sstevel@tonic-gate 5537c478bd9Sstevel@tonic-gate /* 5547c478bd9Sstevel@tonic-gate * zero cpu_intr_weight on all cpus - it is safe to traverse 5557c478bd9Sstevel@tonic-gate * cpu_list since we hold cpu_lock. 5567c478bd9Sstevel@tonic-gate */ 5577c478bd9Sstevel@tonic-gate cp = cpu_list; 5587c478bd9Sstevel@tonic-gate do { 5597c478bd9Sstevel@tonic-gate cp->cpu_intr_weight = 0; 5607c478bd9Sstevel@tonic-gate } while ((cp = cp->cpu_next) != cpu_list); 5617c478bd9Sstevel@tonic-gate 5627c478bd9Sstevel@tonic-gate /* 5637c478bd9Sstevel@tonic-gate * Assume that this redistribution may encounter a device weight 5647c478bd9Sstevel@tonic-gate * via driver.conf tuning of "ddi-intr-weight" that is at most 5657c478bd9Sstevel@tonic-gate * intr_dist_weight_maxfactor times larger. 5667c478bd9Sstevel@tonic-gate */ 5677c478bd9Sstevel@tonic-gate max_weight = intr_dist_weight_max * intr_dist_weight_maxfactor; 5687c478bd9Sstevel@tonic-gate if (max_weight > intr_dist_weight_maxmax) 5697c478bd9Sstevel@tonic-gate max_weight = intr_dist_weight_maxmax; 5707c478bd9Sstevel@tonic-gate intr_dist_weight_max = 1; 5717c478bd9Sstevel@tonic-gate 5727c478bd9Sstevel@tonic-gate INTR_DEBUG((CE_CONT, "intr_dist: " 5737c478bd9Sstevel@tonic-gate "intr_redist_all_cpus: %d-0\n", max_weight)); 5747c478bd9Sstevel@tonic-gate 5757c478bd9Sstevel@tonic-gate /* 5767c478bd9Sstevel@tonic-gate * Redistribute weighted, from heavy to light. The callback that 5777c478bd9Sstevel@tonic-gate * specifies a weight equal to weight_max should redirect all 5787c478bd9Sstevel@tonic-gate * interrupts of weight weight_max or greater [weight_max, inf.). 5797c478bd9Sstevel@tonic-gate * Interrupts of lesser weight should be processed on the call with 5807c478bd9Sstevel@tonic-gate * the matching weight. This allows all the heaver weight interrupts 5817c478bd9Sstevel@tonic-gate * on all weighted busses (multiple pci busses) to be redirected prior 5827c478bd9Sstevel@tonic-gate * to any lesser weight interrupts. 5837c478bd9Sstevel@tonic-gate */ 5847c478bd9Sstevel@tonic-gate for (weight = max_weight; weight >= 0; weight--) 5857c478bd9Sstevel@tonic-gate for (iptr = intr_dist_whead; iptr != NULL; iptr = iptr->next) 5867c478bd9Sstevel@tonic-gate ((void (*)(void *, int32_t, int32_t))iptr->func) 5877c478bd9Sstevel@tonic-gate (iptr->arg, max_weight, weight); 5887c478bd9Sstevel@tonic-gate 5897c478bd9Sstevel@tonic-gate /* redistribute normal (non-weighted) interrupts */ 5907c478bd9Sstevel@tonic-gate for (iptr = intr_dist_head; iptr != NULL; iptr = iptr->next) 5917c478bd9Sstevel@tonic-gate ((void (*)(void *))iptr->func)(iptr->arg); 5927c478bd9Sstevel@tonic-gate mutex_exit(&intr_dist_lock); 5937c478bd9Sstevel@tonic-gate } 5947c478bd9Sstevel@tonic-gate 5957c478bd9Sstevel@tonic-gate void 5967c478bd9Sstevel@tonic-gate intr_redist_all_cpus_shutdown(void) 5977c478bd9Sstevel@tonic-gate { 5987c478bd9Sstevel@tonic-gate intr_policy = INTR_CURRENT_CPU; 5997c478bd9Sstevel@tonic-gate intr_redist_all_cpus(); 6007c478bd9Sstevel@tonic-gate } 6017c478bd9Sstevel@tonic-gate 6027c478bd9Sstevel@tonic-gate /* 6037c478bd9Sstevel@tonic-gate * Determine what CPU to target, based on interrupt policy. 6047c478bd9Sstevel@tonic-gate * 6057c478bd9Sstevel@tonic-gate * INTR_FLAT_DIST: hold a current CPU pointer in a static variable and 6067c478bd9Sstevel@tonic-gate * advance through interrupt enabled cpus (round-robin). 6077c478bd9Sstevel@tonic-gate * 6087c478bd9Sstevel@tonic-gate * INTR_WEIGHTED_DIST: search for an enabled CPU with the lowest 6097c478bd9Sstevel@tonic-gate * cpu_intr_weight, round robin when all equal. 6107c478bd9Sstevel@tonic-gate * 6117c478bd9Sstevel@tonic-gate * Weighted interrupt distribution provides two things: "spread" of weight 6127c478bd9Sstevel@tonic-gate * (associated with algorithm itself) and "isolation" (associated with a 6137c478bd9Sstevel@tonic-gate * particular device weight). A redistribution is what provides optimal 6147c478bd9Sstevel@tonic-gate * "isolation" of heavy weight interrupts, optimal "spread" of weight 6157c478bd9Sstevel@tonic-gate * (relative to what came before) is always occurring. 6167c478bd9Sstevel@tonic-gate * 6177c478bd9Sstevel@tonic-gate * An interrupt weight is a subjective number that represents the 6187c478bd9Sstevel@tonic-gate * percentage of a CPU required to service a device's interrupts: the 6197c478bd9Sstevel@tonic-gate * default weight is 0% (however the algorithm still maintains 6207c478bd9Sstevel@tonic-gate * round-robin), a network interface controller (NIC) may have a large 6217c478bd9Sstevel@tonic-gate * weight (35%). Interrupt weight only has meaning relative to the 6227c478bd9Sstevel@tonic-gate * interrupt weight of other devices: a CPU can be weighted more than 6237c478bd9Sstevel@tonic-gate * 100%, and a single device might consume more than 100% of a CPU. 6247c478bd9Sstevel@tonic-gate * 6257c478bd9Sstevel@tonic-gate * A coarse interrupt weight can be defined by the parent nexus driver 6267c478bd9Sstevel@tonic-gate * based on bus specific information, like pci class codes. A nexus 6277c478bd9Sstevel@tonic-gate * driver that supports device interrupt weighting for its children 6287c478bd9Sstevel@tonic-gate * should call intr_dist_cpuid_add/rem_device_weight(), which adds 6297c478bd9Sstevel@tonic-gate * and removes the weight of a device from the CPU that an interrupt 6307c478bd9Sstevel@tonic-gate * is directed at. The quality of initialization improves when the 6317c478bd9Sstevel@tonic-gate * device interrupt weights more accuracy reflect actual run-time weights, 6327c478bd9Sstevel@tonic-gate * and as the assignments are ordered from is heavy to light. 6337c478bd9Sstevel@tonic-gate * 6347c478bd9Sstevel@tonic-gate * The implementation also supports interrupt weight being specified in 6357c478bd9Sstevel@tonic-gate * driver.conf files via the property "ddi-intr-weight", which takes 6367c478bd9Sstevel@tonic-gate * precedence over the nexus supplied weight. This support is added to 6377c478bd9Sstevel@tonic-gate * permit possible tweaking in the product in response to customer 6387c478bd9Sstevel@tonic-gate * problems. This is not a formal or committed interface. 6397c478bd9Sstevel@tonic-gate * 6407c478bd9Sstevel@tonic-gate * While a weighted approach chooses the CPU providing the best spread 6417c478bd9Sstevel@tonic-gate * given past weights, less than optimal isolation can result in cases 6427c478bd9Sstevel@tonic-gate * where heavy weight devices show up last. The nexus driver's interrupt 6437c478bd9Sstevel@tonic-gate * redistribution logic should use intr_dist_add/rem_weighted so that 6447c478bd9Sstevel@tonic-gate * interrupts can be redistributed heavy first for optimal isolation. 6457c478bd9Sstevel@tonic-gate */ 6467c478bd9Sstevel@tonic-gate uint32_t 6477c478bd9Sstevel@tonic-gate intr_dist_cpuid(void) 6487c478bd9Sstevel@tonic-gate { 6497c478bd9Sstevel@tonic-gate static struct cpu *curr_cpu; 6507c478bd9Sstevel@tonic-gate struct cpu *start_cpu; 6517c478bd9Sstevel@tonic-gate struct cpu *new_cpu; 6527c478bd9Sstevel@tonic-gate struct cpu *cp; 6537c478bd9Sstevel@tonic-gate int cpuid = -1; 6547c478bd9Sstevel@tonic-gate 6557c478bd9Sstevel@tonic-gate /* Establish exclusion for curr_cpu and cpu_intr_weight manipulation */ 6567c478bd9Sstevel@tonic-gate mutex_enter(&intr_dist_cpu_lock); 6577c478bd9Sstevel@tonic-gate 6587c478bd9Sstevel@tonic-gate switch (intr_policy) { 6597c478bd9Sstevel@tonic-gate case INTR_CURRENT_CPU: 6607c478bd9Sstevel@tonic-gate cpuid = CPU->cpu_id; 6617c478bd9Sstevel@tonic-gate break; 6627c478bd9Sstevel@tonic-gate 6637c478bd9Sstevel@tonic-gate case INTR_BOOT_CPU: 6647c478bd9Sstevel@tonic-gate panic("INTR_BOOT_CPU no longer supported."); 6657c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 6667c478bd9Sstevel@tonic-gate 6677c478bd9Sstevel@tonic-gate case INTR_FLAT_DIST: 6687c478bd9Sstevel@tonic-gate case INTR_WEIGHTED_DIST: 6697c478bd9Sstevel@tonic-gate default: 6707c478bd9Sstevel@tonic-gate /* 6717c478bd9Sstevel@tonic-gate * Ensure that curr_cpu is valid - cpu_next will be NULL if 6727c478bd9Sstevel@tonic-gate * the cpu has been deleted (cpu structs are never freed). 6737c478bd9Sstevel@tonic-gate */ 6747c478bd9Sstevel@tonic-gate if (curr_cpu == NULL || curr_cpu->cpu_next == NULL) 6757c478bd9Sstevel@tonic-gate curr_cpu = CPU; 6767c478bd9Sstevel@tonic-gate 6777c478bd9Sstevel@tonic-gate /* 6787c478bd9Sstevel@tonic-gate * Advance to online CPU after curr_cpu (round-robin). For 6797c478bd9Sstevel@tonic-gate * INTR_WEIGHTED_DIST we choose the cpu with the lightest 6807c478bd9Sstevel@tonic-gate * weight. For a nexus that does not support weight the 6817c478bd9Sstevel@tonic-gate * default weight of zero is used. We degrade to round-robin 6827c478bd9Sstevel@tonic-gate * behavior among equal weightes. The default weight is zero 6837c478bd9Sstevel@tonic-gate * and round-robin behavior continues. 6847c478bd9Sstevel@tonic-gate * 6857c478bd9Sstevel@tonic-gate * Disable preemption while traversing cpu_next_onln to 6867c478bd9Sstevel@tonic-gate * ensure the list does not change. This works because 6877c478bd9Sstevel@tonic-gate * modifiers of this list and other lists in a struct cpu 6887c478bd9Sstevel@tonic-gate * call pause_cpus() before making changes. 6897c478bd9Sstevel@tonic-gate */ 6907c478bd9Sstevel@tonic-gate kpreempt_disable(); 6917c478bd9Sstevel@tonic-gate cp = start_cpu = curr_cpu->cpu_next_onln; 6927c478bd9Sstevel@tonic-gate new_cpu = NULL; 6937c478bd9Sstevel@tonic-gate do { 6947c478bd9Sstevel@tonic-gate /* Skip CPUs with interrupts disabled */ 6957c478bd9Sstevel@tonic-gate if ((cp->cpu_flags & CPU_ENABLE) == 0) 6967c478bd9Sstevel@tonic-gate continue; 6977c478bd9Sstevel@tonic-gate 6987c478bd9Sstevel@tonic-gate if (intr_policy == INTR_FLAT_DIST) { 6997c478bd9Sstevel@tonic-gate /* select CPU */ 7007c478bd9Sstevel@tonic-gate new_cpu = cp; 7017c478bd9Sstevel@tonic-gate break; 7027c478bd9Sstevel@tonic-gate } else if ((new_cpu == NULL) || 7037c478bd9Sstevel@tonic-gate (cp->cpu_intr_weight < new_cpu->cpu_intr_weight)) { 7047c478bd9Sstevel@tonic-gate /* Choose if lighter weight */ 7057c478bd9Sstevel@tonic-gate new_cpu = cp; 7067c478bd9Sstevel@tonic-gate } 7077c478bd9Sstevel@tonic-gate } while ((cp = cp->cpu_next_onln) != start_cpu); 7087c478bd9Sstevel@tonic-gate ASSERT(new_cpu); 7097c478bd9Sstevel@tonic-gate cpuid = new_cpu->cpu_id; 7107c478bd9Sstevel@tonic-gate 7117c478bd9Sstevel@tonic-gate INTR_DEBUG((CE_CONT, "intr_dist: cpu %2d weight %3d: " 7127c478bd9Sstevel@tonic-gate "targeted\n", cpuid, new_cpu->cpu_intr_weight)); 7137c478bd9Sstevel@tonic-gate 7147c478bd9Sstevel@tonic-gate /* update static pointer for next round-robin */ 7157c478bd9Sstevel@tonic-gate curr_cpu = new_cpu; 7167c478bd9Sstevel@tonic-gate kpreempt_enable(); 7177c478bd9Sstevel@tonic-gate break; 7187c478bd9Sstevel@tonic-gate } 7197c478bd9Sstevel@tonic-gate mutex_exit(&intr_dist_cpu_lock); 7207c478bd9Sstevel@tonic-gate return (cpuid); 7217c478bd9Sstevel@tonic-gate } 7227c478bd9Sstevel@tonic-gate 7237c478bd9Sstevel@tonic-gate /* 7247c478bd9Sstevel@tonic-gate * Add or remove the the weight of a device from a CPUs interrupt weight. 7257c478bd9Sstevel@tonic-gate * 7267c478bd9Sstevel@tonic-gate * We expect nexus drivers to call intr_dist_cpuid_add/rem_device_weight for 7277c478bd9Sstevel@tonic-gate * their children to improve the overall quality of interrupt initialization. 7287c478bd9Sstevel@tonic-gate * 7297c478bd9Sstevel@tonic-gate * If a nexues shares the CPU returned by a single intr_dist_cpuid() call 7307c478bd9Sstevel@tonic-gate * among multiple devices (sharing ino) then the nexus should call 7317c478bd9Sstevel@tonic-gate * intr_dist_cpuid_add/rem_device_weight for each device separately. Devices 7327c478bd9Sstevel@tonic-gate * that share must specify the same cpuid. 7337c478bd9Sstevel@tonic-gate * 7347c478bd9Sstevel@tonic-gate * If a nexus driver is unable to determine the cpu at remove_intr time 7357c478bd9Sstevel@tonic-gate * for some of its interrupts, then it should not call add_device_weight - 7367c478bd9Sstevel@tonic-gate * intr_dist_cpuid will still provide round-robin. 7377c478bd9Sstevel@tonic-gate * 7387c478bd9Sstevel@tonic-gate * An established device weight (from dev_info node) takes precedence over 7397c478bd9Sstevel@tonic-gate * the weight passed in. If a device weight is not already established 7407c478bd9Sstevel@tonic-gate * then the passed in nexus weight is established. 7417c478bd9Sstevel@tonic-gate */ 7427c478bd9Sstevel@tonic-gate void 7437c478bd9Sstevel@tonic-gate intr_dist_cpuid_add_device_weight(uint32_t cpuid, 7447c478bd9Sstevel@tonic-gate dev_info_t *dip, int32_t nweight) 7457c478bd9Sstevel@tonic-gate { 7467c478bd9Sstevel@tonic-gate int32_t eweight; 7477c478bd9Sstevel@tonic-gate 7487c478bd9Sstevel@tonic-gate /* 7497c478bd9Sstevel@tonic-gate * For non-weighted policy everything has weight of zero (and we get 7507c478bd9Sstevel@tonic-gate * round-robin distribution from intr_dist_cpuid). 7517c478bd9Sstevel@tonic-gate * NB: intr_policy is limited to this file. A weighted nexus driver is 7527c478bd9Sstevel@tonic-gate * calls this rouitne even if intr_policy has been patched to 7537c478bd9Sstevel@tonic-gate * INTR_FLAG_DIST. 7547c478bd9Sstevel@tonic-gate */ 7557c478bd9Sstevel@tonic-gate ASSERT(dip); 7567c478bd9Sstevel@tonic-gate if (intr_policy != INTR_WEIGHTED_DIST) 7577c478bd9Sstevel@tonic-gate return; 7587c478bd9Sstevel@tonic-gate 7597c478bd9Sstevel@tonic-gate eweight = i_ddi_get_intr_weight(dip); 7607c478bd9Sstevel@tonic-gate INTR_DEBUG((CE_CONT, "intr_dist: cpu %2d weight %3d: +%2d/%2d for " 7617c478bd9Sstevel@tonic-gate "%s#%d/%s#%d\n", cpuid, cpu[cpuid]->cpu_intr_weight, 7627c478bd9Sstevel@tonic-gate nweight, eweight, ddi_driver_name(ddi_get_parent(dip)), 7637c478bd9Sstevel@tonic-gate ddi_get_instance(ddi_get_parent(dip)), 7647c478bd9Sstevel@tonic-gate ddi_driver_name(dip), ddi_get_instance(dip))); 7657c478bd9Sstevel@tonic-gate 7667c478bd9Sstevel@tonic-gate /* if no establish weight, establish nexus weight */ 7677c478bd9Sstevel@tonic-gate if (eweight < 0) { 7687c478bd9Sstevel@tonic-gate if (nweight > 0) 7697c478bd9Sstevel@tonic-gate (void) i_ddi_set_intr_weight(dip, nweight); 7707c478bd9Sstevel@tonic-gate else 7717c478bd9Sstevel@tonic-gate nweight = 0; 7727c478bd9Sstevel@tonic-gate } else 7737c478bd9Sstevel@tonic-gate nweight = eweight; /* use established weight */ 7747c478bd9Sstevel@tonic-gate 7757c478bd9Sstevel@tonic-gate /* Establish exclusion for cpu_intr_weight manipulation */ 7767c478bd9Sstevel@tonic-gate mutex_enter(&intr_dist_cpu_lock); 7777c478bd9Sstevel@tonic-gate cpu[cpuid]->cpu_intr_weight += nweight; 7787c478bd9Sstevel@tonic-gate 7797c478bd9Sstevel@tonic-gate /* update intr_dist_weight_max */ 7807c478bd9Sstevel@tonic-gate if (nweight > intr_dist_weight_max) 7817c478bd9Sstevel@tonic-gate intr_dist_weight_max = nweight; 7827c478bd9Sstevel@tonic-gate mutex_exit(&intr_dist_cpu_lock); 7837c478bd9Sstevel@tonic-gate } 7847c478bd9Sstevel@tonic-gate 7857c478bd9Sstevel@tonic-gate void 7867c478bd9Sstevel@tonic-gate intr_dist_cpuid_rem_device_weight(uint32_t cpuid, dev_info_t *dip) 7877c478bd9Sstevel@tonic-gate { 7887c478bd9Sstevel@tonic-gate struct cpu *cp; 7897c478bd9Sstevel@tonic-gate int32_t weight; 7907c478bd9Sstevel@tonic-gate 7917c478bd9Sstevel@tonic-gate ASSERT(dip); 7927c478bd9Sstevel@tonic-gate if (intr_policy != INTR_WEIGHTED_DIST) 7937c478bd9Sstevel@tonic-gate return; 7947c478bd9Sstevel@tonic-gate 7957c478bd9Sstevel@tonic-gate /* remove weight of device from cpu */ 7967c478bd9Sstevel@tonic-gate weight = i_ddi_get_intr_weight(dip); 7977c478bd9Sstevel@tonic-gate if (weight < 0) 7987c478bd9Sstevel@tonic-gate weight = 0; 7997c478bd9Sstevel@tonic-gate INTR_DEBUG((CE_CONT, "intr_dist: cpu %2d weight %3d: -%2d for " 8007c478bd9Sstevel@tonic-gate "%s#%d/%s#%d\n", cpuid, cpu[cpuid]->cpu_intr_weight, weight, 8017c478bd9Sstevel@tonic-gate ddi_driver_name(ddi_get_parent(dip)), 8027c478bd9Sstevel@tonic-gate ddi_get_instance(ddi_get_parent(dip)), 8037c478bd9Sstevel@tonic-gate ddi_driver_name(dip), ddi_get_instance(dip))); 8047c478bd9Sstevel@tonic-gate 8057c478bd9Sstevel@tonic-gate /* Establish exclusion for cpu_intr_weight manipulation */ 8067c478bd9Sstevel@tonic-gate mutex_enter(&intr_dist_cpu_lock); 8077c478bd9Sstevel@tonic-gate cp = cpu[cpuid]; 8087c478bd9Sstevel@tonic-gate cp->cpu_intr_weight -= weight; 8097c478bd9Sstevel@tonic-gate if (cp->cpu_intr_weight < 0) 8107c478bd9Sstevel@tonic-gate cp->cpu_intr_weight = 0; /* sanity */ 8117c478bd9Sstevel@tonic-gate mutex_exit(&intr_dist_cpu_lock); 8127c478bd9Sstevel@tonic-gate } 813