17ff178cdSJimmy Vetayases /*
27ff178cdSJimmy Vetayases * CDDL HEADER START
37ff178cdSJimmy Vetayases *
47ff178cdSJimmy Vetayases * The contents of this file are subject to the terms of the
57ff178cdSJimmy Vetayases * Common Development and Distribution License (the "License").
67ff178cdSJimmy Vetayases * You may not use this file except in compliance with the License.
77ff178cdSJimmy Vetayases *
87ff178cdSJimmy Vetayases * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97ff178cdSJimmy Vetayases * or http://www.opensolaris.org/os/licensing.
107ff178cdSJimmy Vetayases * See the License for the specific language governing permissions
117ff178cdSJimmy Vetayases * and limitations under the License.
127ff178cdSJimmy Vetayases *
137ff178cdSJimmy Vetayases * When distributing Covered Code, include this CDDL HEADER in each
147ff178cdSJimmy Vetayases * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157ff178cdSJimmy Vetayases * If applicable, add the following below this CDDL HEADER, with the
167ff178cdSJimmy Vetayases * fields enclosed by brackets "[]" replaced with your own identifying
177ff178cdSJimmy Vetayases * information: Portions Copyright [yyyy] [name of copyright owner]
187ff178cdSJimmy Vetayases *
197ff178cdSJimmy Vetayases * CDDL HEADER END
207ff178cdSJimmy Vetayases */
217ff178cdSJimmy Vetayases /*
227ff178cdSJimmy Vetayases * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
2330acb30dSHans Rosenfeld * Copyright 2017 Joyent, Inc.
247ff178cdSJimmy Vetayases */
257ff178cdSJimmy Vetayases /*
267ff178cdSJimmy Vetayases * Copyright (c) 2010, Intel Corporation.
277ff178cdSJimmy Vetayases * All rights reserved.
287ff178cdSJimmy Vetayases */
297ff178cdSJimmy Vetayases
3015c07adcSJohn Levon /*
3115c07adcSJohn Levon * Copyright (c) 2018, Joyent, Inc.
3215c07adcSJohn Levon */
3315c07adcSJohn Levon
347ff178cdSJimmy Vetayases /*
357ff178cdSJimmy Vetayases * PSMI 1.1 extensions are supported only in 2.6 and later versions.
367ff178cdSJimmy Vetayases * PSMI 1.2 extensions are supported only in 2.7 and later versions.
377ff178cdSJimmy Vetayases * PSMI 1.3 and 1.4 extensions are supported in Solaris 10.
387ff178cdSJimmy Vetayases * PSMI 1.5 extensions are supported in Solaris Nevada.
397ff178cdSJimmy Vetayases * PSMI 1.6 extensions are supported in Solaris Nevada.
407ff178cdSJimmy Vetayases * PSMI 1.7 extensions are supported in Solaris Nevada.
417ff178cdSJimmy Vetayases */
427ff178cdSJimmy Vetayases #define PSMI_1_7
437ff178cdSJimmy Vetayases
447ff178cdSJimmy Vetayases #include <sys/processor.h>
457ff178cdSJimmy Vetayases #include <sys/time.h>
467ff178cdSJimmy Vetayases #include <sys/psm.h>
477ff178cdSJimmy Vetayases #include <sys/smp_impldefs.h>
487ff178cdSJimmy Vetayases #include <sys/inttypes.h>
497ff178cdSJimmy Vetayases #include <sys/cram.h>
507ff178cdSJimmy Vetayases #include <sys/acpi/acpi.h>
517ff178cdSJimmy Vetayases #include <sys/acpica.h>
527ff178cdSJimmy Vetayases #include <sys/psm_common.h>
537ff178cdSJimmy Vetayases #include <sys/apic.h>
547ff178cdSJimmy Vetayases #include <sys/apic_common.h>
557ff178cdSJimmy Vetayases #include <sys/pit.h>
567ff178cdSJimmy Vetayases #include <sys/ddi.h>
577ff178cdSJimmy Vetayases #include <sys/sunddi.h>
587ff178cdSJimmy Vetayases #include <sys/ddi_impldefs.h>
597ff178cdSJimmy Vetayases #include <sys/pci.h>
607ff178cdSJimmy Vetayases #include <sys/promif.h>
617ff178cdSJimmy Vetayases #include <sys/x86_archext.h>
627ff178cdSJimmy Vetayases #include <sys/cpc_impl.h>
637ff178cdSJimmy Vetayases #include <sys/uadmin.h>
647ff178cdSJimmy Vetayases #include <sys/panic.h>
657ff178cdSJimmy Vetayases #include <sys/debug.h>
667ff178cdSJimmy Vetayases #include <sys/archsystm.h>
677ff178cdSJimmy Vetayases #include <sys/trap.h>
687ff178cdSJimmy Vetayases #include <sys/machsystm.h>
697ff178cdSJimmy Vetayases #include <sys/cpuvar.h>
707ff178cdSJimmy Vetayases #include <sys/rm_platter.h>
717ff178cdSJimmy Vetayases #include <sys/privregs.h>
727ff178cdSJimmy Vetayases #include <sys/cyclic.h>
737ff178cdSJimmy Vetayases #include <sys/note.h>
747ff178cdSJimmy Vetayases #include <sys/pci_intr_lib.h>
757ff178cdSJimmy Vetayases #include <sys/sunndi.h>
767ff178cdSJimmy Vetayases #include <sys/hpet.h>
777ff178cdSJimmy Vetayases #include <sys/clock.h>
787ff178cdSJimmy Vetayases
797ff178cdSJimmy Vetayases /*
807ff178cdSJimmy Vetayases * Part of mp_platfrom_common.c that's used only by pcplusmp & xpv_psm
817ff178cdSJimmy Vetayases * but not apix.
827ff178cdSJimmy Vetayases * These functions may be moved to xpv_psm later when apix and pcplusmp
837ff178cdSJimmy Vetayases * are merged together
847ff178cdSJimmy Vetayases */
857ff178cdSJimmy Vetayases
867ff178cdSJimmy Vetayases /*
877ff178cdSJimmy Vetayases * Local Function Prototypes
887ff178cdSJimmy Vetayases */
897ff178cdSJimmy Vetayases static void apic_mark_vector(uchar_t oldvector, uchar_t newvector);
907ff178cdSJimmy Vetayases static void apic_xlate_vector_free_timeout_handler(void *arg);
917ff178cdSJimmy Vetayases static int apic_check_stuck_interrupt(apic_irq_t *irq_ptr, int old_bind_cpu,
927ff178cdSJimmy Vetayases int new_bind_cpu, int apicindex, int intin_no, int which_irq,
937ff178cdSJimmy Vetayases struct ioapic_reprogram_data *drep);
947ff178cdSJimmy Vetayases static int apic_setup_irq_table(dev_info_t *dip, int irqno,
957ff178cdSJimmy Vetayases struct apic_io_intr *intrp, struct intrspec *ispec, iflag_t *intr_flagp,
967ff178cdSJimmy Vetayases int type);
977ff178cdSJimmy Vetayases static void apic_try_deferred_reprogram(int ipl, int vect);
987ff178cdSJimmy Vetayases static void delete_defer_repro_ent(int which_irq);
997ff178cdSJimmy Vetayases static void apic_ioapic_wait_pending_clear(int ioapicindex,
1007ff178cdSJimmy Vetayases int intin_no);
1017ff178cdSJimmy Vetayases
1027ff178cdSJimmy Vetayases extern int apic_acpi_translate_pci_irq(dev_info_t *dip, int busid, int devid,
1037ff178cdSJimmy Vetayases int ipin, int *pci_irqp, iflag_t *intr_flagp);
1047ff178cdSJimmy Vetayases extern int apic_handle_pci_pci_bridge(dev_info_t *idip, int child_devno,
1057ff178cdSJimmy Vetayases int child_ipin, struct apic_io_intr **intrp);
1067ff178cdSJimmy Vetayases extern uchar_t acpi_find_ioapic(int irq);
1077ff178cdSJimmy Vetayases extern struct apic_io_intr *apic_find_io_intr_w_busid(int irqno, int busid);
1087ff178cdSJimmy Vetayases extern int apic_find_bus_id(int bustype);
1097ff178cdSJimmy Vetayases extern int apic_find_intin(uchar_t ioapic, uchar_t intin);
1107ff178cdSJimmy Vetayases extern void apic_record_rdt_entry(apic_irq_t *irqptr, int irq);
1117ff178cdSJimmy Vetayases
1127ff178cdSJimmy Vetayases extern int apic_sci_vect;
1137ff178cdSJimmy Vetayases extern iflag_t apic_sci_flags;
1147ff178cdSJimmy Vetayases /* ACPI HPET interrupt configuration; -1 if HPET not used */
1157ff178cdSJimmy Vetayases extern int apic_hpet_vect;
1167ff178cdSJimmy Vetayases extern iflag_t apic_hpet_flags;
1177ff178cdSJimmy Vetayases extern int apic_intr_policy;
1187ff178cdSJimmy Vetayases extern char *psm_name;
1197ff178cdSJimmy Vetayases
1207ff178cdSJimmy Vetayases /*
1217ff178cdSJimmy Vetayases * number of bits per byte, from <sys/param.h>
1227ff178cdSJimmy Vetayases */
1237ff178cdSJimmy Vetayases #define UCHAR_MAX UINT8_MAX
1247ff178cdSJimmy Vetayases
1257ff178cdSJimmy Vetayases /* Max wait time (in repetitions) for flags to clear in an RDT entry. */
1267ff178cdSJimmy Vetayases extern int apic_max_reps_clear_pending;
1277ff178cdSJimmy Vetayases
1287ff178cdSJimmy Vetayases /* The irq # is implicit in the array index: */
1297ff178cdSJimmy Vetayases struct ioapic_reprogram_data apic_reprogram_info[APIC_MAX_VECTOR+1];
1307ff178cdSJimmy Vetayases /*
1317ff178cdSJimmy Vetayases * APIC_MAX_VECTOR + 1 is the maximum # of IRQs as well. ioapic_reprogram_info
1327ff178cdSJimmy Vetayases * is indexed by IRQ number, NOT by vector number.
1337ff178cdSJimmy Vetayases */
1347ff178cdSJimmy Vetayases
1357ff178cdSJimmy Vetayases extern int apic_int_busy_mark;
1367ff178cdSJimmy Vetayases extern int apic_int_free_mark;
1377ff178cdSJimmy Vetayases extern int apic_diff_for_redistribution;
1387ff178cdSJimmy Vetayases extern int apic_sample_factor_redistribution;
1397ff178cdSJimmy Vetayases extern int apic_redist_cpu_skip;
1407ff178cdSJimmy Vetayases extern int apic_num_imbalance;
1417ff178cdSJimmy Vetayases extern int apic_num_rebind;
1427ff178cdSJimmy Vetayases
1437ff178cdSJimmy Vetayases /* timeout for xlate_vector, mark_vector */
1447ff178cdSJimmy Vetayases int apic_revector_timeout = 16 * 10000; /* 160 millisec */
1457ff178cdSJimmy Vetayases
1467ff178cdSJimmy Vetayases extern int apic_defconf;
1477ff178cdSJimmy Vetayases extern int apic_irq_translate;
1487ff178cdSJimmy Vetayases
1497ff178cdSJimmy Vetayases extern int apic_use_acpi_madt_only; /* 1=ONLY use MADT from ACPI */
1507ff178cdSJimmy Vetayases
1517ff178cdSJimmy Vetayases extern uchar_t apic_io_vectbase[MAX_IO_APIC];
1527ff178cdSJimmy Vetayases
1537ff178cdSJimmy Vetayases extern boolean_t ioapic_mask_workaround[MAX_IO_APIC];
1547ff178cdSJimmy Vetayases
1557ff178cdSJimmy Vetayases /*
1567ff178cdSJimmy Vetayases * First available slot to be used as IRQ index into the apic_irq_table
1577ff178cdSJimmy Vetayases * for those interrupts (like MSI/X) that don't have a physical IRQ.
1587ff178cdSJimmy Vetayases */
1597ff178cdSJimmy Vetayases extern int apic_first_avail_irq;
1607ff178cdSJimmy Vetayases
1617ff178cdSJimmy Vetayases /*
1627ff178cdSJimmy Vetayases * apic_defer_reprogram_lock ensures that only one processor is handling
1637ff178cdSJimmy Vetayases * deferred interrupt programming at *_intr_exit time.
1647ff178cdSJimmy Vetayases */
1657ff178cdSJimmy Vetayases static lock_t apic_defer_reprogram_lock;
1667ff178cdSJimmy Vetayases
1677ff178cdSJimmy Vetayases /*
1687ff178cdSJimmy Vetayases * The current number of deferred reprogrammings outstanding
1697ff178cdSJimmy Vetayases */
1707ff178cdSJimmy Vetayases uint_t apic_reprogram_outstanding = 0;
1717ff178cdSJimmy Vetayases
1727ff178cdSJimmy Vetayases #ifdef DEBUG
1737ff178cdSJimmy Vetayases /*
1747ff178cdSJimmy Vetayases * Counters that keep track of deferred reprogramming stats
1757ff178cdSJimmy Vetayases */
1767ff178cdSJimmy Vetayases uint_t apic_intr_deferrals = 0;
1777ff178cdSJimmy Vetayases uint_t apic_intr_deliver_timeouts = 0;
1787ff178cdSJimmy Vetayases uint_t apic_last_ditch_reprogram_failures = 0;
1797ff178cdSJimmy Vetayases uint_t apic_deferred_setup_failures = 0;
1807ff178cdSJimmy Vetayases uint_t apic_defer_repro_total_retries = 0;
1817ff178cdSJimmy Vetayases uint_t apic_defer_repro_successes = 0;
1827ff178cdSJimmy Vetayases uint_t apic_deferred_spurious_enters = 0;
1837ff178cdSJimmy Vetayases #endif
1847ff178cdSJimmy Vetayases
1857ff178cdSJimmy Vetayases extern int apic_io_max;
1867ff178cdSJimmy Vetayases extern struct apic_io_intr *apic_io_intrp;
1877ff178cdSJimmy Vetayases
1887ff178cdSJimmy Vetayases uchar_t apic_vector_to_irq[APIC_MAX_VECTOR+1];
1897ff178cdSJimmy Vetayases
1907ff178cdSJimmy Vetayases extern uint32_t eisa_level_intr_mask;
1917ff178cdSJimmy Vetayases /* At least MSB will be set if EISA bus */
1927ff178cdSJimmy Vetayases
1937ff178cdSJimmy Vetayases extern int apic_pci_bus_total;
1947ff178cdSJimmy Vetayases extern uchar_t apic_single_pci_busid;
1957ff178cdSJimmy Vetayases
1967ff178cdSJimmy Vetayases /*
1977ff178cdSJimmy Vetayases * Following declarations are for revectoring; used when ISRs at different
1987ff178cdSJimmy Vetayases * IPLs share an irq.
1997ff178cdSJimmy Vetayases */
2007ff178cdSJimmy Vetayases static lock_t apic_revector_lock;
2017ff178cdSJimmy Vetayases int apic_revector_pending = 0;
2027ff178cdSJimmy Vetayases static uchar_t *apic_oldvec_to_newvec;
2037ff178cdSJimmy Vetayases static uchar_t *apic_newvec_to_oldvec;
2047ff178cdSJimmy Vetayases
2057ff178cdSJimmy Vetayases /* ACPI Interrupt Source Override Structure ptr */
2067ff178cdSJimmy Vetayases extern ACPI_MADT_INTERRUPT_OVERRIDE *acpi_isop;
2077ff178cdSJimmy Vetayases extern int acpi_iso_cnt;
2087ff178cdSJimmy Vetayases
2097ff178cdSJimmy Vetayases /*
2107ff178cdSJimmy Vetayases * Auto-configuration routines
2117ff178cdSJimmy Vetayases */
2127ff178cdSJimmy Vetayases
2137ff178cdSJimmy Vetayases /*
2147ff178cdSJimmy Vetayases * Initialise vector->ipl and ipl->pri arrays. level_intr and irqtable
2157ff178cdSJimmy Vetayases * are also set to NULL. vector->irq is set to a value which cannot map
2167ff178cdSJimmy Vetayases * to a real irq to show that it is free.
2177ff178cdSJimmy Vetayases */
2187ff178cdSJimmy Vetayases void
apic_init_common(void)2197ff178cdSJimmy Vetayases apic_init_common(void)
2207ff178cdSJimmy Vetayases {
2217ff178cdSJimmy Vetayases int i, j, indx;
2227ff178cdSJimmy Vetayases int *iptr;
2237ff178cdSJimmy Vetayases
2247ff178cdSJimmy Vetayases /*
2257ff178cdSJimmy Vetayases * Initialize apic_ipls from apic_vectortoipl. This array is
2267ff178cdSJimmy Vetayases * used in apic_intr_enter to determine the IPL to use for the
2277ff178cdSJimmy Vetayases * corresponding vector. On some systems, due to hardware errata
2287ff178cdSJimmy Vetayases * and interrupt sharing, the IPL may not correspond to the IPL listed
2297ff178cdSJimmy Vetayases * in apic_vectortoipl (see apic_addspl and apic_delspl).
2307ff178cdSJimmy Vetayases */
2317ff178cdSJimmy Vetayases for (i = 0; i < (APIC_AVAIL_VECTOR / APIC_VECTOR_PER_IPL); i++) {
2327ff178cdSJimmy Vetayases indx = i * APIC_VECTOR_PER_IPL;
2337ff178cdSJimmy Vetayases
2347ff178cdSJimmy Vetayases for (j = 0; j < APIC_VECTOR_PER_IPL; j++, indx++)
2357ff178cdSJimmy Vetayases apic_ipls[indx] = apic_vectortoipl[i];
2367ff178cdSJimmy Vetayases }
2377ff178cdSJimmy Vetayases
2387ff178cdSJimmy Vetayases /* cpu 0 is always up (for now) */
2397ff178cdSJimmy Vetayases apic_cpus[0].aci_status = APIC_CPU_ONLINE | APIC_CPU_INTR_ENABLE;
2407ff178cdSJimmy Vetayases
2417ff178cdSJimmy Vetayases iptr = (int *)&apic_irq_table[0];
2427ff178cdSJimmy Vetayases for (i = 0; i <= APIC_MAX_VECTOR; i++) {
2437ff178cdSJimmy Vetayases apic_level_intr[i] = 0;
244*c91bd91cSToomas Soome *iptr++ = 0;
2457ff178cdSJimmy Vetayases apic_vector_to_irq[i] = APIC_RESV_IRQ;
2467ff178cdSJimmy Vetayases
2477ff178cdSJimmy Vetayases /* These *must* be initted to B_TRUE! */
2487ff178cdSJimmy Vetayases apic_reprogram_info[i].done = B_TRUE;
2497ff178cdSJimmy Vetayases apic_reprogram_info[i].irqp = NULL;
2507ff178cdSJimmy Vetayases apic_reprogram_info[i].tries = 0;
2517ff178cdSJimmy Vetayases apic_reprogram_info[i].bindcpu = 0;
2527ff178cdSJimmy Vetayases }
2537ff178cdSJimmy Vetayases
2547ff178cdSJimmy Vetayases /*
2557ff178cdSJimmy Vetayases * Allocate a dummy irq table entry for the reserved entry.
2567ff178cdSJimmy Vetayases * This takes care of the race between removing an irq and
2577ff178cdSJimmy Vetayases * clock detecting a CPU in that irq during interrupt load
2587ff178cdSJimmy Vetayases * sampling.
2597ff178cdSJimmy Vetayases */
2607ff178cdSJimmy Vetayases apic_irq_table[APIC_RESV_IRQ] =
2617ff178cdSJimmy Vetayases kmem_zalloc(sizeof (apic_irq_t), KM_SLEEP);
2627ff178cdSJimmy Vetayases
2637ff178cdSJimmy Vetayases mutex_init(&airq_mutex, NULL, MUTEX_DEFAULT, NULL);
2647ff178cdSJimmy Vetayases }
2657ff178cdSJimmy Vetayases
2667ff178cdSJimmy Vetayases void
ioapic_init_intr(int mask_apic)2677ff178cdSJimmy Vetayases ioapic_init_intr(int mask_apic)
2687ff178cdSJimmy Vetayases {
2697ff178cdSJimmy Vetayases int ioapic_ix;
2707ff178cdSJimmy Vetayases struct intrspec ispec;
2717ff178cdSJimmy Vetayases apic_irq_t *irqptr;
2727ff178cdSJimmy Vetayases int i, j;
2737ff178cdSJimmy Vetayases ulong_t iflag;
2747ff178cdSJimmy Vetayases
2757ff178cdSJimmy Vetayases LOCK_INIT_CLEAR(&apic_revector_lock);
2767ff178cdSJimmy Vetayases LOCK_INIT_CLEAR(&apic_defer_reprogram_lock);
2777ff178cdSJimmy Vetayases
2787ff178cdSJimmy Vetayases /* mask interrupt vectors */
2797ff178cdSJimmy Vetayases for (j = 0; j < apic_io_max && mask_apic; j++) {
2807ff178cdSJimmy Vetayases int intin_max;
2817ff178cdSJimmy Vetayases
2827ff178cdSJimmy Vetayases ioapic_ix = j;
2837ff178cdSJimmy Vetayases /* Bits 23-16 define the maximum redirection entries */
2847ff178cdSJimmy Vetayases intin_max = (ioapic_read(ioapic_ix, APIC_VERS_CMD) >> 16)
2857ff178cdSJimmy Vetayases & 0xff;
2867ff178cdSJimmy Vetayases for (i = 0; i <= intin_max; i++)
2877ff178cdSJimmy Vetayases ioapic_write(ioapic_ix, APIC_RDT_CMD + 2 * i, AV_MASK);
2887ff178cdSJimmy Vetayases }
2897ff178cdSJimmy Vetayases
2907ff178cdSJimmy Vetayases /*
2917ff178cdSJimmy Vetayases * Hack alert: deal with ACPI SCI interrupt chicken/egg here
2927ff178cdSJimmy Vetayases */
2937ff178cdSJimmy Vetayases if (apic_sci_vect > 0) {
2947ff178cdSJimmy Vetayases /*
2957ff178cdSJimmy Vetayases * acpica has already done add_avintr(); we just
2967ff178cdSJimmy Vetayases * to finish the job by mimicing translate_irq()
2977ff178cdSJimmy Vetayases *
2987ff178cdSJimmy Vetayases * Fake up an intrspec and setup the tables
2997ff178cdSJimmy Vetayases */
3007ff178cdSJimmy Vetayases ispec.intrspec_vec = apic_sci_vect;
3017ff178cdSJimmy Vetayases ispec.intrspec_pri = SCI_IPL;
3027ff178cdSJimmy Vetayases
3037ff178cdSJimmy Vetayases if (apic_setup_irq_table(NULL, apic_sci_vect, NULL,
3047ff178cdSJimmy Vetayases &ispec, &apic_sci_flags, DDI_INTR_TYPE_FIXED) < 0) {
3057ff178cdSJimmy Vetayases cmn_err(CE_WARN, "!apic: SCI setup failed");
3067ff178cdSJimmy Vetayases return;
3077ff178cdSJimmy Vetayases }
3087ff178cdSJimmy Vetayases irqptr = apic_irq_table[apic_sci_vect];
3097ff178cdSJimmy Vetayases
3107ff178cdSJimmy Vetayases iflag = intr_clear();
3117ff178cdSJimmy Vetayases lock_set(&apic_ioapic_lock);
3127ff178cdSJimmy Vetayases
3137ff178cdSJimmy Vetayases /* Program I/O APIC */
3147ff178cdSJimmy Vetayases (void) apic_setup_io_intr(irqptr, apic_sci_vect, B_FALSE);
3157ff178cdSJimmy Vetayases
3167ff178cdSJimmy Vetayases lock_clear(&apic_ioapic_lock);
3177ff178cdSJimmy Vetayases intr_restore(iflag);
3187ff178cdSJimmy Vetayases
3197ff178cdSJimmy Vetayases irqptr->airq_share++;
3207ff178cdSJimmy Vetayases }
3217ff178cdSJimmy Vetayases
3227ff178cdSJimmy Vetayases /*
3237ff178cdSJimmy Vetayases * Hack alert: deal with ACPI HPET interrupt chicken/egg here.
3247ff178cdSJimmy Vetayases */
3257ff178cdSJimmy Vetayases if (apic_hpet_vect > 0) {
3267ff178cdSJimmy Vetayases /*
3277ff178cdSJimmy Vetayases * hpet has already done add_avintr(); we just need
3287ff178cdSJimmy Vetayases * to finish the job by mimicing translate_irq()
3297ff178cdSJimmy Vetayases *
3307ff178cdSJimmy Vetayases * Fake up an intrspec and setup the tables
3317ff178cdSJimmy Vetayases */
3327ff178cdSJimmy Vetayases ispec.intrspec_vec = apic_hpet_vect;
3337ff178cdSJimmy Vetayases ispec.intrspec_pri = CBE_HIGH_PIL;
3347ff178cdSJimmy Vetayases
3357ff178cdSJimmy Vetayases if (apic_setup_irq_table(NULL, apic_hpet_vect, NULL,
3367ff178cdSJimmy Vetayases &ispec, &apic_hpet_flags, DDI_INTR_TYPE_FIXED) < 0) {
3377ff178cdSJimmy Vetayases cmn_err(CE_WARN, "!apic: HPET setup failed");
3387ff178cdSJimmy Vetayases return;
3397ff178cdSJimmy Vetayases }
3407ff178cdSJimmy Vetayases irqptr = apic_irq_table[apic_hpet_vect];
3417ff178cdSJimmy Vetayases
3427ff178cdSJimmy Vetayases iflag = intr_clear();
3437ff178cdSJimmy Vetayases lock_set(&apic_ioapic_lock);
3447ff178cdSJimmy Vetayases
3457ff178cdSJimmy Vetayases /* Program I/O APIC */
3467ff178cdSJimmy Vetayases (void) apic_setup_io_intr(irqptr, apic_hpet_vect, B_FALSE);
3477ff178cdSJimmy Vetayases
3487ff178cdSJimmy Vetayases lock_clear(&apic_ioapic_lock);
3497ff178cdSJimmy Vetayases intr_restore(iflag);
3507ff178cdSJimmy Vetayases
3517ff178cdSJimmy Vetayases irqptr->airq_share++;
3527ff178cdSJimmy Vetayases }
3537ff178cdSJimmy Vetayases }
3547ff178cdSJimmy Vetayases
3557ff178cdSJimmy Vetayases /*
3567ff178cdSJimmy Vetayases * Add mask bits to disable interrupt vector from happening
3577ff178cdSJimmy Vetayases * at or above IPL. In addition, it should remove mask bits
3587ff178cdSJimmy Vetayases * to enable interrupt vectors below the given IPL.
3597ff178cdSJimmy Vetayases *
3607ff178cdSJimmy Vetayases * Both add and delspl are complicated by the fact that different interrupts
3617ff178cdSJimmy Vetayases * may share IRQs. This can happen in two ways.
3627ff178cdSJimmy Vetayases * 1. The same H/W line is shared by more than 1 device
3637ff178cdSJimmy Vetayases * 1a. with interrupts at different IPLs
3647ff178cdSJimmy Vetayases * 1b. with interrupts at same IPL
3657ff178cdSJimmy Vetayases * 2. We ran out of vectors at a given IPL and started sharing vectors.
3667ff178cdSJimmy Vetayases * 1b and 2 should be handled gracefully, except for the fact some ISRs
3677ff178cdSJimmy Vetayases * will get called often when no interrupt is pending for the device.
3687ff178cdSJimmy Vetayases * For 1a, we handle it at the higher IPL.
3697ff178cdSJimmy Vetayases */
3707ff178cdSJimmy Vetayases /*ARGSUSED*/
3717ff178cdSJimmy Vetayases int
apic_addspl_common(int irqno,int ipl,int min_ipl,int max_ipl)3727ff178cdSJimmy Vetayases apic_addspl_common(int irqno, int ipl, int min_ipl, int max_ipl)
3737ff178cdSJimmy Vetayases {
3747ff178cdSJimmy Vetayases uchar_t vector;
3757ff178cdSJimmy Vetayases ulong_t iflag;
3767ff178cdSJimmy Vetayases apic_irq_t *irqptr, *irqheadptr;
3777ff178cdSJimmy Vetayases int irqindex;
3787ff178cdSJimmy Vetayases
3797ff178cdSJimmy Vetayases ASSERT(max_ipl <= UCHAR_MAX);
3807ff178cdSJimmy Vetayases irqindex = IRQINDEX(irqno);
3817ff178cdSJimmy Vetayases
3827ff178cdSJimmy Vetayases if ((irqindex == -1) || (!apic_irq_table[irqindex]))
3837ff178cdSJimmy Vetayases return (PSM_FAILURE);
3847ff178cdSJimmy Vetayases
3857ff178cdSJimmy Vetayases mutex_enter(&airq_mutex);
3867ff178cdSJimmy Vetayases irqptr = irqheadptr = apic_irq_table[irqindex];
3877ff178cdSJimmy Vetayases
3887ff178cdSJimmy Vetayases DDI_INTR_IMPLDBG((CE_CONT, "apic_addspl: dip=0x%p type=%d irqno=0x%x "
3897ff178cdSJimmy Vetayases "vector=0x%x\n", (void *)irqptr->airq_dip,
3907ff178cdSJimmy Vetayases irqptr->airq_mps_intr_index, irqno, irqptr->airq_vector));
3917ff178cdSJimmy Vetayases
3927ff178cdSJimmy Vetayases while (irqptr) {
3937ff178cdSJimmy Vetayases if (VIRTIRQ(irqindex, irqptr->airq_share_id) == irqno)
3947ff178cdSJimmy Vetayases break;
3957ff178cdSJimmy Vetayases irqptr = irqptr->airq_next;
3967ff178cdSJimmy Vetayases }
3977ff178cdSJimmy Vetayases irqptr->airq_share++;
3987ff178cdSJimmy Vetayases
3997ff178cdSJimmy Vetayases mutex_exit(&airq_mutex);
4007ff178cdSJimmy Vetayases
4017ff178cdSJimmy Vetayases /* return if it is not hardware interrupt */
4027ff178cdSJimmy Vetayases if (irqptr->airq_mps_intr_index == RESERVE_INDEX)
4037ff178cdSJimmy Vetayases return (PSM_SUCCESS);
4047ff178cdSJimmy Vetayases
4057ff178cdSJimmy Vetayases /* Or if there are more interupts at a higher IPL */
4067ff178cdSJimmy Vetayases if (ipl != max_ipl)
4077ff178cdSJimmy Vetayases return (PSM_SUCCESS);
4087ff178cdSJimmy Vetayases
4097ff178cdSJimmy Vetayases /*
4107ff178cdSJimmy Vetayases * if apic_picinit() has not been called yet, just return.
4117ff178cdSJimmy Vetayases * At the end of apic_picinit(), we will call setup_io_intr().
4127ff178cdSJimmy Vetayases */
4137ff178cdSJimmy Vetayases
4147ff178cdSJimmy Vetayases if (!apic_picinit_called)
4157ff178cdSJimmy Vetayases return (PSM_SUCCESS);
4167ff178cdSJimmy Vetayases
4177ff178cdSJimmy Vetayases /*
4187ff178cdSJimmy Vetayases * Upgrade vector if max_ipl is not earlier ipl. If we cannot allocate,
4197ff178cdSJimmy Vetayases * return failure.
4207ff178cdSJimmy Vetayases */
4217ff178cdSJimmy Vetayases if (irqptr->airq_ipl != max_ipl &&
4227ff178cdSJimmy Vetayases !ioapic_mask_workaround[irqptr->airq_ioapicindex]) {
4237ff178cdSJimmy Vetayases
4247ff178cdSJimmy Vetayases vector = apic_allocate_vector(max_ipl, irqindex, 1);
4257ff178cdSJimmy Vetayases if (vector == 0) {
4267ff178cdSJimmy Vetayases irqptr->airq_share--;
4277ff178cdSJimmy Vetayases return (PSM_FAILURE);
4287ff178cdSJimmy Vetayases }
4297ff178cdSJimmy Vetayases irqptr = irqheadptr;
4307ff178cdSJimmy Vetayases apic_mark_vector(irqptr->airq_vector, vector);
4317ff178cdSJimmy Vetayases while (irqptr) {
4327ff178cdSJimmy Vetayases irqptr->airq_vector = vector;
4337ff178cdSJimmy Vetayases irqptr->airq_ipl = (uchar_t)max_ipl;
4347ff178cdSJimmy Vetayases /*
4357ff178cdSJimmy Vetayases * reprogram irq being added and every one else
4367ff178cdSJimmy Vetayases * who is not in the UNINIT state
4377ff178cdSJimmy Vetayases */
4387ff178cdSJimmy Vetayases if ((VIRTIRQ(irqindex, irqptr->airq_share_id) ==
4397ff178cdSJimmy Vetayases irqno) || (irqptr->airq_temp_cpu != IRQ_UNINIT)) {
4407ff178cdSJimmy Vetayases apic_record_rdt_entry(irqptr, irqindex);
4417ff178cdSJimmy Vetayases
4427ff178cdSJimmy Vetayases iflag = intr_clear();
4437ff178cdSJimmy Vetayases lock_set(&apic_ioapic_lock);
4447ff178cdSJimmy Vetayases
4457ff178cdSJimmy Vetayases (void) apic_setup_io_intr(irqptr, irqindex,
4467ff178cdSJimmy Vetayases B_FALSE);
4477ff178cdSJimmy Vetayases
4487ff178cdSJimmy Vetayases lock_clear(&apic_ioapic_lock);
4497ff178cdSJimmy Vetayases intr_restore(iflag);
4507ff178cdSJimmy Vetayases }
4517ff178cdSJimmy Vetayases irqptr = irqptr->airq_next;
4527ff178cdSJimmy Vetayases }
4537ff178cdSJimmy Vetayases return (PSM_SUCCESS);
4547ff178cdSJimmy Vetayases
4557ff178cdSJimmy Vetayases } else if (irqptr->airq_ipl != max_ipl &&
4567ff178cdSJimmy Vetayases ioapic_mask_workaround[irqptr->airq_ioapicindex]) {
4577ff178cdSJimmy Vetayases /*
4587ff178cdSJimmy Vetayases * We cannot upgrade the vector, but we can change
4597ff178cdSJimmy Vetayases * the IPL that this vector induces.
4607ff178cdSJimmy Vetayases *
4617ff178cdSJimmy Vetayases * Note that we subtract APIC_BASE_VECT from the vector
4627ff178cdSJimmy Vetayases * here because this array is used in apic_intr_enter
4637ff178cdSJimmy Vetayases * (no need to add APIC_BASE_VECT in that hot code
4647ff178cdSJimmy Vetayases * path since we can do it in the rarely-executed path
4657ff178cdSJimmy Vetayases * here).
4667ff178cdSJimmy Vetayases */
4677ff178cdSJimmy Vetayases apic_ipls[irqptr->airq_vector - APIC_BASE_VECT] =
4687ff178cdSJimmy Vetayases (uchar_t)max_ipl;
4697ff178cdSJimmy Vetayases
4707ff178cdSJimmy Vetayases irqptr = irqheadptr;
4717ff178cdSJimmy Vetayases while (irqptr) {
4727ff178cdSJimmy Vetayases irqptr->airq_ipl = (uchar_t)max_ipl;
4737ff178cdSJimmy Vetayases irqptr = irqptr->airq_next;
4747ff178cdSJimmy Vetayases }
4757ff178cdSJimmy Vetayases
4767ff178cdSJimmy Vetayases return (PSM_SUCCESS);
4777ff178cdSJimmy Vetayases }
4787ff178cdSJimmy Vetayases
4797ff178cdSJimmy Vetayases ASSERT(irqptr);
4807ff178cdSJimmy Vetayases
4817ff178cdSJimmy Vetayases iflag = intr_clear();
4827ff178cdSJimmy Vetayases lock_set(&apic_ioapic_lock);
4837ff178cdSJimmy Vetayases
4847ff178cdSJimmy Vetayases (void) apic_setup_io_intr(irqptr, irqindex, B_FALSE);
4857ff178cdSJimmy Vetayases
4867ff178cdSJimmy Vetayases lock_clear(&apic_ioapic_lock);
4877ff178cdSJimmy Vetayases intr_restore(iflag);
4887ff178cdSJimmy Vetayases
4897ff178cdSJimmy Vetayases return (PSM_SUCCESS);
4907ff178cdSJimmy Vetayases }
4917ff178cdSJimmy Vetayases
4927ff178cdSJimmy Vetayases /*
4937ff178cdSJimmy Vetayases * Recompute mask bits for the given interrupt vector.
4947ff178cdSJimmy Vetayases * If there is no interrupt servicing routine for this
4957ff178cdSJimmy Vetayases * vector, this function should disable interrupt vector
4967ff178cdSJimmy Vetayases * from happening at all IPLs. If there are still
4977ff178cdSJimmy Vetayases * handlers using the given vector, this function should
4987ff178cdSJimmy Vetayases * disable the given vector from happening below the lowest
4997ff178cdSJimmy Vetayases * IPL of the remaining hadlers.
5007ff178cdSJimmy Vetayases */
5017ff178cdSJimmy Vetayases /*ARGSUSED*/
5027ff178cdSJimmy Vetayases int
apic_delspl_common(int irqno,int ipl,int min_ipl,int max_ipl)5037ff178cdSJimmy Vetayases apic_delspl_common(int irqno, int ipl, int min_ipl, int max_ipl)
5047ff178cdSJimmy Vetayases {
5057ff178cdSJimmy Vetayases uchar_t vector;
5067ff178cdSJimmy Vetayases uint32_t bind_cpu;
5077ff178cdSJimmy Vetayases int intin, irqindex;
5087ff178cdSJimmy Vetayases int ioapic_ix;
5097ff178cdSJimmy Vetayases apic_irq_t *irqptr, *preirqptr, *irqheadptr, *irqp;
5107ff178cdSJimmy Vetayases ulong_t iflag;
5117ff178cdSJimmy Vetayases
5127ff178cdSJimmy Vetayases mutex_enter(&airq_mutex);
5137ff178cdSJimmy Vetayases irqindex = IRQINDEX(irqno);
5147ff178cdSJimmy Vetayases irqptr = preirqptr = irqheadptr = apic_irq_table[irqindex];
5157ff178cdSJimmy Vetayases
5167ff178cdSJimmy Vetayases DDI_INTR_IMPLDBG((CE_CONT, "apic_delspl: dip=0x%p type=%d irqno=0x%x "
5177ff178cdSJimmy Vetayases "vector=0x%x\n", (void *)irqptr->airq_dip,
5187ff178cdSJimmy Vetayases irqptr->airq_mps_intr_index, irqno, irqptr->airq_vector));
5197ff178cdSJimmy Vetayases
5207ff178cdSJimmy Vetayases while (irqptr) {
5217ff178cdSJimmy Vetayases if (VIRTIRQ(irqindex, irqptr->airq_share_id) == irqno)
5227ff178cdSJimmy Vetayases break;
5237ff178cdSJimmy Vetayases preirqptr = irqptr;
5247ff178cdSJimmy Vetayases irqptr = irqptr->airq_next;
5257ff178cdSJimmy Vetayases }
5267ff178cdSJimmy Vetayases ASSERT(irqptr);
5277ff178cdSJimmy Vetayases
5287ff178cdSJimmy Vetayases irqptr->airq_share--;
5297ff178cdSJimmy Vetayases
5307ff178cdSJimmy Vetayases mutex_exit(&airq_mutex);
5317ff178cdSJimmy Vetayases
5327ff178cdSJimmy Vetayases /*
5337ff178cdSJimmy Vetayases * If there are more interrupts at a higher IPL, we don't need
5347ff178cdSJimmy Vetayases * to disable anything.
5357ff178cdSJimmy Vetayases */
5367ff178cdSJimmy Vetayases if (ipl < max_ipl)
5377ff178cdSJimmy Vetayases return (PSM_SUCCESS);
5387ff178cdSJimmy Vetayases
5397ff178cdSJimmy Vetayases /* return if it is not hardware interrupt */
5407ff178cdSJimmy Vetayases if (irqptr->airq_mps_intr_index == RESERVE_INDEX)
5417ff178cdSJimmy Vetayases return (PSM_SUCCESS);
5427ff178cdSJimmy Vetayases
5437ff178cdSJimmy Vetayases if (!apic_picinit_called) {
5447ff178cdSJimmy Vetayases /*
5457ff178cdSJimmy Vetayases * Clear irq_struct. If two devices shared an intpt
5467ff178cdSJimmy Vetayases * line & 1 unloaded before picinit, we are hosed. But, then
5477ff178cdSJimmy Vetayases * we hope the machine survive.
5487ff178cdSJimmy Vetayases */
5497ff178cdSJimmy Vetayases irqptr->airq_mps_intr_index = FREE_INDEX;
5507ff178cdSJimmy Vetayases irqptr->airq_temp_cpu = IRQ_UNINIT;
5517ff178cdSJimmy Vetayases apic_free_vector(irqptr->airq_vector);
5527ff178cdSJimmy Vetayases return (PSM_SUCCESS);
5537ff178cdSJimmy Vetayases }
5547ff178cdSJimmy Vetayases /*
5557ff178cdSJimmy Vetayases * Downgrade vector to new max_ipl if needed. If we cannot allocate,
5567ff178cdSJimmy Vetayases * use old IPL. Not very elegant, but it should work.
5577ff178cdSJimmy Vetayases */
5587ff178cdSJimmy Vetayases if ((irqptr->airq_ipl != max_ipl) && (max_ipl != PSM_INVALID_IPL) &&
5597ff178cdSJimmy Vetayases !ioapic_mask_workaround[irqptr->airq_ioapicindex]) {
5607ff178cdSJimmy Vetayases apic_irq_t *irqp;
561a833a696SJosef 'Jeff' Sipek if ((vector = apic_allocate_vector(max_ipl, irqno, 1))) {
5627ff178cdSJimmy Vetayases apic_mark_vector(irqheadptr->airq_vector, vector);
5637ff178cdSJimmy Vetayases irqp = irqheadptr;
5647ff178cdSJimmy Vetayases while (irqp) {
5657ff178cdSJimmy Vetayases irqp->airq_vector = vector;
5667ff178cdSJimmy Vetayases irqp->airq_ipl = (uchar_t)max_ipl;
5677ff178cdSJimmy Vetayases if (irqp->airq_temp_cpu != IRQ_UNINIT) {
5687ff178cdSJimmy Vetayases apic_record_rdt_entry(irqp, irqindex);
5697ff178cdSJimmy Vetayases
5707ff178cdSJimmy Vetayases iflag = intr_clear();
5717ff178cdSJimmy Vetayases lock_set(&apic_ioapic_lock);
5727ff178cdSJimmy Vetayases
5737ff178cdSJimmy Vetayases (void) apic_setup_io_intr(irqp,
5747ff178cdSJimmy Vetayases irqindex, B_FALSE);
5757ff178cdSJimmy Vetayases
5767ff178cdSJimmy Vetayases lock_clear(&apic_ioapic_lock);
5777ff178cdSJimmy Vetayases intr_restore(iflag);
5787ff178cdSJimmy Vetayases }
5797ff178cdSJimmy Vetayases irqp = irqp->airq_next;
5807ff178cdSJimmy Vetayases }
5817ff178cdSJimmy Vetayases }
5827ff178cdSJimmy Vetayases
5837ff178cdSJimmy Vetayases } else if (irqptr->airq_ipl != max_ipl &&
5847ff178cdSJimmy Vetayases max_ipl != PSM_INVALID_IPL &&
5857ff178cdSJimmy Vetayases ioapic_mask_workaround[irqptr->airq_ioapicindex]) {
5867ff178cdSJimmy Vetayases
5877ff178cdSJimmy Vetayases /*
5887ff178cdSJimmy Vetayases * We cannot downgrade the IPL of the vector below the vector's
5897ff178cdSJimmy Vetayases * hardware priority. If we did, it would be possible for a
5907ff178cdSJimmy Vetayases * higher-priority hardware vector to interrupt a CPU running at an IPL
5917ff178cdSJimmy Vetayases * lower than the hardware priority of the interrupting vector (but
5927ff178cdSJimmy Vetayases * higher than the soft IPL of this IRQ). When this happens, we would
5937ff178cdSJimmy Vetayases * then try to drop the IPL BELOW what it was (effectively dropping
5947ff178cdSJimmy Vetayases * below base_spl) which would be potentially catastrophic.
5957ff178cdSJimmy Vetayases *
5967ff178cdSJimmy Vetayases * (e.g. Suppose the hardware vector associated with this IRQ is 0x40
5977ff178cdSJimmy Vetayases * (hardware IPL of 4). Further assume that the old IPL of this IRQ
5987ff178cdSJimmy Vetayases * was 4, but the new IPL is 1. If we forced vector 0x40 to result in
5997ff178cdSJimmy Vetayases * an IPL of 1, it would be possible for the processor to be executing
6007ff178cdSJimmy Vetayases * at IPL 3 and for an interrupt to come in on vector 0x40, interrupting
6017ff178cdSJimmy Vetayases * the currently-executing ISR. When apic_intr_enter consults
6027ff178cdSJimmy Vetayases * apic_irqs[], it will return 1, bringing the IPL of the CPU down to 1
6037ff178cdSJimmy Vetayases * so even though the processor was running at IPL 4, an IPL 1
6047ff178cdSJimmy Vetayases * interrupt will have interrupted it, which must not happen)).
6057ff178cdSJimmy Vetayases *
6067ff178cdSJimmy Vetayases * Effectively, this means that the hardware priority corresponding to
6077ff178cdSJimmy Vetayases * the IRQ's IPL (in apic_ipls[]) cannot be lower than the vector's
6087ff178cdSJimmy Vetayases * hardware priority.
6097ff178cdSJimmy Vetayases *
6107ff178cdSJimmy Vetayases * (In the above example, then, after removal of the IPL 4 device's
6117ff178cdSJimmy Vetayases * interrupt handler, the new IPL will continue to be 4 because the
6127ff178cdSJimmy Vetayases * hardware priority that IPL 1 implies is lower than the hardware
6137ff178cdSJimmy Vetayases * priority of the vector used.)
6147ff178cdSJimmy Vetayases */
6157ff178cdSJimmy Vetayases /* apic_ipls is indexed by vector, starting at APIC_BASE_VECT */
6167ff178cdSJimmy Vetayases const int apic_ipls_index = irqptr->airq_vector -
6177ff178cdSJimmy Vetayases APIC_BASE_VECT;
6187ff178cdSJimmy Vetayases const int vect_inherent_hwpri = irqptr->airq_vector >>
6197ff178cdSJimmy Vetayases APIC_IPL_SHIFT;
6207ff178cdSJimmy Vetayases
6217ff178cdSJimmy Vetayases /*
6227ff178cdSJimmy Vetayases * If there are still devices using this IRQ, determine the
6237ff178cdSJimmy Vetayases * new ipl to use.
6247ff178cdSJimmy Vetayases */
6257ff178cdSJimmy Vetayases if (irqptr->airq_share) {
6267ff178cdSJimmy Vetayases int vect_desired_hwpri, hwpri;
6277ff178cdSJimmy Vetayases
6287ff178cdSJimmy Vetayases ASSERT(max_ipl < MAXIPL);
6297ff178cdSJimmy Vetayases vect_desired_hwpri = apic_ipltopri[max_ipl] >>
6307ff178cdSJimmy Vetayases APIC_IPL_SHIFT;
6317ff178cdSJimmy Vetayases
6327ff178cdSJimmy Vetayases /*
6337ff178cdSJimmy Vetayases * If the desired IPL's hardware priority is lower
6347ff178cdSJimmy Vetayases * than that of the vector, use the hardware priority
6357ff178cdSJimmy Vetayases * of the vector to determine the new IPL.
6367ff178cdSJimmy Vetayases */
6377ff178cdSJimmy Vetayases hwpri = (vect_desired_hwpri < vect_inherent_hwpri) ?
6387ff178cdSJimmy Vetayases vect_inherent_hwpri : vect_desired_hwpri;
6397ff178cdSJimmy Vetayases
6407ff178cdSJimmy Vetayases /*
6417ff178cdSJimmy Vetayases * Now, to get the right index for apic_vectortoipl,
6427ff178cdSJimmy Vetayases * we need to subtract APIC_BASE_VECT from the
6437ff178cdSJimmy Vetayases * hardware-vector-equivalent (in hwpri). Since hwpri
6447ff178cdSJimmy Vetayases * is already shifted, we shift APIC_BASE_VECT before
6457ff178cdSJimmy Vetayases * doing the subtraction.
6467ff178cdSJimmy Vetayases */
6477ff178cdSJimmy Vetayases hwpri -= (APIC_BASE_VECT >> APIC_IPL_SHIFT);
6487ff178cdSJimmy Vetayases
6497ff178cdSJimmy Vetayases ASSERT(hwpri >= 0);
6507ff178cdSJimmy Vetayases ASSERT(hwpri < MAXIPL);
6517ff178cdSJimmy Vetayases max_ipl = apic_vectortoipl[hwpri];
65230acb30dSHans Rosenfeld apic_ipls[apic_ipls_index] = (uchar_t)max_ipl;
6537ff178cdSJimmy Vetayases
6547ff178cdSJimmy Vetayases irqp = irqheadptr;
6557ff178cdSJimmy Vetayases while (irqp) {
6567ff178cdSJimmy Vetayases irqp->airq_ipl = (uchar_t)max_ipl;
6577ff178cdSJimmy Vetayases irqp = irqp->airq_next;
6587ff178cdSJimmy Vetayases }
6597ff178cdSJimmy Vetayases } else {
6607ff178cdSJimmy Vetayases /*
6617ff178cdSJimmy Vetayases * No more devices on this IRQ, so reset this vector's
6627ff178cdSJimmy Vetayases * element in apic_ipls to the original IPL for this
6637ff178cdSJimmy Vetayases * vector
6647ff178cdSJimmy Vetayases */
6657ff178cdSJimmy Vetayases apic_ipls[apic_ipls_index] =
6667ff178cdSJimmy Vetayases apic_vectortoipl[vect_inherent_hwpri];
6677ff178cdSJimmy Vetayases }
6687ff178cdSJimmy Vetayases }
6697ff178cdSJimmy Vetayases
6707ff178cdSJimmy Vetayases /*
6717ff178cdSJimmy Vetayases * If there are still active interrupts, we are done.
6727ff178cdSJimmy Vetayases */
6737ff178cdSJimmy Vetayases if (irqptr->airq_share)
6747ff178cdSJimmy Vetayases return (PSM_SUCCESS);
6757ff178cdSJimmy Vetayases
6767ff178cdSJimmy Vetayases iflag = intr_clear();
6777ff178cdSJimmy Vetayases lock_set(&apic_ioapic_lock);
6787ff178cdSJimmy Vetayases
6797ff178cdSJimmy Vetayases if (irqptr->airq_mps_intr_index == MSI_INDEX) {
6807ff178cdSJimmy Vetayases /*
6817ff178cdSJimmy Vetayases * Disable the MSI vector
6827ff178cdSJimmy Vetayases * Make sure we only disable on the last
6837ff178cdSJimmy Vetayases * of the multi-MSI support
6847ff178cdSJimmy Vetayases */
6857ff178cdSJimmy Vetayases if (i_ddi_intr_get_current_nenables(irqptr->airq_dip) == 1) {
6867ff178cdSJimmy Vetayases apic_pci_msi_disable_mode(irqptr->airq_dip,
6877ff178cdSJimmy Vetayases DDI_INTR_TYPE_MSI);
6887ff178cdSJimmy Vetayases }
6897ff178cdSJimmy Vetayases } else if (irqptr->airq_mps_intr_index == MSIX_INDEX) {
6907ff178cdSJimmy Vetayases /*
6917ff178cdSJimmy Vetayases * Disable the MSI-X vector
6927ff178cdSJimmy Vetayases * needs to clear its mask and addr/data for each MSI-X
6937ff178cdSJimmy Vetayases */
6947ff178cdSJimmy Vetayases apic_pci_msi_unconfigure(irqptr->airq_dip, DDI_INTR_TYPE_MSIX,
6957ff178cdSJimmy Vetayases irqptr->airq_origirq);
6967ff178cdSJimmy Vetayases /*
6977ff178cdSJimmy Vetayases * Make sure we only disable on the last MSI-X
6987ff178cdSJimmy Vetayases */
6997ff178cdSJimmy Vetayases if (i_ddi_intr_get_current_nenables(irqptr->airq_dip) == 1) {
7007ff178cdSJimmy Vetayases apic_pci_msi_disable_mode(irqptr->airq_dip,
7017ff178cdSJimmy Vetayases DDI_INTR_TYPE_MSIX);
7027ff178cdSJimmy Vetayases }
7037ff178cdSJimmy Vetayases } else {
7047ff178cdSJimmy Vetayases /*
7057ff178cdSJimmy Vetayases * The assumption here is that this is safe, even for
7067ff178cdSJimmy Vetayases * systems with IOAPICs that suffer from the hardware
7077ff178cdSJimmy Vetayases * erratum because all devices have been quiesced before
7087ff178cdSJimmy Vetayases * they unregister their interrupt handlers. If that
7097ff178cdSJimmy Vetayases * assumption turns out to be false, this mask operation
7107ff178cdSJimmy Vetayases * can induce the same erratum result we're trying to
7117ff178cdSJimmy Vetayases * avoid.
7127ff178cdSJimmy Vetayases */
7137ff178cdSJimmy Vetayases ioapic_ix = irqptr->airq_ioapicindex;
7147ff178cdSJimmy Vetayases intin = irqptr->airq_intin_no;
7157ff178cdSJimmy Vetayases ioapic_write(ioapic_ix, APIC_RDT_CMD + 2 * intin, AV_MASK);
7167ff178cdSJimmy Vetayases }
7177ff178cdSJimmy Vetayases
7187ff178cdSJimmy Vetayases apic_vt_ops->apic_intrmap_free_entry(&irqptr->airq_intrmap_private);
7197ff178cdSJimmy Vetayases
7207ff178cdSJimmy Vetayases /*
7217ff178cdSJimmy Vetayases * This irq entry is the only one in the chain.
7227ff178cdSJimmy Vetayases */
7237ff178cdSJimmy Vetayases if (irqheadptr->airq_next == NULL) {
7247ff178cdSJimmy Vetayases ASSERT(irqheadptr == irqptr);
7257ff178cdSJimmy Vetayases bind_cpu = irqptr->airq_temp_cpu;
7267ff178cdSJimmy Vetayases if (((uint32_t)bind_cpu != IRQ_UNBOUND) &&
7277ff178cdSJimmy Vetayases ((uint32_t)bind_cpu != IRQ_UNINIT)) {
7287ff178cdSJimmy Vetayases ASSERT(apic_cpu_in_range(bind_cpu));
7297ff178cdSJimmy Vetayases if (bind_cpu & IRQ_USER_BOUND) {
7307ff178cdSJimmy Vetayases /* If hardbound, temp_cpu == cpu */
7317ff178cdSJimmy Vetayases bind_cpu &= ~IRQ_USER_BOUND;
7327ff178cdSJimmy Vetayases apic_cpus[bind_cpu].aci_bound--;
7337ff178cdSJimmy Vetayases } else
7347ff178cdSJimmy Vetayases apic_cpus[bind_cpu].aci_temp_bound--;
7357ff178cdSJimmy Vetayases }
7367ff178cdSJimmy Vetayases irqptr->airq_temp_cpu = IRQ_UNINIT;
7377ff178cdSJimmy Vetayases irqptr->airq_mps_intr_index = FREE_INDEX;
7387ff178cdSJimmy Vetayases lock_clear(&apic_ioapic_lock);
7397ff178cdSJimmy Vetayases intr_restore(iflag);
7407ff178cdSJimmy Vetayases apic_free_vector(irqptr->airq_vector);
7417ff178cdSJimmy Vetayases return (PSM_SUCCESS);
7427ff178cdSJimmy Vetayases }
7437ff178cdSJimmy Vetayases
7447ff178cdSJimmy Vetayases /*
7457ff178cdSJimmy Vetayases * If we get here, we are sharing the vector and there are more than
7467ff178cdSJimmy Vetayases * one active irq entries in the chain.
7477ff178cdSJimmy Vetayases */
7487ff178cdSJimmy Vetayases lock_clear(&apic_ioapic_lock);
7497ff178cdSJimmy Vetayases intr_restore(iflag);
7507ff178cdSJimmy Vetayases
7517ff178cdSJimmy Vetayases mutex_enter(&airq_mutex);
7527ff178cdSJimmy Vetayases /* Remove the irq entry from the chain */
7537ff178cdSJimmy Vetayases if (irqptr == irqheadptr) { /* The irq entry is at the head */
7547ff178cdSJimmy Vetayases apic_irq_table[irqindex] = irqptr->airq_next;
7557ff178cdSJimmy Vetayases } else {
7567ff178cdSJimmy Vetayases preirqptr->airq_next = irqptr->airq_next;
7577ff178cdSJimmy Vetayases }
7587ff178cdSJimmy Vetayases /* Free the irq entry */
7597ff178cdSJimmy Vetayases kmem_free(irqptr, sizeof (apic_irq_t));
7607ff178cdSJimmy Vetayases mutex_exit(&airq_mutex);
7617ff178cdSJimmy Vetayases
7627ff178cdSJimmy Vetayases return (PSM_SUCCESS);
7637ff178cdSJimmy Vetayases }
7647ff178cdSJimmy Vetayases
7657ff178cdSJimmy Vetayases /*
7667ff178cdSJimmy Vetayases * apic_introp_xlate() replaces apic_translate_irq() and is
7677ff178cdSJimmy Vetayases * called only from apic_intr_ops(). With the new ADII framework,
7687ff178cdSJimmy Vetayases * the priority can no longer be retrieved through i_ddi_get_intrspec().
7697ff178cdSJimmy Vetayases * It has to be passed in from the caller.
7707ff178cdSJimmy Vetayases *
7717ff178cdSJimmy Vetayases * Return value:
7727ff178cdSJimmy Vetayases * Success: irqno for the given device
7737ff178cdSJimmy Vetayases * Failure: -1
7747ff178cdSJimmy Vetayases */
7757ff178cdSJimmy Vetayases int
apic_introp_xlate(dev_info_t * dip,struct intrspec * ispec,int type)7767ff178cdSJimmy Vetayases apic_introp_xlate(dev_info_t *dip, struct intrspec *ispec, int type)
7777ff178cdSJimmy Vetayases {
7787ff178cdSJimmy Vetayases char dev_type[16];
7797ff178cdSJimmy Vetayases int dev_len, pci_irq, newirq, bustype, devid, busid, i;
7807ff178cdSJimmy Vetayases int irqno = ispec->intrspec_vec;
7817ff178cdSJimmy Vetayases ddi_acc_handle_t cfg_handle;
7827ff178cdSJimmy Vetayases uchar_t ipin;
7837ff178cdSJimmy Vetayases struct apic_io_intr *intrp;
7847ff178cdSJimmy Vetayases iflag_t intr_flag;
7857ff178cdSJimmy Vetayases ACPI_SUBTABLE_HEADER *hp;
7867ff178cdSJimmy Vetayases ACPI_MADT_INTERRUPT_OVERRIDE *isop;
7877ff178cdSJimmy Vetayases apic_irq_t *airqp;
7887ff178cdSJimmy Vetayases int parent_is_pci_or_pciex = 0;
7897ff178cdSJimmy Vetayases int child_is_pciex = 0;
7907ff178cdSJimmy Vetayases
7917ff178cdSJimmy Vetayases DDI_INTR_IMPLDBG((CE_CONT, "apic_introp_xlate: dip=0x%p name=%s "
7927ff178cdSJimmy Vetayases "type=%d irqno=0x%x\n", (void *)dip, ddi_get_name(dip), type,
7937ff178cdSJimmy Vetayases irqno));
7947ff178cdSJimmy Vetayases
7957ff178cdSJimmy Vetayases dev_len = sizeof (dev_type);
7967ff178cdSJimmy Vetayases if (ddi_getlongprop_buf(DDI_DEV_T_ANY, ddi_get_parent(dip),
7977ff178cdSJimmy Vetayases DDI_PROP_DONTPASS, "device_type", (caddr_t)dev_type,
7987ff178cdSJimmy Vetayases &dev_len) == DDI_PROP_SUCCESS) {
7997ff178cdSJimmy Vetayases if ((strcmp(dev_type, "pci") == 0) ||
8007ff178cdSJimmy Vetayases (strcmp(dev_type, "pciex") == 0))
8017ff178cdSJimmy Vetayases parent_is_pci_or_pciex = 1;
8027ff178cdSJimmy Vetayases }
8037ff178cdSJimmy Vetayases
8047ff178cdSJimmy Vetayases if (ddi_getlongprop_buf(DDI_DEV_T_ANY, dip,
8057ff178cdSJimmy Vetayases DDI_PROP_DONTPASS, "compatible", (caddr_t)dev_type,
8067ff178cdSJimmy Vetayases &dev_len) == DDI_PROP_SUCCESS) {
8077ff178cdSJimmy Vetayases if (strstr(dev_type, "pciex"))
8087ff178cdSJimmy Vetayases child_is_pciex = 1;
8097ff178cdSJimmy Vetayases }
8107ff178cdSJimmy Vetayases
8117ff178cdSJimmy Vetayases if (DDI_INTR_IS_MSI_OR_MSIX(type)) {
8127ff178cdSJimmy Vetayases if ((airqp = apic_find_irq(dip, ispec, type)) != NULL) {
8137ff178cdSJimmy Vetayases airqp->airq_iflag.bustype =
8147ff178cdSJimmy Vetayases child_is_pciex ? BUS_PCIE : BUS_PCI;
8157ff178cdSJimmy Vetayases return (apic_vector_to_irq[airqp->airq_vector]);
8167ff178cdSJimmy Vetayases }
8177ff178cdSJimmy Vetayases return (apic_setup_irq_table(dip, irqno, NULL, ispec,
8187ff178cdSJimmy Vetayases NULL, type));
8197ff178cdSJimmy Vetayases }
8207ff178cdSJimmy Vetayases
8217ff178cdSJimmy Vetayases bustype = 0;
8227ff178cdSJimmy Vetayases
8237ff178cdSJimmy Vetayases /* check if we have already translated this irq */
8247ff178cdSJimmy Vetayases mutex_enter(&airq_mutex);
8257ff178cdSJimmy Vetayases newirq = apic_min_device_irq;
8267ff178cdSJimmy Vetayases for (; newirq <= apic_max_device_irq; newirq++) {
8277ff178cdSJimmy Vetayases airqp = apic_irq_table[newirq];
8287ff178cdSJimmy Vetayases while (airqp) {
8297ff178cdSJimmy Vetayases if ((airqp->airq_dip == dip) &&
8307ff178cdSJimmy Vetayases (airqp->airq_origirq == irqno) &&
8317ff178cdSJimmy Vetayases (airqp->airq_mps_intr_index != FREE_INDEX)) {
8327ff178cdSJimmy Vetayases
8337ff178cdSJimmy Vetayases mutex_exit(&airq_mutex);
8347ff178cdSJimmy Vetayases return (VIRTIRQ(newirq, airqp->airq_share_id));
8357ff178cdSJimmy Vetayases }
8367ff178cdSJimmy Vetayases airqp = airqp->airq_next;
8377ff178cdSJimmy Vetayases }
8387ff178cdSJimmy Vetayases }
8397ff178cdSJimmy Vetayases mutex_exit(&airq_mutex);
8407ff178cdSJimmy Vetayases
8417ff178cdSJimmy Vetayases if (apic_defconf)
8427ff178cdSJimmy Vetayases goto defconf;
8437ff178cdSJimmy Vetayases
8447ff178cdSJimmy Vetayases if ((dip == NULL) || (!apic_irq_translate && !apic_enable_acpi))
8457ff178cdSJimmy Vetayases goto nonpci;
8467ff178cdSJimmy Vetayases
8477ff178cdSJimmy Vetayases if (parent_is_pci_or_pciex) {
8487ff178cdSJimmy Vetayases /* pci device */
8497ff178cdSJimmy Vetayases if (acpica_get_bdf(dip, &busid, &devid, NULL) != 0)
8507ff178cdSJimmy Vetayases goto nonpci;
8517ff178cdSJimmy Vetayases if (busid == 0 && apic_pci_bus_total == 1)
8527ff178cdSJimmy Vetayases busid = (int)apic_single_pci_busid;
8537ff178cdSJimmy Vetayases
8547ff178cdSJimmy Vetayases if (pci_config_setup(dip, &cfg_handle) != DDI_SUCCESS)
8557ff178cdSJimmy Vetayases return (-1);
8567ff178cdSJimmy Vetayases ipin = pci_config_get8(cfg_handle, PCI_CONF_IPIN) - PCI_INTA;
8577ff178cdSJimmy Vetayases pci_config_teardown(&cfg_handle);
8587ff178cdSJimmy Vetayases if (apic_enable_acpi && !apic_use_acpi_madt_only) {
8597ff178cdSJimmy Vetayases if (apic_acpi_translate_pci_irq(dip, busid, devid,
8607ff178cdSJimmy Vetayases ipin, &pci_irq, &intr_flag) != ACPI_PSM_SUCCESS)
8617ff178cdSJimmy Vetayases return (-1);
8627ff178cdSJimmy Vetayases
8637ff178cdSJimmy Vetayases intr_flag.bustype = child_is_pciex ? BUS_PCIE : BUS_PCI;
8647ff178cdSJimmy Vetayases return (apic_setup_irq_table(dip, pci_irq, NULL, ispec,
8657ff178cdSJimmy Vetayases &intr_flag, type));
8667ff178cdSJimmy Vetayases } else {
8677ff178cdSJimmy Vetayases pci_irq = ((devid & 0x1f) << 2) | (ipin & 0x3);
8687ff178cdSJimmy Vetayases if ((intrp = apic_find_io_intr_w_busid(pci_irq, busid))
8697ff178cdSJimmy Vetayases == NULL) {
8707ff178cdSJimmy Vetayases if ((pci_irq = apic_handle_pci_pci_bridge(dip,
8717ff178cdSJimmy Vetayases devid, ipin, &intrp)) == -1)
8727ff178cdSJimmy Vetayases return (-1);
8737ff178cdSJimmy Vetayases }
8747ff178cdSJimmy Vetayases return (apic_setup_irq_table(dip, pci_irq, intrp, ispec,
8757ff178cdSJimmy Vetayases NULL, type));
8767ff178cdSJimmy Vetayases }
8777ff178cdSJimmy Vetayases } else if (strcmp(dev_type, "isa") == 0)
8787ff178cdSJimmy Vetayases bustype = BUS_ISA;
8797ff178cdSJimmy Vetayases else if (strcmp(dev_type, "eisa") == 0)
8807ff178cdSJimmy Vetayases bustype = BUS_EISA;
8817ff178cdSJimmy Vetayases
8827ff178cdSJimmy Vetayases nonpci:
8837ff178cdSJimmy Vetayases if (apic_enable_acpi && !apic_use_acpi_madt_only) {
8847ff178cdSJimmy Vetayases /* search iso entries first */
8857ff178cdSJimmy Vetayases if (acpi_iso_cnt != 0) {
8867ff178cdSJimmy Vetayases hp = (ACPI_SUBTABLE_HEADER *)acpi_isop;
8877ff178cdSJimmy Vetayases i = 0;
8887ff178cdSJimmy Vetayases while (i < acpi_iso_cnt) {
8897ff178cdSJimmy Vetayases if (hp->Type ==
8907ff178cdSJimmy Vetayases ACPI_MADT_TYPE_INTERRUPT_OVERRIDE) {
8917ff178cdSJimmy Vetayases isop =
8927ff178cdSJimmy Vetayases (ACPI_MADT_INTERRUPT_OVERRIDE *) hp;
8937ff178cdSJimmy Vetayases if (isop->Bus == 0 &&
8947ff178cdSJimmy Vetayases isop->SourceIrq == irqno) {
8957ff178cdSJimmy Vetayases newirq = isop->GlobalIrq;
8967ff178cdSJimmy Vetayases intr_flag.intr_po =
8977ff178cdSJimmy Vetayases isop->IntiFlags &
8987ff178cdSJimmy Vetayases ACPI_MADT_POLARITY_MASK;
8997ff178cdSJimmy Vetayases intr_flag.intr_el =
9007ff178cdSJimmy Vetayases (isop->IntiFlags &
9017ff178cdSJimmy Vetayases ACPI_MADT_TRIGGER_MASK)
9027ff178cdSJimmy Vetayases >> 2;
9037ff178cdSJimmy Vetayases intr_flag.bustype = BUS_ISA;
9047ff178cdSJimmy Vetayases
9057ff178cdSJimmy Vetayases return (apic_setup_irq_table(
9067ff178cdSJimmy Vetayases dip, newirq, NULL, ispec,
9077ff178cdSJimmy Vetayases &intr_flag, type));
9087ff178cdSJimmy Vetayases
9097ff178cdSJimmy Vetayases }
9107ff178cdSJimmy Vetayases i++;
9117ff178cdSJimmy Vetayases }
9127ff178cdSJimmy Vetayases hp = (ACPI_SUBTABLE_HEADER *)(((char *)hp) +
9137ff178cdSJimmy Vetayases hp->Length);
9147ff178cdSJimmy Vetayases }
9157ff178cdSJimmy Vetayases }
9167ff178cdSJimmy Vetayases intr_flag.intr_po = INTR_PO_ACTIVE_HIGH;
9177ff178cdSJimmy Vetayases intr_flag.intr_el = INTR_EL_EDGE;
9187ff178cdSJimmy Vetayases intr_flag.bustype = BUS_ISA;
9197ff178cdSJimmy Vetayases return (apic_setup_irq_table(dip, irqno, NULL, ispec,
9207ff178cdSJimmy Vetayases &intr_flag, type));
9217ff178cdSJimmy Vetayases } else {
9227ff178cdSJimmy Vetayases if (bustype == 0) /* not initialized */
9237ff178cdSJimmy Vetayases bustype = eisa_level_intr_mask ? BUS_EISA : BUS_ISA;
9247ff178cdSJimmy Vetayases for (i = 0; i < 2; i++) {
9257ff178cdSJimmy Vetayases if (((busid = apic_find_bus_id(bustype)) != -1) &&
9267ff178cdSJimmy Vetayases ((intrp = apic_find_io_intr_w_busid(irqno, busid))
9277ff178cdSJimmy Vetayases != NULL)) {
9287ff178cdSJimmy Vetayases if ((newirq = apic_setup_irq_table(dip, irqno,
9297ff178cdSJimmy Vetayases intrp, ispec, NULL, type)) != -1) {
9307ff178cdSJimmy Vetayases return (newirq);
9317ff178cdSJimmy Vetayases }
9327ff178cdSJimmy Vetayases goto defconf;
9337ff178cdSJimmy Vetayases }
9347ff178cdSJimmy Vetayases bustype = (bustype == BUS_EISA) ? BUS_ISA : BUS_EISA;
9357ff178cdSJimmy Vetayases }
9367ff178cdSJimmy Vetayases }
9377ff178cdSJimmy Vetayases
9387ff178cdSJimmy Vetayases /* MPS default configuration */
9397ff178cdSJimmy Vetayases defconf:
9407ff178cdSJimmy Vetayases newirq = apic_setup_irq_table(dip, irqno, NULL, ispec, NULL, type);
9417ff178cdSJimmy Vetayases if (newirq == -1)
9427ff178cdSJimmy Vetayases return (-1);
9437ff178cdSJimmy Vetayases ASSERT(IRQINDEX(newirq) == irqno);
9447ff178cdSJimmy Vetayases ASSERT(apic_irq_table[irqno]);
9457ff178cdSJimmy Vetayases return (newirq);
9467ff178cdSJimmy Vetayases }
9477ff178cdSJimmy Vetayases
9487ff178cdSJimmy Vetayases /*
9497ff178cdSJimmy Vetayases * Attempt to share vector with someone else
9507ff178cdSJimmy Vetayases */
9517ff178cdSJimmy Vetayases static int
apic_share_vector(int irqno,iflag_t * intr_flagp,short intr_index,int ipl,uchar_t ioapicindex,uchar_t ipin,apic_irq_t ** irqptrp)9527ff178cdSJimmy Vetayases apic_share_vector(int irqno, iflag_t *intr_flagp, short intr_index, int ipl,
95330acb30dSHans Rosenfeld uchar_t ioapicindex, uchar_t ipin, apic_irq_t **irqptrp)
9547ff178cdSJimmy Vetayases {
9557ff178cdSJimmy Vetayases #ifdef DEBUG
9567ff178cdSJimmy Vetayases apic_irq_t *tmpirqp = NULL;
9577ff178cdSJimmy Vetayases #endif /* DEBUG */
9587ff178cdSJimmy Vetayases apic_irq_t *irqptr, dummyirq;
9597ff178cdSJimmy Vetayases int newirq, chosen_irq = -1, share = 127;
9607ff178cdSJimmy Vetayases int lowest, highest, i;
9617ff178cdSJimmy Vetayases uchar_t share_id;
9627ff178cdSJimmy Vetayases
9637ff178cdSJimmy Vetayases DDI_INTR_IMPLDBG((CE_CONT, "apic_share_vector: irqno=0x%x "
9647ff178cdSJimmy Vetayases "intr_index=0x%x ipl=0x%x\n", irqno, intr_index, ipl));
9657ff178cdSJimmy Vetayases
9667ff178cdSJimmy Vetayases highest = apic_ipltopri[ipl] + APIC_VECTOR_MASK;
9677ff178cdSJimmy Vetayases lowest = apic_ipltopri[ipl-1] + APIC_VECTOR_PER_IPL;
9687ff178cdSJimmy Vetayases
9697ff178cdSJimmy Vetayases if (highest < lowest) /* Both ipl and ipl-1 map to same pri */
9707ff178cdSJimmy Vetayases lowest -= APIC_VECTOR_PER_IPL;
9717ff178cdSJimmy Vetayases dummyirq.airq_mps_intr_index = intr_index;
9727ff178cdSJimmy Vetayases dummyirq.airq_ioapicindex = ioapicindex;
9737ff178cdSJimmy Vetayases dummyirq.airq_intin_no = ipin;
9747ff178cdSJimmy Vetayases if (intr_flagp)
9757ff178cdSJimmy Vetayases dummyirq.airq_iflag = *intr_flagp;
9767ff178cdSJimmy Vetayases apic_record_rdt_entry(&dummyirq, irqno);
9777ff178cdSJimmy Vetayases for (i = lowest; i <= highest; i++) {
9787ff178cdSJimmy Vetayases newirq = apic_vector_to_irq[i];
9797ff178cdSJimmy Vetayases if (newirq == APIC_RESV_IRQ)
9807ff178cdSJimmy Vetayases continue;
9817ff178cdSJimmy Vetayases irqptr = apic_irq_table[newirq];
9827ff178cdSJimmy Vetayases
9837ff178cdSJimmy Vetayases if ((dummyirq.airq_rdt_entry & 0xFF00) !=
9847ff178cdSJimmy Vetayases (irqptr->airq_rdt_entry & 0xFF00))
9857ff178cdSJimmy Vetayases /* not compatible */
9867ff178cdSJimmy Vetayases continue;
9877ff178cdSJimmy Vetayases
9887ff178cdSJimmy Vetayases if (irqptr->airq_share < share) {
9897ff178cdSJimmy Vetayases share = irqptr->airq_share;
9907ff178cdSJimmy Vetayases chosen_irq = newirq;
9917ff178cdSJimmy Vetayases }
9927ff178cdSJimmy Vetayases }
9937ff178cdSJimmy Vetayases if (chosen_irq != -1) {
9947ff178cdSJimmy Vetayases /*
9957ff178cdSJimmy Vetayases * Assign a share id which is free or which is larger
9967ff178cdSJimmy Vetayases * than the largest one.
9977ff178cdSJimmy Vetayases */
9987ff178cdSJimmy Vetayases share_id = 1;
9997ff178cdSJimmy Vetayases mutex_enter(&airq_mutex);
10007ff178cdSJimmy Vetayases irqptr = apic_irq_table[chosen_irq];
10017ff178cdSJimmy Vetayases while (irqptr) {
10027ff178cdSJimmy Vetayases if (irqptr->airq_mps_intr_index == FREE_INDEX) {
10037ff178cdSJimmy Vetayases share_id = irqptr->airq_share_id;
10047ff178cdSJimmy Vetayases break;
10057ff178cdSJimmy Vetayases }
10067ff178cdSJimmy Vetayases if (share_id <= irqptr->airq_share_id)
10077ff178cdSJimmy Vetayases share_id = irqptr->airq_share_id + 1;
10087ff178cdSJimmy Vetayases #ifdef DEBUG
10097ff178cdSJimmy Vetayases tmpirqp = irqptr;
10107ff178cdSJimmy Vetayases #endif /* DEBUG */
10117ff178cdSJimmy Vetayases irqptr = irqptr->airq_next;
10127ff178cdSJimmy Vetayases }
10137ff178cdSJimmy Vetayases if (!irqptr) {
10147ff178cdSJimmy Vetayases irqptr = kmem_zalloc(sizeof (apic_irq_t), KM_SLEEP);
10157ff178cdSJimmy Vetayases irqptr->airq_temp_cpu = IRQ_UNINIT;
10167ff178cdSJimmy Vetayases irqptr->airq_next =
10177ff178cdSJimmy Vetayases apic_irq_table[chosen_irq]->airq_next;
10187ff178cdSJimmy Vetayases apic_irq_table[chosen_irq]->airq_next = irqptr;
10197ff178cdSJimmy Vetayases #ifdef DEBUG
10207ff178cdSJimmy Vetayases tmpirqp = apic_irq_table[chosen_irq];
10217ff178cdSJimmy Vetayases #endif /* DEBUG */
10227ff178cdSJimmy Vetayases }
10237ff178cdSJimmy Vetayases irqptr->airq_mps_intr_index = intr_index;
10247ff178cdSJimmy Vetayases irqptr->airq_ioapicindex = ioapicindex;
10257ff178cdSJimmy Vetayases irqptr->airq_intin_no = ipin;
10267ff178cdSJimmy Vetayases if (intr_flagp)
10277ff178cdSJimmy Vetayases irqptr->airq_iflag = *intr_flagp;
10287ff178cdSJimmy Vetayases irqptr->airq_vector = apic_irq_table[chosen_irq]->airq_vector;
10297ff178cdSJimmy Vetayases irqptr->airq_share_id = share_id;
10307ff178cdSJimmy Vetayases apic_record_rdt_entry(irqptr, irqno);
10317ff178cdSJimmy Vetayases *irqptrp = irqptr;
10327ff178cdSJimmy Vetayases #ifdef DEBUG
10337ff178cdSJimmy Vetayases /* shuffle the pointers to test apic_delspl path */
10347ff178cdSJimmy Vetayases if (tmpirqp) {
10357ff178cdSJimmy Vetayases tmpirqp->airq_next = irqptr->airq_next;
10367ff178cdSJimmy Vetayases irqptr->airq_next = apic_irq_table[chosen_irq];
10377ff178cdSJimmy Vetayases apic_irq_table[chosen_irq] = irqptr;
10387ff178cdSJimmy Vetayases }
10397ff178cdSJimmy Vetayases #endif /* DEBUG */
10407ff178cdSJimmy Vetayases mutex_exit(&airq_mutex);
10417ff178cdSJimmy Vetayases return (VIRTIRQ(chosen_irq, share_id));
10427ff178cdSJimmy Vetayases }
10437ff178cdSJimmy Vetayases return (-1);
10447ff178cdSJimmy Vetayases }
10457ff178cdSJimmy Vetayases
10467ff178cdSJimmy Vetayases /*
10477ff178cdSJimmy Vetayases * Allocate/Initialize the apic_irq_table[] entry for given irqno. If the entry
10487ff178cdSJimmy Vetayases * is used already, we will try to allocate a new irqno.
10497ff178cdSJimmy Vetayases *
10507ff178cdSJimmy Vetayases * Return value:
10517ff178cdSJimmy Vetayases * Success: irqno
10527ff178cdSJimmy Vetayases * Failure: -1
10537ff178cdSJimmy Vetayases */
10547ff178cdSJimmy Vetayases static int
apic_setup_irq_table(dev_info_t * dip,int irqno,struct apic_io_intr * intrp,struct intrspec * ispec,iflag_t * intr_flagp,int type)10557ff178cdSJimmy Vetayases apic_setup_irq_table(dev_info_t *dip, int irqno, struct apic_io_intr *intrp,
10567ff178cdSJimmy Vetayases struct intrspec *ispec, iflag_t *intr_flagp, int type)
10577ff178cdSJimmy Vetayases {
105830acb30dSHans Rosenfeld int origirq;
105930acb30dSHans Rosenfeld uchar_t ipl;
10607ff178cdSJimmy Vetayases int newirq, intr_index;
10617ff178cdSJimmy Vetayases uchar_t ipin, ioapic, ioapicindex, vector;
10627ff178cdSJimmy Vetayases apic_irq_t *irqptr;
10637ff178cdSJimmy Vetayases major_t major;
10647ff178cdSJimmy Vetayases dev_info_t *sdip;
10657ff178cdSJimmy Vetayases
106630acb30dSHans Rosenfeld ASSERT(ispec != NULL);
106730acb30dSHans Rosenfeld
106830acb30dSHans Rosenfeld origirq = ispec->intrspec_vec;
106930acb30dSHans Rosenfeld ipl = ispec->intrspec_pri;
107030acb30dSHans Rosenfeld
10717ff178cdSJimmy Vetayases DDI_INTR_IMPLDBG((CE_CONT, "apic_setup_irq_table: dip=0x%p type=%d "
10727ff178cdSJimmy Vetayases "irqno=0x%x origirq=0x%x\n", (void *)dip, type, irqno, origirq));
10737ff178cdSJimmy Vetayases
10747ff178cdSJimmy Vetayases major = (dip != NULL) ? ddi_driver_major(dip) : 0;
10757ff178cdSJimmy Vetayases
10767ff178cdSJimmy Vetayases if (DDI_INTR_IS_MSI_OR_MSIX(type)) {
10777ff178cdSJimmy Vetayases /* MSI/X doesn't need to setup ioapic stuffs */
10787ff178cdSJimmy Vetayases ioapicindex = 0xff;
10797ff178cdSJimmy Vetayases ioapic = 0xff;
10807ff178cdSJimmy Vetayases ipin = (uchar_t)0xff;
10817ff178cdSJimmy Vetayases intr_index = (type == DDI_INTR_TYPE_MSI) ? MSI_INDEX :
10827ff178cdSJimmy Vetayases MSIX_INDEX;
10837ff178cdSJimmy Vetayases mutex_enter(&airq_mutex);
10847ff178cdSJimmy Vetayases if ((irqno = apic_allocate_irq(apic_first_avail_irq)) == -1) {
10857ff178cdSJimmy Vetayases mutex_exit(&airq_mutex);
10867ff178cdSJimmy Vetayases /* need an irq for MSI/X to index into autovect[] */
10877ff178cdSJimmy Vetayases cmn_err(CE_WARN, "No interrupt irq: %s instance %d",
10887ff178cdSJimmy Vetayases ddi_get_name(dip), ddi_get_instance(dip));
10897ff178cdSJimmy Vetayases return (-1);
10907ff178cdSJimmy Vetayases }
10917ff178cdSJimmy Vetayases mutex_exit(&airq_mutex);
10927ff178cdSJimmy Vetayases
10937ff178cdSJimmy Vetayases } else if (intrp != NULL) {
10947ff178cdSJimmy Vetayases intr_index = (int)(intrp - apic_io_intrp);
10957ff178cdSJimmy Vetayases ioapic = intrp->intr_destid;
10967ff178cdSJimmy Vetayases ipin = intrp->intr_destintin;
10977ff178cdSJimmy Vetayases /* Find ioapicindex. If destid was ALL, we will exit with 0. */
10987ff178cdSJimmy Vetayases for (ioapicindex = apic_io_max - 1; ioapicindex; ioapicindex--)
10997ff178cdSJimmy Vetayases if (apic_io_id[ioapicindex] == ioapic)
11007ff178cdSJimmy Vetayases break;
11017ff178cdSJimmy Vetayases ASSERT((ioapic == apic_io_id[ioapicindex]) ||
11027ff178cdSJimmy Vetayases (ioapic == INTR_ALL_APIC));
11037ff178cdSJimmy Vetayases
11047ff178cdSJimmy Vetayases /* check whether this intin# has been used by another irqno */
11057ff178cdSJimmy Vetayases if ((newirq = apic_find_intin(ioapicindex, ipin)) != -1) {
11067ff178cdSJimmy Vetayases return (newirq);
11077ff178cdSJimmy Vetayases }
11087ff178cdSJimmy Vetayases
11097ff178cdSJimmy Vetayases } else if (intr_flagp != NULL) {
11107ff178cdSJimmy Vetayases /* ACPI case */
11117ff178cdSJimmy Vetayases intr_index = ACPI_INDEX;
11127ff178cdSJimmy Vetayases ioapicindex = acpi_find_ioapic(irqno);
11137ff178cdSJimmy Vetayases ASSERT(ioapicindex != 0xFF);
11147ff178cdSJimmy Vetayases ioapic = apic_io_id[ioapicindex];
11157ff178cdSJimmy Vetayases ipin = irqno - apic_io_vectbase[ioapicindex];
11167ff178cdSJimmy Vetayases if (apic_irq_table[irqno] &&
11177ff178cdSJimmy Vetayases apic_irq_table[irqno]->airq_mps_intr_index == ACPI_INDEX) {
11187ff178cdSJimmy Vetayases ASSERT(apic_irq_table[irqno]->airq_intin_no == ipin &&
11197ff178cdSJimmy Vetayases apic_irq_table[irqno]->airq_ioapicindex ==
11207ff178cdSJimmy Vetayases ioapicindex);
11217ff178cdSJimmy Vetayases return (irqno);
11227ff178cdSJimmy Vetayases }
11237ff178cdSJimmy Vetayases
11247ff178cdSJimmy Vetayases } else {
11257ff178cdSJimmy Vetayases /* default configuration */
11267ff178cdSJimmy Vetayases ioapicindex = 0;
11277ff178cdSJimmy Vetayases ioapic = apic_io_id[ioapicindex];
11287ff178cdSJimmy Vetayases ipin = (uchar_t)irqno;
11297ff178cdSJimmy Vetayases intr_index = DEFAULT_INDEX;
11307ff178cdSJimmy Vetayases }
11317ff178cdSJimmy Vetayases
113230acb30dSHans Rosenfeld if ((vector = apic_allocate_vector(ipl, irqno, 0)) == 0) {
11337ff178cdSJimmy Vetayases if ((newirq = apic_share_vector(irqno, intr_flagp, intr_index,
11347ff178cdSJimmy Vetayases ipl, ioapicindex, ipin, &irqptr)) != -1) {
11357ff178cdSJimmy Vetayases irqptr->airq_ipl = ipl;
11367ff178cdSJimmy Vetayases irqptr->airq_origirq = (uchar_t)origirq;
11377ff178cdSJimmy Vetayases irqptr->airq_dip = dip;
11387ff178cdSJimmy Vetayases irqptr->airq_major = major;
11397ff178cdSJimmy Vetayases sdip = apic_irq_table[IRQINDEX(newirq)]->airq_dip;
11407ff178cdSJimmy Vetayases /* This is OK to do really */
11417ff178cdSJimmy Vetayases if (sdip == NULL) {
11427ff178cdSJimmy Vetayases cmn_err(CE_WARN, "Sharing vectors: %s"
11437ff178cdSJimmy Vetayases " instance %d and SCI",
11447ff178cdSJimmy Vetayases ddi_get_name(dip), ddi_get_instance(dip));
11457ff178cdSJimmy Vetayases } else {
11467ff178cdSJimmy Vetayases cmn_err(CE_WARN, "Sharing vectors: %s"
11477ff178cdSJimmy Vetayases " instance %d and %s instance %d",
11487ff178cdSJimmy Vetayases ddi_get_name(sdip), ddi_get_instance(sdip),
11497ff178cdSJimmy Vetayases ddi_get_name(dip), ddi_get_instance(dip));
11507ff178cdSJimmy Vetayases }
11517ff178cdSJimmy Vetayases return (newirq);
11527ff178cdSJimmy Vetayases }
11537ff178cdSJimmy Vetayases /* try high priority allocation now that share has failed */
11547ff178cdSJimmy Vetayases if ((vector = apic_allocate_vector(ipl, irqno, 1)) == 0) {
11557ff178cdSJimmy Vetayases cmn_err(CE_WARN, "No interrupt vector: %s instance %d",
11567ff178cdSJimmy Vetayases ddi_get_name(dip), ddi_get_instance(dip));
11577ff178cdSJimmy Vetayases return (-1);
11587ff178cdSJimmy Vetayases }
11597ff178cdSJimmy Vetayases }
11607ff178cdSJimmy Vetayases
11617ff178cdSJimmy Vetayases mutex_enter(&airq_mutex);
11627ff178cdSJimmy Vetayases if (apic_irq_table[irqno] == NULL) {
11637ff178cdSJimmy Vetayases irqptr = kmem_zalloc(sizeof (apic_irq_t), KM_SLEEP);
11647ff178cdSJimmy Vetayases irqptr->airq_temp_cpu = IRQ_UNINIT;
11657ff178cdSJimmy Vetayases apic_irq_table[irqno] = irqptr;
11667ff178cdSJimmy Vetayases } else {
11677ff178cdSJimmy Vetayases irqptr = apic_irq_table[irqno];
11687ff178cdSJimmy Vetayases if (irqptr->airq_mps_intr_index != FREE_INDEX) {
11697ff178cdSJimmy Vetayases /*
11707ff178cdSJimmy Vetayases * The slot is used by another irqno, so allocate
11717ff178cdSJimmy Vetayases * a free irqno for this interrupt
11727ff178cdSJimmy Vetayases */
11737ff178cdSJimmy Vetayases newirq = apic_allocate_irq(apic_first_avail_irq);
11747ff178cdSJimmy Vetayases if (newirq == -1) {
11757ff178cdSJimmy Vetayases mutex_exit(&airq_mutex);
11767ff178cdSJimmy Vetayases return (-1);
11777ff178cdSJimmy Vetayases }
11787ff178cdSJimmy Vetayases irqno = newirq;
11797ff178cdSJimmy Vetayases irqptr = apic_irq_table[irqno];
11807ff178cdSJimmy Vetayases if (irqptr == NULL) {
11817ff178cdSJimmy Vetayases irqptr = kmem_zalloc(sizeof (apic_irq_t),
11827ff178cdSJimmy Vetayases KM_SLEEP);
11837ff178cdSJimmy Vetayases irqptr->airq_temp_cpu = IRQ_UNINIT;
11847ff178cdSJimmy Vetayases apic_irq_table[irqno] = irqptr;
11857ff178cdSJimmy Vetayases }
11867ff178cdSJimmy Vetayases vector = apic_modify_vector(vector, newirq);
11877ff178cdSJimmy Vetayases }
11887ff178cdSJimmy Vetayases }
11897ff178cdSJimmy Vetayases apic_max_device_irq = max(irqno, apic_max_device_irq);
11907ff178cdSJimmy Vetayases apic_min_device_irq = min(irqno, apic_min_device_irq);
11917ff178cdSJimmy Vetayases mutex_exit(&airq_mutex);
11927ff178cdSJimmy Vetayases irqptr->airq_ioapicindex = ioapicindex;
11937ff178cdSJimmy Vetayases irqptr->airq_intin_no = ipin;
11947ff178cdSJimmy Vetayases irqptr->airq_ipl = ipl;
11957ff178cdSJimmy Vetayases irqptr->airq_vector = vector;
11967ff178cdSJimmy Vetayases irqptr->airq_origirq = (uchar_t)origirq;
11977ff178cdSJimmy Vetayases irqptr->airq_share_id = 0;
11987ff178cdSJimmy Vetayases irqptr->airq_mps_intr_index = (short)intr_index;
11997ff178cdSJimmy Vetayases irqptr->airq_dip = dip;
12007ff178cdSJimmy Vetayases irqptr->airq_major = major;
12017ff178cdSJimmy Vetayases irqptr->airq_cpu = apic_bind_intr(dip, irqno, ioapic, ipin);
12027ff178cdSJimmy Vetayases if (intr_flagp)
12037ff178cdSJimmy Vetayases irqptr->airq_iflag = *intr_flagp;
12047ff178cdSJimmy Vetayases
12057ff178cdSJimmy Vetayases if (!DDI_INTR_IS_MSI_OR_MSIX(type)) {
12067ff178cdSJimmy Vetayases /* setup I/O APIC entry for non-MSI/X interrupts */
12077ff178cdSJimmy Vetayases apic_record_rdt_entry(irqptr, irqno);
12087ff178cdSJimmy Vetayases }
12097ff178cdSJimmy Vetayases return (irqno);
12107ff178cdSJimmy Vetayases }
12117ff178cdSJimmy Vetayases
12127ff178cdSJimmy Vetayases /*
12137ff178cdSJimmy Vetayases * return the cpu to which this intr should be bound.
12147ff178cdSJimmy Vetayases * Check properties or any other mechanism to see if user wants it
12157ff178cdSJimmy Vetayases * bound to a specific CPU. If so, return the cpu id with high bit set.
12167ff178cdSJimmy Vetayases * If not, use the policy to choose a cpu and return the id.
12177ff178cdSJimmy Vetayases */
12187ff178cdSJimmy Vetayases uint32_t
apic_bind_intr(dev_info_t * dip,int irq,uchar_t ioapicid,uchar_t intin)12197ff178cdSJimmy Vetayases apic_bind_intr(dev_info_t *dip, int irq, uchar_t ioapicid, uchar_t intin)
12207ff178cdSJimmy Vetayases {
12217ff178cdSJimmy Vetayases int instance, instno, prop_len, bind_cpu, count;
12227ff178cdSJimmy Vetayases uint_t i, rc;
12237ff178cdSJimmy Vetayases uint32_t cpu;
12247ff178cdSJimmy Vetayases major_t major;
12257ff178cdSJimmy Vetayases char *name, *drv_name, *prop_val, *cptr;
12267ff178cdSJimmy Vetayases char prop_name[32];
12277ff178cdSJimmy Vetayases ulong_t iflag;
12287ff178cdSJimmy Vetayases
12297ff178cdSJimmy Vetayases
12307ff178cdSJimmy Vetayases if (apic_intr_policy == INTR_LOWEST_PRIORITY)
12317ff178cdSJimmy Vetayases return (IRQ_UNBOUND);
12327ff178cdSJimmy Vetayases
12337ff178cdSJimmy Vetayases if (apic_nproc == 1)
12347ff178cdSJimmy Vetayases return (0);
12357ff178cdSJimmy Vetayases
123630acb30dSHans Rosenfeld /*
123730acb30dSHans Rosenfeld * dip may be NULL for interrupts not associated with a device driver,
123830acb30dSHans Rosenfeld * such as the ACPI SCI or HPET interrupts. In that case just use the
123930acb30dSHans Rosenfeld * next CPU and return.
124030acb30dSHans Rosenfeld */
124130acb30dSHans Rosenfeld if (dip == NULL) {
124230acb30dSHans Rosenfeld iflag = intr_clear();
124330acb30dSHans Rosenfeld lock_set(&apic_ioapic_lock);
124430acb30dSHans Rosenfeld bind_cpu = apic_get_next_bind_cpu();
124530acb30dSHans Rosenfeld lock_clear(&apic_ioapic_lock);
124630acb30dSHans Rosenfeld intr_restore(iflag);
124730acb30dSHans Rosenfeld
124830acb30dSHans Rosenfeld cmn_err(CE_CONT, "!%s: irq 0x%x "
124930acb30dSHans Rosenfeld "vector 0x%x ioapic 0x%x intin 0x%x is bound to cpu %d\n",
125030acb30dSHans Rosenfeld psm_name, irq, apic_irq_table[irq]->airq_vector, ioapicid,
125130acb30dSHans Rosenfeld intin, bind_cpu & ~IRQ_USER_BOUND);
125230acb30dSHans Rosenfeld
125330acb30dSHans Rosenfeld return ((uint32_t)bind_cpu);
125430acb30dSHans Rosenfeld }
125530acb30dSHans Rosenfeld
125630acb30dSHans Rosenfeld name = ddi_get_name(dip);
125730acb30dSHans Rosenfeld major = ddi_name_to_major(name);
125830acb30dSHans Rosenfeld drv_name = ddi_major_to_name(major);
125930acb30dSHans Rosenfeld instance = ddi_get_instance(dip);
126030acb30dSHans Rosenfeld if (apic_intr_policy == INTR_ROUND_ROBIN_WITH_AFFINITY) {
126130acb30dSHans Rosenfeld i = apic_min_device_irq;
126230acb30dSHans Rosenfeld for (; i <= apic_max_device_irq; i++) {
126330acb30dSHans Rosenfeld if ((i == irq) || (apic_irq_table[i] == NULL) ||
126430acb30dSHans Rosenfeld (apic_irq_table[i]->airq_mps_intr_index
126530acb30dSHans Rosenfeld == FREE_INDEX))
126630acb30dSHans Rosenfeld continue;
126730acb30dSHans Rosenfeld
126830acb30dSHans Rosenfeld if ((apic_irq_table[i]->airq_major == major) &&
126930acb30dSHans Rosenfeld (!(apic_irq_table[i]->airq_cpu & IRQ_USER_BOUND))) {
127030acb30dSHans Rosenfeld cpu = apic_irq_table[i]->airq_cpu;
127130acb30dSHans Rosenfeld
127230acb30dSHans Rosenfeld cmn_err(CE_CONT,
127330acb30dSHans Rosenfeld "!%s: %s (%s) instance #%d "
127430acb30dSHans Rosenfeld "irq 0x%x vector 0x%x ioapic 0x%x "
127530acb30dSHans Rosenfeld "intin 0x%x is bound to cpu %d\n",
127630acb30dSHans Rosenfeld psm_name,
127730acb30dSHans Rosenfeld name, drv_name, instance, irq,
127830acb30dSHans Rosenfeld apic_irq_table[irq]->airq_vector,
127930acb30dSHans Rosenfeld ioapicid, intin, cpu);
128030acb30dSHans Rosenfeld return (cpu);
12817ff178cdSJimmy Vetayases }
12827ff178cdSJimmy Vetayases }
128330acb30dSHans Rosenfeld }
128430acb30dSHans Rosenfeld /*
128530acb30dSHans Rosenfeld * search for "drvname"_intpt_bind_cpus property first, the
128630acb30dSHans Rosenfeld * syntax of the property should be "a[,b,c,...]" where
128730acb30dSHans Rosenfeld * instance 0 binds to cpu a, instance 1 binds to cpu b,
128830acb30dSHans Rosenfeld * instance 3 binds to cpu c...
128930acb30dSHans Rosenfeld * ddi_getlongprop() will search /option first, then /
129030acb30dSHans Rosenfeld * if "drvname"_intpt_bind_cpus doesn't exist, then find
129130acb30dSHans Rosenfeld * intpt_bind_cpus property. The syntax is the same, and
129230acb30dSHans Rosenfeld * it applies to all the devices if its "drvname" specific
129330acb30dSHans Rosenfeld * property doesn't exist
129430acb30dSHans Rosenfeld */
129530acb30dSHans Rosenfeld (void) strcpy(prop_name, drv_name);
129630acb30dSHans Rosenfeld (void) strcat(prop_name, "_intpt_bind_cpus");
129730acb30dSHans Rosenfeld rc = ddi_getlongprop(DDI_DEV_T_ANY, dip, 0, prop_name,
129830acb30dSHans Rosenfeld (caddr_t)&prop_val, &prop_len);
129930acb30dSHans Rosenfeld if (rc != DDI_PROP_SUCCESS) {
130030acb30dSHans Rosenfeld rc = ddi_getlongprop(DDI_DEV_T_ANY, dip, 0,
130130acb30dSHans Rosenfeld "intpt_bind_cpus", (caddr_t)&prop_val, &prop_len);
13027ff178cdSJimmy Vetayases }
13037ff178cdSJimmy Vetayases if (rc == DDI_PROP_SUCCESS) {
13047ff178cdSJimmy Vetayases for (i = count = 0; i < (prop_len - 1); i++)
13057ff178cdSJimmy Vetayases if (prop_val[i] == ',')
13067ff178cdSJimmy Vetayases count++;
13077ff178cdSJimmy Vetayases if (prop_val[i-1] != ',')
13087ff178cdSJimmy Vetayases count++;
13097ff178cdSJimmy Vetayases /*
13107ff178cdSJimmy Vetayases * if somehow the binding instances defined in the
13117ff178cdSJimmy Vetayases * property are not enough for this instno., then
13127ff178cdSJimmy Vetayases * reuse the pattern for the next instance until
13137ff178cdSJimmy Vetayases * it reaches the requested instno
13147ff178cdSJimmy Vetayases */
13157ff178cdSJimmy Vetayases instno = instance % count;
13167ff178cdSJimmy Vetayases i = 0;
13177ff178cdSJimmy Vetayases cptr = prop_val;
13187ff178cdSJimmy Vetayases while (i < instno)
13197ff178cdSJimmy Vetayases if (*cptr++ == ',')
13207ff178cdSJimmy Vetayases i++;
13217ff178cdSJimmy Vetayases bind_cpu = stoi(&cptr);
13227ff178cdSJimmy Vetayases kmem_free(prop_val, prop_len);
13237ff178cdSJimmy Vetayases /* if specific CPU is bogus, then default to next cpu */
13247ff178cdSJimmy Vetayases if (!apic_cpu_in_range(bind_cpu)) {
13257ff178cdSJimmy Vetayases cmn_err(CE_WARN, "%s: %s=%s: CPU %d not present",
13267ff178cdSJimmy Vetayases psm_name, prop_name, prop_val, bind_cpu);
13277ff178cdSJimmy Vetayases rc = DDI_PROP_NOT_FOUND;
13287ff178cdSJimmy Vetayases } else {
13297ff178cdSJimmy Vetayases /* indicate that we are bound at user request */
13307ff178cdSJimmy Vetayases bind_cpu |= IRQ_USER_BOUND;
13317ff178cdSJimmy Vetayases }
13327ff178cdSJimmy Vetayases /*
13337ff178cdSJimmy Vetayases * no need to check apic_cpus[].aci_status, if specific CPU is
13347ff178cdSJimmy Vetayases * not up, then post_cpu_start will handle it.
13357ff178cdSJimmy Vetayases */
13367ff178cdSJimmy Vetayases }
133730acb30dSHans Rosenfeld
13387ff178cdSJimmy Vetayases if (rc != DDI_PROP_SUCCESS) {
13397ff178cdSJimmy Vetayases iflag = intr_clear();
13407ff178cdSJimmy Vetayases lock_set(&apic_ioapic_lock);
13417ff178cdSJimmy Vetayases bind_cpu = apic_get_next_bind_cpu();
13427ff178cdSJimmy Vetayases lock_clear(&apic_ioapic_lock);
13437ff178cdSJimmy Vetayases intr_restore(iflag);
13447ff178cdSJimmy Vetayases }
13457ff178cdSJimmy Vetayases
134630acb30dSHans Rosenfeld cmn_err(CE_CONT, "!%s: %s (%s) instance %d irq 0x%x "
134730acb30dSHans Rosenfeld "vector 0x%x ioapic 0x%x intin 0x%x is bound to cpu %d\n",
134830acb30dSHans Rosenfeld psm_name, name, drv_name, instance, irq,
134930acb30dSHans Rosenfeld apic_irq_table[irq]->airq_vector, ioapicid, intin,
135030acb30dSHans Rosenfeld bind_cpu & ~IRQ_USER_BOUND);
13517ff178cdSJimmy Vetayases
13527ff178cdSJimmy Vetayases return ((uint32_t)bind_cpu);
13537ff178cdSJimmy Vetayases }
13547ff178cdSJimmy Vetayases
13557ff178cdSJimmy Vetayases /*
13567ff178cdSJimmy Vetayases * Mark vector as being in the process of being deleted. Interrupts
13577ff178cdSJimmy Vetayases * may still come in on some CPU. The moment an interrupt comes with
13587ff178cdSJimmy Vetayases * the new vector, we know we can free the old one. Called only from
13597ff178cdSJimmy Vetayases * addspl and delspl with interrupts disabled. Because an interrupt
13607ff178cdSJimmy Vetayases * can be shared, but no interrupt from either device may come in,
13617ff178cdSJimmy Vetayases * we also use a timeout mechanism, which we arbitrarily set to
13627ff178cdSJimmy Vetayases * apic_revector_timeout microseconds.
13637ff178cdSJimmy Vetayases */
13647ff178cdSJimmy Vetayases static void
apic_mark_vector(uchar_t oldvector,uchar_t newvector)13657ff178cdSJimmy Vetayases apic_mark_vector(uchar_t oldvector, uchar_t newvector)
13667ff178cdSJimmy Vetayases {
13677ff178cdSJimmy Vetayases ulong_t iflag;
13687ff178cdSJimmy Vetayases
13697ff178cdSJimmy Vetayases iflag = intr_clear();
13707ff178cdSJimmy Vetayases lock_set(&apic_revector_lock);
13717ff178cdSJimmy Vetayases if (!apic_oldvec_to_newvec) {
13727ff178cdSJimmy Vetayases apic_oldvec_to_newvec =
13737ff178cdSJimmy Vetayases kmem_zalloc(sizeof (newvector) * APIC_MAX_VECTOR * 2,
13747ff178cdSJimmy Vetayases KM_NOSLEEP);
13757ff178cdSJimmy Vetayases
13767ff178cdSJimmy Vetayases if (!apic_oldvec_to_newvec) {
13777ff178cdSJimmy Vetayases /*
13787ff178cdSJimmy Vetayases * This failure is not catastrophic.
13797ff178cdSJimmy Vetayases * But, the oldvec will never be freed.
13807ff178cdSJimmy Vetayases */
13817ff178cdSJimmy Vetayases apic_error |= APIC_ERR_MARK_VECTOR_FAIL;
13827ff178cdSJimmy Vetayases lock_clear(&apic_revector_lock);
13837ff178cdSJimmy Vetayases intr_restore(iflag);
13847ff178cdSJimmy Vetayases return;
13857ff178cdSJimmy Vetayases }
13867ff178cdSJimmy Vetayases apic_newvec_to_oldvec = &apic_oldvec_to_newvec[APIC_MAX_VECTOR];
13877ff178cdSJimmy Vetayases }
13887ff178cdSJimmy Vetayases
13897ff178cdSJimmy Vetayases /* See if we already did this for drivers which do double addintrs */
13907ff178cdSJimmy Vetayases if (apic_oldvec_to_newvec[oldvector] != newvector) {
13917ff178cdSJimmy Vetayases apic_oldvec_to_newvec[oldvector] = newvector;
13927ff178cdSJimmy Vetayases apic_newvec_to_oldvec[newvector] = oldvector;
13937ff178cdSJimmy Vetayases apic_revector_pending++;
13947ff178cdSJimmy Vetayases }
13957ff178cdSJimmy Vetayases lock_clear(&apic_revector_lock);
13967ff178cdSJimmy Vetayases intr_restore(iflag);
13977ff178cdSJimmy Vetayases (void) timeout(apic_xlate_vector_free_timeout_handler,
13987ff178cdSJimmy Vetayases (void *)(uintptr_t)oldvector, drv_usectohz(apic_revector_timeout));
13997ff178cdSJimmy Vetayases }
14007ff178cdSJimmy Vetayases
14017ff178cdSJimmy Vetayases /*
14027ff178cdSJimmy Vetayases * xlate_vector is called from intr_enter if revector_pending is set.
14037ff178cdSJimmy Vetayases * It will xlate it if needed and mark the old vector as free.
14047ff178cdSJimmy Vetayases */
14057ff178cdSJimmy Vetayases uchar_t
apic_xlate_vector(uchar_t vector)14067ff178cdSJimmy Vetayases apic_xlate_vector(uchar_t vector)
14077ff178cdSJimmy Vetayases {
14087ff178cdSJimmy Vetayases uchar_t newvector, oldvector = 0;
14097ff178cdSJimmy Vetayases
14107ff178cdSJimmy Vetayases lock_set(&apic_revector_lock);
14117ff178cdSJimmy Vetayases /* Do we really need to do this ? */
14127ff178cdSJimmy Vetayases if (!apic_revector_pending) {
14137ff178cdSJimmy Vetayases lock_clear(&apic_revector_lock);
14147ff178cdSJimmy Vetayases return (vector);
14157ff178cdSJimmy Vetayases }
14167ff178cdSJimmy Vetayases if ((newvector = apic_oldvec_to_newvec[vector]) != 0)
14177ff178cdSJimmy Vetayases oldvector = vector;
14187ff178cdSJimmy Vetayases else {
14197ff178cdSJimmy Vetayases /*
14207ff178cdSJimmy Vetayases * The incoming vector is new . See if a stale entry is
14217ff178cdSJimmy Vetayases * remaining
14227ff178cdSJimmy Vetayases */
14237ff178cdSJimmy Vetayases if ((oldvector = apic_newvec_to_oldvec[vector]) != 0)
14247ff178cdSJimmy Vetayases newvector = vector;
14257ff178cdSJimmy Vetayases }
14267ff178cdSJimmy Vetayases
14277ff178cdSJimmy Vetayases if (oldvector) {
14287ff178cdSJimmy Vetayases apic_revector_pending--;
14297ff178cdSJimmy Vetayases apic_oldvec_to_newvec[oldvector] = 0;
14307ff178cdSJimmy Vetayases apic_newvec_to_oldvec[newvector] = 0;
14317ff178cdSJimmy Vetayases apic_free_vector(oldvector);
14327ff178cdSJimmy Vetayases lock_clear(&apic_revector_lock);
14337ff178cdSJimmy Vetayases /* There could have been more than one reprogramming! */
14347ff178cdSJimmy Vetayases return (apic_xlate_vector(newvector));
14357ff178cdSJimmy Vetayases }
14367ff178cdSJimmy Vetayases lock_clear(&apic_revector_lock);
14377ff178cdSJimmy Vetayases return (vector);
14387ff178cdSJimmy Vetayases }
14397ff178cdSJimmy Vetayases
14407ff178cdSJimmy Vetayases void
apic_xlate_vector_free_timeout_handler(void * arg)14417ff178cdSJimmy Vetayases apic_xlate_vector_free_timeout_handler(void *arg)
14427ff178cdSJimmy Vetayases {
14437ff178cdSJimmy Vetayases ulong_t iflag;
14447ff178cdSJimmy Vetayases uchar_t oldvector, newvector;
14457ff178cdSJimmy Vetayases
14467ff178cdSJimmy Vetayases oldvector = (uchar_t)(uintptr_t)arg;
14477ff178cdSJimmy Vetayases iflag = intr_clear();
14487ff178cdSJimmy Vetayases lock_set(&apic_revector_lock);
14497ff178cdSJimmy Vetayases if ((newvector = apic_oldvec_to_newvec[oldvector]) != 0) {
14507ff178cdSJimmy Vetayases apic_free_vector(oldvector);
14517ff178cdSJimmy Vetayases apic_oldvec_to_newvec[oldvector] = 0;
14527ff178cdSJimmy Vetayases apic_newvec_to_oldvec[newvector] = 0;
14537ff178cdSJimmy Vetayases apic_revector_pending--;
14547ff178cdSJimmy Vetayases }
14557ff178cdSJimmy Vetayases
14567ff178cdSJimmy Vetayases lock_clear(&apic_revector_lock);
14577ff178cdSJimmy Vetayases intr_restore(iflag);
14587ff178cdSJimmy Vetayases }
14597ff178cdSJimmy Vetayases
14607ff178cdSJimmy Vetayases /*
14617ff178cdSJimmy Vetayases * Bind interrupt corresponding to irq_ptr to bind_cpu.
14627ff178cdSJimmy Vetayases * Must be called with interrupts disabled and apic_ioapic_lock held
14637ff178cdSJimmy Vetayases */
14647ff178cdSJimmy Vetayases int
apic_rebind(apic_irq_t * irq_ptr,int bind_cpu,struct ioapic_reprogram_data * drep)14657ff178cdSJimmy Vetayases apic_rebind(apic_irq_t *irq_ptr, int bind_cpu,
14667ff178cdSJimmy Vetayases struct ioapic_reprogram_data *drep)
14677ff178cdSJimmy Vetayases {
14687ff178cdSJimmy Vetayases int ioapicindex, intin_no;
14697ff178cdSJimmy Vetayases uint32_t airq_temp_cpu;
14707ff178cdSJimmy Vetayases apic_cpus_info_t *cpu_infop;
14717ff178cdSJimmy Vetayases uint32_t rdt_entry;
14727ff178cdSJimmy Vetayases int which_irq;
14737ff178cdSJimmy Vetayases ioapic_rdt_t irdt;
14747ff178cdSJimmy Vetayases
14757ff178cdSJimmy Vetayases which_irq = apic_vector_to_irq[irq_ptr->airq_vector];
14767ff178cdSJimmy Vetayases
14777ff178cdSJimmy Vetayases intin_no = irq_ptr->airq_intin_no;
14787ff178cdSJimmy Vetayases ioapicindex = irq_ptr->airq_ioapicindex;
14797ff178cdSJimmy Vetayases airq_temp_cpu = irq_ptr->airq_temp_cpu;
14807ff178cdSJimmy Vetayases if (airq_temp_cpu != IRQ_UNINIT && airq_temp_cpu != IRQ_UNBOUND) {
14817ff178cdSJimmy Vetayases if (airq_temp_cpu & IRQ_USER_BOUND)
14827ff178cdSJimmy Vetayases /* Mask off high bit so it can be used as array index */
14837ff178cdSJimmy Vetayases airq_temp_cpu &= ~IRQ_USER_BOUND;
14847ff178cdSJimmy Vetayases
14857ff178cdSJimmy Vetayases ASSERT(apic_cpu_in_range(airq_temp_cpu));
14867ff178cdSJimmy Vetayases }
14877ff178cdSJimmy Vetayases
14887ff178cdSJimmy Vetayases /*
14897ff178cdSJimmy Vetayases * Can't bind to a CPU that's not accepting interrupts:
14907ff178cdSJimmy Vetayases */
14917ff178cdSJimmy Vetayases cpu_infop = &apic_cpus[bind_cpu & ~IRQ_USER_BOUND];
14927ff178cdSJimmy Vetayases if (!(cpu_infop->aci_status & APIC_CPU_INTR_ENABLE))
14937ff178cdSJimmy Vetayases return (1);
14947ff178cdSJimmy Vetayases
14957ff178cdSJimmy Vetayases /*
14967ff178cdSJimmy Vetayases * If we are about to change the interrupt vector for this interrupt,
14977ff178cdSJimmy Vetayases * and this interrupt is level-triggered, attached to an IOAPIC,
14987ff178cdSJimmy Vetayases * has been delivered to a CPU and that CPU has not handled it
14997ff178cdSJimmy Vetayases * yet, we cannot reprogram the IOAPIC now.
15007ff178cdSJimmy Vetayases */
15017ff178cdSJimmy Vetayases if (!APIC_IS_MSI_OR_MSIX_INDEX(irq_ptr->airq_mps_intr_index)) {
15027ff178cdSJimmy Vetayases
15037ff178cdSJimmy Vetayases rdt_entry = READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapicindex,
15047ff178cdSJimmy Vetayases intin_no);
15057ff178cdSJimmy Vetayases
15067ff178cdSJimmy Vetayases if ((irq_ptr->airq_vector != RDT_VECTOR(rdt_entry)) &&
15077ff178cdSJimmy Vetayases apic_check_stuck_interrupt(irq_ptr, airq_temp_cpu,
15087ff178cdSJimmy Vetayases bind_cpu, ioapicindex, intin_no, which_irq, drep) != 0) {
15097ff178cdSJimmy Vetayases
15107ff178cdSJimmy Vetayases return (0);
15117ff178cdSJimmy Vetayases }
15127ff178cdSJimmy Vetayases
15137ff178cdSJimmy Vetayases /*
15147ff178cdSJimmy Vetayases * NOTE: We do not unmask the RDT here, as an interrupt MAY
15157ff178cdSJimmy Vetayases * still come in before we have a chance to reprogram it below.
15167ff178cdSJimmy Vetayases * The reprogramming below will simultaneously change and
15177ff178cdSJimmy Vetayases * unmask the RDT entry.
15187ff178cdSJimmy Vetayases */
15197ff178cdSJimmy Vetayases
15207ff178cdSJimmy Vetayases if ((uint32_t)bind_cpu == IRQ_UNBOUND) {
15217ff178cdSJimmy Vetayases irdt.ir_lo = AV_LDEST | AV_LOPRI |
15227ff178cdSJimmy Vetayases irq_ptr->airq_rdt_entry;
15237ff178cdSJimmy Vetayases
15247ff178cdSJimmy Vetayases irdt.ir_hi = AV_TOALL >> APIC_ID_BIT_OFFSET;
15257ff178cdSJimmy Vetayases
15267ff178cdSJimmy Vetayases apic_vt_ops->apic_intrmap_alloc_entry(
15277ff178cdSJimmy Vetayases &irq_ptr->airq_intrmap_private, NULL,
15287ff178cdSJimmy Vetayases DDI_INTR_TYPE_FIXED, 1, ioapicindex);
15297ff178cdSJimmy Vetayases apic_vt_ops->apic_intrmap_map_entry(
15307ff178cdSJimmy Vetayases irq_ptr->airq_intrmap_private, (void *)&irdt,
15317ff178cdSJimmy Vetayases DDI_INTR_TYPE_FIXED, 1);
15327ff178cdSJimmy Vetayases apic_vt_ops->apic_intrmap_record_rdt(
15337ff178cdSJimmy Vetayases irq_ptr->airq_intrmap_private, &irdt);
15347ff178cdSJimmy Vetayases
15357ff178cdSJimmy Vetayases /* Write the RDT entry -- no specific CPU binding */
15367ff178cdSJimmy Vetayases WRITE_IOAPIC_RDT_ENTRY_HIGH_DWORD(ioapicindex, intin_no,
15377ff178cdSJimmy Vetayases irdt.ir_hi | AV_TOALL);
15387ff178cdSJimmy Vetayases
15397ff178cdSJimmy Vetayases if (airq_temp_cpu != IRQ_UNINIT && airq_temp_cpu !=
15407ff178cdSJimmy Vetayases IRQ_UNBOUND)
15417ff178cdSJimmy Vetayases apic_cpus[airq_temp_cpu].aci_temp_bound--;
15427ff178cdSJimmy Vetayases
15437ff178cdSJimmy Vetayases /*
15447ff178cdSJimmy Vetayases * Write the vector, trigger, and polarity portion of
15457ff178cdSJimmy Vetayases * the RDT
15467ff178cdSJimmy Vetayases */
15477ff178cdSJimmy Vetayases WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapicindex, intin_no,
15487ff178cdSJimmy Vetayases irdt.ir_lo);
15497ff178cdSJimmy Vetayases
15507ff178cdSJimmy Vetayases irq_ptr->airq_temp_cpu = IRQ_UNBOUND;
15517ff178cdSJimmy Vetayases return (0);
15527ff178cdSJimmy Vetayases }
15537ff178cdSJimmy Vetayases }
15547ff178cdSJimmy Vetayases
15557ff178cdSJimmy Vetayases if (bind_cpu & IRQ_USER_BOUND) {
15567ff178cdSJimmy Vetayases cpu_infop->aci_bound++;
15577ff178cdSJimmy Vetayases } else {
15587ff178cdSJimmy Vetayases cpu_infop->aci_temp_bound++;
15597ff178cdSJimmy Vetayases }
15607ff178cdSJimmy Vetayases ASSERT(apic_cpu_in_range(bind_cpu));
15617ff178cdSJimmy Vetayases
15627ff178cdSJimmy Vetayases if ((airq_temp_cpu != IRQ_UNBOUND) && (airq_temp_cpu != IRQ_UNINIT)) {
15637ff178cdSJimmy Vetayases apic_cpus[airq_temp_cpu].aci_temp_bound--;
15647ff178cdSJimmy Vetayases }
15657ff178cdSJimmy Vetayases if (!APIC_IS_MSI_OR_MSIX_INDEX(irq_ptr->airq_mps_intr_index)) {
15667ff178cdSJimmy Vetayases
15677ff178cdSJimmy Vetayases irdt.ir_lo = AV_PDEST | AV_FIXED | irq_ptr->airq_rdt_entry;
15687ff178cdSJimmy Vetayases irdt.ir_hi = cpu_infop->aci_local_id;
15697ff178cdSJimmy Vetayases
15707ff178cdSJimmy Vetayases apic_vt_ops->apic_intrmap_alloc_entry(
15717ff178cdSJimmy Vetayases &irq_ptr->airq_intrmap_private, NULL, DDI_INTR_TYPE_FIXED,
15727ff178cdSJimmy Vetayases 1, ioapicindex);
15737ff178cdSJimmy Vetayases apic_vt_ops->apic_intrmap_map_entry(
15747ff178cdSJimmy Vetayases irq_ptr->airq_intrmap_private,
15757ff178cdSJimmy Vetayases (void *)&irdt, DDI_INTR_TYPE_FIXED, 1);
15767ff178cdSJimmy Vetayases apic_vt_ops->apic_intrmap_record_rdt(
15777ff178cdSJimmy Vetayases irq_ptr->airq_intrmap_private, &irdt);
15787ff178cdSJimmy Vetayases
15797ff178cdSJimmy Vetayases /* Write the RDT entry -- bind to a specific CPU: */
15807ff178cdSJimmy Vetayases WRITE_IOAPIC_RDT_ENTRY_HIGH_DWORD(ioapicindex, intin_no,
15817ff178cdSJimmy Vetayases irdt.ir_hi);
15827ff178cdSJimmy Vetayases
15837ff178cdSJimmy Vetayases /* Write the vector, trigger, and polarity portion of the RDT */
15847ff178cdSJimmy Vetayases WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapicindex, intin_no,
15857ff178cdSJimmy Vetayases irdt.ir_lo);
15867ff178cdSJimmy Vetayases
15877ff178cdSJimmy Vetayases } else {
15887ff178cdSJimmy Vetayases int type = (irq_ptr->airq_mps_intr_index == MSI_INDEX) ?
15897ff178cdSJimmy Vetayases DDI_INTR_TYPE_MSI : DDI_INTR_TYPE_MSIX;
15907ff178cdSJimmy Vetayases if (type == DDI_INTR_TYPE_MSI) {
15917ff178cdSJimmy Vetayases if (irq_ptr->airq_ioapicindex ==
15927ff178cdSJimmy Vetayases irq_ptr->airq_origirq) {
15937ff178cdSJimmy Vetayases /* first one */
15947ff178cdSJimmy Vetayases DDI_INTR_IMPLDBG((CE_CONT, "apic_rebind: call "
15957ff178cdSJimmy Vetayases "apic_pci_msi_enable_vector\n"));
15967ff178cdSJimmy Vetayases apic_pci_msi_enable_vector(irq_ptr,
15977ff178cdSJimmy Vetayases type, which_irq, irq_ptr->airq_vector,
15987ff178cdSJimmy Vetayases irq_ptr->airq_intin_no,
15997ff178cdSJimmy Vetayases cpu_infop->aci_local_id);
16007ff178cdSJimmy Vetayases }
16017ff178cdSJimmy Vetayases if ((irq_ptr->airq_ioapicindex +
16027ff178cdSJimmy Vetayases irq_ptr->airq_intin_no - 1) ==
16037ff178cdSJimmy Vetayases irq_ptr->airq_origirq) { /* last one */
16047ff178cdSJimmy Vetayases DDI_INTR_IMPLDBG((CE_CONT, "apic_rebind: call "
16057ff178cdSJimmy Vetayases "apic_pci_msi_enable_mode\n"));
16067ff178cdSJimmy Vetayases apic_pci_msi_enable_mode(irq_ptr->airq_dip,
16077ff178cdSJimmy Vetayases type, which_irq);
16087ff178cdSJimmy Vetayases }
16097ff178cdSJimmy Vetayases } else { /* MSI-X */
16107ff178cdSJimmy Vetayases apic_pci_msi_enable_vector(irq_ptr, type,
16117ff178cdSJimmy Vetayases irq_ptr->airq_origirq, irq_ptr->airq_vector, 1,
16127ff178cdSJimmy Vetayases cpu_infop->aci_local_id);
16137ff178cdSJimmy Vetayases apic_pci_msi_enable_mode(irq_ptr->airq_dip, type,
16147ff178cdSJimmy Vetayases irq_ptr->airq_origirq);
16157ff178cdSJimmy Vetayases }
16167ff178cdSJimmy Vetayases }
16177ff178cdSJimmy Vetayases irq_ptr->airq_temp_cpu = (uint32_t)bind_cpu;
16187ff178cdSJimmy Vetayases apic_redist_cpu_skip &= ~(1 << (bind_cpu & ~IRQ_USER_BOUND));
16197ff178cdSJimmy Vetayases return (0);
16207ff178cdSJimmy Vetayases }
16217ff178cdSJimmy Vetayases
16227ff178cdSJimmy Vetayases static void
apic_last_ditch_clear_remote_irr(int ioapic_ix,int intin_no)16237ff178cdSJimmy Vetayases apic_last_ditch_clear_remote_irr(int ioapic_ix, int intin_no)
16247ff178cdSJimmy Vetayases {
16257ff178cdSJimmy Vetayases if ((READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix, intin_no)
16267ff178cdSJimmy Vetayases & AV_REMOTE_IRR) != 0) {
16277ff178cdSJimmy Vetayases /*
16287ff178cdSJimmy Vetayases * Trying to clear the bit through normal
16297ff178cdSJimmy Vetayases * channels has failed. So as a last-ditch
16307ff178cdSJimmy Vetayases * effort, try to set the trigger mode to
16317ff178cdSJimmy Vetayases * edge, then to level. This has been
16327ff178cdSJimmy Vetayases * observed to work on many systems.
16337ff178cdSJimmy Vetayases */
16347ff178cdSJimmy Vetayases WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
16357ff178cdSJimmy Vetayases intin_no,
16367ff178cdSJimmy Vetayases READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
16377ff178cdSJimmy Vetayases intin_no) & ~AV_LEVEL);
16387ff178cdSJimmy Vetayases
16397ff178cdSJimmy Vetayases WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
16407ff178cdSJimmy Vetayases intin_no,
16417ff178cdSJimmy Vetayases READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
16427ff178cdSJimmy Vetayases intin_no) | AV_LEVEL);
16437ff178cdSJimmy Vetayases
16447ff178cdSJimmy Vetayases /*
16457ff178cdSJimmy Vetayases * If the bit's STILL set, this interrupt may
16467ff178cdSJimmy Vetayases * be hosed.
16477ff178cdSJimmy Vetayases */
16487ff178cdSJimmy Vetayases if ((READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
16497ff178cdSJimmy Vetayases intin_no) & AV_REMOTE_IRR) != 0) {
16507ff178cdSJimmy Vetayases
16517ff178cdSJimmy Vetayases prom_printf("%s: Remote IRR still "
16527ff178cdSJimmy Vetayases "not clear for IOAPIC %d intin %d.\n"
16537ff178cdSJimmy Vetayases "\tInterrupts to this pin may cease "
16547ff178cdSJimmy Vetayases "functioning.\n", psm_name, ioapic_ix,
16557ff178cdSJimmy Vetayases intin_no);
16567ff178cdSJimmy Vetayases #ifdef DEBUG
16577ff178cdSJimmy Vetayases apic_last_ditch_reprogram_failures++;
16587ff178cdSJimmy Vetayases #endif
16597ff178cdSJimmy Vetayases }
16607ff178cdSJimmy Vetayases }
16617ff178cdSJimmy Vetayases }
16627ff178cdSJimmy Vetayases
16637ff178cdSJimmy Vetayases /*
16647ff178cdSJimmy Vetayases * This function is protected by apic_ioapic_lock coupled with the
16657ff178cdSJimmy Vetayases * fact that interrupts are disabled.
16667ff178cdSJimmy Vetayases */
16677ff178cdSJimmy Vetayases static void
delete_defer_repro_ent(int which_irq)16687ff178cdSJimmy Vetayases delete_defer_repro_ent(int which_irq)
16697ff178cdSJimmy Vetayases {
16707ff178cdSJimmy Vetayases ASSERT(which_irq >= 0);
16717ff178cdSJimmy Vetayases ASSERT(which_irq <= 255);
16727ff178cdSJimmy Vetayases ASSERT(LOCK_HELD(&apic_ioapic_lock));
16737ff178cdSJimmy Vetayases
16747ff178cdSJimmy Vetayases if (apic_reprogram_info[which_irq].done)
16757ff178cdSJimmy Vetayases return;
16767ff178cdSJimmy Vetayases
16777ff178cdSJimmy Vetayases apic_reprogram_info[which_irq].done = B_TRUE;
16787ff178cdSJimmy Vetayases
16797ff178cdSJimmy Vetayases #ifdef DEBUG
16807ff178cdSJimmy Vetayases apic_defer_repro_total_retries +=
16817ff178cdSJimmy Vetayases apic_reprogram_info[which_irq].tries;
16827ff178cdSJimmy Vetayases
16837ff178cdSJimmy Vetayases apic_defer_repro_successes++;
16847ff178cdSJimmy Vetayases #endif
16857ff178cdSJimmy Vetayases
16867ff178cdSJimmy Vetayases if (--apic_reprogram_outstanding == 0) {
16877ff178cdSJimmy Vetayases
16887ff178cdSJimmy Vetayases setlvlx = psm_intr_exit_fn();
16897ff178cdSJimmy Vetayases }
16907ff178cdSJimmy Vetayases }
16917ff178cdSJimmy Vetayases
16927ff178cdSJimmy Vetayases
16937ff178cdSJimmy Vetayases /*
16947ff178cdSJimmy Vetayases * Interrupts must be disabled during this function to prevent
16957ff178cdSJimmy Vetayases * self-deadlock. Interrupts are disabled because this function
16967ff178cdSJimmy Vetayases * is called from apic_check_stuck_interrupt(), which is called
16977ff178cdSJimmy Vetayases * from apic_rebind(), which requires its caller to disable interrupts.
16987ff178cdSJimmy Vetayases */
16997ff178cdSJimmy Vetayases static void
add_defer_repro_ent(apic_irq_t * irq_ptr,int which_irq,int new_bind_cpu)17007ff178cdSJimmy Vetayases add_defer_repro_ent(apic_irq_t *irq_ptr, int which_irq, int new_bind_cpu)
17017ff178cdSJimmy Vetayases {
17027ff178cdSJimmy Vetayases ASSERT(which_irq >= 0);
17037ff178cdSJimmy Vetayases ASSERT(which_irq <= 255);
17047ff178cdSJimmy Vetayases ASSERT(!interrupts_enabled());
17057ff178cdSJimmy Vetayases
17067ff178cdSJimmy Vetayases /*
17077ff178cdSJimmy Vetayases * On the off-chance that there's already a deferred
17087ff178cdSJimmy Vetayases * reprogramming on this irq, check, and if so, just update the
17097ff178cdSJimmy Vetayases * CPU and irq pointer to which the interrupt is targeted, then return.
17107ff178cdSJimmy Vetayases */
17117ff178cdSJimmy Vetayases if (!apic_reprogram_info[which_irq].done) {
17127ff178cdSJimmy Vetayases apic_reprogram_info[which_irq].bindcpu = new_bind_cpu;
17137ff178cdSJimmy Vetayases apic_reprogram_info[which_irq].irqp = irq_ptr;
17147ff178cdSJimmy Vetayases return;
17157ff178cdSJimmy Vetayases }
17167ff178cdSJimmy Vetayases
17177ff178cdSJimmy Vetayases apic_reprogram_info[which_irq].irqp = irq_ptr;
17187ff178cdSJimmy Vetayases apic_reprogram_info[which_irq].bindcpu = new_bind_cpu;
17197ff178cdSJimmy Vetayases apic_reprogram_info[which_irq].tries = 0;
17207ff178cdSJimmy Vetayases /*
17217ff178cdSJimmy Vetayases * This must be the last thing set, since we're not
17227ff178cdSJimmy Vetayases * grabbing any locks, apic_try_deferred_reprogram() will
17237ff178cdSJimmy Vetayases * make its decision about using this entry iff done
17247ff178cdSJimmy Vetayases * is false.
17257ff178cdSJimmy Vetayases */
17267ff178cdSJimmy Vetayases apic_reprogram_info[which_irq].done = B_FALSE;
17277ff178cdSJimmy Vetayases
17287ff178cdSJimmy Vetayases /*
17297ff178cdSJimmy Vetayases * If there were previously no deferred reprogrammings, change
17307ff178cdSJimmy Vetayases * setlvlx to call apic_try_deferred_reprogram()
17317ff178cdSJimmy Vetayases */
17327ff178cdSJimmy Vetayases if (++apic_reprogram_outstanding == 1) {
17337ff178cdSJimmy Vetayases
17347ff178cdSJimmy Vetayases setlvlx = apic_try_deferred_reprogram;
17357ff178cdSJimmy Vetayases }
17367ff178cdSJimmy Vetayases }
17377ff178cdSJimmy Vetayases
17387ff178cdSJimmy Vetayases static void
apic_try_deferred_reprogram(int prev_ipl,int irq)17397ff178cdSJimmy Vetayases apic_try_deferred_reprogram(int prev_ipl, int irq)
17407ff178cdSJimmy Vetayases {
17417ff178cdSJimmy Vetayases int reproirq;
17427ff178cdSJimmy Vetayases ulong_t iflag;
17437ff178cdSJimmy Vetayases struct ioapic_reprogram_data *drep;
17447ff178cdSJimmy Vetayases
17457ff178cdSJimmy Vetayases (*psm_intr_exit_fn())(prev_ipl, irq);
17467ff178cdSJimmy Vetayases
17477ff178cdSJimmy Vetayases if (!lock_try(&apic_defer_reprogram_lock)) {
17487ff178cdSJimmy Vetayases return;
17497ff178cdSJimmy Vetayases }
17507ff178cdSJimmy Vetayases
17517ff178cdSJimmy Vetayases /*
17527ff178cdSJimmy Vetayases * Acquire the apic_ioapic_lock so that any other operations that
17537ff178cdSJimmy Vetayases * may affect the apic_reprogram_info state are serialized.
17547ff178cdSJimmy Vetayases * It's still possible for the last deferred reprogramming to clear
17557ff178cdSJimmy Vetayases * between the time we entered this function and the time we get to
17567ff178cdSJimmy Vetayases * the for loop below. In that case, *setlvlx will have been set
17577ff178cdSJimmy Vetayases * back to *_intr_exit and drep will be NULL. (There's no way to
17587ff178cdSJimmy Vetayases * stop that from happening -- we would need to grab a lock before
17597ff178cdSJimmy Vetayases * calling *setlvlx, which is neither realistic nor prudent).
17607ff178cdSJimmy Vetayases */
17617ff178cdSJimmy Vetayases iflag = intr_clear();
17627ff178cdSJimmy Vetayases lock_set(&apic_ioapic_lock);
17637ff178cdSJimmy Vetayases
17647ff178cdSJimmy Vetayases /*
17657ff178cdSJimmy Vetayases * For each deferred RDT entry, try to reprogram it now. Note that
17667ff178cdSJimmy Vetayases * there is no lock acquisition to read apic_reprogram_info because
17677ff178cdSJimmy Vetayases * '.done' is set only after the other fields in the structure are set.
17687ff178cdSJimmy Vetayases */
17697ff178cdSJimmy Vetayases
17707ff178cdSJimmy Vetayases drep = NULL;
17717ff178cdSJimmy Vetayases for (reproirq = 0; reproirq <= APIC_MAX_VECTOR; reproirq++) {
17727ff178cdSJimmy Vetayases if (apic_reprogram_info[reproirq].done == B_FALSE) {
17737ff178cdSJimmy Vetayases drep = &apic_reprogram_info[reproirq];
17747ff178cdSJimmy Vetayases break;
17757ff178cdSJimmy Vetayases }
17767ff178cdSJimmy Vetayases }
17777ff178cdSJimmy Vetayases
17787ff178cdSJimmy Vetayases /*
17797ff178cdSJimmy Vetayases * Either we found a deferred action to perform, or
17807ff178cdSJimmy Vetayases * we entered this function spuriously, after *setlvlx
17817ff178cdSJimmy Vetayases * was restored to point to *_intr_exit. Any other
17827ff178cdSJimmy Vetayases * permutation is invalid.
17837ff178cdSJimmy Vetayases */
17847ff178cdSJimmy Vetayases ASSERT(drep != NULL || *setlvlx == psm_intr_exit_fn());
17857ff178cdSJimmy Vetayases
17867ff178cdSJimmy Vetayases /*
17877ff178cdSJimmy Vetayases * Though we can't really do anything about errors
17887ff178cdSJimmy Vetayases * at this point, keep track of them for reporting.
17897ff178cdSJimmy Vetayases * Note that it is very possible for apic_setup_io_intr
17907ff178cdSJimmy Vetayases * to re-register this very timeout if the Remote IRR bit
17917ff178cdSJimmy Vetayases * has not yet cleared.
17927ff178cdSJimmy Vetayases */
17937ff178cdSJimmy Vetayases
17947ff178cdSJimmy Vetayases #ifdef DEBUG
17957ff178cdSJimmy Vetayases if (drep != NULL) {
17967ff178cdSJimmy Vetayases if (apic_setup_io_intr(drep, reproirq, B_TRUE) != 0) {
17977ff178cdSJimmy Vetayases apic_deferred_setup_failures++;
17987ff178cdSJimmy Vetayases }
17997ff178cdSJimmy Vetayases } else {
18007ff178cdSJimmy Vetayases apic_deferred_spurious_enters++;
18017ff178cdSJimmy Vetayases }
18027ff178cdSJimmy Vetayases #else
18037ff178cdSJimmy Vetayases if (drep != NULL)
18047ff178cdSJimmy Vetayases (void) apic_setup_io_intr(drep, reproirq, B_TRUE);
18057ff178cdSJimmy Vetayases #endif
18067ff178cdSJimmy Vetayases
18077ff178cdSJimmy Vetayases lock_clear(&apic_ioapic_lock);
18087ff178cdSJimmy Vetayases intr_restore(iflag);
18097ff178cdSJimmy Vetayases
18107ff178cdSJimmy Vetayases lock_clear(&apic_defer_reprogram_lock);
18117ff178cdSJimmy Vetayases }
18127ff178cdSJimmy Vetayases
18137ff178cdSJimmy Vetayases static void
apic_ioapic_wait_pending_clear(int ioapic_ix,int intin_no)18147ff178cdSJimmy Vetayases apic_ioapic_wait_pending_clear(int ioapic_ix, int intin_no)
18157ff178cdSJimmy Vetayases {
18167ff178cdSJimmy Vetayases int waited;
18177ff178cdSJimmy Vetayases
18187ff178cdSJimmy Vetayases /*
18197ff178cdSJimmy Vetayases * Wait for the delivery pending bit to clear.
18207ff178cdSJimmy Vetayases */
18217ff178cdSJimmy Vetayases if ((READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix, intin_no) &
18227ff178cdSJimmy Vetayases (AV_LEVEL|AV_PENDING)) == (AV_LEVEL|AV_PENDING)) {
18237ff178cdSJimmy Vetayases
18247ff178cdSJimmy Vetayases /*
18257ff178cdSJimmy Vetayases * If we're still waiting on the delivery of this interrupt,
18267ff178cdSJimmy Vetayases * continue to wait here until it is delivered (this should be
18277ff178cdSJimmy Vetayases * a very small amount of time, but include a timeout just in
18287ff178cdSJimmy Vetayases * case).
18297ff178cdSJimmy Vetayases */
18307ff178cdSJimmy Vetayases for (waited = 0; waited < apic_max_reps_clear_pending;
18317ff178cdSJimmy Vetayases waited++) {
18327ff178cdSJimmy Vetayases if ((READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
18337ff178cdSJimmy Vetayases intin_no) & AV_PENDING) == 0) {
18347ff178cdSJimmy Vetayases break;
18357ff178cdSJimmy Vetayases }
18367ff178cdSJimmy Vetayases }
18377ff178cdSJimmy Vetayases }
18387ff178cdSJimmy Vetayases }
18397ff178cdSJimmy Vetayases
18407ff178cdSJimmy Vetayases
18417ff178cdSJimmy Vetayases /*
18427ff178cdSJimmy Vetayases * Checks to see if the IOAPIC interrupt entry specified has its Remote IRR
18437ff178cdSJimmy Vetayases * bit set. Calls functions that modify the function that setlvlx points to,
18447ff178cdSJimmy Vetayases * so that the reprogramming can be retried very shortly.
18457ff178cdSJimmy Vetayases *
18467ff178cdSJimmy Vetayases * This function will mask the RDT entry if the interrupt is level-triggered.
18477ff178cdSJimmy Vetayases * (The caller is responsible for unmasking the RDT entry.)
18487ff178cdSJimmy Vetayases *
18497ff178cdSJimmy Vetayases * Returns non-zero if the caller should defer IOAPIC reprogramming.
18507ff178cdSJimmy Vetayases */
18517ff178cdSJimmy Vetayases static int
apic_check_stuck_interrupt(apic_irq_t * irq_ptr,int old_bind_cpu,int new_bind_cpu,int ioapic_ix,int intin_no,int which_irq,struct ioapic_reprogram_data * drep)18527ff178cdSJimmy Vetayases apic_check_stuck_interrupt(apic_irq_t *irq_ptr, int old_bind_cpu,
18537ff178cdSJimmy Vetayases int new_bind_cpu, int ioapic_ix, int intin_no, int which_irq,
18547ff178cdSJimmy Vetayases struct ioapic_reprogram_data *drep)
18557ff178cdSJimmy Vetayases {
18567ff178cdSJimmy Vetayases int32_t rdt_entry;
18577ff178cdSJimmy Vetayases int waited;
18587ff178cdSJimmy Vetayases int reps = 0;
18597ff178cdSJimmy Vetayases
18607ff178cdSJimmy Vetayases /*
18617ff178cdSJimmy Vetayases * Wait for the delivery pending bit to clear.
18627ff178cdSJimmy Vetayases */
18637ff178cdSJimmy Vetayases do {
18647ff178cdSJimmy Vetayases ++reps;
18657ff178cdSJimmy Vetayases
18667ff178cdSJimmy Vetayases apic_ioapic_wait_pending_clear(ioapic_ix, intin_no);
18677ff178cdSJimmy Vetayases
18687ff178cdSJimmy Vetayases /*
18697ff178cdSJimmy Vetayases * Mask the RDT entry, but only if it's a level-triggered
18707ff178cdSJimmy Vetayases * interrupt
18717ff178cdSJimmy Vetayases */
18727ff178cdSJimmy Vetayases rdt_entry = READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
18737ff178cdSJimmy Vetayases intin_no);
18747ff178cdSJimmy Vetayases if ((rdt_entry & (AV_LEVEL|AV_MASK)) == AV_LEVEL) {
18757ff178cdSJimmy Vetayases
18767ff178cdSJimmy Vetayases /* Mask it */
18777ff178cdSJimmy Vetayases WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix, intin_no,
18787ff178cdSJimmy Vetayases AV_MASK | rdt_entry);
18797ff178cdSJimmy Vetayases }
18807ff178cdSJimmy Vetayases
18817ff178cdSJimmy Vetayases if ((rdt_entry & AV_LEVEL) == AV_LEVEL) {
18827ff178cdSJimmy Vetayases /*
18837ff178cdSJimmy Vetayases * If there was a race and an interrupt was injected
18847ff178cdSJimmy Vetayases * just before we masked, check for that case here.
18857ff178cdSJimmy Vetayases * Then, unmask the RDT entry and try again. If we're
18867ff178cdSJimmy Vetayases * on our last try, don't unmask (because we want the
18877ff178cdSJimmy Vetayases * RDT entry to remain masked for the rest of the
18887ff178cdSJimmy Vetayases * function).
18897ff178cdSJimmy Vetayases */
18907ff178cdSJimmy Vetayases rdt_entry = READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
18917ff178cdSJimmy Vetayases intin_no);
18927ff178cdSJimmy Vetayases if ((rdt_entry & AV_PENDING) &&
18937ff178cdSJimmy Vetayases (reps < apic_max_reps_clear_pending)) {
18947ff178cdSJimmy Vetayases /* Unmask it */
18957ff178cdSJimmy Vetayases WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
18967ff178cdSJimmy Vetayases intin_no, rdt_entry & ~AV_MASK);
18977ff178cdSJimmy Vetayases }
18987ff178cdSJimmy Vetayases }
18997ff178cdSJimmy Vetayases
19007ff178cdSJimmy Vetayases } while ((rdt_entry & AV_PENDING) &&
19017ff178cdSJimmy Vetayases (reps < apic_max_reps_clear_pending));
19027ff178cdSJimmy Vetayases
19037ff178cdSJimmy Vetayases #ifdef DEBUG
190415c07adcSJohn Levon if (rdt_entry & AV_PENDING)
190515c07adcSJohn Levon apic_intr_deliver_timeouts++;
19067ff178cdSJimmy Vetayases #endif
19077ff178cdSJimmy Vetayases
19087ff178cdSJimmy Vetayases /*
19097ff178cdSJimmy Vetayases * If the remote IRR bit is set, then the interrupt has been sent
19107ff178cdSJimmy Vetayases * to a CPU for processing. We have no choice but to wait for
19117ff178cdSJimmy Vetayases * that CPU to process the interrupt, at which point the remote IRR
19127ff178cdSJimmy Vetayases * bit will be cleared.
19137ff178cdSJimmy Vetayases */
19147ff178cdSJimmy Vetayases if ((READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix, intin_no) &
19157ff178cdSJimmy Vetayases (AV_LEVEL|AV_REMOTE_IRR)) == (AV_LEVEL|AV_REMOTE_IRR)) {
19167ff178cdSJimmy Vetayases
19177ff178cdSJimmy Vetayases /*
19187ff178cdSJimmy Vetayases * If the CPU that this RDT is bound to is NOT the current
19197ff178cdSJimmy Vetayases * CPU, wait until that CPU handles the interrupt and ACKs
19207ff178cdSJimmy Vetayases * it. If this interrupt is not bound to any CPU (that is,
19217ff178cdSJimmy Vetayases * if it's bound to the logical destination of "anyone"), it
19227ff178cdSJimmy Vetayases * may have been delivered to the current CPU so handle that
19237ff178cdSJimmy Vetayases * case by deferring the reprogramming (below).
19247ff178cdSJimmy Vetayases */
19257ff178cdSJimmy Vetayases if ((old_bind_cpu != IRQ_UNBOUND) &&
19267ff178cdSJimmy Vetayases (old_bind_cpu != IRQ_UNINIT) &&
19277ff178cdSJimmy Vetayases (old_bind_cpu != psm_get_cpu_id())) {
19287ff178cdSJimmy Vetayases for (waited = 0; waited < apic_max_reps_clear_pending;
19297ff178cdSJimmy Vetayases waited++) {
19307ff178cdSJimmy Vetayases if ((READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
19317ff178cdSJimmy Vetayases intin_no) & AV_REMOTE_IRR) == 0) {
19327ff178cdSJimmy Vetayases
19337ff178cdSJimmy Vetayases delete_defer_repro_ent(which_irq);
19347ff178cdSJimmy Vetayases
19357ff178cdSJimmy Vetayases /* Remote IRR has cleared! */
19367ff178cdSJimmy Vetayases return (0);
19377ff178cdSJimmy Vetayases }
19387ff178cdSJimmy Vetayases }
19397ff178cdSJimmy Vetayases }
19407ff178cdSJimmy Vetayases
19417ff178cdSJimmy Vetayases /*
19427ff178cdSJimmy Vetayases * If we waited and the Remote IRR bit is still not cleared,
19437ff178cdSJimmy Vetayases * AND if we've invoked the timeout APIC_REPROGRAM_MAX_TIMEOUTS
19447ff178cdSJimmy Vetayases * times for this interrupt, try the last-ditch workaround:
19457ff178cdSJimmy Vetayases */
19467ff178cdSJimmy Vetayases if (drep && drep->tries >= APIC_REPROGRAM_MAX_TRIES) {
19477ff178cdSJimmy Vetayases
19487ff178cdSJimmy Vetayases apic_last_ditch_clear_remote_irr(ioapic_ix, intin_no);
19497ff178cdSJimmy Vetayases
19507ff178cdSJimmy Vetayases /* Mark this one as reprogrammed: */
19517ff178cdSJimmy Vetayases delete_defer_repro_ent(which_irq);
19527ff178cdSJimmy Vetayases
19537ff178cdSJimmy Vetayases return (0);
19547ff178cdSJimmy Vetayases } else {
19557ff178cdSJimmy Vetayases #ifdef DEBUG
19567ff178cdSJimmy Vetayases apic_intr_deferrals++;
19577ff178cdSJimmy Vetayases #endif
19587ff178cdSJimmy Vetayases
19597ff178cdSJimmy Vetayases /*
19607ff178cdSJimmy Vetayases * If waiting for the Remote IRR bit (above) didn't
19617ff178cdSJimmy Vetayases * allow it to clear, defer the reprogramming.
19627ff178cdSJimmy Vetayases * Add a new deferred-programming entry if the
19637ff178cdSJimmy Vetayases * caller passed a NULL one (and update the existing one
19647ff178cdSJimmy Vetayases * in case anything changed).
19657ff178cdSJimmy Vetayases */
19667ff178cdSJimmy Vetayases add_defer_repro_ent(irq_ptr, which_irq, new_bind_cpu);
19677ff178cdSJimmy Vetayases if (drep)
19687ff178cdSJimmy Vetayases drep->tries++;
19697ff178cdSJimmy Vetayases
19707ff178cdSJimmy Vetayases /* Inform caller to defer IOAPIC programming: */
19717ff178cdSJimmy Vetayases return (1);
19727ff178cdSJimmy Vetayases }
19737ff178cdSJimmy Vetayases
19747ff178cdSJimmy Vetayases }
19757ff178cdSJimmy Vetayases
19767ff178cdSJimmy Vetayases /* Remote IRR is clear */
19777ff178cdSJimmy Vetayases delete_defer_repro_ent(which_irq);
19787ff178cdSJimmy Vetayases
19797ff178cdSJimmy Vetayases return (0);
19807ff178cdSJimmy Vetayases }
19817ff178cdSJimmy Vetayases
19827ff178cdSJimmy Vetayases /*
19837ff178cdSJimmy Vetayases * Called to migrate all interrupts at an irq to another cpu.
19847ff178cdSJimmy Vetayases * Must be called with interrupts disabled and apic_ioapic_lock held
19857ff178cdSJimmy Vetayases */
19867ff178cdSJimmy Vetayases int
apic_rebind_all(apic_irq_t * irq_ptr,int bind_cpu)19877ff178cdSJimmy Vetayases apic_rebind_all(apic_irq_t *irq_ptr, int bind_cpu)
19887ff178cdSJimmy Vetayases {
19897ff178cdSJimmy Vetayases apic_irq_t *irqptr = irq_ptr;
19907ff178cdSJimmy Vetayases int retval = 0;
19917ff178cdSJimmy Vetayases
19927ff178cdSJimmy Vetayases while (irqptr) {
19937ff178cdSJimmy Vetayases if (irqptr->airq_temp_cpu != IRQ_UNINIT)
19947ff178cdSJimmy Vetayases retval |= apic_rebind(irqptr, bind_cpu, NULL);
19957ff178cdSJimmy Vetayases irqptr = irqptr->airq_next;
19967ff178cdSJimmy Vetayases }
19977ff178cdSJimmy Vetayases
19987ff178cdSJimmy Vetayases return (retval);
19997ff178cdSJimmy Vetayases }
20007ff178cdSJimmy Vetayases
20017ff178cdSJimmy Vetayases /*
20027ff178cdSJimmy Vetayases * apic_intr_redistribute does all the messy computations for identifying
20037ff178cdSJimmy Vetayases * which interrupt to move to which CPU. Currently we do just one interrupt
20047ff178cdSJimmy Vetayases * at a time. This reduces the time we spent doing all this within clock
20057ff178cdSJimmy Vetayases * interrupt. When it is done in idle, we could do more than 1.
20067ff178cdSJimmy Vetayases * First we find the most busy and the most free CPU (time in ISR only)
20077ff178cdSJimmy Vetayases * skipping those CPUs that has been identified as being ineligible (cpu_skip)
20087ff178cdSJimmy Vetayases * Then we look for IRQs which are closest to the difference between the
20097ff178cdSJimmy Vetayases * most busy CPU and the average ISR load. We try to find one whose load
20107ff178cdSJimmy Vetayases * is less than difference.If none exists, then we chose one larger than the
20117ff178cdSJimmy Vetayases * difference, provided it does not make the most idle CPU worse than the
20127ff178cdSJimmy Vetayases * most busy one. In the end, we clear all the busy fields for CPUs. For
20137ff178cdSJimmy Vetayases * IRQs, they are cleared as they are scanned.
20147ff178cdSJimmy Vetayases */
20157ff178cdSJimmy Vetayases void
apic_intr_redistribute(void)20167ff178cdSJimmy Vetayases apic_intr_redistribute(void)
20177ff178cdSJimmy Vetayases {
20187ff178cdSJimmy Vetayases int busiest_cpu, most_free_cpu;
20197ff178cdSJimmy Vetayases int cpu_free, cpu_busy, max_busy, min_busy;
20207ff178cdSJimmy Vetayases int min_free, diff;
20217ff178cdSJimmy Vetayases int average_busy, cpus_online;
20227ff178cdSJimmy Vetayases int i, busy;
20237ff178cdSJimmy Vetayases ulong_t iflag;
20247ff178cdSJimmy Vetayases apic_cpus_info_t *cpu_infop;
20257ff178cdSJimmy Vetayases apic_irq_t *min_busy_irq = NULL;
20267ff178cdSJimmy Vetayases apic_irq_t *max_busy_irq = NULL;
20277ff178cdSJimmy Vetayases
20287ff178cdSJimmy Vetayases busiest_cpu = most_free_cpu = -1;
20297ff178cdSJimmy Vetayases cpu_free = cpu_busy = max_busy = average_busy = 0;
20307ff178cdSJimmy Vetayases min_free = apic_sample_factor_redistribution;
20317ff178cdSJimmy Vetayases cpus_online = 0;
20327ff178cdSJimmy Vetayases /*
20337ff178cdSJimmy Vetayases * Below we will check for CPU_INTR_ENABLE, bound, temp_bound, temp_cpu
20347ff178cdSJimmy Vetayases * without ioapic_lock. That is OK as we are just doing statistical
20357ff178cdSJimmy Vetayases * sampling anyway and any inaccuracy now will get corrected next time
20367ff178cdSJimmy Vetayases * The call to rebind which actually changes things will make sure
20377ff178cdSJimmy Vetayases * we are consistent.
20387ff178cdSJimmy Vetayases */
20397ff178cdSJimmy Vetayases for (i = 0; i < apic_nproc; i++) {
20407ff178cdSJimmy Vetayases if (apic_cpu_in_range(i) &&
20417ff178cdSJimmy Vetayases !(apic_redist_cpu_skip & (1 << i)) &&
20427ff178cdSJimmy Vetayases (apic_cpus[i].aci_status & APIC_CPU_INTR_ENABLE)) {
20437ff178cdSJimmy Vetayases
20447ff178cdSJimmy Vetayases cpu_infop = &apic_cpus[i];
20457ff178cdSJimmy Vetayases /*
20467ff178cdSJimmy Vetayases * If no unbound interrupts or only 1 total on this
20477ff178cdSJimmy Vetayases * CPU, skip
20487ff178cdSJimmy Vetayases */
20497ff178cdSJimmy Vetayases if (!cpu_infop->aci_temp_bound ||
20507ff178cdSJimmy Vetayases (cpu_infop->aci_bound + cpu_infop->aci_temp_bound)
20517ff178cdSJimmy Vetayases == 1) {
20527ff178cdSJimmy Vetayases apic_redist_cpu_skip |= 1 << i;
20537ff178cdSJimmy Vetayases continue;
20547ff178cdSJimmy Vetayases }
20557ff178cdSJimmy Vetayases
20567ff178cdSJimmy Vetayases busy = cpu_infop->aci_busy;
20577ff178cdSJimmy Vetayases average_busy += busy;
20587ff178cdSJimmy Vetayases cpus_online++;
20597ff178cdSJimmy Vetayases if (max_busy < busy) {
20607ff178cdSJimmy Vetayases max_busy = busy;
20617ff178cdSJimmy Vetayases busiest_cpu = i;
20627ff178cdSJimmy Vetayases }
20637ff178cdSJimmy Vetayases if (min_free > busy) {
20647ff178cdSJimmy Vetayases min_free = busy;
20657ff178cdSJimmy Vetayases most_free_cpu = i;
20667ff178cdSJimmy Vetayases }
20677ff178cdSJimmy Vetayases if (busy > apic_int_busy_mark) {
20687ff178cdSJimmy Vetayases cpu_busy |= 1 << i;
20697ff178cdSJimmy Vetayases } else {
20707ff178cdSJimmy Vetayases if (busy < apic_int_free_mark)
20717ff178cdSJimmy Vetayases cpu_free |= 1 << i;
20727ff178cdSJimmy Vetayases }
20737ff178cdSJimmy Vetayases }
20747ff178cdSJimmy Vetayases }
20757ff178cdSJimmy Vetayases if ((cpu_busy && cpu_free) ||
20767ff178cdSJimmy Vetayases (max_busy >= (min_free + apic_diff_for_redistribution))) {
20777ff178cdSJimmy Vetayases
20787ff178cdSJimmy Vetayases apic_num_imbalance++;
20797ff178cdSJimmy Vetayases #ifdef DEBUG
20807ff178cdSJimmy Vetayases if (apic_verbose & APIC_VERBOSE_IOAPIC_FLAG) {
20817ff178cdSJimmy Vetayases prom_printf(
20827ff178cdSJimmy Vetayases "redistribute busy=%x free=%x max=%x min=%x",
20837ff178cdSJimmy Vetayases cpu_busy, cpu_free, max_busy, min_free);
20847ff178cdSJimmy Vetayases }
20857ff178cdSJimmy Vetayases #endif /* DEBUG */
20867ff178cdSJimmy Vetayases
20877ff178cdSJimmy Vetayases
20887ff178cdSJimmy Vetayases average_busy /= cpus_online;
20897ff178cdSJimmy Vetayases
20907ff178cdSJimmy Vetayases diff = max_busy - average_busy;
20917ff178cdSJimmy Vetayases min_busy = max_busy; /* start with the max possible value */
20927ff178cdSJimmy Vetayases max_busy = 0;
20937ff178cdSJimmy Vetayases min_busy_irq = max_busy_irq = NULL;
20947ff178cdSJimmy Vetayases i = apic_min_device_irq;
20957ff178cdSJimmy Vetayases for (; i <= apic_max_device_irq; i++) {
20967ff178cdSJimmy Vetayases apic_irq_t *irq_ptr;
20977ff178cdSJimmy Vetayases /* Change to linked list per CPU ? */
20987ff178cdSJimmy Vetayases if ((irq_ptr = apic_irq_table[i]) == NULL)
20997ff178cdSJimmy Vetayases continue;
21007ff178cdSJimmy Vetayases /* Check for irq_busy & decide which one to move */
21017ff178cdSJimmy Vetayases /* Also zero them for next round */
21027ff178cdSJimmy Vetayases if ((irq_ptr->airq_temp_cpu == busiest_cpu) &&
21037ff178cdSJimmy Vetayases irq_ptr->airq_busy) {
21047ff178cdSJimmy Vetayases if (irq_ptr->airq_busy < diff) {
21057ff178cdSJimmy Vetayases /*
21067ff178cdSJimmy Vetayases * Check for least busy CPU,
21077ff178cdSJimmy Vetayases * best fit or what ?
21087ff178cdSJimmy Vetayases */
21097ff178cdSJimmy Vetayases if (max_busy < irq_ptr->airq_busy) {
21107ff178cdSJimmy Vetayases /*
21117ff178cdSJimmy Vetayases * Most busy within the
21127ff178cdSJimmy Vetayases * required differential
21137ff178cdSJimmy Vetayases */
21147ff178cdSJimmy Vetayases max_busy = irq_ptr->airq_busy;
21157ff178cdSJimmy Vetayases max_busy_irq = irq_ptr;
21167ff178cdSJimmy Vetayases }
21177ff178cdSJimmy Vetayases } else {
21187ff178cdSJimmy Vetayases if (min_busy > irq_ptr->airq_busy) {
21197ff178cdSJimmy Vetayases /*
21207ff178cdSJimmy Vetayases * least busy, but more than
21217ff178cdSJimmy Vetayases * the reqd diff
21227ff178cdSJimmy Vetayases */
21237ff178cdSJimmy Vetayases if (min_busy <
21247ff178cdSJimmy Vetayases (diff + average_busy -
21257ff178cdSJimmy Vetayases min_free)) {
21267ff178cdSJimmy Vetayases /*
21277ff178cdSJimmy Vetayases * Making sure new cpu
21287ff178cdSJimmy Vetayases * will not end up
21297ff178cdSJimmy Vetayases * worse
21307ff178cdSJimmy Vetayases */
21317ff178cdSJimmy Vetayases min_busy =
21327ff178cdSJimmy Vetayases irq_ptr->airq_busy;
21337ff178cdSJimmy Vetayases
21347ff178cdSJimmy Vetayases min_busy_irq = irq_ptr;
21357ff178cdSJimmy Vetayases }
21367ff178cdSJimmy Vetayases }
21377ff178cdSJimmy Vetayases }
21387ff178cdSJimmy Vetayases }
21397ff178cdSJimmy Vetayases irq_ptr->airq_busy = 0;
21407ff178cdSJimmy Vetayases }
21417ff178cdSJimmy Vetayases
21427ff178cdSJimmy Vetayases if (max_busy_irq != NULL) {
21437ff178cdSJimmy Vetayases #ifdef DEBUG
21447ff178cdSJimmy Vetayases if (apic_verbose & APIC_VERBOSE_IOAPIC_FLAG) {
21457ff178cdSJimmy Vetayases prom_printf("rebinding %x to %x",
21467ff178cdSJimmy Vetayases max_busy_irq->airq_vector, most_free_cpu);
21477ff178cdSJimmy Vetayases }
21487ff178cdSJimmy Vetayases #endif /* DEBUG */
21497ff178cdSJimmy Vetayases iflag = intr_clear();
21507ff178cdSJimmy Vetayases if (lock_try(&apic_ioapic_lock)) {
21517ff178cdSJimmy Vetayases if (apic_rebind_all(max_busy_irq,
21527ff178cdSJimmy Vetayases most_free_cpu) == 0) {
21537ff178cdSJimmy Vetayases /* Make change permenant */
21547ff178cdSJimmy Vetayases max_busy_irq->airq_cpu =
21557ff178cdSJimmy Vetayases (uint32_t)most_free_cpu;
21567ff178cdSJimmy Vetayases }
21577ff178cdSJimmy Vetayases lock_clear(&apic_ioapic_lock);
21587ff178cdSJimmy Vetayases }
21597ff178cdSJimmy Vetayases intr_restore(iflag);
21607ff178cdSJimmy Vetayases
21617ff178cdSJimmy Vetayases } else if (min_busy_irq != NULL) {
21627ff178cdSJimmy Vetayases #ifdef DEBUG
21637ff178cdSJimmy Vetayases if (apic_verbose & APIC_VERBOSE_IOAPIC_FLAG) {
21647ff178cdSJimmy Vetayases prom_printf("rebinding %x to %x",
21657ff178cdSJimmy Vetayases min_busy_irq->airq_vector, most_free_cpu);
21667ff178cdSJimmy Vetayases }
21677ff178cdSJimmy Vetayases #endif /* DEBUG */
21687ff178cdSJimmy Vetayases
21697ff178cdSJimmy Vetayases iflag = intr_clear();
21707ff178cdSJimmy Vetayases if (lock_try(&apic_ioapic_lock)) {
21717ff178cdSJimmy Vetayases if (apic_rebind_all(min_busy_irq,
21727ff178cdSJimmy Vetayases most_free_cpu) == 0) {
21737ff178cdSJimmy Vetayases /* Make change permenant */
21747ff178cdSJimmy Vetayases min_busy_irq->airq_cpu =
21757ff178cdSJimmy Vetayases (uint32_t)most_free_cpu;
21767ff178cdSJimmy Vetayases }
21777ff178cdSJimmy Vetayases lock_clear(&apic_ioapic_lock);
21787ff178cdSJimmy Vetayases }
21797ff178cdSJimmy Vetayases intr_restore(iflag);
21807ff178cdSJimmy Vetayases
21817ff178cdSJimmy Vetayases } else {
21827ff178cdSJimmy Vetayases if (cpu_busy != (1 << busiest_cpu)) {
21837ff178cdSJimmy Vetayases apic_redist_cpu_skip |= 1 << busiest_cpu;
21847ff178cdSJimmy Vetayases /*
21857ff178cdSJimmy Vetayases * We leave cpu_skip set so that next time we
21867ff178cdSJimmy Vetayases * can choose another cpu
21877ff178cdSJimmy Vetayases */
21887ff178cdSJimmy Vetayases }
21897ff178cdSJimmy Vetayases }
21907ff178cdSJimmy Vetayases apic_num_rebind++;
21917ff178cdSJimmy Vetayases } else {
21927ff178cdSJimmy Vetayases /*
21937ff178cdSJimmy Vetayases * found nothing. Could be that we skipped over valid CPUs
21947ff178cdSJimmy Vetayases * or we have balanced everything. If we had a variable
21957ff178cdSJimmy Vetayases * ticks_for_redistribution, it could be increased here.
21967ff178cdSJimmy Vetayases * apic_int_busy, int_free etc would also need to be
21977ff178cdSJimmy Vetayases * changed.
21987ff178cdSJimmy Vetayases */
21997ff178cdSJimmy Vetayases if (apic_redist_cpu_skip)
22007ff178cdSJimmy Vetayases apic_redist_cpu_skip = 0;
22017ff178cdSJimmy Vetayases }
22027ff178cdSJimmy Vetayases for (i = 0; i < apic_nproc; i++) {
22037ff178cdSJimmy Vetayases if (apic_cpu_in_range(i)) {
22047ff178cdSJimmy Vetayases apic_cpus[i].aci_busy = 0;
22057ff178cdSJimmy Vetayases }
22067ff178cdSJimmy Vetayases }
22077ff178cdSJimmy Vetayases }
22087ff178cdSJimmy Vetayases
22097ff178cdSJimmy Vetayases void
apic_cleanup_busy(void)22107ff178cdSJimmy Vetayases apic_cleanup_busy(void)
22117ff178cdSJimmy Vetayases {
22127ff178cdSJimmy Vetayases int i;
22137ff178cdSJimmy Vetayases apic_irq_t *irq_ptr;
22147ff178cdSJimmy Vetayases
22157ff178cdSJimmy Vetayases for (i = 0; i < apic_nproc; i++) {
22167ff178cdSJimmy Vetayases if (apic_cpu_in_range(i)) {
22177ff178cdSJimmy Vetayases apic_cpus[i].aci_busy = 0;
22187ff178cdSJimmy Vetayases }
22197ff178cdSJimmy Vetayases }
22207ff178cdSJimmy Vetayases
22217ff178cdSJimmy Vetayases for (i = apic_min_device_irq; i <= apic_max_device_irq; i++) {
22227ff178cdSJimmy Vetayases if ((irq_ptr = apic_irq_table[i]) != NULL)
22237ff178cdSJimmy Vetayases irq_ptr->airq_busy = 0;
22247ff178cdSJimmy Vetayases }
22257ff178cdSJimmy Vetayases }
2226