17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate * CDDL HEADER START
37c478bd9Sstevel@tonic-gate *
47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the
57c478bd9Sstevel@tonic-gate * Common Development and Distribution License, Version 1.0 only
67c478bd9Sstevel@tonic-gate * (the "License"). You may not use this file except in compliance
77c478bd9Sstevel@tonic-gate * with the License.
87c478bd9Sstevel@tonic-gate *
97c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
107c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing.
117c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions
127c478bd9Sstevel@tonic-gate * and limitations under the License.
137c478bd9Sstevel@tonic-gate *
147c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each
157c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
167c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the
177c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying
187c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner]
197c478bd9Sstevel@tonic-gate *
207c478bd9Sstevel@tonic-gate * CDDL HEADER END
217c478bd9Sstevel@tonic-gate */
227c478bd9Sstevel@tonic-gate /*
237c478bd9Sstevel@tonic-gate * Copyright 2005 Sun Microsystems, Inc. All rights reserved.
247c478bd9Sstevel@tonic-gate * Use is subject to license terms.
257c478bd9Sstevel@tonic-gate */
26*2a1fd0ffSPeter Tribble /*
27*2a1fd0ffSPeter Tribble * Copyright 2019 Peter Tribble.
28*2a1fd0ffSPeter Tribble */
297c478bd9Sstevel@tonic-gate
307c478bd9Sstevel@tonic-gate /*
317c478bd9Sstevel@tonic-gate * PCI Control Block object
327c478bd9Sstevel@tonic-gate */
337c478bd9Sstevel@tonic-gate #include <sys/types.h>
347c478bd9Sstevel@tonic-gate #include <sys/kmem.h>
357c478bd9Sstevel@tonic-gate #include <sys/systm.h> /* timeout() */
367c478bd9Sstevel@tonic-gate #include <sys/async.h>
377c478bd9Sstevel@tonic-gate #include <sys/sunddi.h>
387c478bd9Sstevel@tonic-gate #include <sys/ddi_impldefs.h>
397c478bd9Sstevel@tonic-gate #include <sys/pci/pci_obj.h>
407c478bd9Sstevel@tonic-gate #include <sys/machsystm.h>
417c478bd9Sstevel@tonic-gate
427c478bd9Sstevel@tonic-gate /*LINTLIBRARY*/
437c478bd9Sstevel@tonic-gate
447c478bd9Sstevel@tonic-gate void
cb_create(pci_t * pci_p)457c478bd9Sstevel@tonic-gate cb_create(pci_t *pci_p)
467c478bd9Sstevel@tonic-gate {
477c478bd9Sstevel@tonic-gate cb_t *cb_p = (cb_t *)kmem_zalloc(sizeof (cb_t), KM_SLEEP);
487c478bd9Sstevel@tonic-gate
497c478bd9Sstevel@tonic-gate mutex_init(&cb_p->cb_intr_lock, NULL, MUTEX_DRIVER, NULL);
507c478bd9Sstevel@tonic-gate pci_p->pci_cb_p = cb_p;
517c478bd9Sstevel@tonic-gate cb_p->cb_pci_cmn_p = pci_p->pci_common_p;
527c478bd9Sstevel@tonic-gate
537c478bd9Sstevel@tonic-gate pci_cb_setup(pci_p);
547c478bd9Sstevel@tonic-gate }
557c478bd9Sstevel@tonic-gate
567c478bd9Sstevel@tonic-gate void
cb_destroy(pci_t * pci_p)577c478bd9Sstevel@tonic-gate cb_destroy(pci_t *pci_p)
587c478bd9Sstevel@tonic-gate {
597c478bd9Sstevel@tonic-gate cb_t *cb_p = pci_p->pci_cb_p;
607c478bd9Sstevel@tonic-gate
617c478bd9Sstevel@tonic-gate intr_dist_rem(cb_intr_dist, cb_p);
627c478bd9Sstevel@tonic-gate pci_cb_teardown(pci_p);
637c478bd9Sstevel@tonic-gate pci_p->pci_cb_p = NULL;
647c478bd9Sstevel@tonic-gate mutex_destroy(&cb_p->cb_intr_lock);
657c478bd9Sstevel@tonic-gate kmem_free(cb_p, sizeof (cb_t));
667c478bd9Sstevel@tonic-gate }
677c478bd9Sstevel@tonic-gate
687c478bd9Sstevel@tonic-gate static void
cb_set_nintr_reg(cb_t * cb_p,ib_ino_t ino,uint64_t value)697c478bd9Sstevel@tonic-gate cb_set_nintr_reg(cb_t *cb_p, ib_ino_t ino, uint64_t value)
707c478bd9Sstevel@tonic-gate {
717c478bd9Sstevel@tonic-gate uint64_t pa = cb_ino_to_clr_pa(cb_p, ino);
727c478bd9Sstevel@tonic-gate
737c478bd9Sstevel@tonic-gate DEBUG3(DBG_CB|DBG_CONT, NULL,
747c478bd9Sstevel@tonic-gate "pci-%x cb_set_nintr_reg: ino=%x PA=%016llx\n",
757c478bd9Sstevel@tonic-gate cb_p->cb_pci_cmn_p->pci_common_id, ino, pa);
767c478bd9Sstevel@tonic-gate
777c478bd9Sstevel@tonic-gate stdphysio(pa, value);
787c478bd9Sstevel@tonic-gate (void) lddphysio(pa); /* flush the previous write */
797c478bd9Sstevel@tonic-gate }
807c478bd9Sstevel@tonic-gate
817c478bd9Sstevel@tonic-gate /*
827c478bd9Sstevel@tonic-gate * enable an internal interrupt source:
837c478bd9Sstevel@tonic-gate * if an interrupt is shared by both sides, record it in cb_inos[] and
847c478bd9Sstevel@tonic-gate * cb will own its distribution.
857c478bd9Sstevel@tonic-gate */
867c478bd9Sstevel@tonic-gate void
cb_enable_nintr(pci_t * pci_p,enum cb_nintr_index idx)877c478bd9Sstevel@tonic-gate cb_enable_nintr(pci_t *pci_p, enum cb_nintr_index idx)
887c478bd9Sstevel@tonic-gate {
897c478bd9Sstevel@tonic-gate cb_t *cb_p = pci_p->pci_cb_p;
907c478bd9Sstevel@tonic-gate ib_ino_t ino = IB_MONDO_TO_INO(pci_p->pci_inos[idx]);
917c478bd9Sstevel@tonic-gate ib_mondo_t mondo = CB_INO_TO_MONDO(cb_p, ino);
927c478bd9Sstevel@tonic-gate uint32_t cpu_id;
937c478bd9Sstevel@tonic-gate uint64_t reg, pa;
947c478bd9Sstevel@tonic-gate
957c478bd9Sstevel@tonic-gate ASSERT(idx < CBNINTR_MAX);
967c478bd9Sstevel@tonic-gate pa = cb_ino_to_map_pa(cb_p, ino);
977c478bd9Sstevel@tonic-gate
987c478bd9Sstevel@tonic-gate mutex_enter(&cb_p->cb_intr_lock);
997c478bd9Sstevel@tonic-gate cpu_id = intr_dist_cpuid();
1007c478bd9Sstevel@tonic-gate
1017c478bd9Sstevel@tonic-gate reg = ib_get_map_reg(mondo, cpu_id);
1027c478bd9Sstevel@tonic-gate stdphysio(pa, reg);
1037c478bd9Sstevel@tonic-gate
1047c478bd9Sstevel@tonic-gate ASSERT(cb_p->cb_inos[idx] == 0);
1057c478bd9Sstevel@tonic-gate cb_p->cb_inos[idx] = ino;
1067c478bd9Sstevel@tonic-gate
1077c478bd9Sstevel@tonic-gate cb_set_nintr_reg(cb_p, ino, COMMON_CLEAR_INTR_REG_IDLE);
1087c478bd9Sstevel@tonic-gate mutex_exit(&cb_p->cb_intr_lock);
1097c478bd9Sstevel@tonic-gate
1107c478bd9Sstevel@tonic-gate DEBUG3(DBG_CB|DBG_CONT, NULL,
1117c478bd9Sstevel@tonic-gate "pci-%x cb_enable_nintr: ino=%x cpu_id=%x\n",
1127c478bd9Sstevel@tonic-gate pci_p->pci_id, ino, cpu_id);
1137c478bd9Sstevel@tonic-gate DEBUG2(DBG_CB|DBG_CONT, NULL, "\tPA=%016llx data=%016llx\n", pa, reg);
1147c478bd9Sstevel@tonic-gate }
1157c478bd9Sstevel@tonic-gate
1167c478bd9Sstevel@tonic-gate static void
cb_disable_nintr_reg(cb_t * cb_p,ib_ino_t ino,int wait)1177c478bd9Sstevel@tonic-gate cb_disable_nintr_reg(cb_t *cb_p, ib_ino_t ino, int wait)
1187c478bd9Sstevel@tonic-gate {
1197c478bd9Sstevel@tonic-gate uint64_t tmp, map_reg_pa = cb_ino_to_map_pa(cb_p, ino);
1207c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cb_p->cb_intr_lock));
1217c478bd9Sstevel@tonic-gate
1227c478bd9Sstevel@tonic-gate /* mark interrupt invalid in mapping register */
1237c478bd9Sstevel@tonic-gate tmp = lddphysio(map_reg_pa) & ~COMMON_INTR_MAP_REG_VALID;
1247c478bd9Sstevel@tonic-gate stdphysio(map_reg_pa, tmp);
1257c478bd9Sstevel@tonic-gate (void) lddphysio(map_reg_pa); /* flush previous write */
1267c478bd9Sstevel@tonic-gate
1277c478bd9Sstevel@tonic-gate if (wait) {
1287c478bd9Sstevel@tonic-gate hrtime_t start_time;
1297c478bd9Sstevel@tonic-gate uint64_t state_reg_pa = cb_p->cb_obsta_pa;
1307c478bd9Sstevel@tonic-gate uint_t shift = (ino & 0x1f) << 1;
1317c478bd9Sstevel@tonic-gate
1327c478bd9Sstevel@tonic-gate /* busy wait if there is interrupt being processed */
1337c478bd9Sstevel@tonic-gate /* unless panic or timeout for interrupt pending is reached */
1347c478bd9Sstevel@tonic-gate start_time = gethrtime();
1357c478bd9Sstevel@tonic-gate while ((((lddphysio(state_reg_pa) >> shift) &
1367c478bd9Sstevel@tonic-gate COMMON_CLEAR_INTR_REG_MASK) ==
1377c478bd9Sstevel@tonic-gate COMMON_CLEAR_INTR_REG_PENDING) && !panicstr) {
1387c478bd9Sstevel@tonic-gate if (gethrtime() - start_time > pci_intrpend_timeout) {
1397c478bd9Sstevel@tonic-gate cmn_err(CE_WARN,
140f47a9c50Smathue "pci@%x cb_disable_nintr_reg(%lx,%x) timeout",
1417c478bd9Sstevel@tonic-gate cb_p->cb_pci_cmn_p->pci_common_id,
1427c478bd9Sstevel@tonic-gate map_reg_pa,
1437c478bd9Sstevel@tonic-gate CB_INO_TO_MONDO(cb_p, ino));
1447c478bd9Sstevel@tonic-gate break;
1457c478bd9Sstevel@tonic-gate }
1467c478bd9Sstevel@tonic-gate }
1477c478bd9Sstevel@tonic-gate }
1487c478bd9Sstevel@tonic-gate }
1497c478bd9Sstevel@tonic-gate
1507c478bd9Sstevel@tonic-gate void
cb_disable_nintr(cb_t * cb_p,enum cb_nintr_index idx,int wait)1517c478bd9Sstevel@tonic-gate cb_disable_nintr(cb_t *cb_p, enum cb_nintr_index idx, int wait)
1527c478bd9Sstevel@tonic-gate {
1537c478bd9Sstevel@tonic-gate ib_ino_t ino = cb_p->cb_inos[idx];
1547c478bd9Sstevel@tonic-gate ASSERT(idx < CBNINTR_MAX);
1557c478bd9Sstevel@tonic-gate ASSERT(ino);
1567c478bd9Sstevel@tonic-gate
1577c478bd9Sstevel@tonic-gate mutex_enter(&cb_p->cb_intr_lock);
1587c478bd9Sstevel@tonic-gate cb_disable_nintr_reg(cb_p, ino, wait);
1597c478bd9Sstevel@tonic-gate cb_set_nintr_reg(cb_p, ino, COMMON_CLEAR_INTR_REG_PENDING);
1607c478bd9Sstevel@tonic-gate cb_p->cb_inos[idx] = 0;
1617c478bd9Sstevel@tonic-gate mutex_exit(&cb_p->cb_intr_lock);
1627c478bd9Sstevel@tonic-gate }
1637c478bd9Sstevel@tonic-gate
1647c478bd9Sstevel@tonic-gate void
cb_clear_nintr(cb_t * cb_p,enum cb_nintr_index idx)1657c478bd9Sstevel@tonic-gate cb_clear_nintr(cb_t *cb_p, enum cb_nintr_index idx)
1667c478bd9Sstevel@tonic-gate {
1677c478bd9Sstevel@tonic-gate ib_ino_t ino = cb_p->cb_inos[idx];
1687c478bd9Sstevel@tonic-gate ASSERT(idx < CBNINTR_MAX);
1697c478bd9Sstevel@tonic-gate ASSERT(ino);
1707c478bd9Sstevel@tonic-gate cb_set_nintr_reg(cb_p, ino, COMMON_CLEAR_INTR_REG_IDLE);
1717c478bd9Sstevel@tonic-gate }
1727c478bd9Sstevel@tonic-gate
1737c478bd9Sstevel@tonic-gate void
cb_intr_dist(void * arg)1747c478bd9Sstevel@tonic-gate cb_intr_dist(void *arg)
1757c478bd9Sstevel@tonic-gate {
1767c478bd9Sstevel@tonic-gate int i;
1777c478bd9Sstevel@tonic-gate cb_t *cb_p = (cb_t *)arg;
1787c478bd9Sstevel@tonic-gate
1797c478bd9Sstevel@tonic-gate mutex_enter(&cb_p->cb_intr_lock);
1807c478bd9Sstevel@tonic-gate for (i = 0; i < cb_p->cb_no_of_inos; i++) {
1817c478bd9Sstevel@tonic-gate uint64_t mr_pa;
1827c478bd9Sstevel@tonic-gate volatile uint64_t imr;
1837c478bd9Sstevel@tonic-gate ib_mondo_t mondo;
1847c478bd9Sstevel@tonic-gate uint32_t cpu_id;
1857c478bd9Sstevel@tonic-gate
1867c478bd9Sstevel@tonic-gate ib_ino_t ino = cb_p->cb_inos[i];
1877c478bd9Sstevel@tonic-gate if (!ino) /* skip non-shared interrupts */
1887c478bd9Sstevel@tonic-gate continue;
1897c478bd9Sstevel@tonic-gate
1907c478bd9Sstevel@tonic-gate mr_pa = cb_ino_to_map_pa(cb_p, ino);
1917c478bd9Sstevel@tonic-gate imr = lddphysio(mr_pa);
1927c478bd9Sstevel@tonic-gate if (!IB_INO_INTR_ISON(imr))
1937c478bd9Sstevel@tonic-gate continue;
1947c478bd9Sstevel@tonic-gate
1957c478bd9Sstevel@tonic-gate mondo = CB_INO_TO_MONDO(cb_p, ino);
1967c478bd9Sstevel@tonic-gate cpu_id = intr_dist_cpuid();
1977c478bd9Sstevel@tonic-gate if (ib_map_reg_get_cpu(imr) == cpu_id)
1987c478bd9Sstevel@tonic-gate continue; /* same cpu target, no re-program */
1997c478bd9Sstevel@tonic-gate cb_disable_nintr_reg(cb_p, ino, IB_INTR_WAIT);
2007c478bd9Sstevel@tonic-gate stdphysio(mr_pa, ib_get_map_reg(mondo, cpu_id));
2017c478bd9Sstevel@tonic-gate (void) lddphysio(mr_pa); /* flush previous write */
2027c478bd9Sstevel@tonic-gate }
2037c478bd9Sstevel@tonic-gate mutex_exit(&cb_p->cb_intr_lock);
2047c478bd9Sstevel@tonic-gate }
2057c478bd9Sstevel@tonic-gate
2067c478bd9Sstevel@tonic-gate void
cb_suspend(cb_t * cb_p)2077c478bd9Sstevel@tonic-gate cb_suspend(cb_t *cb_p)
2087c478bd9Sstevel@tonic-gate {
2097c478bd9Sstevel@tonic-gate int i, inos = cb_p->cb_no_of_inos;
2107c478bd9Sstevel@tonic-gate ASSERT(!cb_p->cb_imr_save);
2117c478bd9Sstevel@tonic-gate cb_p->cb_imr_save = kmem_alloc(inos * sizeof (uint64_t), KM_SLEEP);
2127c478bd9Sstevel@tonic-gate
2137c478bd9Sstevel@tonic-gate /*
2147c478bd9Sstevel@tonic-gate * save the internal interrupts' mapping registers content
2157c478bd9Sstevel@tonic-gate *
2167c478bd9Sstevel@tonic-gate * The PBM IMR really doesn't need to be saved, as it is
2177c478bd9Sstevel@tonic-gate * different per side and is handled by pbm_suspend/resume.
2187c478bd9Sstevel@tonic-gate * But it complicates the logic.
2197c478bd9Sstevel@tonic-gate */
2207c478bd9Sstevel@tonic-gate for (i = 0; i < inos; i++) {
2217c478bd9Sstevel@tonic-gate uint64_t pa;
2227c478bd9Sstevel@tonic-gate ib_ino_t ino = cb_p->cb_inos[i];
2237c478bd9Sstevel@tonic-gate if (!ino)
2247c478bd9Sstevel@tonic-gate continue;
2257c478bd9Sstevel@tonic-gate pa = cb_ino_to_map_pa(cb_p, ino);
2267c478bd9Sstevel@tonic-gate cb_p->cb_imr_save[i] = lddphysio(pa);
2277c478bd9Sstevel@tonic-gate }
2287c478bd9Sstevel@tonic-gate }
2297c478bd9Sstevel@tonic-gate
2307c478bd9Sstevel@tonic-gate void
cb_resume(cb_t * cb_p)2317c478bd9Sstevel@tonic-gate cb_resume(cb_t *cb_p)
2327c478bd9Sstevel@tonic-gate {
2337c478bd9Sstevel@tonic-gate int i;
2347c478bd9Sstevel@tonic-gate for (i = 0; i < cb_p->cb_no_of_inos; i++) {
2357c478bd9Sstevel@tonic-gate uint64_t pa;
2367c478bd9Sstevel@tonic-gate ib_ino_t ino = cb_p->cb_inos[i];
2377c478bd9Sstevel@tonic-gate if (!ino)
2387c478bd9Sstevel@tonic-gate continue;
2397c478bd9Sstevel@tonic-gate pa = cb_ino_to_map_pa(cb_p, ino);
2407c478bd9Sstevel@tonic-gate cb_set_nintr_reg(cb_p, ino, COMMON_CLEAR_INTR_REG_IDLE);
2417c478bd9Sstevel@tonic-gate stdphysio(pa, cb_p->cb_imr_save[i]); /* restore IMR */
2427c478bd9Sstevel@tonic-gate }
2437c478bd9Sstevel@tonic-gate kmem_free(cb_p->cb_imr_save, cb_p->cb_no_of_inos * sizeof (uint64_t));
2447c478bd9Sstevel@tonic-gate cb_p->cb_imr_save = NULL;
2457c478bd9Sstevel@tonic-gate }
246