xref: /illumos-gate/usr/src/uts/sun4u/io/pci/pcipsy.c (revision a195726fa33097e56cf1c25c31feddb827e140f0)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License, Version 1.0 only
6  * (the "License").  You may not use this file except in compliance
7  * with the License.
8  *
9  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10  * or http://www.opensolaris.org/os/licensing.
11  * See the License for the specific language governing permissions
12  * and limitations under the License.
13  *
14  * When distributing Covered Code, include this CDDL HEADER in each
15  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16  * If applicable, add the following below this CDDL HEADER, with the
17  * fields enclosed by brackets "[]" replaced with your own identifying
18  * information: Portions Copyright [yyyy] [name of copyright owner]
19  *
20  * CDDL HEADER END
21  */
22 /*
23  * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 #pragma ident	"%Z%%M%	%I%	%E% SMI"
28 
29 /*
30  * Psycho+ specifics implementation:
31  *	interrupt mapping register
32  *	PBM configuration
33  *	ECC and PBM error handling
34  *	Iommu mapping handling
35  *	Streaming Cache flushing
36  */
37 
38 #include <sys/types.h>
39 #include <sys/kmem.h>
40 #include <sys/sysmacros.h>
41 #include <sys/async.h>
42 #include <sys/ivintr.h>
43 #include <sys/systm.h>
44 #include <sys/intreg.h>		/* UPAID_TO_IGN() */
45 #include <sys/intr.h>
46 #include <sys/sunddi.h>
47 #include <sys/sunndi.h>
48 #include <sys/machsystm.h>
49 #include <sys/fm/util.h>
50 #include <sys/ddi_impldefs.h>
51 #include <sys/iommutsb.h>
52 #include <sys/spl.h>
53 #include <sys/fm/util.h>
54 #include <sys/fm/protocol.h>
55 #include <sys/fm/io/pci.h>
56 #include <sys/fm/io/sun4upci.h>
57 #include <sys/pci/pci_obj.h>
58 #include <sys/pci/pcipsy.h>
59 
60 #ifdef _STARFIRE
61 #include <sys/starfire.h>
62 #endif /* _STARFIRE */
63 
64 static uint32_t pci_identity_init(pci_t *pci_p);
65 static int pci_intr_setup(pci_t *pci_p);
66 static void pci_pbm_errstate_get(pci_t *pci_p, pbm_errstate_t *pbm_err_p);
67 
68 static pci_ksinfo_t	*pci_name_kstat;
69 
70 /*LINTLIBRARY*/
71 /* called by pci_attach() DDI_ATTACH to initialize pci objects */
72 int
73 pci_obj_setup(pci_t *pci_p)
74 {
75 	pci_common_t *cmn_p;
76 	int ret;
77 
78 	mutex_enter(&pci_global_mutex);
79 	cmn_p = get_pci_common_soft_state(pci_p->pci_id);
80 	if (cmn_p == NULL) {
81 		uint_t id = pci_p->pci_id;
82 		if (alloc_pci_common_soft_state(id) != DDI_SUCCESS) {
83 			mutex_exit(&pci_global_mutex);
84 			return (DDI_FAILURE);
85 		}
86 		cmn_p = get_pci_common_soft_state(id);
87 		cmn_p->pci_common_id = id;
88 	}
89 
90 	ASSERT((pci_p->pci_side == 0) || (pci_p->pci_side == 1));
91 	if (cmn_p->pci_p[pci_p->pci_side]) {
92 		/* second side attach */
93 		pci_p->pci_side = PCI_OTHER_SIDE(pci_p->pci_side);
94 		ASSERT(cmn_p->pci_p[pci_p->pci_side] == NULL);
95 	}
96 
97 	cmn_p->pci_p[pci_p->pci_side] = pci_p;
98 	pci_p->pci_common_p = cmn_p;
99 
100 	if (cmn_p->pci_common_refcnt == 0) {
101 		/* Perform allocation first to avoid delicate unwinding. */
102 		if (pci_alloc_tsb(pci_p) != DDI_SUCCESS) {
103 			cmn_p->pci_p[pci_p->pci_side] = NULL;
104 			pci_p->pci_common_p = NULL;
105 			free_pci_common_soft_state(cmn_p->pci_common_id);
106 			mutex_exit(&pci_global_mutex);
107 			return (DDI_FAILURE);
108 		}
109 		cmn_p->pci_common_tsb_cookie = pci_p->pci_tsb_cookie;
110 		cmn_p->pci_chip_id = pci_identity_init(pci_p);
111 
112 		ib_create(pci_p);
113 		cmn_p->pci_common_ib_p = pci_p->pci_ib_p;
114 
115 		cb_create(pci_p);
116 		cmn_p->pci_common_cb_p = pci_p->pci_cb_p;
117 
118 		iommu_create(pci_p);
119 		cmn_p->pci_common_iommu_p = pci_p->pci_iommu_p;
120 
121 		ecc_create(pci_p);
122 		cmn_p->pci_common_ecc_p = pci_p->pci_ecc_p;
123 	} else {
124 		ASSERT(cmn_p->pci_common_refcnt == 1);
125 
126 		pci_p->pci_tsb_cookie = cmn_p->pci_common_tsb_cookie;
127 		pci_p->pci_ib_p = cmn_p->pci_common_ib_p;
128 		pci_p->pci_cb_p = cmn_p->pci_common_cb_p;
129 		pci_p->pci_iommu_p = cmn_p->pci_common_iommu_p;
130 		pci_p->pci_ecc_p = cmn_p->pci_common_ecc_p;
131 	}
132 
133 	pbm_create(pci_p);
134 	sc_create(pci_p);
135 
136 	pci_fm_create(pci_p);
137 
138 	if ((ret = pci_intr_setup(pci_p)) != DDI_SUCCESS)
139 		goto done;
140 	if (CHIP_TYPE(pci_p) == PCI_CHIP_PSYCHO)
141 		pci_kstat_create(pci_p);
142 
143 	cmn_p->pci_common_attachcnt++;
144 	cmn_p->pci_common_refcnt++;
145 done:
146 	mutex_exit(&pci_global_mutex);
147 	if (ret != DDI_SUCCESS)
148 		cmn_err(CE_NOTE, "Interrupt register failure, returning 0x%x\n",
149 			ret);
150 	return (ret);
151 }
152 
153 /* called by pci_detach() DDI_DETACH to destroy pci objects */
154 void
155 pci_obj_destroy(pci_t *pci_p)
156 {
157 	pci_common_t *cmn_p;
158 
159 	mutex_enter(&pci_global_mutex);
160 
161 	cmn_p = pci_p->pci_common_p;
162 	cmn_p->pci_common_refcnt--;
163 	cmn_p->pci_common_attachcnt--;
164 
165 	pci_kstat_destroy(pci_p);
166 
167 	sc_destroy(pci_p);
168 	pbm_destroy(pci_p);
169 	pci_fm_destroy(pci_p);
170 
171 	if (cmn_p->pci_common_refcnt != 0) {
172 		cmn_p->pci_p[pci_p->pci_side] = NULL;
173 		mutex_exit(&pci_global_mutex);
174 		return;
175 	}
176 
177 	ecc_destroy(pci_p);
178 	iommu_destroy(pci_p);
179 	cb_destroy(pci_p);
180 	ib_destroy(pci_p);
181 
182 	free_pci_common_soft_state(cmn_p->pci_common_id);
183 	pci_intr_teardown(pci_p);
184 	mutex_exit(&pci_global_mutex);
185 }
186 
187 /* called by pci_attach() DDI_RESUME to (re)initialize pci objects */
188 void
189 pci_obj_resume(pci_t *pci_p)
190 {
191 	pci_common_t *cmn_p = pci_p->pci_common_p;
192 
193 	mutex_enter(&pci_global_mutex);
194 
195 	if (cmn_p->pci_common_attachcnt == 0) {
196 		ib_configure(pci_p->pci_ib_p);
197 		iommu_configure(pci_p->pci_iommu_p);
198 		ecc_configure(pci_p);
199 		ib_resume(pci_p->pci_ib_p);
200 	}
201 
202 	pbm_configure(pci_p->pci_pbm_p);
203 	sc_configure(pci_p->pci_sc_p);
204 
205 	if (cmn_p->pci_common_attachcnt == 0)
206 		cb_resume(pci_p->pci_cb_p);
207 
208 	pbm_resume(pci_p->pci_pbm_p);
209 
210 	cmn_p->pci_common_attachcnt++;
211 	mutex_exit(&pci_global_mutex);
212 }
213 
214 /* called by pci_detach() DDI_SUSPEND to suspend pci objects */
215 void
216 pci_obj_suspend(pci_t *pci_p)
217 {
218 	mutex_enter(&pci_global_mutex);
219 
220 	pbm_suspend(pci_p->pci_pbm_p);
221 	if (!--pci_p->pci_common_p->pci_common_attachcnt) {
222 		ib_suspend(pci_p->pci_ib_p);
223 		cb_suspend(pci_p->pci_cb_p);
224 	}
225 
226 	mutex_exit(&pci_global_mutex);
227 }
228 
229 static uint32_t javelin_prom_fix[] = {0xfff800, 0, 0, 0x3f};
230 static int
231 pci_intr_setup(pci_t *pci_p)
232 {
233 	extern char *platform;
234 	dev_info_t *dip = pci_p->pci_dip;
235 	pbm_t *pbm_p = pci_p->pci_pbm_p;
236 	cb_t *cb_p = pci_p->pci_cb_p;
237 	int i, no_of_intrs;
238 
239 	/*
240 	 * This is a hack to fix a broken imap entry in the javelin PROM.
241 	 * see bugid 4226603
242 	 */
243 	if (strcmp((const char *)&platform, "SUNW,Ultra-250") == 0)
244 		(void) ddi_prop_create(DDI_DEV_T_NONE, dip, DDI_PROP_CANSLEEP,
245 		    "interrupt-map-mask", (caddr_t)javelin_prom_fix,
246 		    sizeof (javelin_prom_fix));
247 
248 	/*
249 	 * Get the interrupts property.
250 	 */
251 	if (ddi_getlongprop(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
252 		"interrupts", (caddr_t)&pci_p->pci_inos,
253 		&pci_p->pci_inos_len) != DDI_SUCCESS)
254 		cmn_err(CE_PANIC, "%s%d: no interrupts property\n",
255 			ddi_driver_name(dip), ddi_get_instance(dip));
256 
257 	/*
258 	 * figure out number of interrupts in the "interrupts" property
259 	 * and convert them all into ino.
260 	 */
261 	i = ddi_getprop(DDI_DEV_T_ANY, dip, 0, "#interrupt-cells", 1);
262 	i = CELLS_1275_TO_BYTES(i);
263 	no_of_intrs = pci_p->pci_inos_len / i;
264 	for (i = 0; i < no_of_intrs; i++)
265 		pci_p->pci_inos[i] = IB_MONDO_TO_INO(pci_p->pci_inos[i]);
266 
267 	if (pci_p->pci_common_p->pci_common_refcnt == 0) {
268 		cb_p->cb_no_of_inos = no_of_intrs;
269 		if (i = cb_register_intr(pci_p))
270 			goto teardown;
271 		if (i = ecc_register_intr(pci_p))
272 			goto teardown;
273 
274 		intr_dist_add(cb_intr_dist, cb_p);
275 		cb_enable_intr(pci_p);
276 		ecc_enable_intr(pci_p);
277 	}
278 
279 	if (i = pbm_register_intr(pbm_p)) {
280 		if (pci_p->pci_common_p->pci_common_refcnt == 0)
281 			intr_dist_rem(cb_intr_dist, cb_p);
282 		goto teardown;
283 	}
284 	intr_dist_add(pbm_intr_dist, pbm_p);
285 	ib_intr_enable(pci_p, pci_p->pci_inos[CBNINTR_PBM]);
286 
287 	if (pci_p->pci_common_p->pci_common_refcnt == 0)
288 		intr_dist_add_weighted(ib_intr_dist_all, pci_p->pci_ib_p);
289 	return (DDI_SUCCESS);
290 teardown:
291 	pci_intr_teardown(pci_p);
292 	return (i);
293 }
294 
295 /*
296  * pci_fix_ranges - fixes the config space entry of the "ranges"
297  *	property on psycho+ platforms
298  */
299 void
300 pci_fix_ranges(pci_ranges_t *rng_p, int rng_entries)
301 {
302 	int i;
303 	for (i = 0; i < rng_entries; i++, rng_p++)
304 		if ((rng_p->child_high & PCI_REG_ADDR_M) == PCI_ADDR_CONFIG)
305 			rng_p->parent_low |= rng_p->child_high;
306 }
307 
308 /*
309  * map_pci_registers
310  *
311  * This function is called from the attach routine to map the registers
312  * accessed by this driver.
313  *
314  * used by: pci_attach()
315  *
316  * return value: DDI_FAILURE on failure
317  */
318 int
319 map_pci_registers(pci_t *pci_p, dev_info_t *dip)
320 {
321 	ddi_device_acc_attr_t attr;
322 
323 	attr.devacc_attr_version = DDI_DEVICE_ATTR_V0;
324 	attr.devacc_attr_dataorder = DDI_STRICTORDER_ACC;
325 
326 	attr.devacc_attr_endian_flags = DDI_NEVERSWAP_ACC;
327 	if (ddi_regs_map_setup(dip, 0, &pci_p->pci_address[0], 0, 0,
328 	    &attr, &pci_p->pci_ac[0]) != DDI_SUCCESS) {
329 		cmn_err(CE_WARN, "%s%d: unable to map reg entry 0\n",
330 			ddi_driver_name(dip), ddi_get_instance(dip));
331 		return (DDI_FAILURE);
332 	}
333 	/*
334 	 * if we don't have streaming buffer, then we don't have
335 	 * pci_address[2].
336 	 */
337 	if (pci_stream_buf_exists &&
338 	    ddi_regs_map_setup(dip, 2, &pci_p->pci_address[2], 0, 0,
339 	    &attr, &pci_p->pci_ac[2]) != DDI_SUCCESS) {
340 		cmn_err(CE_WARN, "%s%d: unable to map reg entry 2\n",
341 			ddi_driver_name(dip), ddi_get_instance(dip));
342 		ddi_regs_map_free(&pci_p->pci_ac[0]);
343 		return (DDI_FAILURE);
344 	}
345 
346 	/*
347 	 * The second register set contains the bridge's configuration
348 	 * header.  This header is at the very beginning of the bridge's
349 	 * configuration space.  This space has litte-endian byte order.
350 	 */
351 	attr.devacc_attr_endian_flags = DDI_STRUCTURE_LE_ACC;
352 	if (ddi_regs_map_setup(dip, 1, &pci_p->pci_address[1], 0,
353 	    PCI_CONF_HDR_SIZE, &attr, &pci_p->pci_ac[1]) != DDI_SUCCESS) {
354 
355 		cmn_err(CE_WARN, "%s%d: unable to map reg entry 1\n",
356 			ddi_driver_name(dip), ddi_get_instance(dip));
357 		ddi_regs_map_free(&pci_p->pci_ac[0]);
358 		if (pci_stream_buf_exists)
359 			ddi_regs_map_free(&pci_p->pci_ac[2]);
360 		return (DDI_FAILURE);
361 	}
362 	DEBUG3(DBG_ATTACH, dip, "address (%p,%p,%p)\n",
363 	    pci_p->pci_address[0], pci_p->pci_address[1],
364 	    pci_p->pci_address[2]);
365 
366 	return (DDI_SUCCESS);
367 }
368 
369 /*
370  * unmap_pci_registers:
371  *
372  * This routine unmap the registers mapped by map_pci_registers.
373  *
374  * used by: pci_detach()
375  *
376  * return value: none
377  */
378 void
379 unmap_pci_registers(pci_t *pci_p)
380 {
381 	ddi_regs_map_free(&pci_p->pci_ac[0]);
382 	ddi_regs_map_free(&pci_p->pci_ac[1]);
383 	if (pci_stream_buf_exists)
384 		ddi_regs_map_free(&pci_p->pci_ac[2]);
385 }
386 
387 /*
388  * These convenience wrappers relies on map_pci_registers() to setup
389  * pci_address[0-2] correctly at first.
390  */
391 /* The psycho+ reg base is at 1fe.0000.0000 */
392 static uintptr_t
393 get_reg_base(pci_t *pci_p)
394 {
395 	return ((uintptr_t)pci_p->pci_address[pci_stream_buf_exists ? 2 : 0]);
396 }
397 
398 /* The psycho+ config reg base is always the 2nd reg entry */
399 static uintptr_t
400 get_config_reg_base(pci_t *pci_p)
401 {
402 	return ((uintptr_t)(pci_p->pci_address[1]));
403 }
404 
405 uint64_t
406 ib_get_map_reg(ib_mondo_t mondo, uint32_t cpu_id)
407 {
408 	return ((mondo) | (cpu_id << COMMON_INTR_MAP_REG_TID_SHIFT) |
409 	    COMMON_INTR_MAP_REG_VALID);
410 
411 }
412 
413 uint32_t
414 ib_map_reg_get_cpu(volatile uint64_t reg)
415 {
416 	return ((reg & COMMON_INTR_MAP_REG_TID) >>
417 	    COMMON_INTR_MAP_REG_TID_SHIFT);
418 }
419 
420 uint64_t *
421 ib_intr_map_reg_addr(ib_t *ib_p, ib_ino_t ino)
422 {
423 	uint64_t *addr;
424 
425 	if (ino & 0x20)
426 		addr = (uint64_t *)(ib_p->ib_obio_intr_map_regs +
427 		    (((uint_t)ino & 0x1f) << 3));
428 	else
429 		addr = (uint64_t *)(ib_p->ib_slot_intr_map_regs +
430 		    (((uint_t)ino & 0x3c) << 1));
431 	return (addr);
432 }
433 
434 uint64_t *
435 ib_clear_intr_reg_addr(ib_t *ib_p, ib_ino_t ino)
436 {
437 	uint64_t *addr;
438 
439 	if (ino & 0x20)
440 		addr = (uint64_t *)(ib_p->ib_obio_clear_intr_regs +
441 		    (((uint_t)ino & 0x1f) << 3));
442 	else
443 		addr = (uint64_t *)(ib_p->ib_slot_clear_intr_regs +
444 		    (((uint_t)ino & 0x1f) << 3));
445 	return (addr);
446 }
447 
448 /*
449  * psycho have one mapping register per slot
450  */
451 void
452 ib_ino_map_reg_share(ib_t *ib_p, ib_ino_t ino, ib_ino_info_t *ino_p)
453 {
454 	if (!IB_IS_OBIO_INO(ino)) {
455 		ASSERT(ino_p->ino_slot_no < 8);
456 		ib_p->ib_map_reg_counters[ino_p->ino_slot_no]++;
457 	}
458 }
459 
460 /*
461  * return true if the ino shares mapping register with other interrupts
462  * of the same slot, or is still shared by other On-board devices.
463  */
464 int
465 ib_ino_map_reg_unshare(ib_t *ib_p, ib_ino_t ino, ib_ino_info_t *ino_p)
466 {
467 	ASSERT(IB_IS_OBIO_INO(ino) || ino_p->ino_slot_no < 8);
468 
469 	if (IB_IS_OBIO_INO(ino))
470 		return (ino_p->ino_ih_size);
471 	else
472 		return (--ib_p->ib_map_reg_counters[ino_p->ino_slot_no]);
473 }
474 
475 /*ARGSUSED*/
476 void
477 pci_pbm_intr_dist(pbm_t *pbm_p)
478 {
479 }
480 
481 uintptr_t
482 pci_ib_setup(ib_t *ib_p)
483 {
484 	pci_t *pci_p = ib_p->ib_pci_p;
485 	uintptr_t a = get_reg_base(pci_p);
486 
487 	ib_p->ib_ign = PCI_ID_TO_IGN(pci_p->pci_id);
488 	ib_p->ib_max_ino = PSYCHO_MAX_INO;
489 	ib_p->ib_slot_intr_map_regs = a + PSYCHO_IB_SLOT_INTR_MAP_REG_OFFSET;
490 	ib_p->ib_obio_intr_map_regs = a + PSYCHO_IB_OBIO_INTR_MAP_REG_OFFSET;
491 	ib_p->ib_obio_clear_intr_regs =
492 		a + PSYCHO_IB_OBIO_CLEAR_INTR_REG_OFFSET;
493 	return (a);
494 }
495 
496 uint32_t
497 pci_xlate_intr(dev_info_t *dip, dev_info_t *rdip, ib_t *ib_p, uint32_t intr)
498 {
499 	int32_t len;
500 	dev_info_t *cdip;
501 	pci_regspec_t *pci_rp;
502 	uint32_t bus, dev, phys_hi;
503 
504 	if ((intr > PCI_INTD) || (intr < PCI_INTA))
505 		goto done;
506 	if (ddi_prop_exists(DDI_DEV_T_ANY, rdip, NULL, "interrupt-map"))
507 		goto done;
508 	/*
509 	 * Hack for pre 1275 imap machines e.g. quark & tazmo
510 	 * We need to turn any PCI interrupts into ino interrupts.  machines
511 	 * supporting imap will have this done in the map.
512 	 */
513 	cdip = get_my_childs_dip(dip, rdip);
514 	if (ddi_getlongprop(DDI_DEV_T_ANY, cdip, DDI_PROP_DONTPASS, "reg",
515 		(caddr_t)&pci_rp, &len) != DDI_SUCCESS)
516 		return (0);
517 	phys_hi = pci_rp->pci_phys_hi;
518 	kmem_free(pci_rp, len);
519 
520 	bus = PCI_REG_BUS_G(phys_hi);
521 	dev = PCI_REG_DEV_G(phys_hi);
522 
523 	/*
524 	 * The ino for a given device id is derived as 0BSSNN where
525 	 *
526 	 *	B = 0 for bus A, 1 for bus B
527 	 *	SS = dev - 1 for bus A, dev - 2 for bus B
528 	 *	NN = 00 for INTA#, 01 for INTB#, 10 for INTC#, 11 for INTD#
529 	 *
530 	 * if pci bus number > 0x80, then devices are located on the A side(66)
531 	 */
532 	DEBUG3(DBG_IB, dip, "pci_xlate_intr: bus=%x, dev=%x, intr=%x\n",
533 		bus, dev, intr);
534 	intr--;
535 	intr |= (bus & 0x80) ? ((dev - 1) << 2) : (0x10 | ((dev - 2) << 2));
536 
537 	DEBUG1(DBG_IB, dip, "pci_xlate_intr: done ino=%x\n", intr);
538 done:
539 	return (IB_INO_TO_MONDO(ib_p, intr));
540 }
541 
542 /*
543  * Return the cpuid to to be used for an ino. Psycho has special slot-cpu
544  * constraints on cpu assignment:
545  *
546  * On multi-function pci cards, functions have separate devinfo nodes and
547  * interrupts. Some pci support hardware, such as the psycho/pcipsy chip,
548  * control interrupt-to-cpu binding on a per pci-slot basis instead of per
549  * function.  For hardware like this, if an interrupt for one function has
550  * already been directed to a particular cpu, we can't choose a different
551  * cpu for another function implemented in the same pci-slot - if we did
552  * we would be redirecting the first function too (which causes problems
553  * for consistent interrupt distribution).
554  *
555  * This function determines if there is already an established slot-oriented
556  * interrupt-to-cpu binding established, if there is then it returns that
557  * cpu.  Otherwise a new cpu is selected by intr_dist_cpuid().
558  *
559  * The devinfo node we are trying to associate a cpu with is
560  * ino_p->ino_ih_head->ih_dip.
561  */
562 uint32_t
563 pci_intr_dist_cpuid(ib_t *ib_p, ib_ino_info_t *ino_p)
564 {
565 	dev_info_t	*rdip = ino_p->ino_ih_head->ih_dip;
566 	dev_info_t	*prdip = ddi_get_parent(rdip);
567 	ib_ino_info_t	*sino_p;
568 	dev_info_t	*sdip;
569 	dev_info_t	*psdip;
570 	char		*buf1 = NULL, *buf2 = NULL;
571 	char		*s1, *s2, *s3;
572 	int		l2;
573 	int		cpu_id;
574 
575 	/* must be psycho driver parent (not ebus) */
576 	if (strcmp(ddi_driver_name(prdip), "pcipsy") != 0)
577 		goto newcpu;
578 
579 	/*
580 	 * From PCI 1275 binding: 2.2.1.3 Unit Address representation:
581 	 *   Since the "unit-number" is the address that appears in on Open
582 	 *   Firmware 'device path', it follows that only the DD and DD,FF
583 	 *   forms of the text representation can appear in a 'device path'.
584 	 *
585 	 * The rdip unit address is of the form "DD[,FF]".  Define two
586 	 * unit address strings that represent same-slot use: "DD" and "DD,".
587 	 * The first compare uses strcmp, the second uses strncmp.
588 	 */
589 	s1 = ddi_get_name_addr(rdip);
590 	if (s1 == NULL)
591 		goto newcpu;
592 
593 	buf1 = kmem_alloc(MAXNAMELEN, KM_SLEEP);	/* strcmp */
594 	buf2 = kmem_alloc(MAXNAMELEN, KM_SLEEP);	/* strncmp */
595 	s1 = strcpy(buf1, s1);
596 	s2 = strcpy(buf2, s1);
597 
598 	s1 = strrchr(s1, ',');
599 	if (s1) {
600 		*s1 = '\0';			/* have "DD,FF" */
601 		s1 = buf1;			/* search via strcmp "DD" */
602 
603 		s2 = strrchr(s2, ',');
604 		*(s2 + 1) = '\0';
605 		s2 = buf2;
606 		l2 = strlen(s2);		/* search via strncmp "DD," */
607 	} else {
608 		(void) strcat(s2, ",");		/* have "DD" */
609 		l2 = strlen(s2);		/* search via strncmp "DD," */
610 	}
611 
612 	/*
613 	 * Search the established ino list for devinfo nodes bound
614 	 * to an ino that matches one of the slot use strings.
615 	 */
616 	ASSERT(MUTEX_HELD(&ib_p->ib_ino_lst_mutex));
617 	for (sino_p = ib_p->ib_ino_lst; sino_p; sino_p = sino_p->ino_next) {
618 		/* skip self and non-established */
619 		if ((sino_p == ino_p) || (sino_p->ino_established == 0))
620 			continue;
621 
622 		/* skip non-siblings */
623 		sdip = sino_p->ino_ih_head->ih_dip;
624 		psdip = ddi_get_parent(sdip);
625 		if (psdip != prdip)
626 			continue;
627 
628 		/* must be psycho driver parent (not ebus) */
629 		if (strcmp(ddi_driver_name(psdip), "pcipsy") != 0)
630 			continue;
631 
632 		s3 = ddi_get_name_addr(sdip);
633 		if ((s1 && (strcmp(s1, s3) == 0)) ||
634 		    (strncmp(s2, s3, l2) == 0)) {
635 			extern int intr_dist_debug;
636 
637 			if (intr_dist_debug)
638 				cmn_err(CE_CONT, "intr_dist: "
639 				    "pcipsy`pci_intr_dist_cpuid "
640 				    "%s#%d %s: cpu %d established "
641 				    "by %s#%d %s\n", ddi_driver_name(rdip),
642 				    ddi_get_instance(rdip),
643 				    ddi_deviname(rdip, buf1), sino_p->ino_cpuid,
644 				    ddi_driver_name(sdip),
645 				    ddi_get_instance(sdip),
646 				    ddi_deviname(sdip, buf2));
647 			break;
648 		}
649 	}
650 
651 	/* If a slot use match is found then use established cpu */
652 	if (sino_p) {
653 		cpu_id = sino_p->ino_cpuid;	/* target established cpu */
654 		goto out;
655 	}
656 
657 newcpu:	cpu_id = intr_dist_cpuid();		/* target new cpu */
658 
659 out:	if (buf1)
660 		kmem_free(buf1, MAXNAMELEN);
661 	if (buf2)
662 		kmem_free(buf2, MAXNAMELEN);
663 	return (cpu_id);
664 }
665 
666 
667 /*ARGSUSED*/
668 uint_t
669 cb_thermal_intr(caddr_t a)
670 {
671 	cmn_err(CE_WARN, "pci: Thermal warning detected!\n");
672 	if (pci_thermal_intr_fatal) {
673 		do_shutdown();
674 
675 		/*
676 		 * In case do_shutdown() fails to halt the system.
677 		 */
678 		(void) timeout((void(*)(void *))power_down, NULL,
679 		    thermal_powerdown_delay * hz);
680 	}
681 	return (DDI_INTR_CLAIMED);
682 }
683 
684 void
685 pci_cb_teardown(pci_t *pci_p)
686 {
687 	cb_t	*cb_p = pci_p->pci_cb_p;
688 	uint32_t mondo;
689 
690 	if (pci_p->pci_thermal_interrupt != -1) {
691 		mondo = ((pci_p->pci_cb_p->cb_ign  << PCI_INO_BITS) |
692 		    pci_p->pci_inos[CBNINTR_THERMAL]);
693 		mondo = CB_MONDO_TO_XMONDO(pci_p->pci_cb_p, mondo);
694 
695 		cb_disable_nintr(cb_p, CBNINTR_THERMAL, IB_INTR_WAIT);
696 		rem_ivintr(mondo, NULL);
697 	}
698 #ifdef _STARFIRE
699 	pc_ittrans_uninit(cb_p->cb_ittrans_cookie);
700 #endif /* _STARFIRE */
701 }
702 
703 int
704 cb_register_intr(pci_t *pci_p)
705 {
706 	uint32_t mondo;
707 
708 	if (pci_p->pci_thermal_interrupt == -1)
709 		return (DDI_SUCCESS);
710 
711 	mondo = ((pci_p->pci_cb_p->cb_ign << PCI_INO_BITS) |
712 	    pci_p->pci_inos[CBNINTR_THERMAL]);
713 	mondo = CB_MONDO_TO_XMONDO(pci_p->pci_cb_p, mondo);
714 
715 	VERIFY(add_ivintr(mondo, pci_pil[CBNINTR_THERMAL],
716 	    cb_thermal_intr, (caddr_t)pci_p->pci_cb_p, NULL) == 0);
717 
718 	return (PCI_ATTACH_RETCODE(PCI_CB_OBJ, PCI_OBJ_INTR_ADD, DDI_SUCCESS));
719 }
720 
721 void
722 cb_enable_intr(pci_t *pci_p)
723 {
724 	if (pci_p->pci_thermal_interrupt != -1)
725 		cb_enable_nintr(pci_p, CBNINTR_THERMAL);
726 }
727 
728 uint64_t
729 cb_ino_to_map_pa(cb_t *cb_p, ib_ino_t ino)
730 {
731 	return (cb_p->cb_map_pa + ((ino & 0x1f) << 3));
732 }
733 
734 uint64_t
735 cb_ino_to_clr_pa(cb_t *cb_p, ib_ino_t ino)
736 {
737 	return (cb_p->cb_clr_pa + ((ino & 0x1f) << 3));
738 }
739 
740 /*
741  * allow removal of exported/shared thermal interrupt
742  */
743 int
744 cb_remove_xintr(pci_t *pci_p, dev_info_t *dip, dev_info_t *rdip,
745 	ib_ino_t ino, ib_mondo_t mondo)
746 {
747 	if (ino != pci_p->pci_inos[CBNINTR_THERMAL])
748 		return (DDI_FAILURE);
749 
750 	cb_disable_nintr(pci_p->pci_cb_p, CBNINTR_THERMAL, IB_INTR_WAIT);
751 	rem_ivintr(mondo, NULL);
752 
753 	DEBUG1(DBG_R_INTX, dip, "remove xintr %x\n", ino);
754 	return (DDI_SUCCESS);
755 }
756 
757 int
758 pci_ecc_add_intr(pci_t *pci_p, int inum, ecc_intr_info_t *eii_p)
759 {
760 	uint32_t mondo;
761 
762 	mondo = ((pci_p->pci_cb_p->cb_ign << PCI_INO_BITS) |
763 	    pci_p->pci_inos[inum]);
764 	mondo = CB_MONDO_TO_XMONDO(pci_p->pci_cb_p, mondo);
765 
766 	VERIFY(add_ivintr(mondo, pci_pil[inum], ecc_intr,
767 	    (caddr_t)eii_p, NULL) == 0);
768 
769 	return (PCI_ATTACH_RETCODE(PCI_ECC_OBJ, PCI_OBJ_INTR_ADD, DDI_SUCCESS));
770 }
771 
772 void
773 pci_ecc_rem_intr(pci_t *pci_p, int inum, ecc_intr_info_t *eii_p)
774 {
775 	uint32_t mondo;
776 
777 	mondo = ((pci_p->pci_cb_p->cb_ign << PCI_INO_BITS) |
778 	    pci_p->pci_inos[inum]);
779 	mondo = CB_MONDO_TO_XMONDO(pci_p->pci_cb_p, mondo);
780 
781 	rem_ivintr(mondo, NULL);
782 }
783 
784 static int pbm_has_pass_1_cheerio(pci_t *pci_p);
785 
786 void
787 pbm_configure(pbm_t *pbm_p)
788 {
789 	pci_t *pci_p = pbm_p->pbm_pci_p;
790 	cb_t *cb_p = pci_p->pci_cb_p;
791 	dev_info_t *dip = pci_p->pci_dip;
792 	int instance = ddi_get_instance(dip);
793 	uint32_t mask = 1 << instance;
794 	uint64_t l;
795 	uint16_t s = 0;
796 
797 	/*
798 	 * Workarounds for hardware bugs:
799 	 *
800 	 * bus parking
801 	 *
802 	 *	Pass 2 psycho parts have a bug that requires bus
803 	 *	parking to be disabled.
804 	 *
805 	 *	Pass 1 cheerio parts have a bug which prevents them
806 	 *	from working on a PBM with bus parking enabled.
807 	 *
808 	 * rerun disable
809 	 *
810 	 *	Pass 1 and 2 psycho's require that the rerun's be
811 	 *	enabled.
812 	 *
813 	 * retry limit
814 	 *
815 	 *	For pass 1 and pass 2 psycho parts we disable the
816 	 *	retry limit.  This is because the limit of 16 seems
817 	 *	too restrictive for devices that are children of pci
818 	 *	to pci bridges.  For pass 3 this limit will be 64.
819 	 *
820 	 * DMA write/PIO read sync
821 	 *
822 	 *	For pass 2 psycho, the disable this feature.
823 	 */
824 	l = lddphysio(cb_p->cb_base_pa + PSYCHO_CB_CONTROL_STATUS_REG_OFFSET);
825 	l &= PSYCHO_CB_CONTROL_STATUS_VER;
826 	l >>= PSYCHO_CB_CONTROL_STATUS_VER_SHIFT;
827 
828 	DEBUG2(DBG_ATTACH, dip, "cb_create: ver=%d, mask=%x\n", l, mask);
829 	pci_rerun_disable = (uint32_t)-1;
830 
831 	switch (l) {
832 	case 0:
833 		DEBUG0(DBG_ATTACH, dip, "cb_create: psycho pass 1\n");
834 		if (!pci_disable_pass1_workarounds) {
835 			if (pbm_has_pass_1_cheerio(pci_p))
836 				pci_bus_parking_enable &= ~mask;
837 			pci_rerun_disable &= ~mask;
838 			pci_retry_disable |= mask;
839 		}
840 		break;
841 	case 1:
842 		if (!pci_disable_pass2_workarounds) {
843 			pci_bus_parking_enable &= ~mask;
844 			pci_rerun_disable &= ~mask;
845 			pci_retry_disable |= mask;
846 			pci_dwsync_disable |= mask;
847 		}
848 		break;
849 	case 2:
850 		if (!pci_disable_pass3_workarounds) {
851 			pci_dwsync_disable |= mask;
852 			if (pbm_has_pass_1_cheerio(pci_p))
853 				pci_bus_parking_enable &= ~mask;
854 		}
855 		break;
856 	case 3:
857 		if (!pci_disable_plus_workarounds) {
858 			pci_dwsync_disable |= mask;
859 			if (pbm_has_pass_1_cheerio(pci_p))
860 				pci_bus_parking_enable &= ~mask;
861 		}
862 		break;
863 	default:
864 		if (!pci_disable_default_workarounds) {
865 			pci_dwsync_disable |= mask;
866 			if (pbm_has_pass_1_cheerio(pci_p))
867 				pci_bus_parking_enable &= ~mask;
868 		}
869 		break;
870 	}
871 
872 	/*
873 	 * Clear any PBM errors.
874 	 */
875 	l = (PSYCHO_PCI_AFSR_E_MASK << PSYCHO_PCI_AFSR_PE_SHIFT) |
876 		(PSYCHO_PCI_AFSR_E_MASK << PSYCHO_PCI_AFSR_SE_SHIFT);
877 	*pbm_p->pbm_async_flt_status_reg = l;
878 
879 	/*
880 	 * Clear error bits in configuration status register.
881 	 */
882 	s = PCI_STAT_PERROR | PCI_STAT_S_PERROR |
883 		PCI_STAT_R_MAST_AB | PCI_STAT_R_TARG_AB |
884 		PCI_STAT_S_TARG_AB | PCI_STAT_S_PERROR;
885 	DEBUG1(DBG_ATTACH, dip, "pbm_configure: conf status reg=%x\n", s);
886 	pbm_p->pbm_config_header->ch_status_reg = s;
887 	DEBUG1(DBG_ATTACH, dip, "pbm_configure: conf status reg==%x\n",
888 		pbm_p->pbm_config_header->ch_status_reg);
889 
890 	l = *pbm_p->pbm_ctrl_reg;	/* save control register state */
891 	DEBUG1(DBG_ATTACH, dip, "pbm_configure: ctrl reg==%llx\n", l);
892 
893 	/*
894 	 * See if any SERR# signals are asserted.  We'll clear them later.
895 	 */
896 	if (l & COMMON_PCI_CTRL_SERR)
897 		cmn_err(CE_WARN, "%s%d: SERR asserted on pci bus\n",
898 		    ddi_driver_name(dip), instance);
899 
900 	/*
901 	 * Determine if PCI bus is running at 33 or 66 mhz.
902 	 */
903 	if (l & COMMON_PCI_CTRL_SPEED)
904 		pbm_p->pbm_speed = PBM_SPEED_66MHZ;
905 	else
906 		pbm_p->pbm_speed = PBM_SPEED_33MHZ;
907 	DEBUG1(DBG_ATTACH, dip, "pbm_configure: %d mhz\n",
908 	    pbm_p->pbm_speed  == PBM_SPEED_66MHZ ? 66 : 33);
909 
910 	/*
911 	 * Enable error interrupts.
912 	 */
913 	if (pci_error_intr_enable & mask)
914 		l |= PSYCHO_PCI_CTRL_ERR_INT_EN;
915 	else
916 		l &= ~PSYCHO_PCI_CTRL_ERR_INT_EN;
917 
918 	/*
919 	 * Disable pci streaming byte errors and error interrupts.
920 	 */
921 	pci_sbh_error_intr_enable &= ~mask;
922 	l &= ~PSYCHO_PCI_CTRL_SBH_INT_EN;
923 
924 	/*
925 	 * Enable/disable bus parking.
926 	 */
927 	if ((pci_bus_parking_enable & mask) &&
928 	    !ddi_prop_exists(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
929 	    "no-bus-parking"))
930 		l |= PSYCHO_PCI_CTRL_ARB_PARK;
931 	else
932 		l &= ~PSYCHO_PCI_CTRL_ARB_PARK;
933 
934 	/*
935 	 * Enable arbitration.
936 	 */
937 	if (pci_p->pci_side == B)
938 		l = (l & ~PSYCHO_PCI_CTRL_ARB_EN_MASK) | pci_b_arb_enable;
939 	else
940 		l = (l & ~PSYCHO_PCI_CTRL_ARB_EN_MASK) | pci_a_arb_enable;
941 
942 	/*
943 	 * Make sure SERR is clear
944 	 */
945 	l |= COMMON_PCI_CTRL_SERR;
946 
947 	/*
948 	 * Make sure power management interrupt is disabled.
949 	 */
950 	l &= ~PSYCHO_PCI_CTRL_WAKEUP_EN;
951 
952 #ifdef _STARFIRE
953 	/*
954 	 * Hack to determine whether we do Starfire special handling
955 	 * For starfire, we simply program a constant odd-value
956 	 * (0x1D) in the MID field.
957 	 *
958 	 * Zero out the MID field before ORing. We leave the LSB of
959 	 * the MID field intact since we cannot have a zero (even)
960 	 * MID value.
961 	 */
962 	l &= 0xFF0FFFFFFFFFFFFFULL;
963 	l |= 0x1DULL << 51;
964 
965 	/*
966 	 * Program in the Interrupt Group Number.  Here we have to
967 	 * convert the starfire 7bit upaid into a 5bit value.
968 	 */
969 	l |= (uint64_t)STARFIRE_UPAID2HWIGN(pbm_p->pbm_pci_p->pci_id)
970 		<< COMMON_CB_CONTROL_STATUS_IGN_SHIFT;
971 #endif /* _STARFIRE */
972 
973 	/*
974 	 * Now finally write the control register with the appropriate value.
975 	 */
976 	DEBUG1(DBG_ATTACH, dip, "pbm_configure: ctrl reg=%llx\n", l);
977 	*pbm_p->pbm_ctrl_reg = l;
978 
979 	/*
980 	 * Allow the diag register to be set based upon variable that
981 	 * can be configured via /etc/system.
982 	 */
983 	l = *pbm_p->pbm_diag_reg;
984 	DEBUG1(DBG_ATTACH, dip, "pbm_configure: PCI diag reg==%llx\n", l);
985 	if (pci_retry_disable & mask)
986 		l |= COMMON_PCI_DIAG_DIS_RETRY;
987 	if (pci_retry_enable & mask)
988 		l &= ~COMMON_PCI_DIAG_DIS_RETRY;
989 	if (pci_intsync_disable & mask)
990 		l |= COMMON_PCI_DIAG_DIS_INTSYNC;
991 	else
992 		l &= ~COMMON_PCI_DIAG_DIS_INTSYNC;
993 	if (pci_dwsync_disable & mask)
994 		l |= PSYCHO_PCI_DIAG_DIS_DWSYNC;
995 	else
996 		l &= ~PSYCHO_PCI_DIAG_DIS_DWSYNC;
997 	DEBUG1(DBG_ATTACH, dip, "pbm_configure: PCI diag reg=%llx\n", l);
998 	*pbm_p->pbm_diag_reg = l;
999 
1000 	/*
1001 	 * Enable SERR# and parity reporting via command register.
1002 	 */
1003 	s = pci_perr_enable & mask ? PCI_COMM_PARITY_DETECT : 0;
1004 	s |= pci_serr_enable & mask ? PCI_COMM_SERR_ENABLE : 0;
1005 	DEBUG1(DBG_ATTACH, dip, "pbm_configure: conf command reg=%x\n", s);
1006 	pbm_p->pbm_config_header->ch_command_reg = s;
1007 	DEBUG1(DBG_ATTACH, dip, "pbm_configure: conf command reg==%x\n",
1008 		pbm_p->pbm_config_header->ch_command_reg);
1009 
1010 	/*
1011 	 * The current versions of the obp are suppose to set the latency
1012 	 * timer register but do not.  Bug 1234181 is open against this
1013 	 * problem.  Until this bug is fixed we check to see if the obp
1014 	 * has attempted to set the latency timer register by checking
1015 	 * for the existence of a "latency-timer" property.
1016 	 */
1017 	if (pci_set_latency_timer_register) {
1018 		DEBUG1(DBG_ATTACH, dip,
1019 		    "pbm_configure: set psycho latency timer to %x\n",
1020 			pci_latency_timer);
1021 		pbm_p->pbm_config_header->ch_latency_timer_reg =
1022 			pci_latency_timer;
1023 	}
1024 
1025 	(void) ndi_prop_update_int(DDI_DEV_T_ANY, dip, "latency-timer",
1026 		(int)pbm_p->pbm_config_header->ch_latency_timer_reg);
1027 }
1028 
1029 uint_t
1030 pbm_disable_pci_errors(pbm_t *pbm_p)
1031 {
1032 	pci_t *pci_p = pbm_p->pbm_pci_p;
1033 	ib_t *ib_p = pci_p->pci_ib_p;
1034 
1035 	/*
1036 	 * Disable error and streaming byte hole interrupts via the
1037 	 * PBM control register.
1038 	 */
1039 	*pbm_p->pbm_ctrl_reg &=
1040 		~(PSYCHO_PCI_CTRL_ERR_INT_EN | PSYCHO_PCI_CTRL_SBH_INT_EN);
1041 
1042 	/*
1043 	 * Disable error interrupts via the interrupt mapping register.
1044 	 */
1045 	ib_intr_disable(ib_p, pci_p->pci_inos[CBNINTR_PBM], IB_INTR_NOWAIT);
1046 	return (BF_NONE);
1047 }
1048 
1049 /*ARGSUSED*/
1050 uint64_t
1051 pci_sc_configure(pci_t *pci_p)
1052 {
1053 	return (0);
1054 }
1055 
1056 /*ARGSUSED*/
1057 void
1058 pci_pbm_dma_sync(pbm_t *pbm_p, ib_ino_t ino)
1059 {
1060 	uint64_t pa = pbm_p->pbm_sync_reg_pa;
1061 	if (pa)
1062 		(void) lddphysio(pa);		/* Load from Sync Register */
1063 }
1064 
1065 /*ARGSUSED*/
1066 dvma_context_t
1067 pci_iommu_get_dvma_context(iommu_t *iommu_p, dvma_addr_t dvma_pg_index)
1068 {
1069 	ASSERT(0);
1070 	return (0);
1071 }
1072 
1073 /*ARGSUSED*/
1074 void
1075 pci_iommu_free_dvma_context(iommu_t *iommu_p, dvma_context_t ctx)
1076 {
1077 	ASSERT(0);
1078 }
1079 
1080 void
1081 pci_iommu_config(iommu_t *iommu_p, uint64_t iommu_ctl, uint64_t cfgpa)
1082 {
1083 	volatile uint64_t *pbm_csr_p = (volatile uint64_t *)
1084 		get_pbm_reg_base(iommu_p->iommu_pci_p);
1085 	volatile uint64_t pbm_ctl = *pbm_csr_p;
1086 
1087 	volatile uint64_t *iommu_ctl_p = iommu_p->iommu_ctrl_reg;
1088 	volatile uint64_t tsb_bar_val = iommu_p->iommu_tsb_paddr;
1089 	volatile uint64_t *tsb_bar_p = iommu_p->iommu_tsb_base_addr_reg;
1090 
1091 	DEBUG2(DBG_ATTACH, iommu_p->iommu_pci_p->pci_dip,
1092 		"\npci_iommu_config: pbm_csr_p=%016llx pbm_ctl=%016llx",
1093 		pbm_csr_p, pbm_ctl);
1094 	DEBUG2(DBG_ATTACH|DBG_CONT, iommu_p->iommu_pci_p->pci_dip,
1095 		"\n\tiommu_ctl_p=%016llx iommu_ctl=%016llx",
1096 		iommu_ctl_p, iommu_ctl);
1097 	DEBUG2(DBG_ATTACH|DBG_CONT, iommu_p->iommu_pci_p->pci_dip,
1098 		"\n\tcfgpa=%016llx tsb_bar_val=%016llx", cfgpa, tsb_bar_val);
1099 
1100 	if (!cfgpa)
1101 		goto reprog;
1102 
1103 	/* disable PBM arbiters - turn off bits 0-7 */
1104 	*pbm_csr_p = (pbm_ctl >> 8) << 8;
1105 
1106 	/* make sure we own the bus by reading any child device config space */
1107 	(void) ldphysio(cfgpa); /* also flushes the prev write */
1108 reprog:
1109 	*tsb_bar_p = tsb_bar_val;
1110 	*iommu_ctl_p = iommu_ctl;
1111 
1112 	*pbm_csr_p = pbm_ctl;	/* re-enable bus arbitration */
1113 	pbm_ctl = *pbm_csr_p;	/* flush all prev writes */
1114 }
1115 
1116 int
1117 pci_sc_ctx_inv(dev_info_t *dip, sc_t *sc_p, ddi_dma_impl_t *mp)
1118 {
1119 	ASSERT(0);
1120 	return (DDI_FAILURE);
1121 }
1122 
1123 void
1124 pci_cb_setup(pci_t *pci_p)
1125 {
1126 	uint64_t csr, csr_pa, pa;
1127 	cb_t *cb_p = pci_p->pci_cb_p;
1128 
1129 	/* cb_p->cb_node_id = 0; */
1130 	cb_p->cb_ign = PCI_ID_TO_IGN(pci_p->pci_id);
1131 	pa = (uint64_t)hat_getpfnum(kas.a_hat, pci_p->pci_address[0]);
1132 	cb_p->cb_base_pa  = pa = pa >> (32 - MMU_PAGESHIFT) << 32;
1133 	cb_p->cb_map_pa = pa + PSYCHO_IB_OBIO_INTR_MAP_REG_OFFSET;
1134 	cb_p->cb_clr_pa = pa + PSYCHO_IB_OBIO_CLEAR_INTR_REG_OFFSET;
1135 	cb_p->cb_obsta_pa = pa + COMMON_IB_OBIO_INTR_STATE_DIAG_REG;
1136 
1137 	csr_pa = pa + PSYCHO_CB_CONTROL_STATUS_REG_OFFSET;
1138 	csr = lddphysio(csr_pa);
1139 
1140 	/*
1141 	 * Clear any pending address parity errors.
1142 	 */
1143 	if (csr & COMMON_CB_CONTROL_STATUS_APERR) {
1144 		csr |= COMMON_CB_CONTROL_STATUS_APERR;
1145 		cmn_err(CE_WARN, "clearing UPA address parity error\n");
1146 	}
1147 	csr |= COMMON_CB_CONTROL_STATUS_APCKEN;
1148 	csr &= ~COMMON_CB_CONTROL_STATUS_IAP;
1149 	stdphysio(csr_pa, csr);
1150 
1151 #ifdef _STARFIRE
1152 	/* Setup Starfire interrupt target translation */
1153 	pc_ittrans_init(pci_p->pci_id, &cb_p->cb_ittrans_cookie);
1154 #endif /* _STARFIRE */
1155 
1156 }
1157 
1158 void
1159 pci_ecc_setup(ecc_t *ecc_p)
1160 {
1161 	ecc_p->ecc_ue.ecc_errpndg_mask = 0;
1162 	ecc_p->ecc_ue.ecc_offset_mask = PSYCHO_ECC_UE_AFSR_DW_OFFSET;
1163 	ecc_p->ecc_ue.ecc_offset_shift = PSYCHO_ECC_UE_AFSR_DW_OFFSET_SHIFT;
1164 	ecc_p->ecc_ue.ecc_size_log2 = 3;
1165 
1166 	ecc_p->ecc_ce.ecc_errpndg_mask = 0;
1167 	ecc_p->ecc_ce.ecc_offset_mask = PSYCHO_ECC_CE_AFSR_DW_OFFSET;
1168 	ecc_p->ecc_ce.ecc_offset_shift = PSYCHO_ECC_CE_AFSR_DW_OFFSET_SHIFT;
1169 	ecc_p->ecc_ce.ecc_size_log2 = 3;
1170 }
1171 
1172 /*
1173  * overwrite dvma end address (only on virtual-dma systems)
1174  * initialize tsb size
1175  * reset context bits
1176  * return: IOMMU CSR bank base address (VA)
1177  */
1178 uintptr_t
1179 pci_iommu_setup(iommu_t *iommu_p)
1180 {
1181 	pci_dvma_range_prop_t *dvma_prop;
1182 	int dvma_prop_len;
1183 
1184 	pci_t *pci_p = iommu_p->iommu_pci_p;
1185 	dev_info_t *dip = pci_p->pci_dip;
1186 	uint_t tsb_size = iommu_tsb_cookie_to_size(pci_p->pci_tsb_cookie);
1187 	uint_t tsb_size_prop;
1188 
1189 	if (ddi_getlongprop(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
1190 		"virtual-dma", (caddr_t)&dvma_prop, &dvma_prop_len) !=
1191 		DDI_PROP_SUCCESS)
1192 		goto tsb_done;
1193 
1194 	if (dvma_prop_len != sizeof (pci_dvma_range_prop_t)) {
1195 		cmn_err(CE_WARN, "%s%d: invalid virtual-dma property",
1196 			ddi_driver_name(dip), ddi_get_instance(dip));
1197 		goto tsb_end;
1198 	}
1199 	iommu_p->iommu_dvma_end = dvma_prop->dvma_base +
1200 		(dvma_prop->dvma_len - 1);
1201 	tsb_size_prop = IOMMU_BTOP(dvma_prop->dvma_len) * sizeof (uint64_t);
1202 	tsb_size = MIN(tsb_size_prop, tsb_size);
1203 tsb_end:
1204 	kmem_free(dvma_prop, dvma_prop_len);
1205 tsb_done:
1206 	iommu_p->iommu_tsb_size = iommu_tsb_size_encode(tsb_size);
1207 
1208 	if (CHIP_TYPE(pci_p) != PCI_CHIP_HUMMINGBIRD)
1209 		pci_preserve_iommu_tsb = 0;
1210 
1211 	/*
1212 	 * Psycho has no context support.
1213 	 */
1214 	iommu_p->iommu_ctx_bitmap = NULL;
1215 	iommu_p->iommu_flush_ctx_reg = NULL;
1216 	pci_use_contexts = 0;
1217 	pci_sc_use_contexts = 0;
1218 
1219 	/*
1220 	 * Determine the virtual address of the register block
1221 	 * containing the iommu control registers.
1222 	 */
1223 	return (get_reg_base(pci_p));
1224 }
1225 
1226 /*ARGSUSED*/
1227 void
1228 pci_iommu_teardown(iommu_t *iommu_p)
1229 {
1230 }
1231 
1232 /* The psycho+ PBM reg base is at 1fe.0000.2000 */
1233 uintptr_t
1234 get_pbm_reg_base(pci_t *pci_p)
1235 {
1236 	return ((uintptr_t)(pci_p->pci_address[0] +
1237 		(pci_stream_buf_exists ? 0 : PSYCHO_PCI_PBM_REG_BASE)));
1238 }
1239 
1240 void
1241 pci_post_uninit_child(pci_t *pci_p)
1242 {
1243 }
1244 
1245 void
1246 pci_pbm_setup(pbm_t *pbm_p)
1247 {
1248 	pci_t *pci_p = pbm_p->pbm_pci_p;
1249 
1250 	/*
1251 	 * Get the base virtual address for the PBM control block.
1252 	 */
1253 	uintptr_t a = get_pbm_reg_base(pci_p);
1254 
1255 	/*
1256 	 * Get the virtual address of the PCI configuration header.
1257 	 * This should be mapped little-endian.
1258 	 */
1259 	pbm_p->pbm_config_header =
1260 		(config_header_t *)get_config_reg_base(pci_p);
1261 
1262 	/*
1263 	 * Get the virtual addresses for control, error and diag
1264 	 * registers.
1265 	 */
1266 	pbm_p->pbm_ctrl_reg = (uint64_t *)(a + PSYCHO_PCI_CTRL_REG_OFFSET);
1267 	pbm_p->pbm_diag_reg = (uint64_t *)(a + PSYCHO_PCI_DIAG_REG_OFFSET);
1268 	pbm_p->pbm_async_flt_status_reg =
1269 		(uint64_t *)(a + PSYCHO_PCI_ASYNC_FLT_STATUS_REG_OFFSET);
1270 	pbm_p->pbm_async_flt_addr_reg =
1271 		(uint64_t *)(a + PSYCHO_PCI_ASYNC_FLT_ADDR_REG_OFFSET);
1272 
1273 	if (CHIP_TYPE(pci_p) >= PCI_CHIP_SABRE)
1274 		pbm_p->pbm_sync_reg_pa =
1275 			pci_p->pci_cb_p->cb_base_pa + DMA_WRITE_SYNC_REG;
1276 }
1277 
1278 /*ARGSUSED*/
1279 void
1280 pci_pbm_teardown(pbm_t *pbm_p)
1281 {
1282 }
1283 
1284 void
1285 pci_sc_setup(sc_t *sc_p)
1286 {
1287 	pci_t *pci_p = sc_p->sc_pci_p;
1288 
1289 	/*
1290 	 * Determine the virtual addresses of the streaming cache
1291 	 * control/status and flush registers.
1292 	 */
1293 	uintptr_t a = get_pbm_reg_base(pci_p);
1294 	sc_p->sc_ctrl_reg = (uint64_t *)(a + PSYCHO_SC_CTRL_REG_OFFSET);
1295 	sc_p->sc_invl_reg = (uint64_t *)(a + PSYCHO_SC_INVL_REG_OFFSET);
1296 	sc_p->sc_sync_reg = (uint64_t *)(a + PSYCHO_SC_SYNC_REG_OFFSET);
1297 
1298 	/*
1299 	 * Determine the virtual addresses of the streaming cache
1300 	 * diagnostic access registers.
1301 	 */
1302 	a = get_reg_base(pci_p);
1303 	if (pci_p->pci_bus_range.lo != 0) {
1304 		sc_p->sc_data_diag_acc = (uint64_t *)
1305 				(a + PSYCHO_SC_A_DATA_DIAG_OFFSET);
1306 		sc_p->sc_tag_diag_acc = (uint64_t *)
1307 				(a + PSYCHO_SC_A_TAG_DIAG_OFFSET);
1308 		sc_p->sc_ltag_diag_acc = (uint64_t *)
1309 				(a + PSYCHO_SC_A_LTAG_DIAG_OFFSET);
1310 	} else {
1311 		sc_p->sc_data_diag_acc = (uint64_t *)
1312 				(a + PSYCHO_SC_B_DATA_DIAG_OFFSET);
1313 		sc_p->sc_tag_diag_acc = (uint64_t *)
1314 				(a + PSYCHO_SC_B_TAG_DIAG_OFFSET);
1315 		sc_p->sc_ltag_diag_acc = (uint64_t *)
1316 				(a + PSYCHO_SC_B_LTAG_DIAG_OFFSET);
1317 	}
1318 }
1319 
1320 int
1321 pci_get_numproxy(dev_info_t *dip)
1322 {
1323 	return (ddi_prop_get_int(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
1324 		"#upa-interrupt-proxies", 1));
1325 }
1326 
1327 int
1328 pci_get_portid(dev_info_t *dip)
1329 {
1330 	return (ddi_getprop(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
1331 	    "upa-portid", -1));
1332 }
1333 
1334 /*
1335  * pbm_has_pass_1_cheerio
1336  *
1337  *
1338  * Given a PBM soft state pointer, this routine scans it child nodes
1339  * to see if one is a pass 1 cheerio.
1340  *
1341  * return value: 1 if pass 1 cheerio is found, 0 otherwise
1342  */
1343 static int
1344 pbm_has_pass_1_cheerio(pci_t *pci_p)
1345 {
1346 	dev_info_t *cdip;
1347 	int found = 0;
1348 	char *s;
1349 	int rev;
1350 
1351 	cdip = ddi_get_child(pci_p->pci_dip);
1352 	while (cdip != NULL && found == 0) {
1353 		s = ddi_get_name(cdip);
1354 		if (strcmp(s, "ebus") == 0 || strcmp(s, "pci108e,1000") == 0) {
1355 			rev =
1356 			    ddi_getprop(DDI_DEV_T_ANY, cdip, DDI_PROP_DONTPASS,
1357 				"revision-id", 0);
1358 			if (rev == 0)
1359 				found = 1;
1360 		}
1361 		cdip = ddi_get_next_sibling(cdip);
1362 	}
1363 	return (found);
1364 }
1365 
1366 /*
1367  * Psycho Performance Events.
1368  */
1369 pci_kev_mask_t
1370 psycho_pci_events[] = {
1371 	{"dvma_stream_rd_a", 0x0},	{"dvma_stream_wr_a", 0x1},
1372 	{"dvma_const_rd_a", 0x2},	{"dvma_const_wr_a", 0x3},
1373 	{"dvma_stream_buf_mis_a", 0x4}, {"dvma_cycles_a", 0x5},
1374 	{"dvma_wd_xfr_a", 0x6},		{"pio_cycles_a", 0x7},
1375 	{"dvma_stream_rd_b", 0x8},	{"dvma_stream_wr_b", 0x9},
1376 	{"dvma_const_rd_b", 0xa},	{"dvma_const_wr_b", 0xb},
1377 	{"dvma_stream_buf_mis_b", 0xc}, {"dvma_cycles_b", 0xd},
1378 	{"dvma_wd_xfr_b", 0xe},		{"pio_cycles_b", 0xf},
1379 	{"dvma_tlb_misses", 0x10},	{"interrupts", 0x11},
1380 	{"upa_inter_nack", 0x12},	{"pio_reads", 0x13},
1381 	{"pio_writes", 0x14},		{"merge_buffer", 0x15},
1382 	{"dma_tbwalk_a", 0x16},		{"dma_stc_a", 0x17},
1383 	{"dma_tbwalk_b", 0x18},		{"dma_stc_b", 0x19},
1384 	{"clear_pic", 0x1f}
1385 };
1386 
1387 /*
1388  * Create the picN kstat's.
1389  */
1390 void
1391 pci_kstat_init()
1392 {
1393 	pci_name_kstat = (pci_ksinfo_t *)kmem_alloc(sizeof (pci_ksinfo_t),
1394 		KM_NOSLEEP);
1395 
1396 	if (pci_name_kstat == NULL) {
1397 		cmn_err(CE_WARN, "pcipsy : no space for kstat\n");
1398 	} else {
1399 		pci_name_kstat->pic_no_evs =
1400 			sizeof (psycho_pci_events) / sizeof (pci_kev_mask_t);
1401 		pci_name_kstat->pic_shift[0] = PSYCHO_SHIFT_PIC0;
1402 		pci_name_kstat->pic_shift[1] = PSYCHO_SHIFT_PIC1;
1403 		pci_create_name_kstat("pcip",
1404 			pci_name_kstat, psycho_pci_events);
1405 	}
1406 }
1407 
1408 /*
1409  * Called from _fini()
1410  */
1411 void
1412 pci_kstat_fini()
1413 {
1414 	if (pci_name_kstat != NULL) {
1415 		pci_delete_name_kstat(pci_name_kstat);
1416 		kmem_free(pci_name_kstat, sizeof (pci_ksinfo_t));
1417 		pci_name_kstat = NULL;
1418 	}
1419 }
1420 
1421 /* ARGSUSED */
1422 void
1423 pci_add_pci_kstat(pci_t *pci_p)
1424 {
1425 }
1426 
1427 /* ARGSUSED */
1428 void
1429 pci_rem_pci_kstat(pci_t *pci_p)
1430 {
1431 }
1432 
1433 /*
1434  * Create the performance 'counters' kstat.
1435  */
1436 void
1437 pci_add_upstream_kstat(pci_t *pci_p)
1438 {
1439 	pci_common_t 	*cmn_p = pci_p->pci_common_p;
1440 	pci_cntr_pa_t	*cntr_pa_p = &cmn_p->pci_cmn_uks_pa;
1441 	uint64_t regbase = va_to_pa((void *)get_reg_base(pci_p));
1442 
1443 	cntr_pa_p->pcr_pa = regbase + PSYCHO_PERF_PCR_OFFSET;
1444 	cntr_pa_p->pic_pa = regbase + PSYCHO_PERF_PIC_OFFSET;
1445 	cmn_p->pci_common_uksp = pci_create_cntr_kstat(pci_p, "pcip",
1446 		NUM_OF_PICS, pci_cntr_kstat_pa_update, cntr_pa_p);
1447 }
1448 
1449 /*
1450  * Extract the drivers binding name to identify which chip
1451  * we're binding to.  Whenever a new bus bridge is created, the driver alias
1452  * entry should be added here to identify the device if needed.  If a device
1453  * isn't added, the identity defaults to PCI_CHIP_UNIDENTIFIED.
1454  */
1455 static uint32_t
1456 pci_identity_init(pci_t *pci_p)
1457 {
1458 	dev_info_t *dip = pci_p->pci_dip;
1459 	char *name = ddi_binding_name(dip);
1460 
1461 	if (strcmp(name, "pci108e,8000") == 0)
1462 		return (CHIP_ID(PCI_CHIP_PSYCHO, 0x00, 0x00));
1463 	if (strcmp(name, "pci108e,a000") == 0)
1464 		return (CHIP_ID(PCI_CHIP_SABRE, 0x00, 0x00));
1465 	if (strcmp(name, "pci108e,a001") == 0)
1466 		return (CHIP_ID(PCI_CHIP_HUMMINGBIRD, 0x00, 0x00));
1467 	cmn_err(CE_CONT, "?%s%d:using default chip identity\n",
1468 		ddi_driver_name(dip), ddi_get_instance(dip));
1469 	return (CHIP_ID(PCI_CHIP_PSYCHO, 0x00, 0x00));
1470 }
1471 
1472 /*ARGSUSED*/
1473 void
1474 pci_post_init_child(pci_t *pci_p, dev_info_t *child)
1475 {
1476 }
1477 
1478 /*ARGSUSED*/
1479 int
1480 pci_pbm_add_intr(pci_t *pci_p)
1481 {
1482 	return (DDI_SUCCESS);
1483 }
1484 
1485 /*ARGSUSED*/
1486 void
1487 pci_pbm_rem_intr(pci_t *pci_p)
1488 {
1489 }
1490 
1491 /*ARGSUSED*/
1492 void
1493 pci_pbm_suspend(pci_t *pci_p)
1494 {
1495 }
1496 
1497 /*ARGSUSED*/
1498 void
1499 pci_pbm_resume(pci_t *pci_p)
1500 {
1501 }
1502 
1503 /*
1504  * pcipsy error handling 101:
1505  *
1506  * The various functions below are responsible for error handling. Given
1507  * a particular error, they must gather the appropriate state, report all
1508  * errors with correct payload, and attempt recovery where ever possible.
1509  *
1510  * Recovery in the context of this driver is being able notify a leaf device
1511  * of the failed transaction. This leaf device may either be the master or
1512  * target for this transaction and may have already received an error
1513  * notification via a PCI interrupt. Notification is done via DMA and access
1514  * handles. If we capture an address for the transaction then we can map it
1515  * to a handle(if the leaf device is fma-compliant) and fault the handle as
1516  * well as call the device driver registered callback.
1517  *
1518  * The hardware can either interrupt or trap upon detection of an error, in
1519  * some rare cases it also causes a fatal reset.
1520  *
1521  * pbm_error_intr() and ecc_intr() are responsible for PCI Block Module
1522  * errors(generic PCI + bridge specific) and ECC errors, respectively. They
1523  * are common between pcisch and pcipsy and therefore exist in pci_pbm.c and
1524  * pci_ecc.c. To support error handling certain chip specific handlers
1525  * must exist and they are defined below.
1526  *
1527  * cpu_deferred_error() and cpu_async_error(), handle the traps that may
1528  * have originated from IO space. They call into the registered IO callbacks
1529  * to report and handle errors that may have caused the trap.
1530  *
1531  * pci_pbm_err_handler() is called by pbm_error_intr() or pci_err_callback()
1532  * (generic fma callback for pcipsy/pcisch, pci_fm.c). pci_err_callback() is
1533  * called when the CPU has trapped because of a possible IO error(TO/BERR/UE).
1534  * It will call pci_pbm_err_handler() to report and handle all PCI/PBM/IOMMU
1535  * related errors which are detected by the chip.
1536  *
1537  * pci_pbm_err_handler() calls a generic interface pbm_afsr_report()(pci_pbm.c)
1538  * to report the pbm specific errors and attempt to map the failed address
1539  * (if captured) to a device instance. pbm_afsr_report() calls a chip specific
1540  * interface to interpret the afsr bits pci_pbm_classify()(pcisch.c/pcipsy.c).
1541  *
1542  * ecc_err_handler()(pci_ecc.c) also calls a chip specific interface to
1543  * interpret the afsr, pci_ecc_classify(). ecc_err_handler() also calls
1544  * pci_pbm_err_handler() and ndi_fm_handler_dispatch() to log any related
1545  * errors.
1546  *
1547  * To make sure that the trap code and the interrupt code are not going
1548  * to step on each others toes we have a per chip pci_fm_mutex. This also
1549  * makes it necessary for us to be cautious while we are at a high PIL, so
1550  * that we do not cause a subsequent trap that causes us to hang.
1551  *
1552  * The attempt to commonize code was meant to keep in line with the current
1553  * pci driver implementation and it was not meant to confuse. If you are
1554  * confused then don't worry, I was too.
1555  */
1556 
1557 /*
1558  * For Psycho, a UE is always fatal, except if it is a translation error on a
1559  * Darwin platform.  We ignore these because they do not cause data corruption.
1560  */
1561 int
1562 ecc_ue_is_fatal(struct async_flt *ecc)
1563 {
1564 	return (((uint_t)(ecc->flt_stat >> SABRE_UE_AFSR_PDTE_SHIFT) &
1565 	    SABRE_UE_AFSR_E_PDTE) == 0);
1566 }
1567 
1568 /*
1569  * pci_ecc_classify, called by ecc_handler to classify ecc errors
1570  * and determine if we should panic or not.
1571  */
1572 void
1573 pci_ecc_classify(uint64_t err, ecc_errstate_t *ecc_err_p)
1574 {
1575 	struct async_flt *ecc = &ecc_err_p->ecc_aflt;
1576 	pci_common_t *cmn_p = ecc_err_p->ecc_ii_p.ecc_p->ecc_pci_cmn_p;
1577 
1578 	ASSERT(MUTEX_HELD(&cmn_p->pci_fm_mutex));
1579 
1580 	ecc_err_p->ecc_bridge_type = PCI_BRIDGE_TYPE(cmn_p);
1581 	/*
1582 	 * Get the parent bus id that caused the error.
1583 	 */
1584 	ecc_err_p->ecc_dev_id = (ecc_err_p->ecc_afsr & PSYCHO_ECC_UE_AFSR_ID)
1585 			>> PSYCHO_ECC_UE_AFSR_ID_SHIFT;
1586 	/*
1587 	 * Determine the doubleword offset of the error.
1588 	 */
1589 	ecc_err_p->ecc_dw_offset = (ecc_err_p->ecc_afsr &
1590 			PSYCHO_ECC_UE_AFSR_DW_OFFSET)
1591 			>> PSYCHO_ECC_UE_AFSR_DW_OFFSET_SHIFT;
1592 	/*
1593 	 * Determine the primary error type.
1594 	 */
1595 	switch (err) {
1596 	case COMMON_ECC_UE_AFSR_E_PIO:
1597 		if (ecc_err_p->ecc_ii_p.ecc_type == CBNINTR_UE) {
1598 			if (ecc_err_p->ecc_pri) {
1599 				ecc->flt_erpt_class = PCI_ECC_PIO_UE;
1600 			} else {
1601 				ecc->flt_erpt_class = PCI_ECC_SEC_PIO_UE;
1602 			}
1603 			ecc->flt_panic = ecc_ue_is_fatal(&ecc_err_p->ecc_aflt);
1604 		} else {
1605 			ecc->flt_erpt_class = ecc_err_p->ecc_pri ?
1606 				PCI_ECC_PIO_CE : PCI_ECC_SEC_PIO_CE;
1607 			return;
1608 		}
1609 		break;
1610 
1611 	case COMMON_ECC_UE_AFSR_E_DRD:
1612 		if (ecc_err_p->ecc_ii_p.ecc_type == CBNINTR_UE) {
1613 			if (ecc_err_p->ecc_pri) {
1614 				ecc->flt_erpt_class = PCI_ECC_DRD_UE;
1615 			} else {
1616 				ecc->flt_erpt_class = PCI_ECC_SEC_DRD_UE;
1617 			}
1618 			ecc->flt_panic = ecc_ue_is_fatal(&ecc_err_p->ecc_aflt);
1619 		} else {
1620 			ecc->flt_erpt_class = ecc_err_p->ecc_pri ?
1621 				PCI_ECC_DRD_CE : PCI_ECC_SEC_DRD_CE;
1622 			return;
1623 		}
1624 		break;
1625 
1626 	case COMMON_ECC_UE_AFSR_E_DWR:
1627 		if (ecc_err_p->ecc_ii_p.ecc_type == CBNINTR_UE) {
1628 			if (ecc_err_p->ecc_pri) {
1629 				ecc->flt_erpt_class = PCI_ECC_DWR_UE;
1630 			} else {
1631 				ecc->flt_erpt_class = PCI_ECC_SEC_DWR_UE;
1632 			}
1633 			ecc->flt_panic = ecc_ue_is_fatal(&ecc_err_p->ecc_aflt);
1634 		} else {
1635 			ecc->flt_erpt_class = ecc_err_p->ecc_pri ?
1636 				PCI_ECC_DWR_CE : PCI_ECC_SEC_DWR_CE;
1637 			return;
1638 		}
1639 		break;
1640 
1641 	default:
1642 		return;
1643 	}
1644 }
1645 
1646 ushort_t
1647 pci_ecc_get_synd(uint64_t afsr)
1648 {
1649 	return ((ushort_t)((afsr & PSYCHO_ECC_CE_AFSR_SYND)
1650 		>> PSYCHO_ECC_CE_AFSR_SYND_SHIFT));
1651 }
1652 
1653 /*
1654  * pci_pbm_classify, called by pbm_afsr_report to classify piow afsr.
1655  */
1656 int
1657 pci_pbm_classify(pbm_errstate_t *pbm_err_p)
1658 {
1659 	uint32_t e;
1660 	int nerr = 0;
1661 	char **tmp_class;
1662 
1663 	if (pbm_err_p->pbm_pri) {
1664 		tmp_class = &pbm_err_p->pbm_pci.pci_err_class;
1665 		e = PBM_AFSR_TO_PRIERR(pbm_err_p->pbm_afsr);
1666 		pbm_err_p->pbm_log = FM_LOG_PCI;
1667 	} else {
1668 		tmp_class = &pbm_err_p->pbm_err_class;
1669 		e = PBM_AFSR_TO_SECERR(pbm_err_p->pbm_afsr);
1670 		pbm_err_p->pbm_log = FM_LOG_PBM;
1671 	}
1672 
1673 	if (e & PSYCHO_PCI_AFSR_E_MA) {
1674 		*tmp_class = pbm_err_p->pbm_pri ? PCI_MA : PCI_SEC_MA;
1675 		nerr++;
1676 	}
1677 	if (e & PSYCHO_PCI_AFSR_E_TA) {
1678 		*tmp_class = pbm_err_p->pbm_pri ? PCI_REC_TA : PCI_SEC_REC_TA;
1679 		nerr++;
1680 	}
1681 	if (e & PSYCHO_PCI_AFSR_E_RTRY) {
1682 		pbm_err_p->pbm_err_class = pbm_err_p->pbm_pri ?
1683 			PCI_PBM_RETRY : PCI_SEC_PBM_RETRY;
1684 		pbm_err_p->pbm_log = FM_LOG_PBM;
1685 		nerr++;
1686 	}
1687 	if (e & PSYCHO_PCI_AFSR_E_PERR) {
1688 		*tmp_class = pbm_err_p->pbm_pri ? PCI_MDPE : PCI_SEC_MDPE;
1689 		nerr++;
1690 	}
1691 	return (nerr);
1692 }
1693 
1694 /*
1695  * Function used to clear PBM/PCI/IOMMU error state after error handling
1696  * is complete. Only clearing error bits which have been logged. Called by
1697  * pci_pbm_err_handler and pci_bus_exit.
1698  */
1699 static void
1700 pci_clear_error(pci_t *pci_p, pbm_errstate_t *pbm_err_p)
1701 {
1702 	pbm_t *pbm_p = pci_p->pci_pbm_p;
1703 
1704 	ASSERT(MUTEX_HELD(&pbm_p->pbm_pci_p->pci_common_p->pci_fm_mutex));
1705 
1706 	*pbm_p->pbm_ctrl_reg = pbm_err_p->pbm_ctl_stat;
1707 	*pbm_p->pbm_async_flt_status_reg = pbm_err_p->pbm_afsr;
1708 	pbm_p->pbm_config_header->ch_status_reg =
1709 		pbm_err_p->pbm_pci.pci_cfg_stat;
1710 }
1711 
1712 /*ARGSUSED*/
1713 int
1714 pci_pbm_err_handler(dev_info_t *dip, ddi_fm_error_t *derr,
1715 		const void *impl_data, int caller)
1716 {
1717 	int fatal = 0;
1718 	int nonfatal = 0;
1719 	int unknown = 0;
1720 	int rserr = 0;
1721 	uint32_t prierr, secerr;
1722 	pbm_errstate_t pbm_err;
1723 	char buf[FM_MAX_CLASS];
1724 	pci_t *pci_p = (pci_t *)impl_data;
1725 	pbm_t *pbm_p = pci_p->pci_pbm_p;
1726 	int ret = 0;
1727 	uint64_t pbm_ctl_stat;
1728 	uint16_t pci_cfg_stat;
1729 
1730 	ASSERT(MUTEX_HELD(&pci_p->pci_common_p->pci_fm_mutex));
1731 	pci_pbm_errstate_get(pci_p, &pbm_err);
1732 
1733 	derr->fme_ena = derr->fme_ena ? derr->fme_ena :
1734 	    fm_ena_generate(0, FM_ENA_FMT1);
1735 
1736 	prierr = PBM_AFSR_TO_PRIERR(pbm_err.pbm_afsr);
1737 	secerr = PBM_AFSR_TO_SECERR(pbm_err.pbm_afsr);
1738 
1739 	if (derr->fme_flag == DDI_FM_ERR_EXPECTED) {
1740 		if (caller == PCI_TRAP_CALL) {
1741 			/*
1742 			 * For ddi_caut_get treat all events as
1743 			 * nonfatal. The trampoline will set
1744 			 * err_ena = 0, err_status = NONFATAL. We only
1745 			 * really call this function so that pci_clear_error()
1746 			 * and ndi_fm_handler_dispatch() will get called.
1747 			 */
1748 			derr->fme_status = DDI_FM_NONFATAL;
1749 			nonfatal++;
1750 			goto done;
1751 		} else {
1752 			/*
1753 			 * For ddi_caut_put treat all events as nonfatal. Here
1754 			 * we have the handle and can call ndi_fm_acc_err_set().
1755 			 */
1756 			derr->fme_status = DDI_FM_NONFATAL;
1757 			ndi_fm_acc_err_set(pbm_p->pbm_excl_handle, derr);
1758 			nonfatal++;
1759 			goto done;
1760 		}
1761 	} else if (derr->fme_flag == DDI_FM_ERR_PEEK) {
1762 		/*
1763 		 * For ddi_peek treat all events as nonfatal. We only
1764 		 * really call this function so that pci_clear_error()
1765 		 * and ndi_fm_handler_dispatch() will get called.
1766 		 */
1767 		nonfatal++;
1768 		goto done;
1769 	} else if (derr->fme_flag == DDI_FM_ERR_POKE) {
1770 		/*
1771 		 * For ddi_poke we can treat as nonfatal if the
1772 		 * following conditions are met :
1773 		 * 1. Make sure only primary error is MA/TA
1774 		 * 2. Make sure no secondary error
1775 		 * 3. check pci config header stat reg to see MA/TA is
1776 		 *    logged. We cannot verify only MA/TA is recorded
1777 		 *    since it gets much more complicated when a
1778 		 *    PCI-to-PCI bridge is present.
1779 		 */
1780 		if ((prierr == PSYCHO_PCI_AFSR_E_MA) && !secerr &&
1781 		    (pbm_err.pbm_pci.pci_cfg_stat & PCI_STAT_R_MAST_AB)) {
1782 			nonfatal++;
1783 			goto done;
1784 		}
1785 		if ((prierr == PSYCHO_PCI_AFSR_E_TA) && !secerr &&
1786 		    (pbm_err.pbm_pci.pci_cfg_stat & PCI_STAT_R_TARG_AB)) {
1787 			nonfatal++;
1788 			goto done;
1789 		}
1790 	}
1791 
1792 	if (prierr || secerr) {
1793 		ret = pbm_afsr_report(dip, derr->fme_ena, &pbm_err);
1794 		if (ret == DDI_FM_FATAL)
1795 			fatal++;
1796 		else
1797 			nonfatal++;
1798 	}
1799 
1800 	ret = pci_cfg_report(dip, derr, &pbm_err.pbm_pci, caller, prierr);
1801 	if (ret == DDI_FM_FATAL)
1802 		fatal++;
1803 	else if (ret == DDI_FM_NONFATAL)
1804 		nonfatal++;
1805 
1806 	pbm_ctl_stat = pbm_err.pbm_ctl_stat;
1807 	pci_cfg_stat = pbm_err.pbm_pci.pci_cfg_stat;
1808 
1809 	/*
1810 	 * PBM Received System Error - During any transaction, or
1811 	 * at any point on the bus, some device may detect a critical
1812 	 * error and signal a system error to the system.
1813 	 */
1814 	if (pbm_ctl_stat & COMMON_PCI_CTRL_SERR) {
1815 		/*
1816 		 * may be expected (master abort from pci-pci bridge during
1817 		 * poke will generate SERR)
1818 		 */
1819 		if (derr->fme_flag != DDI_FM_ERR_POKE) {
1820 			pbm_err.pbm_pci.pci_err_class = PCI_REC_SERR;
1821 			(void) snprintf(buf, FM_MAX_CLASS, "%s.%s",
1822 			    PCI_ERROR_SUBCLASS, pbm_err.pbm_pci.pci_err_class);
1823 			ddi_fm_ereport_post(dip, buf, derr->fme_ena,
1824 			    DDI_NOSLEEP, FM_VERSION, DATA_TYPE_UINT8, 0,
1825 			    PCI_CONFIG_STATUS, DATA_TYPE_UINT16, pci_cfg_stat,
1826 			    PCI_CONFIG_COMMAND, DATA_TYPE_UINT16,
1827 			    pbm_err.pbm_pci.pci_cfg_comm, PCI_PA,
1828 			    DATA_TYPE_UINT64, (uint64_t)0, NULL);
1829 		}
1830 		rserr++;
1831 	}
1832 
1833 	/* Streaming Byte Hole Error */
1834 	if (pbm_ctl_stat & COMMON_PCI_CTRL_SBH_ERR) {
1835 		if (pci_panic_on_sbh_errors)
1836 			fatal++;
1837 		else
1838 			nonfatal++;
1839 		pbm_err.pbm_err_class = PCI_PSY_SBH;
1840 		pbm_ereport_post(dip, derr->fme_ena, &pbm_err);
1841 	}
1842 done:
1843 	ret = ndi_fm_handler_dispatch(dip, NULL, derr);
1844 	if (ret == DDI_FM_FATAL) {
1845 		fatal++;
1846 	} else if (ret == DDI_FM_NONFATAL) {
1847 		nonfatal++;
1848 	} else if (ret == DDI_FM_UNKNOWN) {
1849 		unknown++;
1850 	}
1851 
1852 	/*
1853 	 * rserr not claimed as nonfatal by a child is treated as fatal
1854 	 */
1855 	if (rserr && ret != DDI_FM_NONFATAL)
1856 		fatal++;
1857 
1858 	/* Cleanup and reset error bits */
1859 	pci_clear_error(pci_p, &pbm_err);
1860 
1861 	return (fatal ? DDI_FM_FATAL : (nonfatal ? DDI_FM_NONFATAL :
1862 	    (unknown ? DDI_FM_UNKNOWN : DDI_FM_OK)));
1863 }
1864 
1865 int
1866 pci_check_error(pci_t *pci_p)
1867 {
1868 	pbm_t *pbm_p = pci_p->pci_pbm_p;
1869 	uint16_t pci_cfg_stat;
1870 	uint64_t pbm_ctl_stat, pbm_afsr;
1871 
1872 	ASSERT(MUTEX_HELD(&pci_p->pci_common_p->pci_fm_mutex));
1873 
1874 	pci_cfg_stat = pbm_p->pbm_config_header->ch_status_reg;
1875 	pbm_ctl_stat = *pbm_p->pbm_ctrl_reg;
1876 	pbm_afsr = *pbm_p->pbm_async_flt_status_reg;
1877 
1878 	if ((pci_cfg_stat & (PCI_STAT_S_PERROR | PCI_STAT_S_TARG_AB |
1879 				PCI_STAT_R_TARG_AB | PCI_STAT_R_MAST_AB |
1880 				PCI_STAT_S_SYSERR | PCI_STAT_PERROR)) ||
1881 			(pbm_ctl_stat & (COMMON_PCI_CTRL_SBH_ERR |
1882 				COMMON_PCI_CTRL_SERR)) ||
1883 			(PBM_AFSR_TO_PRIERR(pbm_afsr)))
1884 		return (1);
1885 
1886 	return (0);
1887 
1888 }
1889 
1890 /*
1891  * Function used to gather PBM/PCI error state for the
1892  * pci_pbm_err_handler. This function must be called while pci_fm_mutex
1893  * is held.
1894  */
1895 static void
1896 pci_pbm_errstate_get(pci_t *pci_p, pbm_errstate_t *pbm_err_p)
1897 {
1898 	pbm_t *pbm_p = pci_p->pci_pbm_p;
1899 
1900 	ASSERT(MUTEX_HELD(&pci_p->pci_common_p->pci_fm_mutex));
1901 	bzero(pbm_err_p, sizeof (pbm_errstate_t));
1902 
1903 	/*
1904 	 * Capture all pbm error state for later logging
1905 	 */
1906 	pbm_err_p->pbm_bridge_type = PCI_BRIDGE_TYPE(pci_p->pci_common_p);
1907 	pbm_err_p->pbm_pci.pci_cfg_stat =
1908 		pbm_p->pbm_config_header->ch_status_reg;
1909 	pbm_err_p->pbm_ctl_stat = *pbm_p->pbm_ctrl_reg;
1910 	pbm_err_p->pbm_pci.pci_cfg_comm =
1911 		pbm_p->pbm_config_header->ch_command_reg;
1912 	pbm_err_p->pbm_afsr = *pbm_p->pbm_async_flt_status_reg;
1913 	pbm_err_p->pbm_afar = *pbm_p->pbm_async_flt_addr_reg;
1914 	pbm_err_p->pbm_pci.pci_pa = *pbm_p->pbm_async_flt_addr_reg;
1915 }
1916 
1917 void
1918 pbm_clear_error(pbm_t *pbm_p)
1919 {
1920 	uint64_t pbm_afsr, pbm_ctl_stat;
1921 
1922 	/*
1923 	 * for poke() support - called from POKE_FLUSH. Spin waiting
1924 	 * for MA, TA or SERR to be cleared by a pbm_error_intr().
1925 	 * We have to wait for SERR too in case the device is beyond
1926 	 * a pci-pci bridge.
1927 	 */
1928 	pbm_ctl_stat = *pbm_p->pbm_ctrl_reg;
1929 	pbm_afsr = *pbm_p->pbm_async_flt_status_reg;
1930 	while (((pbm_afsr >> PSYCHO_PCI_AFSR_PE_SHIFT) &
1931 	    (PSYCHO_PCI_AFSR_E_MA | PSYCHO_PCI_AFSR_E_TA)) ||
1932 	    (pbm_ctl_stat & COMMON_PCI_CTRL_SERR)) {
1933 		pbm_ctl_stat = *pbm_p->pbm_ctrl_reg;
1934 		pbm_afsr = *pbm_p->pbm_async_flt_status_reg;
1935 	}
1936 }
1937 
1938 /*ARGSUSED*/
1939 void
1940 pci_format_addr(dev_info_t *dip, uint64_t *afar, uint64_t afsr)
1941 {
1942 	/*
1943 	 * For Psycho the full address is stored in hardware. So
1944 	 * there is no need to format it.
1945 	 */
1946 }
1947 
1948 /*ARGSUSED*/
1949 int
1950 pci_bus_quiesce(pci_t *pci_p, dev_info_t *dip, void *result)
1951 {
1952 	return (DDI_FAILURE);
1953 }
1954 
1955 /*ARGSUSED*/
1956 int
1957 pci_bus_unquiesce(pci_t *pci_p, dev_info_t *dip, void *result)
1958 {
1959 	return (DDI_FAILURE);
1960 }
1961 
1962 void
1963 pci_vmem_free(iommu_t *iommu_p, ddi_dma_impl_t *mp, void *dvma_addr,
1964     size_t npages)
1965 {
1966 	pci_vmem_do_free(iommu_p, dvma_addr, npages,
1967 	    (mp->dmai_flags & DMAI_FLAGS_VMEMCACHE));
1968 }
1969 
1970 
1971 /*
1972  * NOTE: This call is only used by legacy systems (eg. E250 and E450) that
1973  * require unregistering the pci driver's thermal intrerrupt handler before
1974  * they can register their own.
1975  */
1976 void
1977 pci_thermal_rem_intr(dev_info_t *rdip, uint_t inum)
1978 {
1979 	pci_t		*pci_p;
1980 	dev_info_t	*pdip;
1981 	uint32_t	dev_mondo, pci_mondo;
1982 	int 		instance;
1983 
1984 	for (pdip = ddi_get_parent(rdip); pdip; pdip = ddi_get_parent(pdip)) {
1985 		if (strcmp(ddi_driver_name(pdip), "pcipsy") == 0)
1986 			break;
1987 	}
1988 
1989 	if (!pdip) {
1990 		cmn_err(CE_WARN, "pci_thermal_rem_intr() no pcipsy parent\n");
1991 		return;
1992 	}
1993 
1994 	instance = ddi_get_instance(pdip);
1995 	pci_p = get_pci_soft_state(instance);
1996 
1997 	/* Calculate the requesting device's mondo */
1998 	dev_mondo = pci_xlate_intr(pci_p->pci_dip, rdip, pci_p->pci_ib_p,
1999 	    IB_MONDO_TO_INO(i_ddi_get_inum(rdip, inum)));
2000 
2001 	/* get pci's thermal mondo */
2002 	pci_mondo = ((pci_p->pci_cb_p->cb_ign << PCI_INO_BITS) |
2003 	    pci_p->pci_inos[CBNINTR_THERMAL]);
2004 	pci_mondo = CB_MONDO_TO_XMONDO(pci_p->pci_cb_p, pci_mondo);
2005 
2006 	if (pci_mondo == dev_mondo) {
2007 		DEBUG2(DBG_ATTACH, rdip, "pci_thermal_rem_intr unregistered "
2008 		    "for dip=%s%d:", ddi_driver_name(rdip),
2009 		    ddi_get_instance(rdip));
2010 		rem_ivintr(pci_mondo, NULL);
2011 	}
2012 }
2013