xref: /illumos-gate/usr/src/uts/sun4u/io/px/px_lib4u.c (revision d4bc0535efa2c2219e9f83246a5f371dc7f94273)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #include <sys/types.h>
27 #include <sys/kmem.h>
28 #include <sys/conf.h>
29 #include <sys/ddi.h>
30 #include <sys/sunddi.h>
31 #include <sys/sunndi.h>
32 #include <sys/fm/protocol.h>
33 #include <sys/fm/util.h>
34 #include <sys/modctl.h>
35 #include <sys/disp.h>
36 #include <sys/stat.h>
37 #include <sys/ddi_impldefs.h>
38 #include <sys/vmem.h>
39 #include <sys/iommutsb.h>
40 #include <sys/cpuvar.h>
41 #include <sys/ivintr.h>
42 #include <sys/byteorder.h>
43 #include <sys/hotplug/pci/pciehpc.h>
44 #include <sys/spl.h>
45 #include <px_obj.h>
46 #include <sys/pcie_pwr.h>
47 #include "px_tools_var.h"
48 #include <px_regs.h>
49 #include <px_csr.h>
50 #include <sys/machsystm.h>
51 #include "px_lib4u.h"
52 #include "px_err.h"
53 #include "oberon_regs.h"
54 
55 #pragma weak jbus_stst_order
56 
57 extern void jbus_stst_order();
58 
59 ulong_t px_mmu_dvma_end = 0xfffffffful;
60 uint_t px_ranges_phi_mask = 0xfffffffful;
61 uint64_t *px_oberon_ubc_scratch_regs;
62 uint64_t px_paddr_mask;
63 
64 static int px_goto_l23ready(px_t *px_p);
65 static int px_goto_l0(px_t *px_p);
66 static int px_pre_pwron_check(px_t *px_p);
67 static uint32_t px_identity_init(px_t *px_p);
68 static boolean_t px_cpr_callb(void *arg, int code);
69 static uint_t px_cb_intr(caddr_t arg);
70 
71 /*
72  * ACKNAK Latency Threshold Table.
73  * See Fire PRM 2.0 section 1.2.12.2, table 1-17.
74  */
75 int px_acknak_timer_table[LINK_MAX_PKT_ARR_SIZE][LINK_WIDTH_ARR_SIZE] = {
76 	{0xED,   0x49,  0x43,  0x30},
77 	{0x1A0,  0x76,  0x6B,  0x48},
78 	{0x22F,  0x9A,  0x56,  0x56},
79 	{0x42F,  0x11A, 0x96,  0x96},
80 	{0x82F,  0x21A, 0x116, 0x116},
81 	{0x102F, 0x41A, 0x216, 0x216}
82 };
83 
84 /*
85  * TxLink Replay Timer Latency Table
86  * See Fire PRM 2.0 sections 1.2.12.3, table 1-18.
87  */
88 int px_replay_timer_table[LINK_MAX_PKT_ARR_SIZE][LINK_WIDTH_ARR_SIZE] = {
89 	{0x379,  0x112, 0xFC,  0xB4},
90 	{0x618,  0x1BA, 0x192, 0x10E},
91 	{0x831,  0x242, 0x143, 0x143},
92 	{0xFB1,  0x422, 0x233, 0x233},
93 	{0x1EB0, 0x7E1, 0x412, 0x412},
94 	{0x3CB0, 0xF61, 0x7D2, 0x7D2}
95 };
96 /*
97  * px_lib_map_registers
98  *
99  * This function is called from the attach routine to map the registers
100  * accessed by this driver.
101  *
102  * used by: px_attach()
103  *
104  * return value: DDI_FAILURE on failure
105  */
106 int
107 px_lib_map_regs(pxu_t *pxu_p, dev_info_t *dip)
108 {
109 	ddi_device_acc_attr_t	attr;
110 	px_reg_bank_t		reg_bank = PX_REG_CSR;
111 
112 	DBG(DBG_ATTACH, dip, "px_lib_map_regs: pxu_p:0x%p, dip 0x%p\n",
113 	    pxu_p, dip);
114 
115 	attr.devacc_attr_version = DDI_DEVICE_ATTR_V0;
116 	attr.devacc_attr_dataorder = DDI_STRICTORDER_ACC;
117 	attr.devacc_attr_endian_flags = DDI_NEVERSWAP_ACC;
118 
119 	/*
120 	 * PCI CSR Base
121 	 */
122 	if (ddi_regs_map_setup(dip, reg_bank, &pxu_p->px_address[reg_bank],
123 	    0, 0, &attr, &pxu_p->px_ac[reg_bank]) != DDI_SUCCESS) {
124 		goto fail;
125 	}
126 
127 	reg_bank++;
128 
129 	/*
130 	 * XBUS CSR Base
131 	 */
132 	if (ddi_regs_map_setup(dip, reg_bank, &pxu_p->px_address[reg_bank],
133 	    0, 0, &attr, &pxu_p->px_ac[reg_bank]) != DDI_SUCCESS) {
134 		goto fail;
135 	}
136 
137 	pxu_p->px_address[reg_bank] -= FIRE_CONTROL_STATUS;
138 
139 done:
140 	for (; reg_bank >= PX_REG_CSR; reg_bank--) {
141 		DBG(DBG_ATTACH, dip, "reg_bank 0x%x address 0x%p\n",
142 		    reg_bank, pxu_p->px_address[reg_bank]);
143 	}
144 
145 	return (DDI_SUCCESS);
146 
147 fail:
148 	cmn_err(CE_WARN, "%s%d: unable to map reg entry %d\n",
149 	    ddi_driver_name(dip), ddi_get_instance(dip), reg_bank);
150 
151 	for (reg_bank--; reg_bank >= PX_REG_CSR; reg_bank--) {
152 		pxu_p->px_address[reg_bank] = NULL;
153 		ddi_regs_map_free(&pxu_p->px_ac[reg_bank]);
154 	}
155 
156 	return (DDI_FAILURE);
157 }
158 
159 /*
160  * px_lib_unmap_regs:
161  *
162  * This routine unmaps the registers mapped by map_px_registers.
163  *
164  * used by: px_detach(), and error conditions in px_attach()
165  *
166  * return value: none
167  */
168 void
169 px_lib_unmap_regs(pxu_t *pxu_p)
170 {
171 	int i;
172 
173 	for (i = 0; i < PX_REG_MAX; i++) {
174 		if (pxu_p->px_ac[i])
175 			ddi_regs_map_free(&pxu_p->px_ac[i]);
176 	}
177 }
178 
179 int
180 px_lib_dev_init(dev_info_t *dip, devhandle_t *dev_hdl)
181 {
182 
183 	caddr_t			xbc_csr_base, csr_base;
184 	px_dvma_range_prop_t	px_dvma_range;
185 	pxu_t			*pxu_p;
186 	uint8_t			chip_mask;
187 	px_t			*px_p = DIP_TO_STATE(dip);
188 	px_chip_type_t		chip_type = px_identity_init(px_p);
189 
190 	DBG(DBG_ATTACH, dip, "px_lib_dev_init: dip 0x%p", dip);
191 
192 	if (chip_type == PX_CHIP_UNIDENTIFIED) {
193 		cmn_err(CE_WARN, "%s%d: Unrecognized Hardware Version\n",
194 		    NAMEINST(dip));
195 		return (DDI_FAILURE);
196 	}
197 
198 	chip_mask = BITMASK(chip_type);
199 	px_paddr_mask = (chip_type == PX_CHIP_FIRE) ? MMU_FIRE_PADDR_MASK :
200 	    MMU_OBERON_PADDR_MASK;
201 
202 	/*
203 	 * Allocate platform specific structure and link it to
204 	 * the px state structure.
205 	 */
206 	pxu_p = kmem_zalloc(sizeof (pxu_t), KM_SLEEP);
207 	pxu_p->chip_type = chip_type;
208 	pxu_p->portid  = ddi_getprop(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
209 	    "portid", -1);
210 
211 	/* Map in the registers */
212 	if (px_lib_map_regs(pxu_p, dip) == DDI_FAILURE) {
213 		kmem_free(pxu_p, sizeof (pxu_t));
214 
215 		return (DDI_FAILURE);
216 	}
217 
218 	xbc_csr_base = (caddr_t)pxu_p->px_address[PX_REG_XBC];
219 	csr_base = (caddr_t)pxu_p->px_address[PX_REG_CSR];
220 
221 	pxu_p->tsb_cookie = iommu_tsb_alloc(pxu_p->portid);
222 	pxu_p->tsb_size = iommu_tsb_cookie_to_size(pxu_p->tsb_cookie);
223 	pxu_p->tsb_vaddr = iommu_tsb_cookie_to_va(pxu_p->tsb_cookie);
224 
225 	pxu_p->tsb_paddr = va_to_pa(pxu_p->tsb_vaddr);
226 
227 	/*
228 	 * Create "virtual-dma" property to support child devices
229 	 * needing to know DVMA range.
230 	 */
231 	px_dvma_range.dvma_base = (uint32_t)px_mmu_dvma_end + 1
232 	    - ((pxu_p->tsb_size >> 3) << MMU_PAGE_SHIFT);
233 	px_dvma_range.dvma_len = (uint32_t)
234 	    px_mmu_dvma_end - px_dvma_range.dvma_base + 1;
235 
236 	(void) ddi_prop_update_int_array(DDI_DEV_T_NONE, dip,
237 	    "virtual-dma", (int *)&px_dvma_range,
238 	    sizeof (px_dvma_range_prop_t) / sizeof (int));
239 	/*
240 	 * Initilize all fire hardware specific blocks.
241 	 */
242 	hvio_cb_init(xbc_csr_base, pxu_p);
243 	hvio_ib_init(csr_base, pxu_p);
244 	hvio_pec_init(csr_base, pxu_p);
245 	hvio_mmu_init(csr_base, pxu_p);
246 
247 	px_p->px_plat_p = (void *)pxu_p;
248 
249 	/*
250 	 * Initialize all the interrupt handlers
251 	 */
252 	switch (PX_CHIP_TYPE(pxu_p)) {
253 	case PX_CHIP_OBERON:
254 		/*
255 		 * Oberon hotplug uses SPARE3 field in ILU Error Log Enable
256 		 * register to indicate the status of leaf reset,
257 		 * we need to preserve the value of this bit, and keep it in
258 		 * px_ilu_log_mask to reflect the state of the bit
259 		 */
260 		if (CSR_BR(csr_base, ILU_ERROR_LOG_ENABLE, SPARE3))
261 			px_ilu_log_mask |= (1ull <<
262 			    ILU_ERROR_LOG_ENABLE_SPARE3);
263 		else
264 			px_ilu_log_mask &= ~(1ull <<
265 			    ILU_ERROR_LOG_ENABLE_SPARE3);
266 
267 		px_err_reg_setup_pcie(chip_mask, csr_base, PX_ERR_ENABLE);
268 		break;
269 
270 	case PX_CHIP_FIRE:
271 		px_err_reg_setup_pcie(chip_mask, csr_base, PX_ERR_ENABLE);
272 		break;
273 
274 	default:
275 		cmn_err(CE_WARN, "%s%d: PX primary bus Unknown\n",
276 		    ddi_driver_name(dip), ddi_get_instance(dip));
277 		return (DDI_FAILURE);
278 	}
279 
280 	/* Initilize device handle */
281 	*dev_hdl = (devhandle_t)csr_base;
282 
283 	DBG(DBG_ATTACH, dip, "px_lib_dev_init: dev_hdl 0x%llx\n", *dev_hdl);
284 
285 	return (DDI_SUCCESS);
286 }
287 
288 int
289 px_lib_dev_fini(dev_info_t *dip)
290 {
291 	caddr_t			csr_base;
292 	uint8_t			chip_mask;
293 	px_t			*px_p = DIP_TO_STATE(dip);
294 	pxu_t			*pxu_p = (pxu_t *)px_p->px_plat_p;
295 
296 	DBG(DBG_DETACH, dip, "px_lib_dev_fini: dip 0x%p\n", dip);
297 
298 	/*
299 	 * Deinitialize all the interrupt handlers
300 	 */
301 	switch (PX_CHIP_TYPE(pxu_p)) {
302 	case PX_CHIP_OBERON:
303 	case PX_CHIP_FIRE:
304 		chip_mask = BITMASK(PX_CHIP_TYPE(pxu_p));
305 		csr_base = (caddr_t)pxu_p->px_address[PX_REG_CSR];
306 		px_err_reg_setup_pcie(chip_mask, csr_base, PX_ERR_DISABLE);
307 		break;
308 
309 	default:
310 		cmn_err(CE_WARN, "%s%d: PX primary bus Unknown\n",
311 		    ddi_driver_name(dip), ddi_get_instance(dip));
312 		return (DDI_FAILURE);
313 	}
314 
315 	iommu_tsb_free(pxu_p->tsb_cookie);
316 
317 	px_lib_unmap_regs((pxu_t *)px_p->px_plat_p);
318 	kmem_free(px_p->px_plat_p, sizeof (pxu_t));
319 	px_p->px_plat_p = NULL;
320 	(void) ddi_prop_remove(DDI_DEV_T_NONE, dip, "virtual-dma");
321 
322 	return (DDI_SUCCESS);
323 }
324 
325 /*ARGSUSED*/
326 int
327 px_lib_intr_devino_to_sysino(dev_info_t *dip, devino_t devino,
328     sysino_t *sysino)
329 {
330 	px_t	*px_p = DIP_TO_STATE(dip);
331 	pxu_t	*pxu_p = (pxu_t *)px_p->px_plat_p;
332 	uint64_t	ret;
333 
334 	DBG(DBG_LIB_INT, dip, "px_lib_intr_devino_to_sysino: dip 0x%p "
335 	    "devino 0x%x\n", dip, devino);
336 
337 	if ((ret = hvio_intr_devino_to_sysino(DIP_TO_HANDLE(dip),
338 	    pxu_p, devino, sysino)) != H_EOK) {
339 		DBG(DBG_LIB_INT, dip,
340 		    "hvio_intr_devino_to_sysino failed, ret 0x%lx\n", ret);
341 		return (DDI_FAILURE);
342 	}
343 
344 	DBG(DBG_LIB_INT, dip, "px_lib_intr_devino_to_sysino: sysino 0x%llx\n",
345 	    *sysino);
346 
347 	return (DDI_SUCCESS);
348 }
349 
350 /*ARGSUSED*/
351 int
352 px_lib_intr_getvalid(dev_info_t *dip, sysino_t sysino,
353     intr_valid_state_t *intr_valid_state)
354 {
355 	uint64_t	ret;
356 
357 	DBG(DBG_LIB_INT, dip, "px_lib_intr_getvalid: dip 0x%p sysino 0x%llx\n",
358 	    dip, sysino);
359 
360 	if ((ret = hvio_intr_getvalid(DIP_TO_HANDLE(dip),
361 	    sysino, intr_valid_state)) != H_EOK) {
362 		DBG(DBG_LIB_INT, dip, "hvio_intr_getvalid failed, ret 0x%lx\n",
363 		    ret);
364 		return (DDI_FAILURE);
365 	}
366 
367 	DBG(DBG_LIB_INT, dip, "px_lib_intr_getvalid: intr_valid_state 0x%x\n",
368 	    *intr_valid_state);
369 
370 	return (DDI_SUCCESS);
371 }
372 
373 /*ARGSUSED*/
374 int
375 px_lib_intr_setvalid(dev_info_t *dip, sysino_t sysino,
376     intr_valid_state_t intr_valid_state)
377 {
378 	uint64_t	ret;
379 
380 	DBG(DBG_LIB_INT, dip, "px_lib_intr_setvalid: dip 0x%p sysino 0x%llx "
381 	    "intr_valid_state 0x%x\n", dip, sysino, intr_valid_state);
382 
383 	if ((ret = hvio_intr_setvalid(DIP_TO_HANDLE(dip),
384 	    sysino, intr_valid_state)) != H_EOK) {
385 		DBG(DBG_LIB_INT, dip, "hvio_intr_setvalid failed, ret 0x%lx\n",
386 		    ret);
387 		return (DDI_FAILURE);
388 	}
389 
390 	return (DDI_SUCCESS);
391 }
392 
393 /*ARGSUSED*/
394 int
395 px_lib_intr_getstate(dev_info_t *dip, sysino_t sysino,
396     intr_state_t *intr_state)
397 {
398 	uint64_t	ret;
399 
400 	DBG(DBG_LIB_INT, dip, "px_lib_intr_getstate: dip 0x%p sysino 0x%llx\n",
401 	    dip, sysino);
402 
403 	if ((ret = hvio_intr_getstate(DIP_TO_HANDLE(dip),
404 	    sysino, intr_state)) != H_EOK) {
405 		DBG(DBG_LIB_INT, dip, "hvio_intr_getstate failed, ret 0x%lx\n",
406 		    ret);
407 		return (DDI_FAILURE);
408 	}
409 
410 	DBG(DBG_LIB_INT, dip, "px_lib_intr_getstate: intr_state 0x%x\n",
411 	    *intr_state);
412 
413 	return (DDI_SUCCESS);
414 }
415 
416 /*ARGSUSED*/
417 int
418 px_lib_intr_setstate(dev_info_t *dip, sysino_t sysino,
419     intr_state_t intr_state)
420 {
421 	uint64_t	ret;
422 
423 	DBG(DBG_LIB_INT, dip, "px_lib_intr_setstate: dip 0x%p sysino 0x%llx "
424 	    "intr_state 0x%x\n", dip, sysino, intr_state);
425 
426 	if ((ret = hvio_intr_setstate(DIP_TO_HANDLE(dip),
427 	    sysino, intr_state)) != H_EOK) {
428 		DBG(DBG_LIB_INT, dip, "hvio_intr_setstate failed, ret 0x%lx\n",
429 		    ret);
430 		return (DDI_FAILURE);
431 	}
432 
433 	return (DDI_SUCCESS);
434 }
435 
436 /*ARGSUSED*/
437 int
438 px_lib_intr_gettarget(dev_info_t *dip, sysino_t sysino, cpuid_t *cpuid)
439 {
440 	px_t		*px_p = DIP_TO_STATE(dip);
441 	pxu_t		*pxu_p = (pxu_t *)px_p->px_plat_p;
442 	uint64_t	ret;
443 
444 	DBG(DBG_LIB_INT, dip, "px_lib_intr_gettarget: dip 0x%p sysino 0x%llx\n",
445 	    dip, sysino);
446 
447 	if ((ret = hvio_intr_gettarget(DIP_TO_HANDLE(dip), pxu_p,
448 	    sysino, cpuid)) != H_EOK) {
449 		DBG(DBG_LIB_INT, dip, "hvio_intr_gettarget failed, ret 0x%lx\n",
450 		    ret);
451 		return (DDI_FAILURE);
452 	}
453 
454 	DBG(DBG_LIB_INT, dip, "px_lib_intr_gettarget: cpuid 0x%x\n", cpuid);
455 
456 	return (DDI_SUCCESS);
457 }
458 
459 /*ARGSUSED*/
460 int
461 px_lib_intr_settarget(dev_info_t *dip, sysino_t sysino, cpuid_t cpuid)
462 {
463 	px_t		*px_p = DIP_TO_STATE(dip);
464 	pxu_t		*pxu_p = (pxu_t *)px_p->px_plat_p;
465 	uint64_t	ret;
466 
467 	DBG(DBG_LIB_INT, dip, "px_lib_intr_settarget: dip 0x%p sysino 0x%llx "
468 	    "cpuid 0x%x\n", dip, sysino, cpuid);
469 
470 	if ((ret = hvio_intr_settarget(DIP_TO_HANDLE(dip), pxu_p,
471 	    sysino, cpuid)) != H_EOK) {
472 		DBG(DBG_LIB_INT, dip, "hvio_intr_settarget failed, ret 0x%lx\n",
473 		    ret);
474 		return (DDI_FAILURE);
475 	}
476 
477 	return (DDI_SUCCESS);
478 }
479 
480 /*ARGSUSED*/
481 int
482 px_lib_intr_reset(dev_info_t *dip)
483 {
484 	devino_t	ino;
485 	sysino_t	sysino;
486 
487 	DBG(DBG_LIB_INT, dip, "px_lib_intr_reset: dip 0x%p\n", dip);
488 
489 	/* Reset all Interrupts */
490 	for (ino = 0; ino < INTERRUPT_MAPPING_ENTRIES; ino++) {
491 		if (px_lib_intr_devino_to_sysino(dip, ino,
492 		    &sysino) != DDI_SUCCESS)
493 			return (BF_FATAL);
494 
495 		if (px_lib_intr_setstate(dip, sysino,
496 		    INTR_IDLE_STATE) != DDI_SUCCESS)
497 			return (BF_FATAL);
498 	}
499 
500 	return (BF_NONE);
501 }
502 
503 /*ARGSUSED*/
504 int
505 px_lib_iommu_map(dev_info_t *dip, tsbid_t tsbid, pages_t pages,
506     io_attributes_t attr, void *addr, size_t pfn_index, int flags)
507 {
508 	px_t		*px_p = DIP_TO_STATE(dip);
509 	pxu_t		*pxu_p = (pxu_t *)px_p->px_plat_p;
510 	uint64_t	ret;
511 
512 	DBG(DBG_LIB_DMA, dip, "px_lib_iommu_map: dip 0x%p tsbid 0x%llx "
513 	    "pages 0x%x attr 0x%llx addr 0x%p pfn_index 0x%llx flags 0x%x\n",
514 	    dip, tsbid, pages, attr, addr, pfn_index, flags);
515 
516 	if ((ret = hvio_iommu_map(px_p->px_dev_hdl, pxu_p, tsbid, pages,
517 	    attr, addr, pfn_index, flags)) != H_EOK) {
518 		DBG(DBG_LIB_DMA, dip,
519 		    "px_lib_iommu_map failed, ret 0x%lx\n", ret);
520 		return (DDI_FAILURE);
521 	}
522 
523 	return (DDI_SUCCESS);
524 }
525 
526 /*ARGSUSED*/
527 int
528 px_lib_iommu_demap(dev_info_t *dip, tsbid_t tsbid, pages_t pages)
529 {
530 	px_t		*px_p = DIP_TO_STATE(dip);
531 	pxu_t		*pxu_p = (pxu_t *)px_p->px_plat_p;
532 	uint64_t	ret;
533 
534 	DBG(DBG_LIB_DMA, dip, "px_lib_iommu_demap: dip 0x%p tsbid 0x%llx "
535 	    "pages 0x%x\n", dip, tsbid, pages);
536 
537 	if ((ret = hvio_iommu_demap(px_p->px_dev_hdl, pxu_p, tsbid, pages))
538 	    != H_EOK) {
539 		DBG(DBG_LIB_DMA, dip,
540 		    "px_lib_iommu_demap failed, ret 0x%lx\n", ret);
541 
542 		return (DDI_FAILURE);
543 	}
544 
545 	return (DDI_SUCCESS);
546 }
547 
548 /*ARGSUSED*/
549 int
550 px_lib_iommu_getmap(dev_info_t *dip, tsbid_t tsbid, io_attributes_t *attr_p,
551     r_addr_t *r_addr_p)
552 {
553 	px_t	*px_p = DIP_TO_STATE(dip);
554 	pxu_t	*pxu_p = (pxu_t *)px_p->px_plat_p;
555 	uint64_t	ret;
556 
557 	DBG(DBG_LIB_DMA, dip, "px_lib_iommu_getmap: dip 0x%p tsbid 0x%llx\n",
558 	    dip, tsbid);
559 
560 	if ((ret = hvio_iommu_getmap(DIP_TO_HANDLE(dip), pxu_p, tsbid,
561 	    attr_p, r_addr_p)) != H_EOK) {
562 		DBG(DBG_LIB_DMA, dip,
563 		    "hvio_iommu_getmap failed, ret 0x%lx\n", ret);
564 
565 		return ((ret == H_ENOMAP) ? DDI_DMA_NOMAPPING:DDI_FAILURE);
566 	}
567 
568 	DBG(DBG_LIB_DMA, dip, "px_lib_iommu_getmap: attr 0x%llx "
569 	    "r_addr 0x%llx\n", *attr_p, *r_addr_p);
570 
571 	return (DDI_SUCCESS);
572 }
573 
574 
575 /*
576  * Checks dma attributes against system bypass ranges
577  * The bypass range is determined by the hardware. Return them so the
578  * common code can do generic checking against them.
579  */
580 /*ARGSUSED*/
581 int
582 px_lib_dma_bypass_rngchk(dev_info_t *dip, ddi_dma_attr_t *attr_p,
583     uint64_t *lo_p, uint64_t *hi_p)
584 {
585 	px_t	*px_p = DIP_TO_STATE(dip);
586 	pxu_t	*pxu_p = (pxu_t *)px_p->px_plat_p;
587 
588 	*lo_p = hvio_get_bypass_base(pxu_p);
589 	*hi_p = hvio_get_bypass_end(pxu_p);
590 
591 	return (DDI_SUCCESS);
592 }
593 
594 
595 /*ARGSUSED*/
596 int
597 px_lib_iommu_getbypass(dev_info_t *dip, r_addr_t ra, io_attributes_t attr,
598     io_addr_t *io_addr_p)
599 {
600 	uint64_t	ret;
601 	px_t	*px_p = DIP_TO_STATE(dip);
602 	pxu_t	*pxu_p = (pxu_t *)px_p->px_plat_p;
603 
604 	DBG(DBG_LIB_DMA, dip, "px_lib_iommu_getbypass: dip 0x%p ra 0x%llx "
605 	    "attr 0x%llx\n", dip, ra, attr);
606 
607 	if ((ret = hvio_iommu_getbypass(DIP_TO_HANDLE(dip), pxu_p, ra,
608 	    attr, io_addr_p)) != H_EOK) {
609 		DBG(DBG_LIB_DMA, dip,
610 		    "hvio_iommu_getbypass failed, ret 0x%lx\n", ret);
611 		return (DDI_FAILURE);
612 	}
613 
614 	DBG(DBG_LIB_DMA, dip, "px_lib_iommu_getbypass: io_addr 0x%llx\n",
615 	    *io_addr_p);
616 
617 	return (DDI_SUCCESS);
618 }
619 
620 /*
621  * Returns any needed IO address bit(s) for relaxed ordering in IOMMU
622  * bypass mode.
623  */
624 uint64_t
625 px_lib_ro_bypass(dev_info_t *dip, io_attributes_t attr, uint64_t ioaddr)
626 {
627 	px_t	*px_p = DIP_TO_STATE(dip);
628 	pxu_t	*pxu_p = (pxu_t *)px_p->px_plat_p;
629 
630 	if ((PX_CHIP_TYPE(pxu_p) == PX_CHIP_OBERON) && (attr & PCI_MAP_ATTR_RO))
631 		return (MMU_OBERON_BYPASS_RO | ioaddr);
632 	else
633 		return (ioaddr);
634 }
635 
636 /*
637  * bus dma sync entry point.
638  */
639 /*ARGSUSED*/
640 int
641 px_lib_dma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
642     off_t off, size_t len, uint_t cache_flags)
643 {
644 	ddi_dma_impl_t *mp = (ddi_dma_impl_t *)handle;
645 	px_t	*px_p = DIP_TO_STATE(dip);
646 	pxu_t	*pxu_p = (pxu_t *)px_p->px_plat_p;
647 
648 	DBG(DBG_LIB_DMA, dip, "px_lib_dma_sync: dip 0x%p rdip 0x%p "
649 	    "handle 0x%llx off 0x%x len 0x%x flags 0x%x\n",
650 	    dip, rdip, handle, off, len, cache_flags);
651 
652 	/*
653 	 * No flush needed for Oberon
654 	 */
655 	if (PX_CHIP_TYPE(pxu_p) == PX_CHIP_OBERON)
656 		return (DDI_SUCCESS);
657 
658 	/*
659 	 * jbus_stst_order is found only in certain cpu modules.
660 	 * Just return success if not present.
661 	 */
662 	if (&jbus_stst_order == NULL)
663 		return (DDI_SUCCESS);
664 
665 	if (!(mp->dmai_flags & PX_DMAI_FLAGS_INUSE)) {
666 		cmn_err(CE_WARN, "%s%d: Unbound dma handle %p.",
667 		    ddi_driver_name(rdip), ddi_get_instance(rdip), (void *)mp);
668 
669 		return (DDI_FAILURE);
670 	}
671 
672 	if (mp->dmai_flags & PX_DMAI_FLAGS_NOSYNC)
673 		return (DDI_SUCCESS);
674 
675 	/*
676 	 * No flush needed when sending data from memory to device.
677 	 * Nothing to do to "sync" memory to what device would already see.
678 	 */
679 	if (!(mp->dmai_rflags & DDI_DMA_READ) ||
680 	    ((cache_flags & PX_DMA_SYNC_DDI_FLAGS) == DDI_DMA_SYNC_FORDEV))
681 		return (DDI_SUCCESS);
682 
683 	/*
684 	 * Perform necessary cpu workaround to ensure jbus ordering.
685 	 * CPU's internal "invalidate FIFOs" are flushed.
686 	 */
687 
688 #if !defined(lint)
689 	kpreempt_disable();
690 #endif
691 	jbus_stst_order();
692 #if !defined(lint)
693 	kpreempt_enable();
694 #endif
695 	return (DDI_SUCCESS);
696 }
697 
698 /*
699  * MSIQ Functions:
700  */
701 /*ARGSUSED*/
702 int
703 px_lib_msiq_init(dev_info_t *dip)
704 {
705 	px_t		*px_p = DIP_TO_STATE(dip);
706 	pxu_t		*pxu_p = (pxu_t *)px_p->px_plat_p;
707 	px_msiq_state_t	*msiq_state_p = &px_p->px_ib_p->ib_msiq_state;
708 	px_dvma_addr_t	pg_index;
709 	size_t		q_sz = msiq_state_p->msiq_rec_cnt * sizeof (msiq_rec_t);
710 	size_t		size;
711 	int		i, ret;
712 
713 	DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_init: dip 0x%p\n", dip);
714 
715 	/* must aligned on q_sz (happens to be !!! page) boundary */
716 	ASSERT(q_sz == 8 * 1024);
717 
718 	/*
719 	 * Map the EQ memory into the Fire MMU (has to be 512KB aligned)
720 	 * and then initialize the base address register.
721 	 *
722 	 * Allocate entries from Fire IOMMU so that the resulting address
723 	 * is properly aligned.  Calculate the index of the first allocated
724 	 * entry.  Note: The size of the mapping is assumed to be a multiple
725 	 * of the page size.
726 	 */
727 	size = msiq_state_p->msiq_cnt * q_sz;
728 
729 	msiq_state_p->msiq_buf_p = kmem_zalloc(size, KM_SLEEP);
730 
731 	for (i = 0; i < msiq_state_p->msiq_cnt; i++)
732 		msiq_state_p->msiq_p[i].msiq_base_p = (msiqhead_t *)
733 		    ((caddr_t)msiq_state_p->msiq_buf_p + (i * q_sz));
734 
735 	pxu_p->msiq_mapped_p = vmem_xalloc(px_p->px_mmu_p->mmu_dvma_map,
736 	    size, (512 * 1024), 0, 0, NULL, NULL, VM_NOSLEEP | VM_BESTFIT);
737 
738 	if (pxu_p->msiq_mapped_p == NULL)
739 		return (DDI_FAILURE);
740 
741 	pg_index = MMU_PAGE_INDEX(px_p->px_mmu_p,
742 	    MMU_BTOP((ulong_t)pxu_p->msiq_mapped_p));
743 
744 	if ((ret = px_lib_iommu_map(px_p->px_dip, PCI_TSBID(0, pg_index),
745 	    MMU_BTOP(size), PCI_MAP_ATTR_WRITE, msiq_state_p->msiq_buf_p,
746 	    0, MMU_MAP_BUF)) != DDI_SUCCESS) {
747 		DBG(DBG_LIB_MSIQ, dip,
748 		    "px_lib_msiq_init: px_lib_iommu_map failed, "
749 		    "ret 0x%lx\n", ret);
750 
751 		(void) px_lib_msiq_fini(dip);
752 		return (DDI_FAILURE);
753 	}
754 
755 	if ((ret = hvio_msiq_init(DIP_TO_HANDLE(dip),
756 	    pxu_p)) != H_EOK) {
757 		DBG(DBG_LIB_MSIQ, dip,
758 		    "hvio_msiq_init failed, ret 0x%lx\n", ret);
759 
760 		(void) px_lib_msiq_fini(dip);
761 		return (DDI_FAILURE);
762 	}
763 
764 	return (DDI_SUCCESS);
765 }
766 
767 /*ARGSUSED*/
768 int
769 px_lib_msiq_fini(dev_info_t *dip)
770 {
771 	px_t		*px_p = DIP_TO_STATE(dip);
772 	pxu_t		*pxu_p = (pxu_t *)px_p->px_plat_p;
773 	px_msiq_state_t	*msiq_state_p = &px_p->px_ib_p->ib_msiq_state;
774 	px_dvma_addr_t	pg_index;
775 	size_t		size;
776 
777 	DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_fini: dip 0x%p\n", dip);
778 
779 	/*
780 	 * Unmap and free the EQ memory that had been mapped
781 	 * into the Fire IOMMU.
782 	 */
783 	size = msiq_state_p->msiq_cnt *
784 	    msiq_state_p->msiq_rec_cnt * sizeof (msiq_rec_t);
785 
786 	pg_index = MMU_PAGE_INDEX(px_p->px_mmu_p,
787 	    MMU_BTOP((ulong_t)pxu_p->msiq_mapped_p));
788 
789 	(void) px_lib_iommu_demap(px_p->px_dip,
790 	    PCI_TSBID(0, pg_index), MMU_BTOP(size));
791 
792 	/* Free the entries from the Fire MMU */
793 	vmem_xfree(px_p->px_mmu_p->mmu_dvma_map,
794 	    (void *)pxu_p->msiq_mapped_p, size);
795 
796 	kmem_free(msiq_state_p->msiq_buf_p, msiq_state_p->msiq_cnt *
797 	    msiq_state_p->msiq_rec_cnt * sizeof (msiq_rec_t));
798 
799 	return (DDI_SUCCESS);
800 }
801 
802 /*ARGSUSED*/
803 int
804 px_lib_msiq_info(dev_info_t *dip, msiqid_t msiq_id, r_addr_t *ra_p,
805     uint_t *msiq_rec_cnt_p)
806 {
807 	px_t		*px_p = DIP_TO_STATE(dip);
808 	px_msiq_state_t	*msiq_state_p = &px_p->px_ib_p->ib_msiq_state;
809 	size_t		msiq_size;
810 
811 	DBG(DBG_LIB_MSIQ, dip, "px_msiq_info: dip 0x%p msiq_id 0x%x\n",
812 	    dip, msiq_id);
813 
814 	msiq_size = msiq_state_p->msiq_rec_cnt * sizeof (msiq_rec_t);
815 	ra_p = (r_addr_t *)((caddr_t)msiq_state_p->msiq_buf_p +
816 	    (msiq_id * msiq_size));
817 
818 	*msiq_rec_cnt_p = msiq_state_p->msiq_rec_cnt;
819 
820 	DBG(DBG_LIB_MSIQ, dip, "px_msiq_info: ra_p 0x%p msiq_rec_cnt 0x%x\n",
821 	    ra_p, *msiq_rec_cnt_p);
822 
823 	return (DDI_SUCCESS);
824 }
825 
826 /*ARGSUSED*/
827 int
828 px_lib_msiq_getvalid(dev_info_t *dip, msiqid_t msiq_id,
829     pci_msiq_valid_state_t *msiq_valid_state)
830 {
831 	uint64_t	ret;
832 
833 	DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_getvalid: dip 0x%p msiq_id 0x%x\n",
834 	    dip, msiq_id);
835 
836 	if ((ret = hvio_msiq_getvalid(DIP_TO_HANDLE(dip),
837 	    msiq_id, msiq_valid_state)) != H_EOK) {
838 		DBG(DBG_LIB_MSIQ, dip,
839 		    "hvio_msiq_getvalid failed, ret 0x%lx\n", ret);
840 		return (DDI_FAILURE);
841 	}
842 
843 	DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_getvalid: msiq_valid_state 0x%x\n",
844 	    *msiq_valid_state);
845 
846 	return (DDI_SUCCESS);
847 }
848 
849 /*ARGSUSED*/
850 int
851 px_lib_msiq_setvalid(dev_info_t *dip, msiqid_t msiq_id,
852     pci_msiq_valid_state_t msiq_valid_state)
853 {
854 	uint64_t	ret;
855 
856 	DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_setvalid: dip 0x%p msiq_id 0x%x "
857 	    "msiq_valid_state 0x%x\n", dip, msiq_id, msiq_valid_state);
858 
859 	if ((ret = hvio_msiq_setvalid(DIP_TO_HANDLE(dip),
860 	    msiq_id, msiq_valid_state)) != H_EOK) {
861 		DBG(DBG_LIB_MSIQ, dip,
862 		    "hvio_msiq_setvalid failed, ret 0x%lx\n", ret);
863 		return (DDI_FAILURE);
864 	}
865 
866 	return (DDI_SUCCESS);
867 }
868 
869 /*ARGSUSED*/
870 int
871 px_lib_msiq_getstate(dev_info_t *dip, msiqid_t msiq_id,
872     pci_msiq_state_t *msiq_state)
873 {
874 	uint64_t	ret;
875 
876 	DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_getstate: dip 0x%p msiq_id 0x%x\n",
877 	    dip, msiq_id);
878 
879 	if ((ret = hvio_msiq_getstate(DIP_TO_HANDLE(dip),
880 	    msiq_id, msiq_state)) != H_EOK) {
881 		DBG(DBG_LIB_MSIQ, dip,
882 		    "hvio_msiq_getstate failed, ret 0x%lx\n", ret);
883 		return (DDI_FAILURE);
884 	}
885 
886 	DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_getstate: msiq_state 0x%x\n",
887 	    *msiq_state);
888 
889 	return (DDI_SUCCESS);
890 }
891 
892 /*ARGSUSED*/
893 int
894 px_lib_msiq_setstate(dev_info_t *dip, msiqid_t msiq_id,
895     pci_msiq_state_t msiq_state)
896 {
897 	uint64_t	ret;
898 
899 	DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_setstate: dip 0x%p msiq_id 0x%x "
900 	    "msiq_state 0x%x\n", dip, msiq_id, msiq_state);
901 
902 	if ((ret = hvio_msiq_setstate(DIP_TO_HANDLE(dip),
903 	    msiq_id, msiq_state)) != H_EOK) {
904 		DBG(DBG_LIB_MSIQ, dip,
905 		    "hvio_msiq_setstate failed, ret 0x%lx\n", ret);
906 		return (DDI_FAILURE);
907 	}
908 
909 	return (DDI_SUCCESS);
910 }
911 
912 /*ARGSUSED*/
913 int
914 px_lib_msiq_gethead(dev_info_t *dip, msiqid_t msiq_id,
915     msiqhead_t *msiq_head)
916 {
917 	uint64_t	ret;
918 
919 	DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_gethead: dip 0x%p msiq_id 0x%x\n",
920 	    dip, msiq_id);
921 
922 	if ((ret = hvio_msiq_gethead(DIP_TO_HANDLE(dip),
923 	    msiq_id, msiq_head)) != H_EOK) {
924 		DBG(DBG_LIB_MSIQ, dip,
925 		    "hvio_msiq_gethead failed, ret 0x%lx\n", ret);
926 		return (DDI_FAILURE);
927 	}
928 
929 	DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_gethead: msiq_head 0x%x\n",
930 	    *msiq_head);
931 
932 	return (DDI_SUCCESS);
933 }
934 
935 /*ARGSUSED*/
936 int
937 px_lib_msiq_sethead(dev_info_t *dip, msiqid_t msiq_id,
938     msiqhead_t msiq_head)
939 {
940 	uint64_t	ret;
941 
942 	DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_sethead: dip 0x%p msiq_id 0x%x "
943 	    "msiq_head 0x%x\n", dip, msiq_id, msiq_head);
944 
945 	if ((ret = hvio_msiq_sethead(DIP_TO_HANDLE(dip),
946 	    msiq_id, msiq_head)) != H_EOK) {
947 		DBG(DBG_LIB_MSIQ, dip,
948 		    "hvio_msiq_sethead failed, ret 0x%lx\n", ret);
949 		return (DDI_FAILURE);
950 	}
951 
952 	return (DDI_SUCCESS);
953 }
954 
955 /*ARGSUSED*/
956 int
957 px_lib_msiq_gettail(dev_info_t *dip, msiqid_t msiq_id,
958     msiqtail_t *msiq_tail)
959 {
960 	uint64_t	ret;
961 
962 	DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_gettail: dip 0x%p msiq_id 0x%x\n",
963 	    dip, msiq_id);
964 
965 	if ((ret = hvio_msiq_gettail(DIP_TO_HANDLE(dip),
966 	    msiq_id, msiq_tail)) != H_EOK) {
967 		DBG(DBG_LIB_MSIQ, dip,
968 		    "hvio_msiq_gettail failed, ret 0x%lx\n", ret);
969 		return (DDI_FAILURE);
970 	}
971 
972 	DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_gettail: msiq_tail 0x%x\n",
973 	    *msiq_tail);
974 
975 	return (DDI_SUCCESS);
976 }
977 
978 /*ARGSUSED*/
979 void
980 px_lib_get_msiq_rec(dev_info_t *dip, msiqhead_t *msiq_head_p,
981     msiq_rec_t *msiq_rec_p)
982 {
983 	eq_rec_t	*eq_rec_p = (eq_rec_t *)msiq_head_p;
984 
985 	DBG(DBG_LIB_MSIQ, dip, "px_lib_get_msiq_rec: dip 0x%p eq_rec_p 0x%p\n",
986 	    dip, eq_rec_p);
987 
988 	if (!eq_rec_p->eq_rec_fmt_type) {
989 		/* Set msiq_rec_type to zero */
990 		msiq_rec_p->msiq_rec_type = 0;
991 
992 		return;
993 	}
994 
995 	DBG(DBG_LIB_MSIQ, dip, "px_lib_get_msiq_rec: EQ RECORD, "
996 	    "eq_rec_rid 0x%llx eq_rec_fmt_type 0x%llx "
997 	    "eq_rec_len 0x%llx eq_rec_addr0 0x%llx "
998 	    "eq_rec_addr1 0x%llx eq_rec_data0 0x%llx "
999 	    "eq_rec_data1 0x%llx\n", eq_rec_p->eq_rec_rid,
1000 	    eq_rec_p->eq_rec_fmt_type, eq_rec_p->eq_rec_len,
1001 	    eq_rec_p->eq_rec_addr0, eq_rec_p->eq_rec_addr1,
1002 	    eq_rec_p->eq_rec_data0, eq_rec_p->eq_rec_data1);
1003 
1004 	/*
1005 	 * Only upper 4 bits of eq_rec_fmt_type is used
1006 	 * to identify the EQ record type.
1007 	 */
1008 	switch (eq_rec_p->eq_rec_fmt_type >> 3) {
1009 	case EQ_REC_MSI32:
1010 		msiq_rec_p->msiq_rec_type = MSI32_REC;
1011 
1012 		msiq_rec_p->msiq_rec_data.msi.msi_data =
1013 		    eq_rec_p->eq_rec_data0;
1014 		break;
1015 	case EQ_REC_MSI64:
1016 		msiq_rec_p->msiq_rec_type = MSI64_REC;
1017 
1018 		msiq_rec_p->msiq_rec_data.msi.msi_data =
1019 		    eq_rec_p->eq_rec_data0;
1020 		break;
1021 	case EQ_REC_MSG:
1022 		msiq_rec_p->msiq_rec_type = MSG_REC;
1023 
1024 		msiq_rec_p->msiq_rec_data.msg.msg_route =
1025 		    eq_rec_p->eq_rec_fmt_type & 7;
1026 		msiq_rec_p->msiq_rec_data.msg.msg_targ = eq_rec_p->eq_rec_rid;
1027 		msiq_rec_p->msiq_rec_data.msg.msg_code = eq_rec_p->eq_rec_data0;
1028 		break;
1029 	default:
1030 		cmn_err(CE_WARN, "%s%d: px_lib_get_msiq_rec: "
1031 		    "0x%x is an unknown EQ record type",
1032 		    ddi_driver_name(dip), ddi_get_instance(dip),
1033 		    (int)eq_rec_p->eq_rec_fmt_type);
1034 		break;
1035 	}
1036 
1037 	msiq_rec_p->msiq_rec_rid = eq_rec_p->eq_rec_rid;
1038 	msiq_rec_p->msiq_rec_msi_addr = ((eq_rec_p->eq_rec_addr1 << 16) |
1039 	    (eq_rec_p->eq_rec_addr0 << 2));
1040 }
1041 
1042 /*ARGSUSED*/
1043 void
1044 px_lib_clr_msiq_rec(dev_info_t *dip, msiqhead_t *msiq_head_p)
1045 {
1046 	eq_rec_t	*eq_rec_p = (eq_rec_t *)msiq_head_p;
1047 
1048 	DBG(DBG_LIB_MSIQ, dip, "px_lib_clr_msiq_rec: dip 0x%p eq_rec_p 0x%p\n",
1049 	    dip, eq_rec_p);
1050 
1051 	if (eq_rec_p->eq_rec_fmt_type) {
1052 		/* Zero out eq_rec_fmt_type field */
1053 		eq_rec_p->eq_rec_fmt_type = 0;
1054 	}
1055 }
1056 
1057 /*
1058  * MSI Functions:
1059  */
1060 /*ARGSUSED*/
1061 int
1062 px_lib_msi_init(dev_info_t *dip)
1063 {
1064 	px_t		*px_p = DIP_TO_STATE(dip);
1065 	px_msi_state_t	*msi_state_p = &px_p->px_ib_p->ib_msi_state;
1066 	uint64_t	ret;
1067 
1068 	DBG(DBG_LIB_MSI, dip, "px_lib_msi_init: dip 0x%p\n", dip);
1069 
1070 	if ((ret = hvio_msi_init(DIP_TO_HANDLE(dip),
1071 	    msi_state_p->msi_addr32, msi_state_p->msi_addr64)) != H_EOK) {
1072 		DBG(DBG_LIB_MSIQ, dip, "px_lib_msi_init failed, ret 0x%lx\n",
1073 		    ret);
1074 		return (DDI_FAILURE);
1075 	}
1076 
1077 	return (DDI_SUCCESS);
1078 }
1079 
1080 /*ARGSUSED*/
1081 int
1082 px_lib_msi_getmsiq(dev_info_t *dip, msinum_t msi_num,
1083     msiqid_t *msiq_id)
1084 {
1085 	uint64_t	ret;
1086 
1087 	DBG(DBG_LIB_MSI, dip, "px_lib_msi_getmsiq: dip 0x%p msi_num 0x%x\n",
1088 	    dip, msi_num);
1089 
1090 	if ((ret = hvio_msi_getmsiq(DIP_TO_HANDLE(dip),
1091 	    msi_num, msiq_id)) != H_EOK) {
1092 		DBG(DBG_LIB_MSI, dip,
1093 		    "hvio_msi_getmsiq failed, ret 0x%lx\n", ret);
1094 		return (DDI_FAILURE);
1095 	}
1096 
1097 	DBG(DBG_LIB_MSI, dip, "px_lib_msi_getmsiq: msiq_id 0x%x\n",
1098 	    *msiq_id);
1099 
1100 	return (DDI_SUCCESS);
1101 }
1102 
1103 /*ARGSUSED*/
1104 int
1105 px_lib_msi_setmsiq(dev_info_t *dip, msinum_t msi_num,
1106     msiqid_t msiq_id, msi_type_t msitype)
1107 {
1108 	uint64_t	ret;
1109 
1110 	DBG(DBG_LIB_MSI, dip, "px_lib_msi_setmsiq: dip 0x%p msi_num 0x%x "
1111 	    "msq_id 0x%x\n", dip, msi_num, msiq_id);
1112 
1113 	if ((ret = hvio_msi_setmsiq(DIP_TO_HANDLE(dip),
1114 	    msi_num, msiq_id)) != H_EOK) {
1115 		DBG(DBG_LIB_MSI, dip,
1116 		    "hvio_msi_setmsiq failed, ret 0x%lx\n", ret);
1117 		return (DDI_FAILURE);
1118 	}
1119 
1120 	return (DDI_SUCCESS);
1121 }
1122 
1123 /*ARGSUSED*/
1124 int
1125 px_lib_msi_getvalid(dev_info_t *dip, msinum_t msi_num,
1126     pci_msi_valid_state_t *msi_valid_state)
1127 {
1128 	uint64_t	ret;
1129 
1130 	DBG(DBG_LIB_MSI, dip, "px_lib_msi_getvalid: dip 0x%p msi_num 0x%x\n",
1131 	    dip, msi_num);
1132 
1133 	if ((ret = hvio_msi_getvalid(DIP_TO_HANDLE(dip),
1134 	    msi_num, msi_valid_state)) != H_EOK) {
1135 		DBG(DBG_LIB_MSI, dip,
1136 		    "hvio_msi_getvalid failed, ret 0x%lx\n", ret);
1137 		return (DDI_FAILURE);
1138 	}
1139 
1140 	DBG(DBG_LIB_MSI, dip, "px_lib_msi_getvalid: msiq_id 0x%x\n",
1141 	    *msi_valid_state);
1142 
1143 	return (DDI_SUCCESS);
1144 }
1145 
1146 /*ARGSUSED*/
1147 int
1148 px_lib_msi_setvalid(dev_info_t *dip, msinum_t msi_num,
1149     pci_msi_valid_state_t msi_valid_state)
1150 {
1151 	uint64_t	ret;
1152 
1153 	DBG(DBG_LIB_MSI, dip, "px_lib_msi_setvalid: dip 0x%p msi_num 0x%x "
1154 	    "msi_valid_state 0x%x\n", dip, msi_num, msi_valid_state);
1155 
1156 	if ((ret = hvio_msi_setvalid(DIP_TO_HANDLE(dip),
1157 	    msi_num, msi_valid_state)) != H_EOK) {
1158 		DBG(DBG_LIB_MSI, dip,
1159 		    "hvio_msi_setvalid failed, ret 0x%lx\n", ret);
1160 		return (DDI_FAILURE);
1161 	}
1162 
1163 	return (DDI_SUCCESS);
1164 }
1165 
1166 /*ARGSUSED*/
1167 int
1168 px_lib_msi_getstate(dev_info_t *dip, msinum_t msi_num,
1169     pci_msi_state_t *msi_state)
1170 {
1171 	uint64_t	ret;
1172 
1173 	DBG(DBG_LIB_MSI, dip, "px_lib_msi_getstate: dip 0x%p msi_num 0x%x\n",
1174 	    dip, msi_num);
1175 
1176 	if ((ret = hvio_msi_getstate(DIP_TO_HANDLE(dip),
1177 	    msi_num, msi_state)) != H_EOK) {
1178 		DBG(DBG_LIB_MSI, dip,
1179 		    "hvio_msi_getstate failed, ret 0x%lx\n", ret);
1180 		return (DDI_FAILURE);
1181 	}
1182 
1183 	DBG(DBG_LIB_MSI, dip, "px_lib_msi_getstate: msi_state 0x%x\n",
1184 	    *msi_state);
1185 
1186 	return (DDI_SUCCESS);
1187 }
1188 
1189 /*ARGSUSED*/
1190 int
1191 px_lib_msi_setstate(dev_info_t *dip, msinum_t msi_num,
1192     pci_msi_state_t msi_state)
1193 {
1194 	uint64_t	ret;
1195 
1196 	DBG(DBG_LIB_MSI, dip, "px_lib_msi_setstate: dip 0x%p msi_num 0x%x "
1197 	    "msi_state 0x%x\n", dip, msi_num, msi_state);
1198 
1199 	if ((ret = hvio_msi_setstate(DIP_TO_HANDLE(dip),
1200 	    msi_num, msi_state)) != H_EOK) {
1201 		DBG(DBG_LIB_MSI, dip,
1202 		    "hvio_msi_setstate failed, ret 0x%lx\n", ret);
1203 		return (DDI_FAILURE);
1204 	}
1205 
1206 	return (DDI_SUCCESS);
1207 }
1208 
1209 /*
1210  * MSG Functions:
1211  */
1212 /*ARGSUSED*/
1213 int
1214 px_lib_msg_getmsiq(dev_info_t *dip, pcie_msg_type_t msg_type,
1215     msiqid_t *msiq_id)
1216 {
1217 	uint64_t	ret;
1218 
1219 	DBG(DBG_LIB_MSG, dip, "px_lib_msg_getmsiq: dip 0x%p msg_type 0x%x\n",
1220 	    dip, msg_type);
1221 
1222 	if ((ret = hvio_msg_getmsiq(DIP_TO_HANDLE(dip),
1223 	    msg_type, msiq_id)) != H_EOK) {
1224 		DBG(DBG_LIB_MSG, dip,
1225 		    "hvio_msg_getmsiq failed, ret 0x%lx\n", ret);
1226 		return (DDI_FAILURE);
1227 	}
1228 
1229 	DBG(DBG_LIB_MSI, dip, "px_lib_msg_getmsiq: msiq_id 0x%x\n",
1230 	    *msiq_id);
1231 
1232 	return (DDI_SUCCESS);
1233 }
1234 
1235 /*ARGSUSED*/
1236 int
1237 px_lib_msg_setmsiq(dev_info_t *dip, pcie_msg_type_t msg_type,
1238     msiqid_t msiq_id)
1239 {
1240 	uint64_t	ret;
1241 
1242 	DBG(DBG_LIB_MSG, dip, "px_lib_msi_setstate: dip 0x%p msg_type 0x%x "
1243 	    "msiq_id 0x%x\n", dip, msg_type, msiq_id);
1244 
1245 	if ((ret = hvio_msg_setmsiq(DIP_TO_HANDLE(dip),
1246 	    msg_type, msiq_id)) != H_EOK) {
1247 		DBG(DBG_LIB_MSG, dip,
1248 		    "hvio_msg_setmsiq failed, ret 0x%lx\n", ret);
1249 		return (DDI_FAILURE);
1250 	}
1251 
1252 	return (DDI_SUCCESS);
1253 }
1254 
1255 /*ARGSUSED*/
1256 int
1257 px_lib_msg_getvalid(dev_info_t *dip, pcie_msg_type_t msg_type,
1258     pcie_msg_valid_state_t *msg_valid_state)
1259 {
1260 	uint64_t	ret;
1261 
1262 	DBG(DBG_LIB_MSG, dip, "px_lib_msg_getvalid: dip 0x%p msg_type 0x%x\n",
1263 	    dip, msg_type);
1264 
1265 	if ((ret = hvio_msg_getvalid(DIP_TO_HANDLE(dip), msg_type,
1266 	    msg_valid_state)) != H_EOK) {
1267 		DBG(DBG_LIB_MSG, dip,
1268 		    "hvio_msg_getvalid failed, ret 0x%lx\n", ret);
1269 		return (DDI_FAILURE);
1270 	}
1271 
1272 	DBG(DBG_LIB_MSI, dip, "px_lib_msg_getvalid: msg_valid_state 0x%x\n",
1273 	    *msg_valid_state);
1274 
1275 	return (DDI_SUCCESS);
1276 }
1277 
1278 /*ARGSUSED*/
1279 int
1280 px_lib_msg_setvalid(dev_info_t *dip, pcie_msg_type_t msg_type,
1281     pcie_msg_valid_state_t msg_valid_state)
1282 {
1283 	uint64_t	ret;
1284 
1285 	DBG(DBG_LIB_MSG, dip, "px_lib_msg_setvalid: dip 0x%p msg_type 0x%x "
1286 	    "msg_valid_state 0x%x\n", dip, msg_type, msg_valid_state);
1287 
1288 	if ((ret = hvio_msg_setvalid(DIP_TO_HANDLE(dip), msg_type,
1289 	    msg_valid_state)) != H_EOK) {
1290 		DBG(DBG_LIB_MSG, dip,
1291 		    "hvio_msg_setvalid failed, ret 0x%lx\n", ret);
1292 		return (DDI_FAILURE);
1293 	}
1294 
1295 	return (DDI_SUCCESS);
1296 }
1297 
1298 /*
1299  * Suspend/Resume Functions:
1300  * Currently unsupported by hypervisor
1301  */
1302 int
1303 px_lib_suspend(dev_info_t *dip)
1304 {
1305 	px_t		*px_p = DIP_TO_STATE(dip);
1306 	pxu_t		*pxu_p = (pxu_t *)px_p->px_plat_p;
1307 	px_cb_t		*cb_p = PX2CB(px_p);
1308 	devhandle_t	dev_hdl, xbus_dev_hdl;
1309 	uint64_t	ret = H_EOK;
1310 
1311 	DBG(DBG_DETACH, dip, "px_lib_suspend: dip 0x%p\n", dip);
1312 
1313 	dev_hdl = (devhandle_t)pxu_p->px_address[PX_REG_CSR];
1314 	xbus_dev_hdl = (devhandle_t)pxu_p->px_address[PX_REG_XBC];
1315 
1316 	if ((ret = hvio_suspend(dev_hdl, pxu_p)) != H_EOK)
1317 		goto fail;
1318 
1319 	if (--cb_p->attachcnt == 0) {
1320 		ret = hvio_cb_suspend(xbus_dev_hdl, pxu_p);
1321 		if (ret != H_EOK)
1322 			cb_p->attachcnt++;
1323 	}
1324 	pxu_p->cpr_flag = PX_ENTERED_CPR;
1325 
1326 fail:
1327 	return ((ret != H_EOK) ? DDI_FAILURE: DDI_SUCCESS);
1328 }
1329 
1330 void
1331 px_lib_resume(dev_info_t *dip)
1332 {
1333 	px_t		*px_p = DIP_TO_STATE(dip);
1334 	pxu_t		*pxu_p = (pxu_t *)px_p->px_plat_p;
1335 	px_cb_t		*cb_p = PX2CB(px_p);
1336 	devhandle_t	dev_hdl, xbus_dev_hdl;
1337 	devino_t	pec_ino = px_p->px_inos[PX_INTR_PEC];
1338 	devino_t	xbc_ino = px_p->px_inos[PX_INTR_XBC];
1339 
1340 	DBG(DBG_ATTACH, dip, "px_lib_resume: dip 0x%p\n", dip);
1341 
1342 	dev_hdl = (devhandle_t)pxu_p->px_address[PX_REG_CSR];
1343 	xbus_dev_hdl = (devhandle_t)pxu_p->px_address[PX_REG_XBC];
1344 
1345 	if (++cb_p->attachcnt == 1)
1346 		hvio_cb_resume(dev_hdl, xbus_dev_hdl, xbc_ino, pxu_p);
1347 
1348 	hvio_resume(dev_hdl, pec_ino, pxu_p);
1349 }
1350 
1351 /*
1352  * Generate a unique Oberon UBC ID based on the Logicial System Board and
1353  * the IO Channel from the portid property field.
1354  */
1355 static uint64_t
1356 oberon_get_ubc_id(dev_info_t *dip)
1357 {
1358 	px_t	*px_p = DIP_TO_STATE(dip);
1359 	pxu_t	*pxu_p = (pxu_t *)px_p->px_plat_p;
1360 	uint64_t	ubc_id;
1361 
1362 	/*
1363 	 * Generate a unique 6 bit UBC ID using the 2 IO_Channel#[1:0] bits and
1364 	 * the 4 LSB_ID[3:0] bits from the Oberon's portid property.
1365 	 */
1366 	ubc_id = (((pxu_p->portid >> OBERON_PORT_ID_IOC) &
1367 	    OBERON_PORT_ID_IOC_MASK) | (((pxu_p->portid >>
1368 	    OBERON_PORT_ID_LSB) & OBERON_PORT_ID_LSB_MASK)
1369 	    << OBERON_UBC_ID_LSB));
1370 
1371 	return (ubc_id);
1372 }
1373 
1374 /*
1375  * Oberon does not have a UBC scratch register, so alloc an array of scratch
1376  * registers when needed and use a unique UBC ID as an index. This code
1377  * can be simplified if we use a pre-allocated array. They are currently
1378  * being dynamically allocated because it's only needed by the Oberon.
1379  */
1380 static void
1381 oberon_set_cb(dev_info_t *dip, uint64_t val)
1382 {
1383 	uint64_t	ubc_id;
1384 
1385 	if (px_oberon_ubc_scratch_regs == NULL)
1386 		px_oberon_ubc_scratch_regs =
1387 		    (uint64_t *)kmem_zalloc(sizeof (uint64_t)*
1388 		    OBERON_UBC_ID_MAX, KM_SLEEP);
1389 
1390 	ubc_id = oberon_get_ubc_id(dip);
1391 
1392 	px_oberon_ubc_scratch_regs[ubc_id] = val;
1393 
1394 	/*
1395 	 * Check if any scratch registers are still in use. If all scratch
1396 	 * registers are currently set to zero, then deallocate the scratch
1397 	 * register array.
1398 	 */
1399 	for (ubc_id = 0; ubc_id < OBERON_UBC_ID_MAX; ubc_id++) {
1400 		if (px_oberon_ubc_scratch_regs[ubc_id] != NULL)
1401 			return;
1402 	}
1403 
1404 	/*
1405 	 * All scratch registers are set to zero so deallocate the scratch
1406 	 * register array and set the pointer to NULL.
1407 	 */
1408 	kmem_free(px_oberon_ubc_scratch_regs,
1409 	    (sizeof (uint64_t)*OBERON_UBC_ID_MAX));
1410 
1411 	px_oberon_ubc_scratch_regs = NULL;
1412 }
1413 
1414 /*
1415  * Oberon does not have a UBC scratch register, so use an allocated array of
1416  * scratch registers and use the unique UBC ID as an index into that array.
1417  */
1418 static uint64_t
1419 oberon_get_cb(dev_info_t *dip)
1420 {
1421 	uint64_t	ubc_id;
1422 
1423 	if (px_oberon_ubc_scratch_regs == NULL)
1424 		return (0);
1425 
1426 	ubc_id = oberon_get_ubc_id(dip);
1427 
1428 	return (px_oberon_ubc_scratch_regs[ubc_id]);
1429 }
1430 
1431 /*
1432  * Misc Functions:
1433  * Currently unsupported by hypervisor
1434  */
1435 static uint64_t
1436 px_get_cb(dev_info_t *dip)
1437 {
1438 	px_t	*px_p = DIP_TO_STATE(dip);
1439 	pxu_t	*pxu_p = (pxu_t *)px_p->px_plat_p;
1440 
1441 	/*
1442 	 * Oberon does not currently have Scratchpad registers.
1443 	 */
1444 	if (PX_CHIP_TYPE(pxu_p) == PX_CHIP_OBERON)
1445 		return (oberon_get_cb(dip));
1446 
1447 	return (CSR_XR((caddr_t)pxu_p->px_address[PX_REG_XBC], JBUS_SCRATCH_1));
1448 }
1449 
1450 static void
1451 px_set_cb(dev_info_t *dip, uint64_t val)
1452 {
1453 	px_t	*px_p = DIP_TO_STATE(dip);
1454 	pxu_t	*pxu_p = (pxu_t *)px_p->px_plat_p;
1455 
1456 	/*
1457 	 * Oberon does not currently have Scratchpad registers.
1458 	 */
1459 	if (PX_CHIP_TYPE(pxu_p) == PX_CHIP_OBERON) {
1460 		oberon_set_cb(dip, val);
1461 		return;
1462 	}
1463 
1464 	CSR_XS((caddr_t)pxu_p->px_address[PX_REG_XBC], JBUS_SCRATCH_1, val);
1465 }
1466 
1467 /*ARGSUSED*/
1468 int
1469 px_lib_map_vconfig(dev_info_t *dip,
1470 	ddi_map_req_t *mp, pci_config_offset_t off,
1471 		pci_regspec_t *rp, caddr_t *addrp)
1472 {
1473 	/*
1474 	 * No special config space access services in this layer.
1475 	 */
1476 	return (DDI_FAILURE);
1477 }
1478 
1479 void
1480 px_lib_map_attr_check(ddi_map_req_t *mp)
1481 {
1482 	ddi_acc_hdl_t *hp = mp->map_handlep;
1483 
1484 	/* fire does not accept byte masks from PIO store merge */
1485 	if (hp->ah_acc.devacc_attr_dataorder == DDI_STORECACHING_OK_ACC)
1486 		hp->ah_acc.devacc_attr_dataorder = DDI_STRICTORDER_ACC;
1487 }
1488 
1489 /* This function is called only by poke, caut put and pxtool poke. */
1490 void
1491 px_lib_clr_errs(px_t *px_p, dev_info_t *rdip, uint64_t addr)
1492 {
1493 	px_pec_t	*pec_p = px_p->px_pec_p;
1494 	dev_info_t	*rpdip = px_p->px_dip;
1495 	int		rc_err, fab_err, i;
1496 	int		acctype = pec_p->pec_safeacc_type;
1497 	ddi_fm_error_t	derr;
1498 	px_ranges_t	*ranges_p;
1499 	int		range_len;
1500 	uint32_t	addr_high, addr_low;
1501 	pcie_req_id_t	bdf = PCIE_INVALID_BDF;
1502 
1503 	/* Create the derr */
1504 	bzero(&derr, sizeof (ddi_fm_error_t));
1505 	derr.fme_version = DDI_FME_VERSION;
1506 	derr.fme_ena = fm_ena_generate(0, FM_ENA_FMT1);
1507 	derr.fme_flag = acctype;
1508 
1509 	if (acctype == DDI_FM_ERR_EXPECTED) {
1510 		derr.fme_status = DDI_FM_NONFATAL;
1511 		ndi_fm_acc_err_set(pec_p->pec_acc_hdl, &derr);
1512 	}
1513 
1514 	if (px_fm_enter(px_p) != DDI_SUCCESS)
1515 		return;
1516 
1517 	/* send ereport/handle/clear fire registers */
1518 	rc_err = px_err_cmn_intr(px_p, &derr, PX_LIB_CALL, PX_FM_BLOCK_ALL);
1519 
1520 	/* Figure out if this is a cfg or mem32 access */
1521 	addr_high = (uint32_t)(addr >> 32);
1522 	addr_low = (uint32_t)addr;
1523 	range_len = px_p->px_ranges_length / sizeof (px_ranges_t);
1524 	i = 0;
1525 	for (ranges_p = px_p->px_ranges_p; i < range_len; i++, ranges_p++) {
1526 		if (ranges_p->parent_high == addr_high) {
1527 			switch (ranges_p->child_high & PCI_ADDR_MASK) {
1528 			case PCI_ADDR_CONFIG:
1529 				bdf = (pcie_req_id_t)(addr_low >> 12);
1530 				addr_low = 0;
1531 				break;
1532 			case PCI_ADDR_MEM32:
1533 				if (rdip)
1534 					bdf = PCI_GET_BDF(rdip);
1535 				else
1536 					bdf = PCIE_INVALID_BDF;
1537 				break;
1538 			}
1539 			break;
1540 		}
1541 	}
1542 
1543 	px_rp_en_q(px_p, bdf, addr_low, NULL);
1544 
1545 	/*
1546 	 * XXX - Current code scans the fabric for all px_tool accesses.
1547 	 * In future, do not scan fabric for px_tool access to IO Root Nexus
1548 	 */
1549 	fab_err = px_scan_fabric(px_p, rpdip, &derr);
1550 
1551 	px_err_panic(rc_err, PX_RC, fab_err, B_TRUE);
1552 	px_fm_exit(px_p);
1553 	px_err_panic(rc_err, PX_RC, fab_err, B_FALSE);
1554 }
1555 
1556 #ifdef  DEBUG
1557 int	px_peekfault_cnt = 0;
1558 int	px_pokefault_cnt = 0;
1559 #endif  /* DEBUG */
1560 
1561 /*ARGSUSED*/
1562 static int
1563 px_lib_do_poke(dev_info_t *dip, dev_info_t *rdip,
1564     peekpoke_ctlops_t *in_args)
1565 {
1566 	px_t *px_p = DIP_TO_STATE(dip);
1567 	px_pec_t *pec_p = px_p->px_pec_p;
1568 	int err = DDI_SUCCESS;
1569 	on_trap_data_t otd;
1570 
1571 	mutex_enter(&pec_p->pec_pokefault_mutex);
1572 	pec_p->pec_ontrap_data = &otd;
1573 	pec_p->pec_safeacc_type = DDI_FM_ERR_POKE;
1574 
1575 	/* Set up protected environment. */
1576 	if (!on_trap(&otd, OT_DATA_ACCESS)) {
1577 		uintptr_t tramp = otd.ot_trampoline;
1578 
1579 		otd.ot_trampoline = (uintptr_t)&poke_fault;
1580 		err = do_poke(in_args->size, (void *)in_args->dev_addr,
1581 		    (void *)in_args->host_addr);
1582 		otd.ot_trampoline = tramp;
1583 	} else
1584 		err = DDI_FAILURE;
1585 
1586 	px_lib_clr_errs(px_p, rdip, in_args->dev_addr);
1587 
1588 	if (otd.ot_trap & OT_DATA_ACCESS)
1589 		err = DDI_FAILURE;
1590 
1591 	/* Take down protected environment. */
1592 	no_trap();
1593 
1594 	pec_p->pec_ontrap_data = NULL;
1595 	pec_p->pec_safeacc_type = DDI_FM_ERR_UNEXPECTED;
1596 	mutex_exit(&pec_p->pec_pokefault_mutex);
1597 
1598 #ifdef  DEBUG
1599 	if (err == DDI_FAILURE)
1600 		px_pokefault_cnt++;
1601 #endif
1602 	return (err);
1603 }
1604 
1605 /*ARGSUSED*/
1606 static int
1607 px_lib_do_caut_put(dev_info_t *dip, dev_info_t *rdip,
1608     peekpoke_ctlops_t *cautacc_ctlops_arg)
1609 {
1610 	size_t size = cautacc_ctlops_arg->size;
1611 	uintptr_t dev_addr = cautacc_ctlops_arg->dev_addr;
1612 	uintptr_t host_addr = cautacc_ctlops_arg->host_addr;
1613 	ddi_acc_impl_t *hp = (ddi_acc_impl_t *)cautacc_ctlops_arg->handle;
1614 	size_t repcount = cautacc_ctlops_arg->repcount;
1615 	uint_t flags = cautacc_ctlops_arg->flags;
1616 
1617 	px_t *px_p = DIP_TO_STATE(dip);
1618 	px_pec_t *pec_p = px_p->px_pec_p;
1619 	int err = DDI_SUCCESS;
1620 
1621 	/*
1622 	 * Note that i_ndi_busop_access_enter ends up grabbing the pokefault
1623 	 * mutex.
1624 	 */
1625 	i_ndi_busop_access_enter(hp->ahi_common.ah_dip, (ddi_acc_handle_t)hp);
1626 
1627 	pec_p->pec_ontrap_data = (on_trap_data_t *)hp->ahi_err->err_ontrap;
1628 	pec_p->pec_safeacc_type = DDI_FM_ERR_EXPECTED;
1629 	hp->ahi_err->err_expected = DDI_FM_ERR_EXPECTED;
1630 
1631 	if (!i_ddi_ontrap((ddi_acc_handle_t)hp)) {
1632 		for (; repcount; repcount--) {
1633 			switch (size) {
1634 
1635 			case sizeof (uint8_t):
1636 				i_ddi_put8(hp, (uint8_t *)dev_addr,
1637 				    *(uint8_t *)host_addr);
1638 				break;
1639 
1640 			case sizeof (uint16_t):
1641 				i_ddi_put16(hp, (uint16_t *)dev_addr,
1642 				    *(uint16_t *)host_addr);
1643 				break;
1644 
1645 			case sizeof (uint32_t):
1646 				i_ddi_put32(hp, (uint32_t *)dev_addr,
1647 				    *(uint32_t *)host_addr);
1648 				break;
1649 
1650 			case sizeof (uint64_t):
1651 				i_ddi_put64(hp, (uint64_t *)dev_addr,
1652 				    *(uint64_t *)host_addr);
1653 				break;
1654 			}
1655 
1656 			host_addr += size;
1657 
1658 			if (flags == DDI_DEV_AUTOINCR)
1659 				dev_addr += size;
1660 
1661 			px_lib_clr_errs(px_p, rdip, dev_addr);
1662 
1663 			if (pec_p->pec_ontrap_data->ot_trap & OT_DATA_ACCESS) {
1664 				err = DDI_FAILURE;
1665 #ifdef  DEBUG
1666 				px_pokefault_cnt++;
1667 #endif
1668 				break;
1669 			}
1670 		}
1671 	}
1672 
1673 	i_ddi_notrap((ddi_acc_handle_t)hp);
1674 	pec_p->pec_ontrap_data = NULL;
1675 	pec_p->pec_safeacc_type = DDI_FM_ERR_UNEXPECTED;
1676 	i_ndi_busop_access_exit(hp->ahi_common.ah_dip, (ddi_acc_handle_t)hp);
1677 	hp->ahi_err->err_expected = DDI_FM_ERR_UNEXPECTED;
1678 
1679 	return (err);
1680 }
1681 
1682 
1683 int
1684 px_lib_ctlops_poke(dev_info_t *dip, dev_info_t *rdip,
1685     peekpoke_ctlops_t *in_args)
1686 {
1687 	return (in_args->handle ? px_lib_do_caut_put(dip, rdip, in_args) :
1688 	    px_lib_do_poke(dip, rdip, in_args));
1689 }
1690 
1691 
1692 /*ARGSUSED*/
1693 static int
1694 px_lib_do_peek(dev_info_t *dip, peekpoke_ctlops_t *in_args)
1695 {
1696 	px_t *px_p = DIP_TO_STATE(dip);
1697 	px_pec_t *pec_p = px_p->px_pec_p;
1698 	int err = DDI_SUCCESS;
1699 	on_trap_data_t otd;
1700 
1701 	mutex_enter(&pec_p->pec_pokefault_mutex);
1702 	if (px_fm_enter(px_p) != DDI_SUCCESS)
1703 		return (DDI_FAILURE);
1704 	pec_p->pec_safeacc_type = DDI_FM_ERR_PEEK;
1705 	px_fm_exit(px_p);
1706 
1707 	if (!on_trap(&otd, OT_DATA_ACCESS)) {
1708 		uintptr_t tramp = otd.ot_trampoline;
1709 
1710 		otd.ot_trampoline = (uintptr_t)&peek_fault;
1711 		err = do_peek(in_args->size, (void *)in_args->dev_addr,
1712 		    (void *)in_args->host_addr);
1713 		otd.ot_trampoline = tramp;
1714 	} else
1715 		err = DDI_FAILURE;
1716 
1717 	no_trap();
1718 	pec_p->pec_safeacc_type = DDI_FM_ERR_UNEXPECTED;
1719 	mutex_exit(&pec_p->pec_pokefault_mutex);
1720 
1721 #ifdef  DEBUG
1722 	if (err == DDI_FAILURE)
1723 		px_peekfault_cnt++;
1724 #endif
1725 	return (err);
1726 }
1727 
1728 
1729 static int
1730 px_lib_do_caut_get(dev_info_t *dip, peekpoke_ctlops_t *cautacc_ctlops_arg)
1731 {
1732 	size_t size = cautacc_ctlops_arg->size;
1733 	uintptr_t dev_addr = cautacc_ctlops_arg->dev_addr;
1734 	uintptr_t host_addr = cautacc_ctlops_arg->host_addr;
1735 	ddi_acc_impl_t *hp = (ddi_acc_impl_t *)cautacc_ctlops_arg->handle;
1736 	size_t repcount = cautacc_ctlops_arg->repcount;
1737 	uint_t flags = cautacc_ctlops_arg->flags;
1738 
1739 	px_t *px_p = DIP_TO_STATE(dip);
1740 	px_pec_t *pec_p = px_p->px_pec_p;
1741 	int err = DDI_SUCCESS;
1742 
1743 	/*
1744 	 * Note that i_ndi_busop_access_enter ends up grabbing the pokefault
1745 	 * mutex.
1746 	 */
1747 	i_ndi_busop_access_enter(hp->ahi_common.ah_dip, (ddi_acc_handle_t)hp);
1748 
1749 	pec_p->pec_ontrap_data = (on_trap_data_t *)hp->ahi_err->err_ontrap;
1750 	pec_p->pec_safeacc_type = DDI_FM_ERR_EXPECTED;
1751 	hp->ahi_err->err_expected = DDI_FM_ERR_EXPECTED;
1752 
1753 	if (repcount == 1) {
1754 		if (!i_ddi_ontrap((ddi_acc_handle_t)hp)) {
1755 			i_ddi_caut_get(size, (void *)dev_addr,
1756 			    (void *)host_addr);
1757 		} else {
1758 			int i;
1759 			uint8_t *ff_addr = (uint8_t *)host_addr;
1760 			for (i = 0; i < size; i++)
1761 				*ff_addr++ = 0xff;
1762 
1763 			err = DDI_FAILURE;
1764 #ifdef  DEBUG
1765 			px_peekfault_cnt++;
1766 #endif
1767 		}
1768 	} else {
1769 		if (!i_ddi_ontrap((ddi_acc_handle_t)hp)) {
1770 			for (; repcount; repcount--) {
1771 				i_ddi_caut_get(size, (void *)dev_addr,
1772 				    (void *)host_addr);
1773 
1774 				host_addr += size;
1775 
1776 				if (flags == DDI_DEV_AUTOINCR)
1777 					dev_addr += size;
1778 			}
1779 		} else {
1780 			err = DDI_FAILURE;
1781 #ifdef  DEBUG
1782 			px_peekfault_cnt++;
1783 #endif
1784 		}
1785 	}
1786 
1787 	i_ddi_notrap((ddi_acc_handle_t)hp);
1788 	pec_p->pec_ontrap_data = NULL;
1789 	pec_p->pec_safeacc_type = DDI_FM_ERR_UNEXPECTED;
1790 	i_ndi_busop_access_exit(hp->ahi_common.ah_dip, (ddi_acc_handle_t)hp);
1791 	hp->ahi_err->err_expected = DDI_FM_ERR_UNEXPECTED;
1792 
1793 	return (err);
1794 }
1795 
1796 /*ARGSUSED*/
1797 int
1798 px_lib_ctlops_peek(dev_info_t *dip, dev_info_t *rdip,
1799     peekpoke_ctlops_t *in_args, void *result)
1800 {
1801 	result = (void *)in_args->host_addr;
1802 	return (in_args->handle ? px_lib_do_caut_get(dip, in_args) :
1803 	    px_lib_do_peek(dip, in_args));
1804 }
1805 
1806 /*
1807  * implements PPM interface
1808  */
1809 int
1810 px_lib_pmctl(int cmd, px_t *px_p)
1811 {
1812 	ASSERT((cmd & ~PPMREQ_MASK) == PPMREQ);
1813 	switch (cmd) {
1814 	case PPMREQ_PRE_PWR_OFF:
1815 		/*
1816 		 * Currently there is no device power management for
1817 		 * the root complex (fire). When there is we need to make
1818 		 * sure that it is at full power before trying to send the
1819 		 * PME_Turn_Off message.
1820 		 */
1821 		DBG(DBG_PWR, px_p->px_dip,
1822 		    "ioctl: request to send PME_Turn_Off\n");
1823 		return (px_goto_l23ready(px_p));
1824 
1825 	case PPMREQ_PRE_PWR_ON:
1826 		DBG(DBG_PWR, px_p->px_dip, "ioctl: PRE_PWR_ON request\n");
1827 		return (px_pre_pwron_check(px_p));
1828 
1829 	case PPMREQ_POST_PWR_ON:
1830 		DBG(DBG_PWR, px_p->px_dip, "ioctl: POST_PWR_ON request\n");
1831 		return (px_goto_l0(px_p));
1832 
1833 	default:
1834 		return (DDI_FAILURE);
1835 	}
1836 }
1837 
1838 /*
1839  * sends PME_Turn_Off message to put the link in L2/L3 ready state.
1840  * called by px_ioctl.
1841  * returns DDI_SUCCESS or DDI_FAILURE
1842  * 1. Wait for link to be in L1 state (link status reg)
1843  * 2. write to PME_Turn_off reg to boradcast
1844  * 3. set timeout
1845  * 4. If timeout, return failure.
1846  * 5. If PM_TO_Ack, wait till link is in L2/L3 ready
1847  */
1848 static int
1849 px_goto_l23ready(px_t *px_p)
1850 {
1851 	pcie_pwr_t	*pwr_p;
1852 	pxu_t		*pxu_p = (pxu_t *)px_p->px_plat_p;
1853 	caddr_t	csr_base = (caddr_t)pxu_p->px_address[PX_REG_CSR];
1854 	int		ret = DDI_SUCCESS;
1855 	clock_t		end, timeleft;
1856 	int		mutex_held = 1;
1857 
1858 	/* If no PM info, return failure */
1859 	if (!PCIE_PMINFO(px_p->px_dip) ||
1860 	    !(pwr_p = PCIE_NEXUS_PMINFO(px_p->px_dip)))
1861 		return (DDI_FAILURE);
1862 
1863 	mutex_enter(&pwr_p->pwr_lock);
1864 	mutex_enter(&px_p->px_l23ready_lock);
1865 	/* Clear the PME_To_ACK receieved flag */
1866 	px_p->px_pm_flags &= ~PX_PMETOACK_RECVD;
1867 	/*
1868 	 * When P25 is the downstream device, after receiving
1869 	 * PME_To_ACK, fire will go to Detect state, which causes
1870 	 * the link down event. Inform FMA that this is expected.
1871 	 * In case of all other cards complaint with the pci express
1872 	 * spec, this will happen when the power is re-applied. FMA
1873 	 * code will clear this flag after one instance of LDN. Since
1874 	 * there will not be a LDN event for the spec compliant cards,
1875 	 * we need to clear the flag after receiving PME_To_ACK.
1876 	 */
1877 	px_p->px_pm_flags |= PX_LDN_EXPECTED;
1878 	if (px_send_pme_turnoff(csr_base) != DDI_SUCCESS) {
1879 		ret = DDI_FAILURE;
1880 		goto l23ready_done;
1881 	}
1882 	px_p->px_pm_flags |= PX_PME_TURNOFF_PENDING;
1883 
1884 	end = ddi_get_lbolt() + drv_usectohz(px_pme_to_ack_timeout);
1885 	while (!(px_p->px_pm_flags & PX_PMETOACK_RECVD)) {
1886 		timeleft = cv_timedwait(&px_p->px_l23ready_cv,
1887 		    &px_p->px_l23ready_lock, end);
1888 		/*
1889 		 * if cv_timedwait returns -1, it is either
1890 		 * 1) timed out or
1891 		 * 2) there was a pre-mature wakeup but by the time
1892 		 * cv_timedwait is called again end < lbolt i.e.
1893 		 * end is in the past.
1894 		 * 3) By the time we make first cv_timedwait call,
1895 		 * end < lbolt is true.
1896 		 */
1897 		if (timeleft == -1)
1898 			break;
1899 	}
1900 	if (!(px_p->px_pm_flags & PX_PMETOACK_RECVD)) {
1901 		/*
1902 		 * Either timedout or interrupt didn't get a
1903 		 * chance to grab the mutex and set the flag.
1904 		 * release the mutex and delay for sometime.
1905 		 * This will 1) give a chance for interrupt to
1906 		 * set the flag 2) creates a delay between two
1907 		 * consequetive requests.
1908 		 */
1909 		mutex_exit(&px_p->px_l23ready_lock);
1910 		delay(drv_usectohz(50 * PX_MSEC_TO_USEC));
1911 		mutex_held = 0;
1912 		if (!(px_p->px_pm_flags & PX_PMETOACK_RECVD)) {
1913 			ret = DDI_FAILURE;
1914 			DBG(DBG_PWR, px_p->px_dip, " Timed out while waiting"
1915 			    " for PME_TO_ACK\n");
1916 		}
1917 	}
1918 	px_p->px_pm_flags &=
1919 	    ~(PX_PME_TURNOFF_PENDING | PX_PMETOACK_RECVD | PX_LDN_EXPECTED);
1920 
1921 l23ready_done:
1922 	if (mutex_held)
1923 		mutex_exit(&px_p->px_l23ready_lock);
1924 	/*
1925 	 * Wait till link is in L1 idle, if sending PME_Turn_Off
1926 	 * was succesful.
1927 	 */
1928 	if (ret == DDI_SUCCESS) {
1929 		if (px_link_wait4l1idle(csr_base) != DDI_SUCCESS) {
1930 			DBG(DBG_PWR, px_p->px_dip, " Link is not at L1"
1931 			    " even though we received PME_To_ACK.\n");
1932 			/*
1933 			 * Workaround for hardware bug with P25.
1934 			 * Due to a hardware bug with P25, link state
1935 			 * will be Detect state rather than L1 after
1936 			 * link is transitioned to L23Ready state. Since
1937 			 * we don't know whether link is L23ready state
1938 			 * without Fire's state being L1_idle, we delay
1939 			 * here just to make sure that we wait till link
1940 			 * is transitioned to L23Ready state.
1941 			 */
1942 			delay(drv_usectohz(100 * PX_MSEC_TO_USEC));
1943 		}
1944 		pwr_p->pwr_link_lvl = PM_LEVEL_L3;
1945 
1946 	}
1947 	mutex_exit(&pwr_p->pwr_lock);
1948 	return (ret);
1949 }
1950 
1951 /*
1952  * Message interrupt handler intended to be shared for both
1953  * PME and PME_TO_ACK msg handling, currently only handles
1954  * PME_To_ACK message.
1955  */
1956 uint_t
1957 px_pmeq_intr(caddr_t arg)
1958 {
1959 	px_t	*px_p = (px_t *)arg;
1960 
1961 	DBG(DBG_PWR, px_p->px_dip, " PME_To_ACK received \n");
1962 	mutex_enter(&px_p->px_l23ready_lock);
1963 	cv_broadcast(&px_p->px_l23ready_cv);
1964 	if (px_p->px_pm_flags & PX_PME_TURNOFF_PENDING) {
1965 		px_p->px_pm_flags |= PX_PMETOACK_RECVD;
1966 	} else {
1967 		/*
1968 		 * This maybe the second ack received. If so then,
1969 		 * we should be receiving it during wait4L1 stage.
1970 		 */
1971 		px_p->px_pmetoack_ignored++;
1972 	}
1973 	mutex_exit(&px_p->px_l23ready_lock);
1974 	return (DDI_INTR_CLAIMED);
1975 }
1976 
1977 static int
1978 px_pre_pwron_check(px_t *px_p)
1979 {
1980 	pcie_pwr_t	*pwr_p;
1981 
1982 	/* If no PM info, return failure */
1983 	if (!PCIE_PMINFO(px_p->px_dip) ||
1984 	    !(pwr_p = PCIE_NEXUS_PMINFO(px_p->px_dip)))
1985 		return (DDI_FAILURE);
1986 
1987 	/*
1988 	 * For the spec compliant downstream cards link down
1989 	 * is expected when the device is powered on.
1990 	 */
1991 	px_p->px_pm_flags |= PX_LDN_EXPECTED;
1992 	return (pwr_p->pwr_link_lvl == PM_LEVEL_L3 ? DDI_SUCCESS : DDI_FAILURE);
1993 }
1994 
1995 static int
1996 px_goto_l0(px_t *px_p)
1997 {
1998 	pcie_pwr_t	*pwr_p;
1999 	pxu_t		*pxu_p = (pxu_t *)px_p->px_plat_p;
2000 	caddr_t csr_base = (caddr_t)pxu_p->px_address[PX_REG_CSR];
2001 	int		ret = DDI_SUCCESS;
2002 	uint64_t	time_spent = 0;
2003 
2004 	/* If no PM info, return failure */
2005 	if (!PCIE_PMINFO(px_p->px_dip) ||
2006 	    !(pwr_p = PCIE_NEXUS_PMINFO(px_p->px_dip)))
2007 		return (DDI_FAILURE);
2008 
2009 	mutex_enter(&pwr_p->pwr_lock);
2010 	/*
2011 	 * The following link retrain activity will cause LDN and LUP event.
2012 	 * Receiving LDN prior to receiving LUP is expected, not an error in
2013 	 * this case.  Receiving LUP indicates link is fully up to support
2014 	 * powering up down stream device, and of course any further LDN and
2015 	 * LUP outside this context will be error.
2016 	 */
2017 	px_p->px_lup_pending = 1;
2018 	if (px_link_retrain(csr_base) != DDI_SUCCESS) {
2019 		ret = DDI_FAILURE;
2020 		goto l0_done;
2021 	}
2022 
2023 	/* LUP event takes the order of 15ms amount of time to occur */
2024 	for (; px_p->px_lup_pending && (time_spent < px_lup_poll_to);
2025 	    time_spent += px_lup_poll_interval)
2026 		drv_usecwait(px_lup_poll_interval);
2027 	if (px_p->px_lup_pending)
2028 		ret = DDI_FAILURE;
2029 l0_done:
2030 	px_enable_detect_quiet(csr_base);
2031 	if (ret == DDI_SUCCESS)
2032 		pwr_p->pwr_link_lvl = PM_LEVEL_L0;
2033 	mutex_exit(&pwr_p->pwr_lock);
2034 	return (ret);
2035 }
2036 
2037 /*
2038  * Extract the drivers binding name to identify which chip we're binding to.
2039  * Whenever a new bus bridge is created, the driver alias entry should be
2040  * added here to identify the device if needed.  If a device isn't added,
2041  * the identity defaults to PX_CHIP_UNIDENTIFIED.
2042  */
2043 static uint32_t
2044 px_identity_init(px_t *px_p)
2045 {
2046 	dev_info_t	*dip = px_p->px_dip;
2047 	char		*name = ddi_binding_name(dip);
2048 	uint32_t	revision = 0;
2049 
2050 	revision = ddi_prop_get_int(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
2051 	    "module-revision#", 0);
2052 
2053 	/* Check for Fire driver binding name */
2054 	if (strcmp(name, "pciex108e,80f0") == 0) {
2055 		DBG(DBG_ATTACH, dip, "px_identity_init: %s%d: "
2056 		    "(FIRE), module-revision %d\n", NAMEINST(dip),
2057 		    revision);
2058 
2059 		return ((revision >= FIRE_MOD_REV_20) ?
2060 		    PX_CHIP_FIRE : PX_CHIP_UNIDENTIFIED);
2061 	}
2062 
2063 	/* Check for Oberon driver binding name */
2064 	if (strcmp(name, "pciex108e,80f8") == 0) {
2065 		DBG(DBG_ATTACH, dip, "px_identity_init: %s%d: "
2066 		    "(OBERON), module-revision %d\n", NAMEINST(dip),
2067 		    revision);
2068 
2069 		return (PX_CHIP_OBERON);
2070 	}
2071 
2072 	DBG(DBG_ATTACH, dip, "%s%d: Unknown PCI Express Host bridge %s %x\n",
2073 	    ddi_driver_name(dip), ddi_get_instance(dip), name, revision);
2074 
2075 	return (PX_CHIP_UNIDENTIFIED);
2076 }
2077 
2078 int
2079 px_err_add_intr(px_fault_t *px_fault_p)
2080 {
2081 	dev_info_t	*dip = px_fault_p->px_fh_dip;
2082 	px_t		*px_p = DIP_TO_STATE(dip);
2083 
2084 	VERIFY(add_ivintr(px_fault_p->px_fh_sysino, PX_ERR_PIL,
2085 	    (intrfunc)px_fault_p->px_err_func, (caddr_t)px_fault_p,
2086 	    NULL, NULL) == 0);
2087 
2088 	px_ib_intr_enable(px_p, intr_dist_cpuid(), px_fault_p->px_intr_ino);
2089 
2090 	return (DDI_SUCCESS);
2091 }
2092 
2093 void
2094 px_err_rem_intr(px_fault_t *px_fault_p)
2095 {
2096 	dev_info_t	*dip = px_fault_p->px_fh_dip;
2097 	px_t		*px_p = DIP_TO_STATE(dip);
2098 
2099 	px_ib_intr_disable(px_p->px_ib_p, px_fault_p->px_intr_ino,
2100 	    IB_INTR_WAIT);
2101 
2102 	VERIFY(rem_ivintr(px_fault_p->px_fh_sysino, PX_ERR_PIL) == 0);
2103 }
2104 
2105 /*
2106  * px_cb_intr_redist() - sun4u only, CB interrupt redistribution
2107  */
2108 void
2109 px_cb_intr_redist(void *arg)
2110 {
2111 	px_cb_t		*cb_p = (px_cb_t *)arg;
2112 	px_cb_list_t	*pxl;
2113 	px_t		*pxp = NULL;
2114 	px_fault_t	*f_p = NULL;
2115 	uint32_t	new_cpuid;
2116 	intr_valid_state_t	enabled = 0;
2117 
2118 	mutex_enter(&cb_p->cb_mutex);
2119 
2120 	pxl = cb_p->pxl;
2121 	if (!pxl)
2122 		goto cb_done;
2123 
2124 	pxp = pxl->pxp;
2125 	f_p = &pxp->px_cb_fault;
2126 	for (; pxl && (f_p->px_fh_sysino != cb_p->sysino); ) {
2127 		pxl = pxl->next;
2128 		pxp = pxl->pxp;
2129 		f_p = &pxp->px_cb_fault;
2130 	}
2131 	if (pxl == NULL)
2132 		goto cb_done;
2133 
2134 	new_cpuid =  intr_dist_cpuid();
2135 	if (new_cpuid == cb_p->cpuid)
2136 		goto cb_done;
2137 
2138 	if ((px_lib_intr_getvalid(pxp->px_dip, f_p->px_fh_sysino, &enabled)
2139 	    != DDI_SUCCESS) || !enabled) {
2140 		DBG(DBG_IB, pxp->px_dip, "px_cb_intr_redist: CB not enabled, "
2141 		    "sysino(0x%x)\n", f_p->px_fh_sysino);
2142 		goto cb_done;
2143 	}
2144 
2145 	PX_INTR_DISABLE(pxp->px_dip, f_p->px_fh_sysino);
2146 
2147 	cb_p->cpuid = new_cpuid;
2148 	cb_p->sysino = f_p->px_fh_sysino;
2149 	PX_INTR_ENABLE(pxp->px_dip, cb_p->sysino, cb_p->cpuid);
2150 
2151 cb_done:
2152 	mutex_exit(&cb_p->cb_mutex);
2153 }
2154 
2155 /*
2156  * px_cb_add_intr() - Called from attach(9E) to create CB if not yet
2157  * created, to add CB interrupt vector always, but enable only once.
2158  */
2159 int
2160 px_cb_add_intr(px_fault_t *fault_p)
2161 {
2162 	px_t		*px_p = DIP_TO_STATE(fault_p->px_fh_dip);
2163 	pxu_t		*pxu_p = (pxu_t *)px_p->px_plat_p;
2164 	px_cb_t		*cb_p = (px_cb_t *)px_get_cb(fault_p->px_fh_dip);
2165 	px_cb_list_t	*pxl, *pxl_new;
2166 	boolean_t	is_proxy = B_FALSE;
2167 
2168 	/* create cb */
2169 	if (cb_p == NULL) {
2170 		cb_p = kmem_zalloc(sizeof (px_cb_t), KM_SLEEP);
2171 
2172 		mutex_init(&cb_p->cb_mutex, NULL, MUTEX_DRIVER,
2173 		    (void *) ipltospl(FM_ERR_PIL));
2174 
2175 		cb_p->px_cb_func = px_cb_intr;
2176 		pxu_p->px_cb_p = cb_p;
2177 		px_set_cb(fault_p->px_fh_dip, (uint64_t)cb_p);
2178 
2179 		/* px_lib_dev_init allows only FIRE and OBERON */
2180 		px_err_reg_enable(
2181 		    (pxu_p->chip_type == PX_CHIP_FIRE) ?
2182 		    PX_ERR_JBC : PX_ERR_UBC,
2183 		    pxu_p->px_address[PX_REG_XBC]);
2184 	} else
2185 		pxu_p->px_cb_p = cb_p;
2186 
2187 	/* register cb interrupt */
2188 	VERIFY(add_ivintr(fault_p->px_fh_sysino, PX_ERR_PIL,
2189 	    (intrfunc)cb_p->px_cb_func, (caddr_t)cb_p, NULL, NULL) == 0);
2190 
2191 
2192 	/* update cb list */
2193 	mutex_enter(&cb_p->cb_mutex);
2194 	if (cb_p->pxl == NULL) {
2195 		is_proxy = B_TRUE;
2196 		pxl = kmem_zalloc(sizeof (px_cb_list_t), KM_SLEEP);
2197 		pxl->pxp = px_p;
2198 		cb_p->pxl = pxl;
2199 		cb_p->sysino = fault_p->px_fh_sysino;
2200 		cb_p->cpuid = intr_dist_cpuid();
2201 	} else {
2202 		/*
2203 		 * Find the last pxl or
2204 		 * stop short at encountering a redundent entry, or
2205 		 * both.
2206 		 */
2207 		pxl = cb_p->pxl;
2208 		for (; !(pxl->pxp == px_p) && pxl->next; pxl = pxl->next) {};
2209 		ASSERT(pxl->pxp != px_p);
2210 
2211 		/* add to linked list */
2212 		pxl_new = kmem_zalloc(sizeof (px_cb_list_t), KM_SLEEP);
2213 		pxl_new->pxp = px_p;
2214 		pxl->next = pxl_new;
2215 	}
2216 	cb_p->attachcnt++;
2217 	mutex_exit(&cb_p->cb_mutex);
2218 
2219 	if (is_proxy) {
2220 		/* add to interrupt redistribution list */
2221 		intr_dist_add(px_cb_intr_redist, cb_p);
2222 
2223 		/* enable cb hw interrupt */
2224 		px_ib_intr_enable(px_p, cb_p->cpuid, fault_p->px_intr_ino);
2225 	}
2226 
2227 	return (DDI_SUCCESS);
2228 }
2229 
2230 /*
2231  * px_cb_rem_intr() - Called from detach(9E) to remove its CB
2232  * interrupt vector, to shift proxy to the next available px,
2233  * or disable CB interrupt when itself is the last.
2234  */
2235 void
2236 px_cb_rem_intr(px_fault_t *fault_p)
2237 {
2238 	px_t		*px_p = DIP_TO_STATE(fault_p->px_fh_dip), *pxp;
2239 	pxu_t		*pxu_p = (pxu_t *)px_p->px_plat_p;
2240 	px_cb_t		*cb_p = PX2CB(px_p);
2241 	px_cb_list_t	*pxl, *prev;
2242 	px_fault_t	*f_p;
2243 
2244 	ASSERT(cb_p->pxl);
2245 
2246 	/* find and remove this px, and update cb list */
2247 	mutex_enter(&cb_p->cb_mutex);
2248 
2249 	pxl = cb_p->pxl;
2250 	if (pxl->pxp == px_p) {
2251 		cb_p->pxl = pxl->next;
2252 	} else {
2253 		prev = pxl;
2254 		pxl = pxl->next;
2255 		for (; pxl && (pxl->pxp != px_p); prev = pxl, pxl = pxl->next) {
2256 		};
2257 		if (!pxl) {
2258 			cmn_err(CE_WARN, "px_cb_rem_intr: can't find px_p 0x%p "
2259 			    "in registered CB list.", (void *)px_p);
2260 			mutex_exit(&cb_p->cb_mutex);
2261 			return;
2262 		}
2263 		prev->next = pxl->next;
2264 	}
2265 	pxu_p->px_cb_p = NULL;
2266 	cb_p->attachcnt--;
2267 	kmem_free(pxl, sizeof (px_cb_list_t));
2268 	mutex_exit(&cb_p->cb_mutex);
2269 
2270 	/* disable cb hw interrupt */
2271 	if (fault_p->px_fh_sysino == cb_p->sysino)
2272 		px_ib_intr_disable(px_p->px_ib_p, fault_p->px_intr_ino,
2273 		    IB_INTR_WAIT);
2274 
2275 	/* if last px, remove from interrupt redistribution list */
2276 	if (cb_p->pxl == NULL)
2277 		intr_dist_rem(px_cb_intr_redist, cb_p);
2278 
2279 	/* de-register interrupt */
2280 	VERIFY(rem_ivintr(fault_p->px_fh_sysino, PX_ERR_PIL) == 0);
2281 
2282 	/* if not last px, assign next px to manage cb */
2283 	mutex_enter(&cb_p->cb_mutex);
2284 	if (cb_p->pxl) {
2285 		if (fault_p->px_fh_sysino == cb_p->sysino) {
2286 			pxp = cb_p->pxl->pxp;
2287 			f_p = &pxp->px_cb_fault;
2288 			cb_p->sysino = f_p->px_fh_sysino;
2289 
2290 			PX_INTR_ENABLE(pxp->px_dip, cb_p->sysino, cb_p->cpuid);
2291 			(void) px_lib_intr_setstate(pxp->px_dip, cb_p->sysino,
2292 			    INTR_IDLE_STATE);
2293 		}
2294 		mutex_exit(&cb_p->cb_mutex);
2295 		return;
2296 	}
2297 
2298 	/* clean up after the last px */
2299 	mutex_exit(&cb_p->cb_mutex);
2300 
2301 	/* px_lib_dev_init allows only FIRE and OBERON */
2302 	px_err_reg_disable(
2303 	    (pxu_p->chip_type == PX_CHIP_FIRE) ? PX_ERR_JBC : PX_ERR_UBC,
2304 	    pxu_p->px_address[PX_REG_XBC]);
2305 
2306 	mutex_destroy(&cb_p->cb_mutex);
2307 	px_set_cb(fault_p->px_fh_dip, 0ull);
2308 	kmem_free(cb_p, sizeof (px_cb_t));
2309 }
2310 
2311 /*
2312  * px_cb_intr() - sun4u only,  CB interrupt dispatcher
2313  */
2314 uint_t
2315 px_cb_intr(caddr_t arg)
2316 {
2317 	px_cb_t		*cb_p = (px_cb_t *)arg;
2318 	px_t		*pxp;
2319 	px_fault_t	*f_p;
2320 	int		ret;
2321 
2322 	mutex_enter(&cb_p->cb_mutex);
2323 
2324 	if (!cb_p->pxl) {
2325 		mutex_exit(&cb_p->cb_mutex);
2326 		return (DDI_INTR_UNCLAIMED);
2327 	}
2328 
2329 	pxp = cb_p->pxl->pxp;
2330 	f_p = &pxp->px_cb_fault;
2331 
2332 	ret = f_p->px_err_func((caddr_t)f_p);
2333 
2334 	mutex_exit(&cb_p->cb_mutex);
2335 	return (ret);
2336 }
2337 
2338 #ifdef	FMA
2339 void
2340 px_fill_rc_status(px_fault_t *px_fault_p, pciex_rc_error_regs_t *rc_status)
2341 {
2342 	/* populate the rc_status by reading the registers - TBD */
2343 }
2344 #endif /* FMA */
2345 
2346 /*
2347  * Unprotected raw reads/writes of fabric device's config space.
2348  * Only used for temporary PCI-E Fabric Error Handling.
2349  */
2350 uint32_t
2351 px_fab_get(px_t *px_p, pcie_req_id_t bdf, uint16_t offset)
2352 {
2353 	px_ranges_t	*rp = px_p->px_ranges_p;
2354 	uint64_t	range_prop, base_addr;
2355 	int		bank = PCI_REG_ADDR_G(PCI_ADDR_CONFIG);
2356 	uint32_t	val;
2357 
2358 	/* Get Fire's Physical Base Address */
2359 	range_prop = px_get_range_prop(px_p, rp, bank);
2360 
2361 	/* Get config space first. */
2362 	base_addr = range_prop + PX_BDF_TO_CFGADDR(bdf, offset);
2363 
2364 	val = ldphysio(base_addr);
2365 
2366 	return (LE_32(val));
2367 }
2368 
2369 void
2370 px_fab_set(px_t *px_p, pcie_req_id_t bdf, uint16_t offset,
2371     uint32_t val) {
2372 	px_ranges_t	*rp = px_p->px_ranges_p;
2373 	uint64_t	range_prop, base_addr;
2374 	int		bank = PCI_REG_ADDR_G(PCI_ADDR_CONFIG);
2375 
2376 	/* Get Fire's Physical Base Address */
2377 	range_prop = px_get_range_prop(px_p, rp, bank);
2378 
2379 	/* Get config space first. */
2380 	base_addr = range_prop + PX_BDF_TO_CFGADDR(bdf, offset);
2381 
2382 	stphysio(base_addr, LE_32(val));
2383 }
2384 
2385 /*
2386  * cpr callback
2387  *
2388  * disable fabric error msg interrupt prior to suspending
2389  * all device drivers; re-enable fabric error msg interrupt
2390  * after all devices are resumed.
2391  */
2392 static boolean_t
2393 px_cpr_callb(void *arg, int code)
2394 {
2395 	px_t		*px_p = (px_t *)arg;
2396 	px_ib_t		*ib_p = px_p->px_ib_p;
2397 	px_pec_t	*pec_p = px_p->px_pec_p;
2398 	pxu_t		*pxu_p = (pxu_t *)px_p->px_plat_p;
2399 	caddr_t		csr_base;
2400 	devino_t	ce_ino, nf_ino, f_ino;
2401 	px_ino_t	*ce_ino_p, *nf_ino_p, *f_ino_p;
2402 	uint64_t	imu_log_enable, imu_intr_enable;
2403 	uint64_t	imu_log_mask, imu_intr_mask;
2404 
2405 	ce_ino = px_msiqid_to_devino(px_p, pec_p->pec_corr_msg_msiq_id);
2406 	nf_ino = px_msiqid_to_devino(px_p, pec_p->pec_non_fatal_msg_msiq_id);
2407 	f_ino = px_msiqid_to_devino(px_p, pec_p->pec_fatal_msg_msiq_id);
2408 	csr_base = (caddr_t)pxu_p->px_address[PX_REG_CSR];
2409 
2410 	imu_log_enable = CSR_XR(csr_base, IMU_ERROR_LOG_ENABLE);
2411 	imu_intr_enable = CSR_XR(csr_base, IMU_INTERRUPT_ENABLE);
2412 
2413 	imu_log_mask = BITMASK(IMU_ERROR_LOG_ENABLE_FATAL_MES_NOT_EN_LOG_EN) |
2414 	    BITMASK(IMU_ERROR_LOG_ENABLE_NONFATAL_MES_NOT_EN_LOG_EN) |
2415 	    BITMASK(IMU_ERROR_LOG_ENABLE_COR_MES_NOT_EN_LOG_EN);
2416 
2417 	imu_intr_mask =
2418 	    BITMASK(IMU_INTERRUPT_ENABLE_FATAL_MES_NOT_EN_S_INT_EN) |
2419 	    BITMASK(IMU_INTERRUPT_ENABLE_NONFATAL_MES_NOT_EN_S_INT_EN) |
2420 	    BITMASK(IMU_INTERRUPT_ENABLE_COR_MES_NOT_EN_S_INT_EN) |
2421 	    BITMASK(IMU_INTERRUPT_ENABLE_FATAL_MES_NOT_EN_P_INT_EN) |
2422 	    BITMASK(IMU_INTERRUPT_ENABLE_NONFATAL_MES_NOT_EN_P_INT_EN) |
2423 	    BITMASK(IMU_INTERRUPT_ENABLE_COR_MES_NOT_EN_P_INT_EN);
2424 
2425 	switch (code) {
2426 	case CB_CODE_CPR_CHKPT:
2427 		/* disable imu rbne on corr/nonfatal/fatal errors */
2428 		CSR_XS(csr_base, IMU_ERROR_LOG_ENABLE,
2429 		    imu_log_enable & (~imu_log_mask));
2430 
2431 		CSR_XS(csr_base, IMU_INTERRUPT_ENABLE,
2432 		    imu_intr_enable & (~imu_intr_mask));
2433 
2434 		/* disable CORR intr mapping */
2435 		px_ib_intr_disable(ib_p, ce_ino, IB_INTR_NOWAIT);
2436 
2437 		/* disable NON FATAL intr mapping */
2438 		px_ib_intr_disable(ib_p, nf_ino, IB_INTR_NOWAIT);
2439 
2440 		/* disable FATAL intr mapping */
2441 		px_ib_intr_disable(ib_p, f_ino, IB_INTR_NOWAIT);
2442 
2443 		break;
2444 
2445 	case CB_CODE_CPR_RESUME:
2446 		pxu_p->cpr_flag = PX_NOT_CPR;
2447 		mutex_enter(&ib_p->ib_ino_lst_mutex);
2448 
2449 		ce_ino_p = px_ib_locate_ino(ib_p, ce_ino);
2450 		nf_ino_p = px_ib_locate_ino(ib_p, nf_ino);
2451 		f_ino_p = px_ib_locate_ino(ib_p, f_ino);
2452 
2453 		/* enable CORR intr mapping */
2454 		if (ce_ino_p)
2455 			px_ib_intr_enable(px_p, ce_ino_p->ino_cpuid, ce_ino);
2456 		else
2457 			cmn_err(CE_WARN, "px_cpr_callb: RESUME unable to "
2458 			    "reenable PCIe Correctable msg intr.\n");
2459 
2460 		/* enable NON FATAL intr mapping */
2461 		if (nf_ino_p)
2462 			px_ib_intr_enable(px_p, nf_ino_p->ino_cpuid, nf_ino);
2463 		else
2464 			cmn_err(CE_WARN, "px_cpr_callb: RESUME unable to "
2465 			    "reenable PCIe Non Fatal msg intr.\n");
2466 
2467 		/* enable FATAL intr mapping */
2468 		if (f_ino_p)
2469 			px_ib_intr_enable(px_p, f_ino_p->ino_cpuid, f_ino);
2470 		else
2471 			cmn_err(CE_WARN, "px_cpr_callb: RESUME unable to "
2472 			    "reenable PCIe Fatal msg intr.\n");
2473 
2474 		mutex_exit(&ib_p->ib_ino_lst_mutex);
2475 
2476 		/* enable corr/nonfatal/fatal not enable error */
2477 		CSR_XS(csr_base, IMU_ERROR_LOG_ENABLE, (imu_log_enable |
2478 		    (imu_log_mask & px_imu_log_mask)));
2479 		CSR_XS(csr_base, IMU_INTERRUPT_ENABLE, (imu_intr_enable |
2480 		    (imu_intr_mask & px_imu_intr_mask)));
2481 
2482 		break;
2483 	}
2484 
2485 	return (B_TRUE);
2486 }
2487 
2488 uint64_t
2489 px_get_rng_parent_hi_mask(px_t *px_p)
2490 {
2491 	pxu_t *pxu_p = (pxu_t *)px_p->px_plat_p;
2492 	uint64_t mask;
2493 
2494 	switch (PX_CHIP_TYPE(pxu_p)) {
2495 	case PX_CHIP_OBERON:
2496 		mask = OBERON_RANGE_PROP_MASK;
2497 		break;
2498 	case PX_CHIP_FIRE:
2499 		mask = PX_RANGE_PROP_MASK;
2500 		break;
2501 	default:
2502 		mask = PX_RANGE_PROP_MASK;
2503 	}
2504 
2505 	return (mask);
2506 }
2507 
2508 /*
2509  * fetch chip's range propery's value
2510  */
2511 uint64_t
2512 px_get_range_prop(px_t *px_p, px_ranges_t *rp, int bank)
2513 {
2514 	uint64_t mask, range_prop;
2515 
2516 	mask = px_get_rng_parent_hi_mask(px_p);
2517 	range_prop = (((uint64_t)(rp[bank].parent_high & mask)) << 32) |
2518 	    rp[bank].parent_low;
2519 
2520 	return (range_prop);
2521 }
2522 
2523 /*
2524  * add cpr callback
2525  */
2526 void
2527 px_cpr_add_callb(px_t *px_p)
2528 {
2529 	px_p->px_cprcb_id = callb_add(px_cpr_callb, (void *)px_p,
2530 	    CB_CL_CPR_POST_USER, "px_cpr");
2531 }
2532 
2533 /*
2534  * remove cpr callback
2535  */
2536 void
2537 px_cpr_rem_callb(px_t *px_p)
2538 {
2539 	(void) callb_delete(px_p->px_cprcb_id);
2540 }
2541 
2542 /*ARGSUSED*/
2543 static uint_t
2544 px_hp_intr(caddr_t arg1, caddr_t arg2)
2545 {
2546 	px_t	*px_p = (px_t *)arg1;
2547 	pxu_t 	*pxu_p = (pxu_t *)px_p->px_plat_p;
2548 	int	rval;
2549 
2550 	rval = pciehpc_intr(px_p->px_dip);
2551 
2552 #ifdef  DEBUG
2553 	if (rval == DDI_INTR_UNCLAIMED)
2554 		cmn_err(CE_WARN, "%s%d: UNCLAIMED intr\n",
2555 		    ddi_driver_name(px_p->px_dip),
2556 		    ddi_get_instance(px_p->px_dip));
2557 #endif
2558 
2559 	/* Set the interrupt state to idle */
2560 	if (px_lib_intr_setstate(px_p->px_dip,
2561 	    pxu_p->hp_sysino, INTR_IDLE_STATE) != DDI_SUCCESS)
2562 		return (DDI_INTR_UNCLAIMED);
2563 
2564 	return (rval);
2565 }
2566 
2567 int
2568 px_lib_hotplug_init(dev_info_t *dip, void *arg)
2569 {
2570 	px_t	*px_p = DIP_TO_STATE(dip);
2571 	pxu_t 	*pxu_p = (pxu_t *)px_p->px_plat_p;
2572 	uint64_t ret;
2573 
2574 	if ((ret = hvio_hotplug_init(dip, arg)) == DDI_SUCCESS) {
2575 		if (px_lib_intr_devino_to_sysino(px_p->px_dip,
2576 		    px_p->px_inos[PX_INTR_HOTPLUG], &pxu_p->hp_sysino) !=
2577 		    DDI_SUCCESS) {
2578 #ifdef	DEBUG
2579 			cmn_err(CE_WARN, "%s%d: devino_to_sysino fails\n",
2580 			    ddi_driver_name(px_p->px_dip),
2581 			    ddi_get_instance(px_p->px_dip));
2582 #endif
2583 			return (DDI_FAILURE);
2584 		}
2585 
2586 		VERIFY(add_ivintr(pxu_p->hp_sysino, PX_PCIEHP_PIL,
2587 		    (intrfunc)px_hp_intr, (caddr_t)px_p, NULL, NULL) == 0);
2588 
2589 		px_ib_intr_enable(px_p, intr_dist_cpuid(),
2590 		    px_p->px_inos[PX_INTR_HOTPLUG]);
2591 	}
2592 
2593 	return (ret);
2594 }
2595 
2596 void
2597 px_lib_hotplug_uninit(dev_info_t *dip)
2598 {
2599 	if (hvio_hotplug_uninit(dip) == DDI_SUCCESS) {
2600 		px_t	*px_p = DIP_TO_STATE(dip);
2601 		pxu_t 	*pxu_p = (pxu_t *)px_p->px_plat_p;
2602 
2603 		px_ib_intr_disable(px_p->px_ib_p,
2604 		    px_p->px_inos[PX_INTR_HOTPLUG], IB_INTR_WAIT);
2605 
2606 		VERIFY(rem_ivintr(pxu_p->hp_sysino, PX_PCIEHP_PIL) == 0);
2607 	}
2608 }
2609 
2610 /*
2611  * px_hp_intr_redist() - sun4u only, HP interrupt redistribution
2612  */
2613 void
2614 px_hp_intr_redist(px_t *px_p)
2615 {
2616 	if (px_p && (px_p->px_dev_caps & PX_HOTPLUG_CAPABLE)) {
2617 		px_ib_intr_dist_en(px_p->px_dip, intr_dist_cpuid(),
2618 		    px_p->px_inos[PX_INTR_HOTPLUG], B_FALSE);
2619 	}
2620 }
2621 
2622 boolean_t
2623 px_lib_is_in_drain_state(px_t *px_p)
2624 {
2625 	pxu_t 	*pxu_p = (pxu_t *)px_p->px_plat_p;
2626 	caddr_t csr_base = (caddr_t)pxu_p->px_address[PX_REG_CSR];
2627 	uint64_t drain_status;
2628 
2629 	if (PX_CHIP_TYPE(pxu_p) == PX_CHIP_OBERON) {
2630 		drain_status = CSR_BR(csr_base, DRAIN_CONTROL_STATUS, DRAIN);
2631 	} else {
2632 		drain_status = CSR_BR(csr_base, TLU_STATUS, DRAIN);
2633 	}
2634 
2635 	return (drain_status);
2636 }
2637 
2638 pcie_req_id_t
2639 px_lib_get_bdf(px_t *px_p)
2640 {
2641 	pxu_t 	*pxu_p = (pxu_t *)px_p->px_plat_p;
2642 	caddr_t csr_base = (caddr_t)pxu_p->px_address[PX_REG_CSR];
2643 	pcie_req_id_t bdf;
2644 
2645 	bdf = CSR_BR(csr_base, DMC_PCI_EXPRESS_CONFIGURATION, REQ_ID);
2646 
2647 	return (bdf);
2648 }
2649 
2650 /*ARGSUSED*/
2651 int
2652 px_lib_get_root_complex_mps(px_t *px_p, dev_info_t *dip, int *mps)
2653 {
2654 	pxu_t	*pxu_p;
2655 	caddr_t csr_base;
2656 
2657 	pxu_p = (pxu_t *)px_p->px_plat_p;
2658 
2659 	if (pxu_p == NULL)
2660 		return (DDI_FAILURE);
2661 
2662 	csr_base = (caddr_t)pxu_p->px_address[PX_REG_CSR];
2663 
2664 
2665 	*mps = CSR_XR(csr_base, TLU_DEVICE_CAPABILITIES) &
2666 	    TLU_DEVICE_CAPABILITIES_MPS_MASK;
2667 
2668 	return (DDI_SUCCESS);
2669 }
2670 
2671 /*ARGSUSED*/
2672 int
2673 px_lib_set_root_complex_mps(px_t *px_p,  dev_info_t *dip, int mps)
2674 {
2675 	pxu_t	*pxu_p;
2676 	caddr_t csr_base;
2677 	uint64_t dev_ctrl;
2678 	int link_width, val;
2679 	px_chip_type_t chip_type = px_identity_init(px_p);
2680 
2681 	pxu_p = (pxu_t *)px_p->px_plat_p;
2682 
2683 	if (pxu_p == NULL)
2684 		return (DDI_FAILURE);
2685 
2686 	csr_base = (caddr_t)pxu_p->px_address[PX_REG_CSR];
2687 
2688 	dev_ctrl = CSR_XR(csr_base, TLU_DEVICE_CONTROL);
2689 	dev_ctrl |= (mps << TLU_DEVICE_CONTROL_MPS);
2690 
2691 	CSR_XS(csr_base, TLU_DEVICE_CONTROL, dev_ctrl);
2692 
2693 	link_width = CSR_FR(csr_base, TLU_LINK_STATUS, WIDTH);
2694 
2695 	/*
2696 	 * Convert link_width to match timer array configuration.
2697 	 */
2698 	switch (link_width) {
2699 	case 1:
2700 		link_width = 0;
2701 		break;
2702 	case 4:
2703 		link_width = 1;
2704 		break;
2705 	case 8:
2706 		link_width = 2;
2707 		break;
2708 	case 16:
2709 		link_width = 3;
2710 		break;
2711 	default:
2712 		link_width = 0;
2713 	}
2714 
2715 	val = px_replay_timer_table[mps][link_width];
2716 	CSR_XS(csr_base, LPU_TXLINK_REPLAY_TIMER_THRESHOLD, val);
2717 
2718 	if (chip_type == PX_CHIP_OBERON)
2719 		return (DDI_SUCCESS);
2720 
2721 	val = px_acknak_timer_table[mps][link_width];
2722 	CSR_XS(csr_base, LPU_TXLINK_FREQUENT_NAK_LATENCY_TIMER_THRESHOLD, val);
2723 
2724 	return (DDI_SUCCESS);
2725 }
2726