1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21/*
22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23 */
24
25#include <sys/types.h>
26#include <sys/sysmacros.h>
27#include <sys/ddi.h>
28#include <sys/async.h>
29#include <sys/sunddi.h>
30#include <sys/ddifm.h>
31#include <sys/fm/protocol.h>
32#include <sys/vmem.h>
33#include <sys/intr.h>
34#include <sys/ivintr.h>
35#include <sys/errno.h>
36#include <sys/hypervisor_api.h>
37#include <sys/hsvc.h>
38#include <px_obj.h>
39#include <sys/machsystm.h>
40#include <sys/sunndi.h>
41#include <sys/pcie_impl.h>
42#include "px_lib4v.h"
43#include "px_err.h"
44#include <sys/pci_cfgacc.h>
45#include <sys/pci_cfgacc_4v.h>
46
47
48/* mask for the ranges property in calculating the real PFN range */
49uint_t px_ranges_phi_mask = ((1 << 28) -1);
50
51/*
52 * Hypervisor VPCI services information for the px nexus driver.
53 */
54static	uint64_t	px_vpci_maj_ver; /* Negotiated VPCI API major version */
55static	uint64_t	px_vpci_min_ver; /* Negotiated VPCI API minor version */
56static	uint_t		px_vpci_users = 0; /* VPCI API users */
57static	hsvc_info_t px_hsvc_vpci = {
58	HSVC_REV_1, NULL, HSVC_GROUP_VPCI, PX_VPCI_MAJOR_VER,
59	PX_VPCI_MINOR_VER, "PX"
60};
61
62/*
63 * Hypervisor SDIO services information for the px nexus driver.
64 */
65static	uint64_t	px_sdio_min_ver; /* Negotiated SDIO API minor version */
66static	uint_t		px_sdio_users = 0; /* SDIO API users */
67static	hsvc_info_t px_hsvc_sdio = {
68	HSVC_REV_1, NULL, HSVC_GROUP_SDIO, PX_SDIO_MAJOR_VER,
69	PX_SDIO_MINOR_VER, "PX"
70};
71
72/*
73 * Hypervisor SDIO ERR services information for the px nexus driver.
74 */
75static	uint64_t	px_sdio_err_min_ver; /* Negotiated SDIO ERR API */
76						/* minor version */
77static	uint_t		px_sdio_err_users = 0; /* SDIO ERR API users */
78static	hsvc_info_t px_hsvc_sdio_err = {
79	HSVC_REV_1, NULL, HSVC_GROUP_SDIO_ERR, PX_SDIO_ERR_MAJOR_VER,
80	PX_SDIO_ERR_MINOR_VER, "PX"
81};
82
83#define	CHILD_LOANED	"child_loaned"
84static int px_lib_count_waiting_dev(dev_info_t *);
85
86int
87px_lib_dev_init(dev_info_t *dip, devhandle_t *dev_hdl)
88{
89	px_nexus_regspec_t	*rp;
90	uint_t			reglen;
91	int			ret;
92	px_t			*px_p = DIP_TO_STATE(dip);
93	uint64_t mjrnum;
94	uint64_t mnrnum;
95
96	DBG(DBG_ATTACH, dip, "px_lib_dev_init: dip 0x%p\n", dip);
97
98	/*
99	 * Check HV intr group api versioning.
100	 * This driver uses the old interrupt routines which are supported
101	 * in old firmware in the CORE API group and in newer firmware in
102	 * the INTR API group.  Support for these calls will be dropped
103	 * once the INTR API group major goes to 2.
104	 */
105	if ((hsvc_version(HSVC_GROUP_INTR, &mjrnum, &mnrnum) == 0) &&
106	    (mjrnum > 1)) {
107		cmn_err(CE_WARN, "px: unsupported intr api group: "
108		    "maj:0x%lx, min:0x%lx", mjrnum, mnrnum);
109		return (ENOTSUP);
110	}
111
112	ret = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
113	    "reg", (uchar_t **)&rp, &reglen);
114	if (ret != DDI_PROP_SUCCESS) {
115		DBG(DBG_ATTACH, dip, "px_lib_dev_init failed ret=%d\n", ret);
116		return (DDI_FAILURE);
117	}
118
119	/*
120	 * Initilize device handle. The device handle uniquely identifies
121	 * a SUN4V device. It consists of the lower 28-bits of the hi-cell
122	 * of the first entry of the SUN4V device's "reg" property as
123	 * defined by the SUN4V Bus Binding to Open Firmware.
124	 */
125	*dev_hdl = (devhandle_t)((rp->phys_addr >> 32) & DEVHDLE_MASK);
126	ddi_prop_free(rp);
127
128	/*
129	 * hotplug implementation requires this property to be associated with
130	 * any indirect PCI config access services
131	 */
132	(void) ddi_prop_update_int(makedevice(ddi_driver_major(dip),
133	    PCI_MINOR_NUM(ddi_get_instance(dip), PCI_DEVCTL_MINOR)), dip,
134	    PCI_BUS_CONF_MAP_PROP, 1);
135
136	DBG(DBG_ATTACH, dip, "px_lib_dev_init: dev_hdl 0x%llx\n", *dev_hdl);
137
138	/*
139	 * If a /pci node has a pci-intx-not-supported property, this property
140	 * represents that the fabric doesn't support fixed interrupt.
141	 */
142	if (!ddi_prop_exists(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
143	    "pci-intx-not-supported")) {
144		DBG(DBG_ATTACH, dip, "px_lib_dev_init: "
145		    "pci-intx-not-supported is not found, dip=0x%p\n", dip);
146		px_p->px_supp_intr_types |= DDI_INTR_TYPE_FIXED;
147	}
148
149	/*
150	 * Negotiate the API version for VPCI hypervisor services.
151	 */
152	if (px_vpci_users == 0) {
153		if ((ret = hsvc_register(&px_hsvc_vpci, &px_vpci_min_ver))
154		    == 0) {
155			px_vpci_maj_ver = px_hsvc_vpci.hsvc_major;
156			goto hv_negotiation_complete;
157		}
158		/*
159		 * Negotiation with the latest known VPCI hypervisor services
160		 * failed.  Fallback to version 1.0.
161		 */
162		px_hsvc_vpci.hsvc_major = PX_HSVC_MAJOR_VER_1;
163		px_hsvc_vpci.hsvc_minor = PX_HSVC_MINOR_VER_0;
164
165		if ((ret = hsvc_register(&px_hsvc_vpci, &px_vpci_min_ver))
166		    == 0) {
167			px_vpci_maj_ver = px_hsvc_vpci.hsvc_major;
168			goto hv_negotiation_complete;
169		}
170
171		cmn_err(CE_WARN, "%s: cannot negotiate hypervisor services "
172		    "group: 0x%lx major: 0x%lx minor: 0x%lx errno: %d\n",
173		    px_hsvc_vpci.hsvc_modname, px_hsvc_vpci.hsvc_group,
174		    px_hsvc_vpci.hsvc_major, px_hsvc_vpci.hsvc_minor, ret);
175
176		return (DDI_FAILURE);
177	}
178hv_negotiation_complete:
179
180	px_vpci_users++;
181
182	DBG(DBG_ATTACH, dip, "px_lib_dev_init: negotiated VPCI API version, "
183	    "major 0x%lx minor 0x%lx\n", px_vpci_maj_ver,
184	    px_vpci_min_ver);
185
186	/*
187	 * Negotiate the API version for SDIO hypervisor services.
188	 */
189	if ((px_sdio_users == 0) &&
190	    ((ret = hsvc_register(&px_hsvc_sdio, &px_sdio_min_ver)) != 0)) {
191		DBG(DBG_ATTACH, dip, "%s: cannot negotiate hypervisor "
192		    "services group: 0x%lx major: 0x%lx minor: 0x%lx "
193		    "errno: %d\n", px_hsvc_sdio.hsvc_modname,
194		    px_hsvc_sdio.hsvc_group, px_hsvc_sdio.hsvc_major,
195		    px_hsvc_sdio.hsvc_minor, ret);
196	} else {
197		px_sdio_users++;
198		DBG(DBG_ATTACH, dip, "px_lib_dev_init: negotiated SDIO API"
199		    "version, major 0x%lx minor 0x%lx\n",
200		    px_hsvc_sdio.hsvc_major, px_sdio_min_ver);
201	}
202
203	/*
204	 * Negotiate the API version for SDIO ERR hypervisor services.
205	 */
206	if ((px_sdio_err_users == 0) &&
207	    ((ret = hsvc_register(&px_hsvc_sdio_err,
208	    &px_sdio_err_min_ver)) != 0)) {
209		DBG(DBG_ATTACH, dip, "%s: cannot negotiate SDIO ERR hypervisor "
210		    "services group: 0x%lx major: 0x%lx minor: 0x%lx "
211		    "errno: %d\n", px_hsvc_sdio_err.hsvc_modname,
212		    px_hsvc_sdio_err.hsvc_group, px_hsvc_sdio_err.hsvc_major,
213		    px_hsvc_sdio_err.hsvc_minor, ret);
214	} else {
215		px_sdio_err_users++;
216		DBG(DBG_ATTACH, dip, "px_lib_dev_init: negotiated SDIO ERR API "
217		    "version, major 0x%lx minor 0x%lx\n",
218		    px_hsvc_sdio_err.hsvc_major, px_sdio_err_min_ver);
219	}
220
221	/*
222	 * Find out the number of dev we need to wait under this RC
223	 * before we issue fabric sync hypercall
224	 */
225	px_p->px_plat_p = (void *)(uintptr_t)px_lib_count_waiting_dev(dip);
226	DBG(DBG_ATTACH, dip, "Found %d bridges need waiting under RC %p",
227	    (int)(uintptr_t)px_p->px_plat_p, dip);
228	return (DDI_SUCCESS);
229}
230
231/*ARGSUSED*/
232int
233px_lib_dev_fini(dev_info_t *dip)
234{
235	DBG(DBG_DETACH, dip, "px_lib_dev_fini: dip 0x%p\n", dip);
236
237	(void) ddi_prop_remove(makedevice(ddi_driver_major(dip),
238	    PCI_MINOR_NUM(ddi_get_instance(dip), PCI_DEVCTL_MINOR)), dip,
239	    PCI_BUS_CONF_MAP_PROP);
240
241	if (--px_vpci_users == 0)
242		(void) hsvc_unregister(&px_hsvc_vpci);
243
244	if (--px_sdio_users == 0)
245		(void) hsvc_unregister(&px_hsvc_sdio);
246
247	if (--px_sdio_err_users == 0)
248		(void) hsvc_unregister(&px_hsvc_sdio_err);
249
250	return (DDI_SUCCESS);
251}
252
253/*ARGSUSED*/
254int
255px_lib_intr_devino_to_sysino(dev_info_t *dip, devino_t devino,
256    sysino_t *sysino)
257{
258	uint64_t	ret;
259
260	DBG(DBG_LIB_INT, dip, "px_lib_intr_devino_to_sysino: dip 0x%p "
261	    "devino 0x%x\n", dip, devino);
262
263	if ((ret = hvio_intr_devino_to_sysino(DIP_TO_HANDLE(dip),
264	    devino, sysino)) != H_EOK) {
265		DBG(DBG_LIB_INT, dip,
266		    "hvio_intr_devino_to_sysino failed, ret 0x%lx\n", ret);
267		return (DDI_FAILURE);
268	}
269
270	DBG(DBG_LIB_INT, dip, "px_lib_intr_devino_to_sysino: sysino 0x%llx\n",
271	    *sysino);
272
273	return (DDI_SUCCESS);
274}
275
276/*ARGSUSED*/
277int
278px_lib_intr_getvalid(dev_info_t *dip, sysino_t sysino,
279    intr_valid_state_t *intr_valid_state)
280{
281	uint64_t	ret;
282
283	DBG(DBG_LIB_INT, dip, "px_lib_intr_getvalid: dip 0x%p sysino 0x%llx\n",
284	    dip, sysino);
285
286	if ((ret = hvio_intr_getvalid(sysino,
287	    (int *)intr_valid_state)) != H_EOK) {
288		DBG(DBG_LIB_INT, dip, "hvio_intr_getvalid failed, ret 0x%lx\n",
289		    ret);
290		return (DDI_FAILURE);
291	}
292
293	DBG(DBG_LIB_INT, dip, "px_lib_intr_getvalid: intr_valid_state 0x%x\n",
294	    *intr_valid_state);
295
296	return (DDI_SUCCESS);
297}
298
299/*ARGSUSED*/
300int
301px_lib_intr_setvalid(dev_info_t *dip, sysino_t sysino,
302    intr_valid_state_t intr_valid_state)
303{
304	uint64_t	ret;
305
306	DBG(DBG_LIB_INT, dip, "px_lib_intr_setvalid: dip 0x%p sysino 0x%llx "
307	    "intr_valid_state 0x%x\n", dip, sysino, intr_valid_state);
308
309	if ((ret = hvio_intr_setvalid(sysino, intr_valid_state)) != H_EOK) {
310		DBG(DBG_LIB_INT, dip, "hvio_intr_setvalid failed, ret 0x%lx\n",
311		    ret);
312		return (DDI_FAILURE);
313	}
314
315	return (DDI_SUCCESS);
316}
317
318/*ARGSUSED*/
319int
320px_lib_intr_getstate(dev_info_t *dip, sysino_t sysino,
321    intr_state_t *intr_state)
322{
323	uint64_t	ret;
324
325	DBG(DBG_LIB_INT, dip, "px_lib_intr_getstate: dip 0x%p sysino 0x%llx\n",
326	    dip, sysino);
327
328	if ((ret = hvio_intr_getstate(sysino, (int *)intr_state)) != H_EOK) {
329		DBG(DBG_LIB_INT, dip, "hvio_intr_getstate failed, ret 0x%lx\n",
330		    ret);
331		return (DDI_FAILURE);
332	}
333
334	DBG(DBG_LIB_INT, dip, "px_lib_intr_getstate: intr_state 0x%x\n",
335	    *intr_state);
336
337	return (DDI_SUCCESS);
338}
339
340/*ARGSUSED*/
341int
342px_lib_intr_setstate(dev_info_t *dip, sysino_t sysino,
343    intr_state_t intr_state)
344{
345	uint64_t	ret;
346
347	DBG(DBG_LIB_INT, dip, "px_lib_intr_setstate: dip 0x%p sysino 0x%llx "
348	    "intr_state 0x%x\n", dip, sysino, intr_state);
349
350	if ((ret = hvio_intr_setstate(sysino, intr_state)) != H_EOK) {
351		DBG(DBG_LIB_INT, dip, "hvio_intr_setstate failed, ret 0x%lx\n",
352		    ret);
353		return (DDI_FAILURE);
354	}
355
356	return (DDI_SUCCESS);
357}
358
359/*ARGSUSED*/
360int
361px_lib_intr_gettarget(dev_info_t *dip, sysino_t sysino, cpuid_t *cpuid)
362{
363	uint64_t	ret;
364
365	DBG(DBG_LIB_INT, dip, "px_lib_intr_gettarget: dip 0x%p sysino 0x%llx\n",
366	    dip, sysino);
367
368	if ((ret = hvio_intr_gettarget(sysino, cpuid)) != H_EOK) {
369		DBG(DBG_LIB_INT, dip,
370		    "hvio_intr_gettarget failed, ret 0x%lx\n", ret);
371		return (DDI_FAILURE);
372	}
373
374	DBG(DBG_LIB_INT, dip, "px_lib_intr_gettarget: cpuid 0x%x\n", *cpuid);
375
376	return (DDI_SUCCESS);
377}
378
379/*ARGSUSED*/
380int
381px_lib_intr_settarget(dev_info_t *dip, sysino_t sysino, cpuid_t cpuid)
382{
383	uint64_t	ret;
384
385	DBG(DBG_LIB_INT, dip, "px_lib_intr_settarget: dip 0x%p sysino 0x%llx "
386	    "cpuid 0x%x\n", dip, sysino, cpuid);
387
388	ret = hvio_intr_settarget(sysino, cpuid);
389	if (ret == H_ECPUERROR) {
390		cmn_err(CE_PANIC,
391		    "px_lib_intr_settarget: hvio_intr_settarget failed, "
392		    "ret = 0x%lx, cpuid = 0x%x, sysino = 0x%lx\n", ret,
393		    cpuid, sysino);
394	} else if (ret != H_EOK) {
395		DBG(DBG_LIB_INT, dip,
396		    "hvio_intr_settarget failed, ret 0x%lx\n", ret);
397		return (DDI_FAILURE);
398	}
399
400	return (DDI_SUCCESS);
401}
402
403/*ARGSUSED*/
404int
405px_lib_intr_reset(dev_info_t *dip)
406{
407	px_t		*px_p = DIP_TO_STATE(dip);
408	px_ib_t		*ib_p = px_p->px_ib_p;
409	px_ino_t	*ino_p;
410
411	DBG(DBG_LIB_INT, dip, "px_lib_intr_reset: dip 0x%p\n", dip);
412
413	mutex_enter(&ib_p->ib_ino_lst_mutex);
414
415	/* Reset all Interrupts */
416	for (ino_p = ib_p->ib_ino_lst; ino_p; ino_p = ino_p->ino_next_p) {
417		if (px_lib_intr_setstate(dip, ino_p->ino_sysino,
418		    INTR_IDLE_STATE) != DDI_SUCCESS)
419			return (BF_FATAL);
420	}
421
422	mutex_exit(&ib_p->ib_ino_lst_mutex);
423
424	return (BF_NONE);
425}
426
427/*ARGSUSED*/
428int
429px_lib_iommu_map(dev_info_t *dip, tsbid_t tsbid, pages_t pages,
430    io_attributes_t attr, void *addr, size_t pfn_index, int flags)
431{
432	tsbnum_t	tsb_num = PCI_TSBID_TO_TSBNUM(tsbid);
433	tsbindex_t	tsb_index = PCI_TSBID_TO_TSBINDEX(tsbid);
434	io_page_list_t	*pfns, *pfn_p;
435	pages_t		ttes_mapped = 0;
436	int		i, err = DDI_SUCCESS;
437
438	DBG(DBG_LIB_DMA, dip, "px_lib_iommu_map: dip 0x%p tsbid 0x%llx "
439	    "pages 0x%x attr 0x%llx addr 0x%p pfn_index 0x%llx flags 0x%x\n",
440	    dip, tsbid, pages, attr, addr, pfn_index, flags);
441
442	if ((pfns = pfn_p = kmem_zalloc((pages * sizeof (io_page_list_t)),
443	    KM_NOSLEEP)) == NULL) {
444		DBG(DBG_LIB_DMA, dip, "px_lib_iommu_map: kmem_zalloc failed\n");
445		return (DDI_FAILURE);
446	}
447
448	for (i = 0; i < pages; i++)
449		pfns[i] = MMU_PTOB(PX_ADDR2PFN(addr, pfn_index, flags, i));
450
451	/*
452	 * If HV VPCI version is 2.0 and higher, pass BDF, phantom function,
453	 * and relaxed ordering attributes. Otherwise, pass only read or write
454	 * attribute.
455	 */
456	if ((px_vpci_maj_ver == PX_HSVC_MAJOR_VER_1) &&
457	    (px_vpci_min_ver == PX_HSVC_MINOR_VER_0))
458		attr = attr & (PCI_MAP_ATTR_READ | PCI_MAP_ATTR_WRITE);
459
460	while ((ttes_mapped = pfn_p - pfns) < pages) {
461		uintptr_t	ra = va_to_pa(pfn_p);
462		pages_t		ttes2map;
463		uint64_t	ret;
464
465		ttes2map = (MMU_PAGE_SIZE - P2PHASE(ra, MMU_PAGE_SIZE)) >> 3;
466		ra = MMU_PTOB(MMU_BTOP(ra));
467
468		for (ttes2map = MIN(ttes2map, pages - ttes_mapped); ttes2map;
469		    ttes2map -= ttes_mapped, pfn_p += ttes_mapped) {
470
471			ttes_mapped = 0;
472			if ((ret = hvio_iommu_map(DIP_TO_HANDLE(dip),
473			    PCI_TSBID(tsb_num, tsb_index + (pfn_p - pfns)),
474			    ttes2map, attr, (io_page_list_t *)(ra |
475			    ((uintptr_t)pfn_p & MMU_PAGE_OFFSET)),
476			    &ttes_mapped)) != H_EOK) {
477				DBG(DBG_LIB_DMA, dip, "hvio_iommu_map failed "
478				    "ret 0x%lx\n", ret);
479
480				ttes_mapped = pfn_p - pfns;
481				err = DDI_FAILURE;
482				goto cleanup;
483			}
484
485			DBG(DBG_LIB_DMA, dip, "px_lib_iommu_map: tsb_num 0x%x "
486			    "tsb_index 0x%lx ttes_to_map 0x%lx attr 0x%llx "
487			    "ra 0x%p ttes_mapped 0x%x\n", tsb_num,
488			    tsb_index + (pfn_p - pfns), ttes2map, attr,
489			    ra | ((uintptr_t)pfn_p & MMU_PAGE_OFFSET),
490			    ttes_mapped);
491		}
492	}
493
494cleanup:
495	if ((err == DDI_FAILURE) && ttes_mapped)
496		(void) px_lib_iommu_demap(dip, tsbid, ttes_mapped);
497
498	kmem_free(pfns, pages * sizeof (io_page_list_t));
499	return (err);
500}
501
502/*ARGSUSED*/
503int
504px_lib_iommu_demap(dev_info_t *dip, tsbid_t tsbid, pages_t pages)
505{
506	tsbnum_t	tsb_num = PCI_TSBID_TO_TSBNUM(tsbid);
507	tsbindex_t	tsb_index = PCI_TSBID_TO_TSBINDEX(tsbid);
508	pages_t		ttes2demap, ttes_demapped = 0;
509	uint64_t	ret;
510
511	DBG(DBG_LIB_DMA, dip, "px_lib_iommu_demap: dip 0x%p tsbid 0x%llx "
512	    "pages 0x%x\n", dip, tsbid, pages);
513
514	for (ttes2demap = pages; ttes2demap;
515	    ttes2demap -= ttes_demapped, tsb_index += ttes_demapped) {
516		if ((ret = hvio_iommu_demap(DIP_TO_HANDLE(dip),
517		    PCI_TSBID(tsb_num, tsb_index), ttes2demap,
518		    &ttes_demapped)) != H_EOK) {
519			DBG(DBG_LIB_DMA, dip, "hvio_iommu_demap failed, "
520			    "ret 0x%lx\n", ret);
521
522			return (DDI_FAILURE);
523		}
524
525		DBG(DBG_LIB_DMA, dip, "px_lib_iommu_demap: tsb_num 0x%x "
526		    "tsb_index 0x%lx ttes_to_demap 0x%lx ttes_demapped 0x%x\n",
527		    tsb_num, tsb_index, ttes2demap, ttes_demapped);
528	}
529
530	return (DDI_SUCCESS);
531}
532
533/*ARGSUSED*/
534int
535px_lib_iommu_getmap(dev_info_t *dip, tsbid_t tsbid, io_attributes_t *attr_p,
536    r_addr_t *r_addr_p)
537{
538	uint64_t	ret;
539
540	DBG(DBG_LIB_DMA, dip, "px_lib_iommu_getmap: dip 0x%p tsbid 0x%llx\n",
541	    dip, tsbid);
542
543	if ((ret = hvio_iommu_getmap(DIP_TO_HANDLE(dip), tsbid,
544	    attr_p, r_addr_p)) != H_EOK) {
545		DBG(DBG_LIB_DMA, dip,
546		    "hvio_iommu_getmap failed, ret 0x%lx\n", ret);
547
548		return ((ret == H_ENOMAP) ? DDI_DMA_NOMAPPING:DDI_FAILURE);
549	}
550
551	DBG(DBG_LIB_DMA, dip, "px_lib_iommu_getmap: attr 0x%llx "
552	    "r_addr 0x%llx\n", *attr_p, *r_addr_p);
553
554	return (DDI_SUCCESS);
555}
556
557/*ARGSUSED*/
558int
559px_lib_iommu_detach(px_t *px_p)
560{
561	return (DDI_SUCCESS);
562}
563
564/*ARGSUSED*/
565uint64_t
566px_get_rng_parent_hi_mask(px_t *px_p)
567{
568	return (PX_RANGE_PROP_MASK);
569}
570
571/*
572 * Checks dma attributes against system bypass ranges
573 * A sun4v device must be capable of generating the entire 64-bit
574 * address in order to perform bypass DMA.
575 */
576/*ARGSUSED*/
577int
578px_lib_dma_bypass_rngchk(dev_info_t *dip, ddi_dma_attr_t *attr_p,
579    uint64_t *lo_p, uint64_t *hi_p)
580{
581	if ((attr_p->dma_attr_addr_lo != 0ull) ||
582	    (attr_p->dma_attr_addr_hi != UINT64_MAX)) {
583
584		return (DDI_DMA_BADATTR);
585	}
586
587	*lo_p = 0ull;
588	*hi_p = UINT64_MAX;
589
590	return (DDI_SUCCESS);
591}
592
593
594/*ARGSUSED*/
595int
596px_lib_iommu_getbypass(dev_info_t *dip, r_addr_t ra, io_attributes_t attr,
597    io_addr_t *io_addr_p)
598{
599	uint64_t	ret;
600
601	DBG(DBG_LIB_DMA, dip, "px_lib_iommu_getbypass: dip 0x%p ra 0x%llx "
602	    "attr 0x%llx\n", dip, ra, attr);
603	/*
604	 * If HV VPCI version is 2.0 and higher, pass BDF, phantom function,
605	 * and relaxed ordering attributes. Otherwise, pass only read or write
606	 * attribute.
607	 */
608	if ((px_vpci_maj_ver == PX_HSVC_MAJOR_VER_1) &&
609	    (px_vpci_min_ver == PX_HSVC_MINOR_VER_0))
610		attr &= PCI_MAP_ATTR_READ | PCI_MAP_ATTR_WRITE;
611
612	if ((ret = hvio_iommu_getbypass(DIP_TO_HANDLE(dip), ra,
613	    attr, io_addr_p)) != H_EOK) {
614		DBG(DBG_LIB_DMA, dip,
615		    "hvio_iommu_getbypass failed, ret 0x%lx\n", ret);
616		return (ret == H_ENOTSUPPORTED ? DDI_ENOTSUP : DDI_FAILURE);
617	}
618
619	DBG(DBG_LIB_DMA, dip, "px_lib_iommu_getbypass: io_addr 0x%llx\n",
620	    *io_addr_p);
621
622	return (DDI_SUCCESS);
623}
624
625/*
626 * Returns any needed IO address bit(s) for relaxed ordering in IOMMU
627 * bypass mode.
628 */
629/* ARGSUSED */
630uint64_t
631px_lib_ro_bypass(dev_info_t *dip, io_attributes_t attr, uint64_t ioaddr)
632{
633	return (ioaddr);
634}
635
636/*ARGSUSED*/
637int
638px_lib_dma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
639	off_t off, size_t len, uint_t cache_flags)
640{
641	ddi_dma_impl_t *mp = (ddi_dma_impl_t *)handle;
642	uint64_t sync_dir;
643	size_t bytes_synced;
644	int end, idx;
645	off_t pg_off;
646	devhandle_t hdl = DIP_TO_HANDLE(dip); /* need to cache hdl */
647
648	DBG(DBG_LIB_DMA, dip, "px_lib_dma_sync: dip 0x%p rdip 0x%p "
649	    "handle 0x%llx off 0x%x len 0x%x flags 0x%x\n",
650	    dip, rdip, handle, off, len, cache_flags);
651
652	if (!(mp->dmai_flags & PX_DMAI_FLAGS_INUSE)) {
653		cmn_err(CE_WARN, "%s%d: Unbound dma handle %p.",
654		    ddi_driver_name(rdip), ddi_get_instance(rdip), (void *)mp);
655		return (DDI_FAILURE);
656	}
657
658	if (mp->dmai_flags & PX_DMAI_FLAGS_NOSYNC)
659		return (DDI_SUCCESS);
660
661	if (!len)
662		len = mp->dmai_size;
663
664	if (mp->dmai_rflags & DDI_DMA_READ)
665		sync_dir = HVIO_DMA_SYNC_DIR_FROM_DEV;
666	else
667		sync_dir = HVIO_DMA_SYNC_DIR_TO_DEV;
668
669	off += mp->dmai_offset;
670	pg_off = off & MMU_PAGEOFFSET;
671
672	DBG(DBG_LIB_DMA, dip, "px_lib_dma_sync: page offset %x size %x\n",
673	    pg_off, len);
674
675	/* sync on page basis */
676	end = MMU_BTOPR(off + len - 1);
677	for (idx = MMU_BTOP(off); idx < end; idx++,
678	    len -= bytes_synced, pg_off = 0) {
679		size_t bytes_to_sync = bytes_to_sync =
680		    MIN(len, MMU_PAGESIZE - pg_off);
681
682		if (hvio_dma_sync(hdl, MMU_PTOB(PX_GET_MP_PFN(mp, idx)) +
683		    pg_off, bytes_to_sync, sync_dir, &bytes_synced) != H_EOK)
684			break;
685
686		DBG(DBG_LIB_DMA, dip, "px_lib_dma_sync: Called hvio_dma_sync "
687		    "ra = %p bytes to sync = %x bytes synced %x\n",
688		    MMU_PTOB(PX_GET_MP_PFN(mp, idx)) + pg_off, bytes_to_sync,
689		    bytes_synced);
690
691		if (bytes_to_sync != bytes_synced)
692			break;
693	}
694
695	return (len ? DDI_FAILURE : DDI_SUCCESS);
696}
697
698
699/*
700 * MSIQ Functions:
701 */
702
703/*ARGSUSED*/
704int
705px_lib_msiq_init(dev_info_t *dip)
706{
707	px_t		*px_p = DIP_TO_STATE(dip);
708	px_msiq_state_t	*msiq_state_p = &px_p->px_ib_p->ib_msiq_state;
709	r_addr_t	ra;
710	size_t		msiq_size;
711	uint_t		rec_cnt;
712	int		i, err = DDI_SUCCESS;
713	uint64_t	ret;
714
715	DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_init: dip 0x%p\n", dip);
716
717	msiq_size = msiq_state_p->msiq_rec_cnt * sizeof (msiq_rec_t);
718
719	/* sun4v requires all EQ allocation to be on q size boundary */
720	if ((msiq_state_p->msiq_buf_p = contig_mem_alloc_align(
721	    msiq_state_p->msiq_cnt * msiq_size, msiq_size)) == NULL) {
722		DBG(DBG_LIB_MSIQ, dip,
723		    "px_lib_msiq_init: Contig alloc failed\n");
724
725		return (DDI_FAILURE);
726	}
727
728	for (i = 0; i < msiq_state_p->msiq_cnt; i++) {
729		msiq_state_p->msiq_p[i].msiq_base_p = (msiqhead_t *)
730		    ((caddr_t)msiq_state_p->msiq_buf_p + (i * msiq_size));
731
732		ra = (r_addr_t)va_to_pa((caddr_t)msiq_state_p->msiq_buf_p +
733		    (i * msiq_size));
734
735		if ((ret = hvio_msiq_conf(DIP_TO_HANDLE(dip),
736		    (i + msiq_state_p->msiq_1st_msiq_id),
737		    ra, msiq_state_p->msiq_rec_cnt)) != H_EOK) {
738			DBG(DBG_LIB_MSIQ, dip,
739			    "hvio_msiq_conf failed, ret 0x%lx\n", ret);
740			err = DDI_FAILURE;
741			break;
742		}
743
744		if ((err = px_lib_msiq_info(dip,
745		    (i + msiq_state_p->msiq_1st_msiq_id),
746		    &ra, &rec_cnt)) != DDI_SUCCESS) {
747			DBG(DBG_LIB_MSIQ, dip,
748			    "px_lib_msiq_info failed, ret 0x%x\n", err);
749			err = DDI_FAILURE;
750			break;
751		}
752
753		DBG(DBG_LIB_MSIQ, dip,
754		    "px_lib_msiq_init: ra 0x%p rec_cnt 0x%x\n", ra, rec_cnt);
755	}
756
757	return (err);
758}
759
760/*ARGSUSED*/
761int
762px_lib_msiq_fini(dev_info_t *dip)
763{
764	px_t		*px_p = DIP_TO_STATE(dip);
765	px_msiq_state_t	*msiq_state_p = &px_p->px_ib_p->ib_msiq_state;
766	size_t		msiq_size;
767
768	DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_fini: dip 0x%p\n", dip);
769	msiq_size = msiq_state_p->msiq_rec_cnt * sizeof (msiq_rec_t);
770
771	if (msiq_state_p->msiq_buf_p != NULL)
772		contig_mem_free(msiq_state_p->msiq_buf_p,
773		    msiq_state_p->msiq_cnt * msiq_size);
774
775	return (DDI_SUCCESS);
776}
777
778/*ARGSUSED*/
779int
780px_lib_msiq_info(dev_info_t *dip, msiqid_t msiq_id, r_addr_t *ra_p,
781    uint_t *msiq_rec_cnt_p)
782{
783	uint64_t	ret;
784
785	DBG(DBG_LIB_MSIQ, dip, "px_msiq_info: dip 0x%p msiq_id 0x%x\n",
786	    dip, msiq_id);
787
788	if ((ret = hvio_msiq_info(DIP_TO_HANDLE(dip),
789	    msiq_id, ra_p, msiq_rec_cnt_p)) != H_EOK) {
790		DBG(DBG_LIB_MSIQ, dip,
791		    "hvio_msiq_info failed, ret 0x%lx\n", ret);
792		return (DDI_FAILURE);
793	}
794
795	DBG(DBG_LIB_MSIQ, dip, "px_msiq_info: ra_p 0x%p msiq_rec_cnt 0x%x\n",
796	    ra_p, *msiq_rec_cnt_p);
797
798	return (DDI_SUCCESS);
799}
800
801/*ARGSUSED*/
802int
803px_lib_msiq_getvalid(dev_info_t *dip, msiqid_t msiq_id,
804    pci_msiq_valid_state_t *msiq_valid_state)
805{
806	uint64_t	ret;
807
808	DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_getvalid: dip 0x%p msiq_id 0x%x\n",
809	    dip, msiq_id);
810
811	if ((ret = hvio_msiq_getvalid(DIP_TO_HANDLE(dip),
812	    msiq_id, msiq_valid_state)) != H_EOK) {
813		DBG(DBG_LIB_MSIQ, dip,
814		    "hvio_msiq_getvalid failed, ret 0x%lx\n", ret);
815		return (DDI_FAILURE);
816	}
817
818	DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_getvalid: msiq_valid_state 0x%x\n",
819	    *msiq_valid_state);
820
821	return (DDI_SUCCESS);
822}
823
824/*ARGSUSED*/
825int
826px_lib_msiq_setvalid(dev_info_t *dip, msiqid_t msiq_id,
827    pci_msiq_valid_state_t msiq_valid_state)
828{
829	uint64_t	ret;
830
831	DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_setvalid: dip 0x%p msiq_id 0x%x "
832	    "msiq_valid_state 0x%x\n", dip, msiq_id, msiq_valid_state);
833
834	if ((ret = hvio_msiq_setvalid(DIP_TO_HANDLE(dip),
835	    msiq_id, msiq_valid_state)) != H_EOK) {
836		DBG(DBG_LIB_MSIQ, dip,
837		    "hvio_msiq_setvalid failed, ret 0x%lx\n", ret);
838		return (DDI_FAILURE);
839	}
840
841	return (DDI_SUCCESS);
842}
843
844/*ARGSUSED*/
845int
846px_lib_msiq_getstate(dev_info_t *dip, msiqid_t msiq_id,
847    pci_msiq_state_t *msiq_state)
848{
849	uint64_t	ret;
850
851	DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_getstate: dip 0x%p msiq_id 0x%x\n",
852	    dip, msiq_id);
853
854	if ((ret = hvio_msiq_getstate(DIP_TO_HANDLE(dip),
855	    msiq_id, msiq_state)) != H_EOK) {
856		DBG(DBG_LIB_MSIQ, dip,
857		    "hvio_msiq_getstate failed, ret 0x%lx\n", ret);
858		return (DDI_FAILURE);
859	}
860
861	DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_getstate: msiq_state 0x%x\n",
862	    *msiq_state);
863
864	return (DDI_SUCCESS);
865}
866
867/*ARGSUSED*/
868int
869px_lib_msiq_setstate(dev_info_t *dip, msiqid_t msiq_id,
870    pci_msiq_state_t msiq_state)
871{
872	uint64_t	ret;
873
874	DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_setstate: dip 0x%p msiq_id 0x%x "
875	    "msiq_state 0x%x\n", dip, msiq_id, msiq_state);
876
877	if ((ret = hvio_msiq_setstate(DIP_TO_HANDLE(dip),
878	    msiq_id, msiq_state)) != H_EOK) {
879		DBG(DBG_LIB_MSIQ, dip,
880		    "hvio_msiq_setstate failed, ret 0x%lx\n", ret);
881		return (DDI_FAILURE);
882	}
883
884	return (DDI_SUCCESS);
885}
886
887/*ARGSUSED*/
888int
889px_lib_msiq_gethead(dev_info_t *dip, msiqid_t msiq_id,
890    msiqhead_t *msiq_head_p)
891{
892	uint64_t	ret;
893
894	DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_gethead: dip 0x%p msiq_id 0x%x\n",
895	    dip, msiq_id);
896
897	if ((ret = hvio_msiq_gethead(DIP_TO_HANDLE(dip),
898	    msiq_id, msiq_head_p)) != H_EOK) {
899		DBG(DBG_LIB_MSIQ, dip,
900		    "hvio_msiq_gethead failed, ret 0x%lx\n", ret);
901		return (DDI_FAILURE);
902	}
903
904	*msiq_head_p =  (*msiq_head_p / sizeof (msiq_rec_t));
905
906	DBG(DBG_LIB_MSIQ, dip, "px_msiq_gethead: msiq_head 0x%x\n",
907	    *msiq_head_p);
908
909	return (DDI_SUCCESS);
910}
911
912/*ARGSUSED*/
913int
914px_lib_msiq_sethead(dev_info_t *dip, msiqid_t msiq_id,
915    msiqhead_t msiq_head)
916{
917	uint64_t	ret;
918
919	DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_sethead: dip 0x%p msiq_id 0x%x "
920	    "msiq_head 0x%x\n", dip, msiq_id, msiq_head);
921
922	if ((ret = hvio_msiq_sethead(DIP_TO_HANDLE(dip),
923	    msiq_id, msiq_head * sizeof (msiq_rec_t))) != H_EOK) {
924		DBG(DBG_LIB_MSIQ, dip,
925		    "hvio_msiq_sethead failed, ret 0x%lx\n", ret);
926		return (DDI_FAILURE);
927	}
928
929	return (DDI_SUCCESS);
930}
931
932/*ARGSUSED*/
933int
934px_lib_msiq_gettail(dev_info_t *dip, msiqid_t msiq_id,
935    msiqtail_t *msiq_tail_p)
936{
937	uint64_t	ret;
938
939	DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_gettail: dip 0x%p msiq_id 0x%x\n",
940	    dip, msiq_id);
941
942	if ((ret = hvio_msiq_gettail(DIP_TO_HANDLE(dip),
943	    msiq_id, msiq_tail_p)) != H_EOK) {
944		DBG(DBG_LIB_MSIQ, dip,
945		    "hvio_msiq_gettail failed, ret 0x%lx\n", ret);
946		return (DDI_FAILURE);
947	}
948
949	*msiq_tail_p =  (*msiq_tail_p / sizeof (msiq_rec_t));
950	DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_gettail: msiq_tail 0x%x\n",
951	    *msiq_tail_p);
952
953	return (DDI_SUCCESS);
954}
955
956/*ARGSUSED*/
957void
958px_lib_get_msiq_rec(dev_info_t *dip, msiqhead_t *msiq_head_p,
959    msiq_rec_t *msiq_rec_p)
960{
961	msiq_rec_t	*curr_msiq_rec_p = (msiq_rec_t *)msiq_head_p;
962
963	DBG(DBG_LIB_MSIQ, dip, "px_lib_get_msiq_rec: dip 0x%p\n", dip);
964
965	if (!curr_msiq_rec_p->msiq_rec_type) {
966		/* Set msiq_rec_type to zero */
967		msiq_rec_p->msiq_rec_type = 0;
968
969		return;
970	}
971
972	*msiq_rec_p = *curr_msiq_rec_p;
973}
974
975/*ARGSUSED*/
976void
977px_lib_clr_msiq_rec(dev_info_t *dip, msiqhead_t *msiq_head_p)
978{
979	msiq_rec_t	*curr_msiq_rec_p = (msiq_rec_t *)msiq_head_p;
980
981	DBG(DBG_LIB_MSIQ, dip, "px_lib_clr_msiq_rec: dip 0x%p\n", dip);
982
983	/* Zero out msiq_rec_type field */
984	curr_msiq_rec_p->msiq_rec_type  = 0;
985}
986
987/*
988 * MSI Functions:
989 */
990
991/*ARGSUSED*/
992int
993px_lib_msi_init(dev_info_t *dip)
994{
995	DBG(DBG_LIB_MSI, dip, "px_lib_msi_init: dip 0x%p\n", dip);
996
997	/* Noop */
998	return (DDI_SUCCESS);
999}
1000
1001/*ARGSUSED*/
1002int
1003px_lib_msi_getmsiq(dev_info_t *dip, msinum_t msi_num,
1004    msiqid_t *msiq_id)
1005{
1006	uint64_t	ret;
1007
1008	DBG(DBG_LIB_MSI, dip, "px_lib_msi_getmsiq: dip 0x%p msi_num 0x%x\n",
1009	    dip, msi_num);
1010
1011	if ((ret = hvio_msi_getmsiq(DIP_TO_HANDLE(dip),
1012	    msi_num, msiq_id)) != H_EOK) {
1013		DBG(DBG_LIB_MSI, dip,
1014		    "hvio_msi_getmsiq failed, ret 0x%lx\n", ret);
1015		return (DDI_FAILURE);
1016	}
1017
1018	DBG(DBG_LIB_MSI, dip, "px_lib_msi_getmsiq: msiq_id 0x%x\n",
1019	    *msiq_id);
1020
1021	return (DDI_SUCCESS);
1022}
1023
1024/*ARGSUSED*/
1025int
1026px_lib_msi_setmsiq(dev_info_t *dip, msinum_t msi_num,
1027    msiqid_t msiq_id, msi_type_t msitype)
1028{
1029	uint64_t	ret;
1030
1031	DBG(DBG_LIB_MSI, dip, "px_lib_msi_setmsiq: dip 0x%p msi_num 0x%x "
1032	    "msq_id 0x%x\n", dip, msi_num, msiq_id);
1033
1034	if ((ret = hvio_msi_setmsiq(DIP_TO_HANDLE(dip),
1035	    msi_num, msiq_id, msitype)) != H_EOK) {
1036		DBG(DBG_LIB_MSI, dip,
1037		    "hvio_msi_setmsiq failed, ret 0x%lx\n", ret);
1038		return (DDI_FAILURE);
1039	}
1040
1041	return (DDI_SUCCESS);
1042}
1043
1044/*ARGSUSED*/
1045int
1046px_lib_msi_getvalid(dev_info_t *dip, msinum_t msi_num,
1047    pci_msi_valid_state_t *msi_valid_state)
1048{
1049	uint64_t	ret;
1050
1051	DBG(DBG_LIB_MSI, dip, "px_lib_msi_getvalid: dip 0x%p msi_num 0x%x\n",
1052	    dip, msi_num);
1053
1054	if ((ret = hvio_msi_getvalid(DIP_TO_HANDLE(dip),
1055	    msi_num, msi_valid_state)) != H_EOK) {
1056		DBG(DBG_LIB_MSI, dip,
1057		    "hvio_msi_getvalid failed, ret 0x%lx\n", ret);
1058		return (DDI_FAILURE);
1059	}
1060
1061	DBG(DBG_LIB_MSI, dip, "px_lib_msi_getvalid: msiq_id 0x%x\n",
1062	    *msi_valid_state);
1063
1064	return (DDI_SUCCESS);
1065}
1066
1067/*ARGSUSED*/
1068int
1069px_lib_msi_setvalid(dev_info_t *dip, msinum_t msi_num,
1070    pci_msi_valid_state_t msi_valid_state)
1071{
1072	uint64_t	ret;
1073
1074	DBG(DBG_LIB_MSI, dip, "px_lib_msi_setvalid: dip 0x%p msi_num 0x%x "
1075	    "msi_valid_state 0x%x\n", dip, msi_num, msi_valid_state);
1076
1077	if ((ret = hvio_msi_setvalid(DIP_TO_HANDLE(dip),
1078	    msi_num, msi_valid_state)) != H_EOK) {
1079		DBG(DBG_LIB_MSI, dip,
1080		    "hvio_msi_setvalid failed, ret 0x%lx\n", ret);
1081		return (DDI_FAILURE);
1082	}
1083
1084	return (DDI_SUCCESS);
1085}
1086
1087/*ARGSUSED*/
1088int
1089px_lib_msi_getstate(dev_info_t *dip, msinum_t msi_num,
1090    pci_msi_state_t *msi_state)
1091{
1092	uint64_t	ret;
1093
1094	DBG(DBG_LIB_MSI, dip, "px_lib_msi_getstate: dip 0x%p msi_num 0x%x\n",
1095	    dip, msi_num);
1096
1097	if ((ret = hvio_msi_getstate(DIP_TO_HANDLE(dip),
1098	    msi_num, msi_state)) != H_EOK) {
1099		DBG(DBG_LIB_MSI, dip,
1100		    "hvio_msi_getstate failed, ret 0x%lx\n", ret);
1101		return (DDI_FAILURE);
1102	}
1103
1104	DBG(DBG_LIB_MSI, dip, "px_lib_msi_getstate: msi_state 0x%x\n",
1105	    *msi_state);
1106
1107	return (DDI_SUCCESS);
1108}
1109
1110/*ARGSUSED*/
1111int
1112px_lib_msi_setstate(dev_info_t *dip, msinum_t msi_num,
1113    pci_msi_state_t msi_state)
1114{
1115	uint64_t	ret;
1116
1117	DBG(DBG_LIB_MSI, dip, "px_lib_msi_setstate: dip 0x%p msi_num 0x%x "
1118	    "msi_state 0x%x\n", dip, msi_num, msi_state);
1119
1120	if ((ret = hvio_msi_setstate(DIP_TO_HANDLE(dip),
1121	    msi_num, msi_state)) != H_EOK) {
1122		DBG(DBG_LIB_MSI, dip,
1123		    "hvio_msi_setstate failed, ret 0x%lx\n", ret);
1124		return (DDI_FAILURE);
1125	}
1126
1127	return (DDI_SUCCESS);
1128}
1129
1130/*
1131 * MSG Functions:
1132 */
1133
1134/*ARGSUSED*/
1135int
1136px_lib_msg_getmsiq(dev_info_t *dip, pcie_msg_type_t msg_type,
1137    msiqid_t *msiq_id)
1138{
1139	uint64_t	ret;
1140
1141	DBG(DBG_LIB_MSG, dip, "px_lib_msg_getmsiq: dip 0x%p msg_type 0x%x\n",
1142	    dip, msg_type);
1143
1144	if ((ret = hvio_msg_getmsiq(DIP_TO_HANDLE(dip),
1145	    msg_type, msiq_id)) != H_EOK) {
1146		DBG(DBG_LIB_MSG, dip,
1147		    "hvio_msg_getmsiq failed, ret 0x%lx\n", ret);
1148		return (DDI_FAILURE);
1149	}
1150
1151	DBG(DBG_LIB_MSI, dip, "px_lib_msg_getmsiq: msiq_id 0x%x\n",
1152	    *msiq_id);
1153
1154	return (DDI_SUCCESS);
1155}
1156
1157/*ARGSUSED*/
1158int
1159px_lib_msg_setmsiq(dev_info_t *dip, pcie_msg_type_t msg_type,
1160    msiqid_t msiq_id)
1161{
1162	uint64_t	ret;
1163
1164	DBG(DBG_LIB_MSG, dip, "px_lib_msg_setmsiq: dip 0x%p msg_type 0x%x "
1165	    "msq_id 0x%x\n", dip, msg_type, msiq_id);
1166
1167	if ((ret = hvio_msg_setmsiq(DIP_TO_HANDLE(dip),
1168	    msg_type, msiq_id)) != H_EOK) {
1169		DBG(DBG_LIB_MSG, dip,
1170		    "hvio_msg_setmsiq failed, ret 0x%lx\n", ret);
1171		return (DDI_FAILURE);
1172	}
1173
1174	return (DDI_SUCCESS);
1175}
1176
1177/*ARGSUSED*/
1178int
1179px_lib_msg_getvalid(dev_info_t *dip, pcie_msg_type_t msg_type,
1180    pcie_msg_valid_state_t *msg_valid_state)
1181{
1182	uint64_t	ret;
1183
1184	DBG(DBG_LIB_MSG, dip, "px_lib_msg_getvalid: dip 0x%p msg_type 0x%x\n",
1185	    dip, msg_type);
1186
1187	if ((ret = hvio_msg_getvalid(DIP_TO_HANDLE(dip), msg_type,
1188	    msg_valid_state)) != H_EOK) {
1189		DBG(DBG_LIB_MSG, dip,
1190		    "hvio_msg_getvalid failed, ret 0x%lx\n", ret);
1191		return (DDI_FAILURE);
1192	}
1193
1194	DBG(DBG_LIB_MSI, dip, "px_lib_msg_getvalid: msg_valid_state 0x%x\n",
1195	    *msg_valid_state);
1196
1197	return (DDI_SUCCESS);
1198}
1199
1200/*ARGSUSED*/
1201int
1202px_lib_msg_setvalid(dev_info_t *dip, pcie_msg_type_t msg_type,
1203    pcie_msg_valid_state_t msg_valid_state)
1204{
1205	uint64_t	ret;
1206
1207	DBG(DBG_LIB_MSG, dip, "px_lib_msg_setvalid: dip 0x%p msg_type 0x%x "
1208	    "msg_valid_state 0x%x\n", dip, msg_type, msg_valid_state);
1209
1210	if ((ret = hvio_msg_setvalid(DIP_TO_HANDLE(dip), msg_type,
1211	    msg_valid_state)) != H_EOK) {
1212		DBG(DBG_LIB_MSG, dip,
1213		    "hvio_msg_setvalid failed, ret 0x%lx\n", ret);
1214		return (DDI_FAILURE);
1215	}
1216
1217	return (DDI_SUCCESS);
1218}
1219
1220/*
1221 * Suspend/Resume Functions:
1222 * Currently unsupported by hypervisor and all functions are noops.
1223 */
1224/*ARGSUSED*/
1225int
1226px_lib_suspend(dev_info_t *dip)
1227{
1228	DBG(DBG_ATTACH, dip, "px_lib_suspend: Not supported\n");
1229
1230	/* Not supported */
1231	return (DDI_FAILURE);
1232}
1233
1234/*ARGSUSED*/
1235void
1236px_lib_resume(dev_info_t *dip)
1237{
1238	DBG(DBG_ATTACH, dip, "px_lib_resume: Not supported\n");
1239
1240	/* Noop */
1241}
1242
1243/*
1244 * Misc Functions:
1245 * Currently unsupported by hypervisor and all functions are noops.
1246 */
1247/*ARGSUSED*/
1248static int
1249px_lib_config_get(dev_info_t *dip, pci_device_t bdf, pci_config_offset_t off,
1250    uint8_t size, pci_cfg_data_t *data_p)
1251{
1252	uint64_t	ret;
1253
1254	DBG(DBG_LIB_CFG, dip, "px_lib_config_get: dip 0x%p, bdf 0x%llx "
1255	    "off 0x%x size 0x%x\n", dip, bdf, off, size);
1256
1257	if ((ret = hvio_config_get(DIP_TO_HANDLE(dip), bdf, off,
1258	    size, data_p)) != H_EOK) {
1259		DBG(DBG_LIB_CFG, dip,
1260		    "hvio_config_get failed, ret 0x%lx\n", ret);
1261		return (DDI_FAILURE);
1262	}
1263	DBG(DBG_LIB_CFG, dip, "px_config_get: data 0x%x\n", data_p->dw);
1264
1265	return (DDI_SUCCESS);
1266}
1267
1268/*ARGSUSED*/
1269static int
1270px_lib_config_put(dev_info_t *dip, pci_device_t bdf, pci_config_offset_t off,
1271    uint8_t size, pci_cfg_data_t data)
1272{
1273	uint64_t	ret;
1274
1275	DBG(DBG_LIB_CFG, dip, "px_lib_config_put: dip 0x%p, bdf 0x%llx "
1276	    "off 0x%x size 0x%x data 0x%llx\n", dip, bdf, off, size, data.qw);
1277
1278	if ((ret = hvio_config_put(DIP_TO_HANDLE(dip), bdf, off,
1279	    size, data)) != H_EOK) {
1280		DBG(DBG_LIB_CFG, dip,
1281		    "hvio_config_put failed, ret 0x%lx\n", ret);
1282		return (DDI_FAILURE);
1283	}
1284
1285	return (DDI_SUCCESS);
1286}
1287
1288static uint32_t
1289px_pci_config_get(ddi_acc_impl_t *handle, uint32_t *addr, int size)
1290{
1291	px_config_acc_pvt_t *px_pvt = (px_config_acc_pvt_t *)
1292	    handle->ahi_common.ah_bus_private;
1293	pcie_bus_t *busp = NULL;
1294	dev_info_t *cdip = NULL;
1295	uint32_t pci_dev_addr = px_pvt->raddr;
1296	uint32_t vaddr = px_pvt->vaddr;
1297	uint16_t off = (uint16_t)(uintptr_t)(addr - vaddr) & 0xfff;
1298	uint64_t rdata = 0;
1299
1300	if (px_lib_config_get(px_pvt->dip, pci_dev_addr, off,
1301	    size, (pci_cfg_data_t *)&rdata) != DDI_SUCCESS)
1302		/* XXX update error kstats */
1303		return (0xffffffff);
1304
1305	if (cdip = pcie_find_dip_by_bdf(px_pvt->dip, pci_dev_addr >> 8))
1306		busp = PCIE_DIP2BUS(cdip);
1307	/*
1308	 * This can be called early, before busp or busp->bus_dom has
1309	 * been initialized, so check both before invoking
1310	 * PCIE_IS_ASSIGNED.
1311	 */
1312	if (busp && PCIE_BUS2DOM(busp) && PCIE_IS_ASSIGNED(busp)) {
1313		if (off == PCI_CONF_VENID && size == 2)
1314			rdata = busp->bus_dev_ven_id & 0xffff;
1315		else if (off == PCI_CONF_DEVID && size == 2)
1316			rdata = busp->bus_dev_ven_id >> 16;
1317		else if (off == PCI_CONF_VENID && size == 4)
1318			rdata = busp->bus_dev_ven_id;
1319	}
1320	return ((uint32_t)rdata);
1321}
1322
1323static void
1324px_pci_config_put(ddi_acc_impl_t *handle, uint32_t *addr,
1325		int size, pci_cfg_data_t wdata)
1326{
1327	px_config_acc_pvt_t *px_pvt = (px_config_acc_pvt_t *)
1328	    handle->ahi_common.ah_bus_private;
1329	uint32_t pci_dev_addr = px_pvt->raddr;
1330	uint32_t vaddr = px_pvt->vaddr;
1331	uint16_t off = (uint16_t)(uintptr_t)(addr - vaddr) & 0xfff;
1332
1333	if (px_lib_config_put(px_pvt->dip, pci_dev_addr, off,
1334	    size, wdata) != DDI_SUCCESS) {
1335		/*EMPTY*/
1336		/* XXX update error kstats */
1337	}
1338}
1339
1340static uint8_t
1341px_pci_config_get8(ddi_acc_impl_t *handle, uint8_t *addr)
1342{
1343	return ((uint8_t)px_pci_config_get(handle, (uint32_t *)addr, 1));
1344}
1345
1346static uint16_t
1347px_pci_config_get16(ddi_acc_impl_t *handle, uint16_t *addr)
1348{
1349	return ((uint16_t)px_pci_config_get(handle, (uint32_t *)addr, 2));
1350}
1351
1352static uint32_t
1353px_pci_config_get32(ddi_acc_impl_t *handle, uint32_t *addr)
1354{
1355	return ((uint32_t)px_pci_config_get(handle, (uint32_t *)addr, 4));
1356}
1357
1358static uint64_t
1359px_pci_config_get64(ddi_acc_impl_t *handle, uint64_t *addr)
1360{
1361	uint32_t rdatah, rdatal;
1362
1363	rdatal = (uint32_t)px_pci_config_get(handle, (uint32_t *)addr, 4);
1364	rdatah = (uint32_t)px_pci_config_get(handle,
1365	    (uint32_t *)((char *)addr+4), 4);
1366	return (((uint64_t)rdatah << 32) | rdatal);
1367}
1368
1369static void
1370px_pci_config_put8(ddi_acc_impl_t *handle, uint8_t *addr, uint8_t data)
1371{
1372	pci_cfg_data_t wdata = { 0 };
1373
1374	wdata.qw = (uint8_t)data;
1375	px_pci_config_put(handle, (uint32_t *)addr, 1, wdata);
1376}
1377
1378static void
1379px_pci_config_put16(ddi_acc_impl_t *handle, uint16_t *addr, uint16_t data)
1380{
1381	pci_cfg_data_t wdata = { 0 };
1382
1383	wdata.qw = (uint16_t)data;
1384	px_pci_config_put(handle, (uint32_t *)addr, 2, wdata);
1385}
1386
1387static void
1388px_pci_config_put32(ddi_acc_impl_t *handle, uint32_t *addr, uint32_t data)
1389{
1390	pci_cfg_data_t wdata = { 0 };
1391
1392	wdata.qw = (uint32_t)data;
1393	px_pci_config_put(handle, (uint32_t *)addr, 4, wdata);
1394}
1395
1396static void
1397px_pci_config_put64(ddi_acc_impl_t *handle, uint64_t *addr, uint64_t data)
1398{
1399	pci_cfg_data_t wdata = { 0 };
1400
1401	wdata.qw = (uint32_t)(data & 0xffffffff);
1402	px_pci_config_put(handle, (uint32_t *)addr, 4, wdata);
1403	wdata.qw = (uint32_t)((data >> 32) & 0xffffffff);
1404	px_pci_config_put(handle, (uint32_t *)((char *)addr+4), 4, wdata);
1405}
1406
1407static void
1408px_pci_config_rep_get8(ddi_acc_impl_t *handle, uint8_t *host_addr,
1409			uint8_t *dev_addr, size_t repcount, uint_t flags)
1410{
1411	if (flags == DDI_DEV_AUTOINCR)
1412		for (; repcount; repcount--)
1413			*host_addr++ = px_pci_config_get8(handle, dev_addr++);
1414	else
1415		for (; repcount; repcount--)
1416			*host_addr++ = px_pci_config_get8(handle, dev_addr);
1417}
1418
1419/*
1420 * Function to rep read 16 bit data off the PCI configuration space behind
1421 * the 21554's host interface.
1422 */
1423static void
1424px_pci_config_rep_get16(ddi_acc_impl_t *handle, uint16_t *host_addr,
1425			uint16_t *dev_addr, size_t repcount, uint_t flags)
1426{
1427	if (flags == DDI_DEV_AUTOINCR)
1428		for (; repcount; repcount--)
1429			*host_addr++ = px_pci_config_get16(handle, dev_addr++);
1430	else
1431		for (; repcount; repcount--)
1432			*host_addr++ = px_pci_config_get16(handle, dev_addr);
1433}
1434
1435/*
1436 * Function to rep read 32 bit data off the PCI configuration space behind
1437 * the 21554's host interface.
1438 */
1439static void
1440px_pci_config_rep_get32(ddi_acc_impl_t *handle, uint32_t *host_addr,
1441			uint32_t *dev_addr, size_t repcount, uint_t flags)
1442{
1443	if (flags == DDI_DEV_AUTOINCR)
1444		for (; repcount; repcount--)
1445			*host_addr++ = px_pci_config_get32(handle, dev_addr++);
1446	else
1447		for (; repcount; repcount--)
1448			*host_addr++ = px_pci_config_get32(handle, dev_addr);
1449}
1450
1451/*
1452 * Function to rep read 64 bit data off the PCI configuration space behind
1453 * the 21554's host interface.
1454 */
1455static void
1456px_pci_config_rep_get64(ddi_acc_impl_t *handle, uint64_t *host_addr,
1457			uint64_t *dev_addr, size_t repcount, uint_t flags)
1458{
1459	if (flags == DDI_DEV_AUTOINCR)
1460		for (; repcount; repcount--)
1461			*host_addr++ = px_pci_config_get64(handle, dev_addr++);
1462	else
1463		for (; repcount; repcount--)
1464			*host_addr++ = px_pci_config_get64(handle, dev_addr);
1465}
1466
1467/*
1468 * Function to rep write 8 bit data into the PCI configuration space behind
1469 * the 21554's host interface.
1470 */
1471static void
1472px_pci_config_rep_put8(ddi_acc_impl_t *handle, uint8_t *host_addr,
1473			uint8_t *dev_addr, size_t repcount, uint_t flags)
1474{
1475	if (flags == DDI_DEV_AUTOINCR)
1476		for (; repcount; repcount--)
1477			px_pci_config_put8(handle, dev_addr++, *host_addr++);
1478	else
1479		for (; repcount; repcount--)
1480			px_pci_config_put8(handle, dev_addr, *host_addr++);
1481}
1482
1483/*
1484 * Function to rep write 16 bit data into the PCI configuration space behind
1485 * the 21554's host interface.
1486 */
1487static void
1488px_pci_config_rep_put16(ddi_acc_impl_t *handle, uint16_t *host_addr,
1489			uint16_t *dev_addr, size_t repcount, uint_t flags)
1490{
1491	if (flags == DDI_DEV_AUTOINCR)
1492		for (; repcount; repcount--)
1493			px_pci_config_put16(handle, dev_addr++, *host_addr++);
1494	else
1495		for (; repcount; repcount--)
1496			px_pci_config_put16(handle, dev_addr, *host_addr++);
1497}
1498
1499/*
1500 * Function to rep write 32 bit data into the PCI configuration space behind
1501 * the 21554's host interface.
1502 */
1503static void
1504px_pci_config_rep_put32(ddi_acc_impl_t *handle, uint32_t *host_addr,
1505			uint32_t *dev_addr, size_t repcount, uint_t flags)
1506{
1507	if (flags == DDI_DEV_AUTOINCR)
1508		for (; repcount; repcount--)
1509			px_pci_config_put32(handle, dev_addr++, *host_addr++);
1510	else
1511		for (; repcount; repcount--)
1512			px_pci_config_put32(handle, dev_addr, *host_addr++);
1513}
1514
1515/*
1516 * Function to rep write 64 bit data into the PCI configuration space behind
1517 * the 21554's host interface.
1518 */
1519static void
1520px_pci_config_rep_put64(ddi_acc_impl_t *handle, uint64_t *host_addr,
1521			uint64_t *dev_addr, size_t repcount, uint_t flags)
1522{
1523	if (flags == DDI_DEV_AUTOINCR)
1524		for (; repcount; repcount--)
1525			px_pci_config_put64(handle, dev_addr++, *host_addr++);
1526	else
1527		for (; repcount; repcount--)
1528			px_pci_config_put64(handle, dev_addr, *host_addr++);
1529}
1530
1531/*
1532 * Provide a private access handle to route config access calls to Hypervisor.
1533 * Beware: Do all error checking for config space accesses before calling
1534 * this function. ie. do error checking from the calling function.
1535 * Due to a lack of meaningful error code in DDI, the gauranteed return of
1536 * DDI_SUCCESS from here makes the code organization readable/easier from
1537 * the generic code.
1538 */
1539/*ARGSUSED*/
1540int
1541px_lib_map_vconfig(dev_info_t *dip,
1542	ddi_map_req_t *mp, pci_config_offset_t off,
1543	pci_regspec_t *rp, caddr_t *addrp)
1544{
1545	int fmcap;
1546	ndi_err_t *errp;
1547	on_trap_data_t *otp;
1548	ddi_acc_hdl_t *hp;
1549	ddi_acc_impl_t *ap;
1550	uchar_t busnum;	/* bus number */
1551	uchar_t devnum;	/* device number */
1552	uchar_t funcnum; /* function number */
1553	px_config_acc_pvt_t *px_pvt;
1554
1555	hp = (ddi_acc_hdl_t *)mp->map_handlep;
1556	ap = (ddi_acc_impl_t *)hp->ah_platform_private;
1557
1558	/* Check for mapping teardown operation */
1559	if ((mp->map_op == DDI_MO_UNMAP) ||
1560	    (mp->map_op == DDI_MO_UNLOCK)) {
1561		/* free up memory allocated for the private access handle. */
1562		px_pvt = (px_config_acc_pvt_t *)hp->ah_bus_private;
1563		kmem_free((void *)px_pvt, sizeof (px_config_acc_pvt_t));
1564
1565		/* unmap operation of PCI IO/config space. */
1566		return (DDI_SUCCESS);
1567	}
1568
1569	fmcap = ddi_fm_capable(dip);
1570	if (DDI_FM_ACC_ERR_CAP(fmcap)) {
1571		errp = ((ddi_acc_impl_t *)hp)->ahi_err;
1572		otp = (on_trap_data_t *)errp->err_ontrap;
1573		otp->ot_handle = (void *)(hp);
1574		otp->ot_prot = OT_DATA_ACCESS;
1575		errp->err_status = DDI_FM_OK;
1576		errp->err_expected = DDI_FM_ERR_UNEXPECTED;
1577		errp->err_cf = px_err_cfg_hdl_check;
1578	}
1579
1580	ap->ahi_get8 = px_pci_config_get8;
1581	ap->ahi_get16 = px_pci_config_get16;
1582	ap->ahi_get32 = px_pci_config_get32;
1583	ap->ahi_get64 = px_pci_config_get64;
1584	ap->ahi_put8 = px_pci_config_put8;
1585	ap->ahi_put16 = px_pci_config_put16;
1586	ap->ahi_put32 = px_pci_config_put32;
1587	ap->ahi_put64 = px_pci_config_put64;
1588	ap->ahi_rep_get8 = px_pci_config_rep_get8;
1589	ap->ahi_rep_get16 = px_pci_config_rep_get16;
1590	ap->ahi_rep_get32 = px_pci_config_rep_get32;
1591	ap->ahi_rep_get64 = px_pci_config_rep_get64;
1592	ap->ahi_rep_put8 = px_pci_config_rep_put8;
1593	ap->ahi_rep_put16 = px_pci_config_rep_put16;
1594	ap->ahi_rep_put32 = px_pci_config_rep_put32;
1595	ap->ahi_rep_put64 = px_pci_config_rep_put64;
1596
1597	/* Initialize to default check/notify functions */
1598	ap->ahi_fault = 0;
1599	ap->ahi_fault_check = i_ddi_acc_fault_check;
1600	ap->ahi_fault_notify = i_ddi_acc_fault_notify;
1601
1602	/* allocate memory for our private handle */
1603	px_pvt = (px_config_acc_pvt_t *)
1604	    kmem_zalloc(sizeof (px_config_acc_pvt_t), KM_SLEEP);
1605	hp->ah_bus_private = (void *)px_pvt;
1606
1607	busnum = PCI_REG_BUS_G(rp->pci_phys_hi);
1608	devnum = PCI_REG_DEV_G(rp->pci_phys_hi);
1609	funcnum = PCI_REG_FUNC_G(rp->pci_phys_hi);
1610
1611	/* set up private data for use during IO routines */
1612
1613	/* addr needed by the HV APIs */
1614	px_pvt->raddr = busnum << 16 | devnum << 11 | funcnum << 8;
1615	/*
1616	 * Address that specifies the actual offset into the 256MB
1617	 * memory mapped configuration space, 4K per device.
1618	 * First 12bits form the offset into 4K config space.
1619	 * This address is only used during the IO routines to calculate
1620	 * the offset at which the transaction must be performed.
1621	 * Drivers bypassing DDI functions to access PCI config space will
1622	 * panic the system since the following is a bogus virtual address.
1623	 */
1624	px_pvt->vaddr = busnum << 20 | devnum << 15 | funcnum << 12 | off;
1625	px_pvt->dip = dip;
1626
1627	DBG(DBG_LIB_CFG, dip, "px_config_setup: raddr 0x%x, vaddr 0x%x\n",
1628	    px_pvt->raddr, px_pvt->vaddr);
1629	*addrp = (caddr_t)(uintptr_t)px_pvt->vaddr;
1630	return (DDI_SUCCESS);
1631}
1632
1633/*ARGSUSED*/
1634void
1635px_lib_map_attr_check(ddi_map_req_t *mp)
1636{
1637}
1638
1639/*
1640 * px_lib_log_safeacc_err:
1641 * Imitate a cpu/mem trap call when a peek/poke fails.
1642 * This will initiate something similar to px_fm_callback.
1643 */
1644static void
1645px_lib_log_safeacc_err(px_t *px_p, ddi_acc_handle_t handle, int fme_flag,
1646    r_addr_t addr)
1647{
1648	uint32_t	addr_high, addr_low;
1649	pcie_req_id_t	bdf = PCIE_INVALID_BDF;
1650	pci_ranges_t	*ranges_p;
1651	int		range_len, i;
1652	ddi_acc_impl_t *hp = (ddi_acc_impl_t *)handle;
1653	ddi_fm_error_t derr;
1654
1655	if (px_fm_enter(px_p) != DDI_SUCCESS)
1656		return;
1657
1658	derr.fme_status = DDI_FM_NONFATAL;
1659	derr.fme_version = DDI_FME_VERSION;
1660	derr.fme_flag = fme_flag;
1661	derr.fme_ena = fm_ena_generate(0, FM_ENA_FMT1);
1662	derr.fme_acc_handle = handle;
1663	if (hp)
1664		hp->ahi_err->err_expected = DDI_FM_ERR_EXPECTED;
1665
1666	addr_high = (uint32_t)(addr >> 32);
1667	addr_low = (uint32_t)addr;
1668
1669	/*
1670	 * Make sure this failed load came from this PCIe port.  Check by
1671	 * matching the upper 32 bits of the address with the ranges property.
1672	 */
1673	range_len = px_p->px_ranges_length / sizeof (pci_ranges_t);
1674	i = 0;
1675	for (ranges_p = px_p->px_ranges_p; i < range_len; i++, ranges_p++) {
1676		if (ranges_p->parent_high == addr_high) {
1677			switch (ranges_p->child_high & PCI_ADDR_MASK) {
1678			case PCI_ADDR_CONFIG:
1679				bdf = (pcie_req_id_t)(addr_low >> 12);
1680				break;
1681			default:
1682				bdf = PCIE_INVALID_BDF;
1683				break;
1684			}
1685			break;
1686		}
1687	}
1688
1689	(void) px_rp_en_q(px_p, bdf, addr, 0);
1690	(void) px_scan_fabric(px_p, px_p->px_dip, &derr);
1691	px_fm_exit(px_p);
1692}
1693
1694
1695#ifdef  DEBUG
1696int	px_peekfault_cnt = 0;
1697int	px_pokefault_cnt = 0;
1698#endif  /* DEBUG */
1699
1700/*
1701 * Do a safe write to a device.
1702 *
1703 * When this function is given a handle (cautious access), all errors are
1704 * suppressed.
1705 *
1706 * When this function is not given a handle (poke), only Unsupported Request
1707 * and Completer Abort errors are suppressed.
1708 *
1709 * In all cases, all errors are returned in the function return status.
1710 */
1711
1712int
1713px_lib_ctlops_poke(dev_info_t *dip, dev_info_t *rdip,
1714    peekpoke_ctlops_t *in_args)
1715{
1716	px_t *px_p = DIP_TO_STATE(dip);
1717	px_pec_t *pec_p = px_p->px_pec_p;
1718	ddi_acc_impl_t *hp = (ddi_acc_impl_t *)in_args->handle;
1719
1720	size_t repcount = in_args->repcount;
1721	size_t size = in_args->size;
1722	uintptr_t dev_addr = in_args->dev_addr;
1723	uintptr_t host_addr = in_args->host_addr;
1724
1725	int err	= DDI_SUCCESS;
1726	uint64_t hvio_poke_status;
1727	uint32_t wrt_stat;
1728
1729	r_addr_t ra;
1730	uint64_t pokeval;
1731	pcie_req_id_t bdf;
1732
1733	ra = (r_addr_t)va_to_pa((void *)dev_addr);
1734	for (; repcount; repcount--) {
1735
1736		switch (size) {
1737		case sizeof (uint8_t):
1738			pokeval = *(uint8_t *)host_addr;
1739			break;
1740		case sizeof (uint16_t):
1741			pokeval = *(uint16_t *)host_addr;
1742			break;
1743		case sizeof (uint32_t):
1744			pokeval = *(uint32_t *)host_addr;
1745			break;
1746		case sizeof (uint64_t):
1747			pokeval = *(uint64_t *)host_addr;
1748			break;
1749		default:
1750			DBG(DBG_MAP, px_p->px_dip,
1751			    "poke: invalid size %d passed\n", size);
1752			err = DDI_FAILURE;
1753			goto done;
1754		}
1755
1756		/*
1757		 * Grab pokefault mutex since hypervisor does not guarantee
1758		 * poke serialization.
1759		 */
1760		if (hp) {
1761			i_ndi_busop_access_enter(hp->ahi_common.ah_dip,
1762			    (ddi_acc_handle_t)hp);
1763			pec_p->pec_safeacc_type = DDI_FM_ERR_EXPECTED;
1764		} else {
1765			mutex_enter(&pec_p->pec_pokefault_mutex);
1766			pec_p->pec_safeacc_type = DDI_FM_ERR_POKE;
1767		}
1768
1769		if (pcie_get_bdf_from_dip(rdip, &bdf) != DDI_SUCCESS) {
1770			err = DDI_FAILURE;
1771			goto done;
1772		}
1773
1774		hvio_poke_status = hvio_poke(px_p->px_dev_hdl, ra, size,
1775		    pokeval, bdf << 8, &wrt_stat);
1776
1777		if ((hvio_poke_status != H_EOK) || (wrt_stat != H_EOK)) {
1778			err = DDI_FAILURE;
1779#ifdef  DEBUG
1780			px_pokefault_cnt++;
1781#endif
1782			/*
1783			 * For CAUTIOUS and POKE access, notify FMA to
1784			 * cleanup.  Imitate a cpu/mem trap call like in sun4u.
1785			 */
1786			px_lib_log_safeacc_err(px_p, (ddi_acc_handle_t)hp,
1787			    (hp ? DDI_FM_ERR_EXPECTED :
1788			    DDI_FM_ERR_POKE), ra);
1789
1790			pec_p->pec_ontrap_data = NULL;
1791			pec_p->pec_safeacc_type = DDI_FM_ERR_UNEXPECTED;
1792			if (hp) {
1793				i_ndi_busop_access_exit(hp->ahi_common.ah_dip,
1794				    (ddi_acc_handle_t)hp);
1795			} else {
1796				mutex_exit(&pec_p->pec_pokefault_mutex);
1797			}
1798			goto done;
1799		}
1800
1801		pec_p->pec_ontrap_data = NULL;
1802		pec_p->pec_safeacc_type = DDI_FM_ERR_UNEXPECTED;
1803		if (hp) {
1804			i_ndi_busop_access_exit(hp->ahi_common.ah_dip,
1805			    (ddi_acc_handle_t)hp);
1806		} else {
1807			mutex_exit(&pec_p->pec_pokefault_mutex);
1808		}
1809
1810		host_addr += size;
1811
1812		if (in_args->flags == DDI_DEV_AUTOINCR) {
1813			dev_addr += size;
1814			ra = (r_addr_t)va_to_pa((void *)dev_addr);
1815		}
1816	}
1817
1818done:
1819	return (err);
1820}
1821
1822
1823/*ARGSUSED*/
1824int
1825px_lib_ctlops_peek(dev_info_t *dip, dev_info_t *rdip,
1826    peekpoke_ctlops_t *in_args, void *result)
1827{
1828	px_t *px_p = DIP_TO_STATE(dip);
1829	px_pec_t *pec_p = px_p->px_pec_p;
1830	ddi_acc_impl_t *hp = (ddi_acc_impl_t *)in_args->handle;
1831
1832	size_t repcount = in_args->repcount;
1833	uintptr_t dev_addr = in_args->dev_addr;
1834	uintptr_t host_addr = in_args->host_addr;
1835
1836	r_addr_t ra;
1837	uint32_t read_status;
1838	uint64_t hvio_peek_status;
1839	uint64_t peekval;
1840	int err = DDI_SUCCESS;
1841
1842	result = (void *)in_args->host_addr;
1843
1844	ra = (r_addr_t)va_to_pa((void *)dev_addr);
1845	for (; repcount; repcount--) {
1846
1847		/* Lock pokefault mutex so read doesn't mask a poke fault. */
1848		if (hp) {
1849			i_ndi_busop_access_enter(hp->ahi_common.ah_dip,
1850			    (ddi_acc_handle_t)hp);
1851			pec_p->pec_safeacc_type = DDI_FM_ERR_EXPECTED;
1852		} else {
1853			mutex_enter(&pec_p->pec_pokefault_mutex);
1854			pec_p->pec_safeacc_type = DDI_FM_ERR_PEEK;
1855		}
1856
1857		hvio_peek_status = hvio_peek(px_p->px_dev_hdl, ra,
1858		    in_args->size, &read_status, &peekval);
1859
1860		if ((hvio_peek_status != H_EOK) || (read_status != H_EOK)) {
1861			err = DDI_FAILURE;
1862
1863			/*
1864			 * For CAUTIOUS and PEEK access, notify FMA to
1865			 * cleanup.  Imitate a cpu/mem trap call like in sun4u.
1866			 */
1867			px_lib_log_safeacc_err(px_p, (ddi_acc_handle_t)hp,
1868			    (hp ? DDI_FM_ERR_EXPECTED :
1869			    DDI_FM_ERR_PEEK), ra);
1870
1871			/* Stuff FFs in host addr if peek. */
1872			if (hp == NULL) {
1873				int i;
1874				uint8_t *ff_addr = (uint8_t *)host_addr;
1875				for (i = 0; i < in_args->size; i++)
1876					*ff_addr++ = 0xff;
1877			}
1878#ifdef  DEBUG
1879			px_peekfault_cnt++;
1880#endif
1881			pec_p->pec_ontrap_data = NULL;
1882			pec_p->pec_safeacc_type = DDI_FM_ERR_UNEXPECTED;
1883			if (hp) {
1884				i_ndi_busop_access_exit(hp->ahi_common.ah_dip,
1885				    (ddi_acc_handle_t)hp);
1886			} else {
1887				mutex_exit(&pec_p->pec_pokefault_mutex);
1888			}
1889			goto done;
1890
1891		}
1892		pec_p->pec_ontrap_data = NULL;
1893		pec_p->pec_safeacc_type = DDI_FM_ERR_UNEXPECTED;
1894		if (hp) {
1895			i_ndi_busop_access_exit(hp->ahi_common.ah_dip,
1896			    (ddi_acc_handle_t)hp);
1897		} else {
1898			mutex_exit(&pec_p->pec_pokefault_mutex);
1899		}
1900
1901		switch (in_args->size) {
1902		case sizeof (uint8_t):
1903			*(uint8_t *)host_addr = (uint8_t)peekval;
1904			break;
1905		case sizeof (uint16_t):
1906			*(uint16_t *)host_addr = (uint16_t)peekval;
1907			break;
1908		case sizeof (uint32_t):
1909			*(uint32_t *)host_addr = (uint32_t)peekval;
1910			break;
1911		case sizeof (uint64_t):
1912			*(uint64_t *)host_addr = (uint64_t)peekval;
1913			break;
1914		default:
1915			DBG(DBG_MAP, px_p->px_dip,
1916			    "peek: invalid size %d passed\n",
1917			    in_args->size);
1918			err = DDI_FAILURE;
1919			goto done;
1920		}
1921
1922		host_addr += in_args->size;
1923
1924		if (in_args->flags == DDI_DEV_AUTOINCR) {
1925			dev_addr += in_args->size;
1926			ra = (r_addr_t)va_to_pa((void *)dev_addr);
1927		}
1928	}
1929done:
1930	return (err);
1931}
1932
1933
1934/* add interrupt vector */
1935int
1936px_err_add_intr(px_fault_t *px_fault_p)
1937{
1938	px_t	*px_p = DIP_TO_STATE(px_fault_p->px_fh_dip);
1939
1940	DBG(DBG_LIB_INT, px_p->px_dip,
1941	    "px_err_add_intr: calling add_ivintr");
1942
1943	VERIFY(add_ivintr(px_fault_p->px_fh_sysino, PX_ERR_PIL,
1944	    (intrfunc)px_fault_p->px_err_func, (caddr_t)px_fault_p, NULL,
1945	    (caddr_t)&px_fault_p->px_intr_payload[0]) == 0);
1946
1947	DBG(DBG_LIB_INT, px_p->px_dip,
1948	    "px_err_add_intr: ib_intr_enable ");
1949
1950	px_ib_intr_enable(px_p, intr_dist_cpuid(), px_fault_p->px_intr_ino);
1951
1952	return (DDI_SUCCESS);
1953}
1954
1955/* remove interrupt vector */
1956void
1957px_err_rem_intr(px_fault_t *px_fault_p)
1958{
1959	px_t	*px_p = DIP_TO_STATE(px_fault_p->px_fh_dip);
1960
1961	px_ib_intr_disable(px_p->px_ib_p, px_fault_p->px_intr_ino,
1962	    IB_INTR_WAIT);
1963
1964	VERIFY(rem_ivintr(px_fault_p->px_fh_sysino, PX_ERR_PIL) == 0);
1965}
1966
1967void
1968px_cb_intr_redist(void *arg)
1969{
1970	px_t	*px_p = (px_t *)arg;
1971	px_ib_intr_dist_en(px_p->px_dip, intr_dist_cpuid(),
1972	    px_p->px_inos[PX_INTR_XBC], B_FALSE);
1973}
1974
1975int
1976px_cb_add_intr(px_fault_t *f_p)
1977{
1978	px_t	*px_p = DIP_TO_STATE(f_p->px_fh_dip);
1979
1980	DBG(DBG_LIB_INT, px_p->px_dip,
1981	    "px_err_add_intr: calling add_ivintr");
1982
1983	VERIFY(add_ivintr(f_p->px_fh_sysino, PX_ERR_PIL,
1984	    (intrfunc)f_p->px_err_func, (caddr_t)f_p, NULL,
1985	    (caddr_t)&f_p->px_intr_payload[0]) == 0);
1986
1987	intr_dist_add(px_cb_intr_redist, px_p);
1988
1989	DBG(DBG_LIB_INT, px_p->px_dip,
1990	    "px_err_add_intr: ib_intr_enable ");
1991
1992	px_ib_intr_enable(px_p, intr_dist_cpuid(), f_p->px_intr_ino);
1993
1994	return (DDI_SUCCESS);
1995}
1996
1997void
1998px_cb_rem_intr(px_fault_t *f_p)
1999{
2000	intr_dist_rem(px_cb_intr_redist, DIP_TO_STATE(f_p->px_fh_dip));
2001	px_err_rem_intr(f_p);
2002}
2003
2004#ifdef FMA
2005void
2006px_fill_rc_status(px_fault_t *px_fault_p, pciex_rc_error_regs_t *rc_status)
2007{
2008	px_pec_err_t	*err_pkt;
2009
2010	err_pkt = (px_pec_err_t *)px_fault_p->px_intr_payload;
2011
2012	/* initialise all the structure members */
2013	rc_status->status_valid = 0;
2014
2015	if (err_pkt->pec_descr.P) {
2016		/* PCI Status Register */
2017		rc_status->pci_err_status = err_pkt->pci_err_status;
2018		rc_status->status_valid |= PCI_ERR_STATUS_VALID;
2019	}
2020
2021	if (err_pkt->pec_descr.E) {
2022		/* PCIe Status Register */
2023		rc_status->pcie_err_status = err_pkt->pcie_err_status;
2024		rc_status->status_valid |= PCIE_ERR_STATUS_VALID;
2025	}
2026
2027	if (err_pkt->pec_descr.U) {
2028		rc_status->ue_status = err_pkt->ue_reg_status;
2029		rc_status->status_valid |= UE_STATUS_VALID;
2030	}
2031
2032	if (err_pkt->pec_descr.H) {
2033		rc_status->ue_hdr1 = err_pkt->hdr[0];
2034		rc_status->status_valid |= UE_HDR1_VALID;
2035	}
2036
2037	if (err_pkt->pec_descr.I) {
2038		rc_status->ue_hdr2 = err_pkt->hdr[1];
2039		rc_status->status_valid |= UE_HDR2_VALID;
2040	}
2041
2042	/* ue_fst_err_ptr - not available for sun4v?? */
2043
2044
2045	if (err_pkt->pec_descr.S) {
2046		rc_status->source_id = err_pkt->err_src_reg;
2047		rc_status->status_valid |= SOURCE_ID_VALID;
2048	}
2049
2050	if (err_pkt->pec_descr.R) {
2051		rc_status->root_err_status = err_pkt->root_err_status;
2052		rc_status->status_valid |= CE_STATUS_VALID;
2053	}
2054}
2055#endif
2056
2057/*ARGSUSED*/
2058int
2059px_lib_pmctl(int cmd, px_t *px_p)
2060{
2061	return (DDI_FAILURE);
2062}
2063
2064/*ARGSUSED*/
2065uint_t
2066px_pmeq_intr(caddr_t arg)
2067{
2068	return (DDI_INTR_CLAIMED);
2069}
2070
2071/*
2072 * fetch the config space base addr of the root complex
2073 * note this depends on px structure being initialized
2074 */
2075uint64_t
2076px_lib_get_cfgacc_base(dev_info_t *dip)
2077{
2078	int		instance = DIP_TO_INST(dip);
2079	px_t		*px_p = INST_TO_STATE(instance);
2080
2081	return (px_p->px_dev_hdl);
2082}
2083
2084void
2085px_panic_domain(px_t *px_p, pcie_req_id_t bdf)
2086{
2087	uint64_t	ret;
2088	dev_info_t	*dip = px_p->px_dip;
2089
2090	DBG(DBG_ERR_INTR, dip, "px_panic_domain: handle 0x%lx, ino %d, "
2091	    "bdf<<8 0x%lx\n",
2092	    (uint64_t)DIP_TO_HANDLE(dip), px_p->px_cb_fault.px_intr_ino,
2093	    (pci_device_t)bdf << 8);
2094	if ((ret = pci_error_send(DIP_TO_HANDLE(dip),
2095	    px_p->px_cb_fault.px_intr_ino, (pci_device_t)bdf << 8)) != H_EOK) {
2096		DBG(DBG_ERR_INTR, dip, "pci_error_send failed, ret 0x%lx\n",
2097		    ret);
2098	} else
2099		DBG(DBG_ERR_INTR, dip, "pci_error_send worked\n");
2100}
2101
2102/*ARGSUSED*/
2103int
2104px_lib_hotplug_init(dev_info_t *dip, void *arg)
2105{
2106	return (DDI_ENOTSUP);
2107}
2108
2109/*ARGSUSED*/
2110void
2111px_lib_hotplug_uninit(dev_info_t *dip)
2112{
2113}
2114
2115/*ARGSUSED*/
2116void
2117px_hp_intr_redist(px_t *px_p)
2118{
2119}
2120
2121/* Dummy cpr add callback */
2122/*ARGSUSED*/
2123void
2124px_cpr_add_callb(px_t *px_p)
2125{
2126}
2127
2128/* Dummy cpr rem callback */
2129/*ARGSUSED*/
2130void
2131px_cpr_rem_callb(px_t *px_p)
2132{
2133}
2134
2135/*ARGSUSED*/
2136boolean_t
2137px_lib_is_in_drain_state(px_t *px_p)
2138{
2139	return (B_FALSE);
2140}
2141
2142/*
2143 * There is no IOAPI to get the BDF of the pcie root port nexus at this moment.
2144 * Assume it is 0x0000, until otherwise noted.  For now, all sun4v platforms
2145 * have programmed the BDF to be 0x0000.
2146 */
2147/*ARGSUSED*/
2148pcie_req_id_t
2149px_lib_get_bdf(px_t *px_p)
2150{
2151	return (0x0000);
2152}
2153
2154int
2155px_lib_get_root_complex_mps(px_t *px_p, dev_info_t *dip, int *mps)
2156{
2157	pci_device_t	bdf = px_lib_get_bdf(px_p);
2158
2159	if (hvio_get_rp_mps_cap(DIP_TO_HANDLE(dip), bdf, mps) == H_EOK)
2160		return (DDI_SUCCESS);
2161	else
2162		return (DDI_FAILURE);
2163}
2164
2165int
2166px_lib_set_root_complex_mps(px_t *px_p,  dev_info_t *dip, int mps)
2167{
2168	pci_device_t	bdf = px_lib_get_bdf(px_p);
2169
2170	if (hvio_set_rp_mps(DIP_TO_HANDLE(dip), bdf, mps) == H_EOK)
2171		return (DDI_SUCCESS);
2172	else
2173		return (DDI_FAILURE);
2174}
2175
2176static int
2177px_lib_do_count_waiting_dev(dev_info_t *dip, void *arg)
2178{
2179	int *count = (int *)arg;
2180	dev_info_t *cdip = ddi_get_child(dip);
2181
2182	while (cdip != NULL) {
2183		/* check if this is an assigned device */
2184		if (ddi_prop_exists(DDI_DEV_T_NONE, cdip, DDI_PROP_DONTPASS,
2185		    "ddi-assigned")) {
2186			DBG(DBG_ATTACH, dip, "px_lib_do_count_waiting_dev: "
2187			    "Found an assigned dev %p, under bridge %p",
2188			    cdip, dip);
2189
2190			/*
2191			 * Mark this bridge as needing waiting for
2192			 * CHILD_LOANED will be removed after bridge reports
2193			 * its readyness back to px driver
2194			 */
2195			if (ddi_prop_update_int(DDI_DEV_T_NONE, dip,
2196			    CHILD_LOANED, 1) == DDI_PROP_SUCCESS)
2197				(*count)++;
2198			break;
2199		}
2200		cdip = ddi_get_next_sibling(cdip);
2201	}
2202
2203	return (DDI_WALK_CONTINUE);
2204}
2205
2206static int
2207px_lib_count_waiting_dev(dev_info_t *dip)
2208{
2209	int circular_count;
2210	int count = 0;
2211
2212	/* No need to continue if this system is not SDIO capable */
2213	if (px_sdio_users == 0)
2214		return (0);
2215
2216	/* see if px iteslf has assigned children */
2217	(void) px_lib_do_count_waiting_dev(dip, &count);
2218
2219	/* scan dev under this px */
2220	ndi_devi_enter(dip, &circular_count);
2221	ddi_walk_devs(ddi_get_child(dip), px_lib_do_count_waiting_dev, &count);
2222	ndi_devi_exit(dip, circular_count);
2223	return (count);
2224}
2225
2226/* Called from px/bridge driver directly to report its readyness */
2227int
2228px_lib_fabric_sync(dev_info_t *dip)
2229{
2230	px_t *px;
2231	dev_info_t *rcdip;
2232	int waitdev;
2233
2234	/* No need to continue if this system is not SDIO capable */
2235	if (px_sdio_users == 0)
2236		return (DDI_SUCCESS);
2237
2238	/* a valid bridge w/ assigned dev under it? */
2239	if (ddi_prop_remove(DDI_DEV_T_NONE, dip, CHILD_LOANED) !=
2240	    DDI_PROP_SUCCESS)
2241		return (DDI_FAILURE);
2242
2243	/* find out RC dip */
2244	for (rcdip = dip; rcdip != NULL; rcdip = ddi_get_parent(rcdip)) {
2245		if (PCIE_DIP2BUS(rcdip) && PCIE_IS_RC(PCIE_DIP2BUS(rcdip)))
2246			break;
2247	}
2248	if ((rcdip == NULL) || ((px = (px_t *)DIP_TO_STATE(rcdip)) == NULL))
2249		return (DDI_FAILURE);
2250
2251	/* are we ready? */
2252	waitdev = (int)(uintptr_t)px->px_plat_p;
2253	ASSERT(waitdev);
2254	DBG(DBG_CTLOPS, rcdip, "px_lib_fabric_sync: "
2255	    "Px/bridge %p is ready, %d left", rcdip, waitdev - 1);
2256	--waitdev;
2257	px->px_plat_p = (void *)(uintptr_t)waitdev;
2258	if (waitdev != 0)
2259		return (DDI_SUCCESS);
2260
2261	/* notify hpyervisor */
2262	DBG(DBG_CTLOPS, rcdip, "px_lib_fabric_sync: "
2263	    "Notifying HV that RC %p is ready users=%d", rcdip, px_sdio_users);
2264
2265	if (pci_iov_root_configured(px->px_dev_hdl) != H_EOK)
2266		return (DDI_FAILURE);
2267
2268	return (DDI_SUCCESS);
2269}
2270