xref: /illumos-gate/usr/src/uts/i86pc/io/pciex/npe.c (revision ffb64830)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 /*
28  * Copyright 2012 Garrett D'Amore <garrett@damore.org>.  All rights reserved.
29  * Copyright 2019 Joyent, Inc.
30  */
31 
32 /*
33  *	npe (Nexus PCIe driver): Host to PCI-Express local bus driver
34  *
35  *	npe serves as the driver for PCIe Root Complexes and as the nexus driver
36  *	for PCIe devices. See also: npe(7D). For more information about hotplug,
37  *	see the big theory statement at uts/common/os/ddi_hp_impl.c.
38  *
39  *
40  *	NDI EVENT HANDLING SUPPORT
41  *
42  *	npe supports NDI event handling. The only available event is surprise
43  *	removal of a device. Child drivers can register surprise removal event
44  *	callbacks by requesting an event cookie using ddi_get_eventcookie for
45  *	the DDI_DEVI_REMOVE_EVENT and add their callback using
46  *	ddi_add_event_handler. For an example, see the nvme driver in
47  *	uts/common/io/nvme/nvme.c.
48  *
49  *	The NDI events in npe are retrieved using NDI_EVENT_NOPASS, which
50  *	prevent them from being propagated up the tree once they reach the npe's
51  *	bus_get_eventcookie operations. This is important because npe maintains
52  *	the state of PCIe devices and their receptacles, via the PCIe hotplug
53  *	controller driver (pciehpc).
54  *
55  *	Hot removal events are ultimately posted by the PCIe hotplug controller
56  *	interrupt handler for hotplug events. Events are posted using the
57  *	ndi_post_event interface.
58  */
59 
60 #include <sys/conf.h>
61 #include <sys/modctl.h>
62 #include <sys/file.h>
63 #include <sys/pci_impl.h>
64 #include <sys/pcie_impl.h>
65 #include <sys/sysmacros.h>
66 #include <sys/ddi_intr.h>
67 #include <sys/sunndi.h>
68 #include <sys/sunddi.h>
69 #include <sys/ddifm.h>
70 #include <sys/ndifm.h>
71 #include <sys/fm/util.h>
72 #include <sys/hotplug/pci/pcie_hp.h>
73 #include <io/pci/pci_tools_ext.h>
74 #include <io/pci/pci_common.h>
75 #include <io/pciex/pcie_nvidia.h>
76 
77 /*
78  * Helper Macros
79  */
80 #define	NPE_IS_HANDLE_FOR_STDCFG_ACC(hp) \
81 	((hp) != NULL &&						\
82 	((ddi_acc_hdl_t *)(hp))->ah_platform_private != NULL &&		\
83 	(((ddi_acc_impl_t *)((ddi_acc_hdl_t *)(hp))->			\
84 	ah_platform_private)->						\
85 	    ahi_acc_attr &(DDI_ACCATTR_CPU_VADDR|DDI_ACCATTR_CONFIG_SPACE)) \
86 		== DDI_ACCATTR_CONFIG_SPACE)
87 
88 /*
89  * Bus Operation functions
90  */
91 static int	npe_bus_map(dev_info_t *, dev_info_t *, ddi_map_req_t *,
92 		    off_t, off_t, caddr_t *);
93 static int	npe_ctlops(dev_info_t *, dev_info_t *, ddi_ctl_enum_t,
94 		    void *, void *);
95 static int	npe_intr_ops(dev_info_t *, dev_info_t *, ddi_intr_op_t,
96 		    ddi_intr_handle_impl_t *, void *);
97 static int	npe_fm_init(dev_info_t *, dev_info_t *, int,
98 		    ddi_iblock_cookie_t *);
99 static int	npe_bus_get_eventcookie(dev_info_t *, dev_info_t *, char *,
100 		    ddi_eventcookie_t *);
101 static int	npe_bus_add_eventcall(dev_info_t *, dev_info_t *,
102 		    ddi_eventcookie_t, void (*)(dev_info_t *,
103 		    ddi_eventcookie_t, void *, void *),
104 		    void *, ddi_callback_id_t *);
105 static int	npe_bus_remove_eventcall(dev_info_t *, ddi_callback_id_t);
106 static int	npe_bus_post_event(dev_info_t *, dev_info_t *,
107 		    ddi_eventcookie_t, void *);
108 
109 static int	npe_fm_callback(dev_info_t *, ddi_fm_error_t *, const void *);
110 
111 /*
112  * Disable URs and Received MA for all PCIe devices.  Until x86 SW is changed so
113  * that random drivers do not do PIO accesses on devices that it does not own,
114  * these error bits must be disabled.  SERR must also be disabled if URs have
115  * been masked.
116  */
117 uint32_t	npe_aer_uce_mask = PCIE_AER_UCE_UR;
118 uint32_t	npe_aer_ce_mask = 0;
119 uint32_t	npe_aer_suce_mask = PCIE_AER_SUCE_RCVD_MA;
120 
121 struct bus_ops npe_bus_ops = {
122 	BUSO_REV,
123 	npe_bus_map,
124 	NULL,
125 	NULL,
126 	NULL,
127 	i_ddi_map_fault,
128 	NULL,
129 	ddi_dma_allochdl,
130 	ddi_dma_freehdl,
131 	ddi_dma_bindhdl,
132 	ddi_dma_unbindhdl,
133 	ddi_dma_flush,
134 	ddi_dma_win,
135 	ddi_dma_mctl,
136 	npe_ctlops,
137 	ddi_bus_prop_op,
138 	npe_bus_get_eventcookie,
139 	npe_bus_add_eventcall,
140 	npe_bus_remove_eventcall,
141 	npe_bus_post_event,
142 	0,			/* (*bus_intr_ctl)(); */
143 	0,			/* (*bus_config)(); */
144 	0,			/* (*bus_unconfig)(); */
145 	npe_fm_init,		/* (*bus_fm_init)(); */
146 	NULL,			/* (*bus_fm_fini)(); */
147 	NULL,			/* (*bus_fm_access_enter)(); */
148 	NULL,			/* (*bus_fm_access_exit)(); */
149 	NULL,			/* (*bus_power)(); */
150 	npe_intr_ops,		/* (*bus_intr_op)(); */
151 	pcie_hp_common_ops	/* (*bus_hp_op)(); */
152 };
153 
154 static int	npe_open(dev_t *, int, int, cred_t *);
155 static int	npe_close(dev_t, int, int, cred_t *);
156 static int	npe_ioctl(dev_t, int, intptr_t, int, cred_t *, int *);
157 
158 struct cb_ops npe_cb_ops = {
159 	npe_open,			/* open */
160 	npe_close,			/* close */
161 	nodev,				/* strategy */
162 	nodev,				/* print */
163 	nodev,				/* dump */
164 	nodev,				/* read */
165 	nodev,				/* write */
166 	npe_ioctl,			/* ioctl */
167 	nodev,				/* devmap */
168 	nodev,				/* mmap */
169 	nodev,				/* segmap */
170 	nochpoll,			/* poll */
171 	pcie_prop_op,			/* cb_prop_op */
172 	NULL,				/* streamtab */
173 	D_NEW | D_MP | D_HOTPLUG,	/* Driver compatibility flag */
174 	CB_REV,				/* rev */
175 	nodev,				/* int (*cb_aread)() */
176 	nodev				/* int (*cb_awrite)() */
177 };
178 
179 
180 /*
181  * Device Node Operation functions
182  */
183 static int	npe_attach(dev_info_t *devi, ddi_attach_cmd_t cmd);
184 static int	npe_detach(dev_info_t *devi, ddi_detach_cmd_t cmd);
185 static int	npe_info(dev_info_t *, ddi_info_cmd_t, void *, void **);
186 
187 struct dev_ops npe_ops = {
188 	DEVO_REV,		/* devo_rev */
189 	0,			/* refcnt  */
190 	npe_info,		/* info */
191 	nulldev,		/* identify */
192 	nulldev,		/* probe */
193 	npe_attach,		/* attach */
194 	npe_detach,		/* detach */
195 	nulldev,		/* reset */
196 	&npe_cb_ops,		/* driver operations */
197 	&npe_bus_ops,		/* bus operations */
198 	NULL,			/* power */
199 	ddi_quiesce_not_needed,		/* quiesce */
200 };
201 
202 /*
203  * Internal routines in support of particular npe_ctlops.
204  */
205 static int npe_removechild(dev_info_t *child);
206 static int npe_initchild(dev_info_t *child);
207 
208 /*
209  * External support routine
210  */
211 extern void	npe_query_acpi_mcfg(dev_info_t *dip);
212 extern void	npe_ck804_fix_aer_ptr(ddi_acc_handle_t cfg_hdl);
213 extern int	npe_disable_empty_bridges_workaround(dev_info_t *child);
214 extern void	npe_nvidia_error_workaround(ddi_acc_handle_t cfg_hdl);
215 extern void	npe_intel_error_workaround(ddi_acc_handle_t cfg_hdl);
216 extern boolean_t npe_is_mmcfg_supported(dev_info_t *dip);
217 extern void	npe_enable_htmsi_children(dev_info_t *dip);
218 extern int	npe_save_htconfig_children(dev_info_t *dip);
219 extern int	npe_restore_htconfig_children(dev_info_t *dip);
220 
221 /*
222  * Module linkage information for the kernel.
223  */
224 static struct modldrv modldrv = {
225 	&mod_driverops,				/* Type of module */
226 	"Host to PCIe nexus driver",		/* Name of module */
227 	&npe_ops,				/* driver ops */
228 };
229 
230 static struct modlinkage modlinkage = {
231 	MODREV_1,
232 	(void *)&modldrv,
233 	NULL
234 };
235 
236 /* Save minimal state. */
237 void *npe_statep;
238 
239 int
_init(void)240 _init(void)
241 {
242 	int e;
243 
244 	/*
245 	 * Initialize per-pci bus soft state pointer.
246 	 */
247 	e = ddi_soft_state_init(&npe_statep, sizeof (pci_state_t), 1);
248 	if (e != 0)
249 		return (e);
250 
251 	if ((e = mod_install(&modlinkage)) != 0)
252 		ddi_soft_state_fini(&npe_statep);
253 
254 	return (e);
255 }
256 
257 
258 int
_fini(void)259 _fini(void)
260 {
261 	int rc;
262 
263 	rc = mod_remove(&modlinkage);
264 	if (rc != 0)
265 		return (rc);
266 
267 	ddi_soft_state_fini(&npe_statep);
268 	return (rc);
269 }
270 
271 
272 int
_info(struct modinfo * modinfop)273 _info(struct modinfo *modinfop)
274 {
275 	return (mod_info(&modlinkage, modinfop));
276 }
277 
278 /*ARGSUSED*/
279 static int
npe_info(dev_info_t * dip,ddi_info_cmd_t cmd,void * arg,void ** result)280 npe_info(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, void **result)
281 {
282 	minor_t		minor = getminor((dev_t)arg);
283 	int		instance = PCI_MINOR_NUM_TO_INSTANCE(minor);
284 	pci_state_t	*pcip = ddi_get_soft_state(npe_statep, instance);
285 	int		ret = DDI_SUCCESS;
286 
287 	switch (cmd) {
288 	case DDI_INFO_DEVT2INSTANCE:
289 		*result = (void *)(intptr_t)instance;
290 		break;
291 	case DDI_INFO_DEVT2DEVINFO:
292 		if (pcip == NULL) {
293 			ret = DDI_FAILURE;
294 			break;
295 		}
296 
297 		*result = (void *)pcip->pci_dip;
298 		break;
299 	default:
300 		ret = DDI_FAILURE;
301 		break;
302 	}
303 
304 	return (ret);
305 }
306 
307 /*
308  * See big theory statement at the top of this file for more information about
309  * surprise removal events.
310  */
311 #define	NPE_EVENT_TAG_HOT_REMOVAL	0
312 static ndi_event_definition_t npe_ndi_event_defs[1] = {
313 	{NPE_EVENT_TAG_HOT_REMOVAL, DDI_DEVI_REMOVE_EVENT, EPL_KERNEL,
314 	NDI_EVENT_POST_TO_ALL}
315 };
316 
317 static ndi_event_set_t npe_ndi_events = {
318 	NDI_EVENTS_REV1, ARRAY_SIZE(npe_ndi_event_defs), npe_ndi_event_defs
319 };
320 
321 /*ARGSUSED*/
322 static int
npe_attach(dev_info_t * devi,ddi_attach_cmd_t cmd)323 npe_attach(dev_info_t *devi, ddi_attach_cmd_t cmd)
324 {
325 	int		instance = ddi_get_instance(devi);
326 	pci_state_t	*pcip = NULL;
327 	int		ret;
328 
329 	if (cmd == DDI_RESUME) {
330 		/*
331 		 * the system might still be able to resume even if this fails
332 		 */
333 		(void) npe_restore_htconfig_children(devi);
334 		return (DDI_SUCCESS);
335 	}
336 
337 	/*
338 	 * We must do this here in order to ensure that all top level devices
339 	 * get their HyperTransport MSI mapping regs programmed first.
340 	 * "Memory controller" and "hostbridge" class devices are leaf devices
341 	 * that may affect MSI translation functionality for devices
342 	 * connected to the same link/bus.
343 	 *
344 	 * This will also program HT MSI mapping registers on root buses
345 	 * devices (basically sitting on an HT bus) that are not dependent
346 	 * on the aforementioned HT devices for MSI translation.
347 	 */
348 	npe_enable_htmsi_children(devi);
349 
350 	if (ddi_prop_update_string(DDI_DEV_T_NONE, devi, "device_type",
351 	    "pciex") != DDI_PROP_SUCCESS) {
352 		cmn_err(CE_WARN, "npe:  'device_type' prop create failed");
353 	}
354 
355 	if (ddi_soft_state_zalloc(npe_statep, instance) == DDI_SUCCESS)
356 		pcip = ddi_get_soft_state(npe_statep, instance);
357 
358 	if (pcip == NULL)
359 		return (DDI_FAILURE);
360 
361 	pcip->pci_dip = devi;
362 	pcip->pci_soft_state = PCI_SOFT_STATE_CLOSED;
363 
364 	if (pcie_init(devi, NULL) != DDI_SUCCESS)
365 		goto fail1;
366 
367 	ret = ndi_event_alloc_hdl(pcip->pci_dip, NULL, &pcip->pci_ndi_event_hdl,
368 	    NDI_SLEEP);
369 	if (ret == NDI_SUCCESS) {
370 		ret = ndi_event_bind_set(pcip->pci_ndi_event_hdl,
371 		    &npe_ndi_events, NDI_SLEEP);
372 		if (ret != NDI_SUCCESS) {
373 			dev_err(pcip->pci_dip, CE_WARN, "npe:	failed to bind "
374 			    "NDI event set (error=%d)", ret);
375 			goto fail1;
376 		}
377 	} else {
378 		dev_err(pcip->pci_dip, CE_WARN, "npe:	failed to allocate "
379 		    "event handle (error=%d)", ret);
380 		goto fail1;
381 	}
382 
383 	/* Second arg: initialize for pci_express root nexus */
384 	if (pcitool_init(devi, B_TRUE) != DDI_SUCCESS)
385 		goto fail2;
386 
387 	pcip->pci_fmcap = DDI_FM_EREPORT_CAPABLE | DDI_FM_ERRCB_CAPABLE |
388 	    DDI_FM_ACCCHK_CAPABLE | DDI_FM_DMACHK_CAPABLE;
389 	ddi_fm_init(devi, &pcip->pci_fmcap, &pcip->pci_fm_ibc);
390 
391 	if (pcip->pci_fmcap & DDI_FM_ERRCB_CAPABLE) {
392 		ddi_fm_handler_register(devi, npe_fm_callback, NULL);
393 	}
394 
395 	PCIE_DIP2PFD(devi) = kmem_zalloc(sizeof (pf_data_t), KM_SLEEP);
396 	pcie_rc_init_pfd(devi, PCIE_DIP2PFD(devi));
397 
398 	npe_query_acpi_mcfg(devi);
399 	ddi_report_dev(devi);
400 	pcie_fab_init_bus(devi, PCIE_BUS_FINAL);
401 
402 	return (DDI_SUCCESS);
403 
404 fail2:
405 	(void) pcie_uninit(devi);
406 fail1:
407 	pcie_rc_fini_bus(devi);
408 	ddi_soft_state_free(npe_statep, instance);
409 
410 	return (DDI_FAILURE);
411 }
412 
413 /*ARGSUSED*/
414 static int
npe_detach(dev_info_t * devi,ddi_detach_cmd_t cmd)415 npe_detach(dev_info_t *devi, ddi_detach_cmd_t cmd)
416 {
417 	int instance = ddi_get_instance(devi);
418 	pci_state_t *pcip;
419 	int ret;
420 
421 	pcip = ddi_get_soft_state(npe_statep, ddi_get_instance(devi));
422 
423 	switch (cmd) {
424 	case DDI_DETACH:
425 
426 		/*
427 		 * Clean up event handling first, to ensure there are no
428 		 * oustanding callbacks registered.
429 		 */
430 		ret = ndi_event_unbind_set(pcip->pci_ndi_event_hdl,
431 		    &npe_ndi_events, NDI_SLEEP);
432 		if (ret == NDI_SUCCESS) {
433 			/* ndi_event_free_hdl always succeeds. */
434 			(void) ndi_event_free_hdl(pcip->pci_ndi_event_hdl);
435 		} else {
436 			/*
437 			 * The event set will only fail to unbind if there are
438 			 * outstanding callbacks registered for it, which
439 			 * probably means a child driver still has one
440 			 * registered and thus was not cleaned up properly
441 			 * before npe's detach routine was called. Consequently,
442 			 * we should fail the detach here.
443 			 */
444 			dev_err(pcip->pci_dip, CE_WARN, "npe:	failed to "
445 			    "unbind NDI event set (error=%d)", ret);
446 			return (DDI_FAILURE);
447 		}
448 
449 		pcie_fab_fini_bus(devi, PCIE_BUS_INITIAL);
450 
451 		/* Uninitialize pcitool support. */
452 		pcitool_uninit(devi);
453 
454 		if (pcie_uninit(devi) != DDI_SUCCESS)
455 			return (DDI_FAILURE);
456 
457 		if (pcip->pci_fmcap & DDI_FM_ERRCB_CAPABLE)
458 			ddi_fm_handler_unregister(devi);
459 
460 		pcie_rc_fini_pfd(PCIE_DIP2PFD(devi));
461 		kmem_free(PCIE_DIP2PFD(devi), sizeof (pf_data_t));
462 
463 		ddi_fm_fini(devi);
464 		ddi_soft_state_free(npe_statep, instance);
465 
466 		return (DDI_SUCCESS);
467 
468 	case DDI_SUSPEND:
469 		/*
470 		 * the system might still be able to suspend/resume even if
471 		 * this fails
472 		 */
473 		(void) npe_save_htconfig_children(devi);
474 		return (DDI_SUCCESS);
475 	default:
476 		return (DDI_FAILURE);
477 	}
478 }
479 
480 /*
481  * Configure the access handle for standard configuration space
482  * access (see pci_fm_acc_setup for code that initializes the
483  * access-function pointers).
484  */
485 static int
npe_setup_std_pcicfg_acc(dev_info_t * rdip,ddi_map_req_t * mp,ddi_acc_hdl_t * hp,off_t offset,off_t len)486 npe_setup_std_pcicfg_acc(dev_info_t *rdip, ddi_map_req_t *mp,
487     ddi_acc_hdl_t *hp, off_t offset, off_t len)
488 {
489 	int ret;
490 
491 	if ((ret = pci_fm_acc_setup(hp, offset, len)) ==
492 	    DDI_SUCCESS) {
493 		if (DDI_FM_ACC_ERR_CAP(ddi_fm_capable(rdip)) &&
494 		    mp->map_handlep->ah_acc.devacc_attr_access
495 		    != DDI_DEFAULT_ACC) {
496 			ndi_fmc_insert(rdip, ACC_HANDLE,
497 			    (void *)mp->map_handlep, NULL);
498 		}
499 	}
500 	return (ret);
501 }
502 
503 static int
npe_bus_map(dev_info_t * dip,dev_info_t * rdip,ddi_map_req_t * mp,off_t offset,off_t len,caddr_t * vaddrp)504 npe_bus_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp,
505     off_t offset, off_t len, caddr_t *vaddrp)
506 {
507 	int		rnumber;
508 	int		space;
509 	ddi_acc_impl_t	*ap;
510 	ddi_acc_hdl_t	*hp;
511 	ddi_map_req_t	mr;
512 	pci_regspec_t	pci_reg;
513 	pci_regspec_t	*pci_rp;
514 	struct regspec64 reg;
515 	pci_acc_cfblk_t	*cfp;
516 	int		retval;
517 	int64_t		*ecfginfo;
518 	uint_t		nelem;
519 	uint64_t	pci_rlength;
520 
521 	mr = *mp; /* Get private copy of request */
522 	mp = &mr;
523 
524 	/*
525 	 * check for register number
526 	 */
527 	switch (mp->map_type) {
528 	case DDI_MT_REGSPEC:
529 		pci_reg = *(pci_regspec_t *)(mp->map_obj.rp);
530 		pci_rp = &pci_reg;
531 		if (pci_common_get_reg_prop(rdip, pci_rp) != DDI_SUCCESS)
532 			return (DDI_FAILURE);
533 		break;
534 	case DDI_MT_RNUMBER:
535 		rnumber = mp->map_obj.rnumber;
536 		/*
537 		 * get ALL "reg" properties for dip, select the one of
538 		 * of interest. In x86, "assigned-addresses" property
539 		 * is identical to the "reg" property, so there is no
540 		 * need to cross check the two to determine the physical
541 		 * address of the registers.
542 		 * This routine still performs some validity checks to
543 		 * make sure that everything is okay.
544 		 */
545 		if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, rdip,
546 		    DDI_PROP_DONTPASS, "reg", (int **)&pci_rp, &nelem) !=
547 		    DDI_PROP_SUCCESS)
548 			return (DDI_FAILURE);
549 
550 		/*
551 		 * validate the register number.
552 		 */
553 		nelem /= (sizeof (pci_regspec_t) / sizeof (int));
554 		if (rnumber >= nelem) {
555 			ddi_prop_free(pci_rp);
556 			return (DDI_FAILURE);
557 		}
558 
559 		/*
560 		 * copy the required entry.
561 		 */
562 		pci_reg = pci_rp[rnumber];
563 
564 		/*
565 		 * free the memory allocated by ddi_prop_lookup_int_array
566 		 */
567 		ddi_prop_free(pci_rp);
568 
569 		pci_rp = &pci_reg;
570 		if (pci_common_get_reg_prop(rdip, pci_rp) != DDI_SUCCESS)
571 			return (DDI_FAILURE);
572 		mp->map_type = DDI_MT_REGSPEC;
573 		break;
574 	default:
575 		return (DDI_ME_INVAL);
576 	}
577 
578 	space = pci_rp->pci_phys_hi & PCI_REG_ADDR_M;
579 
580 	/*
581 	 * check for unmap and unlock of address space
582 	 */
583 	if ((mp->map_op == DDI_MO_UNMAP) || (mp->map_op == DDI_MO_UNLOCK)) {
584 		switch (space) {
585 		case PCI_ADDR_IO:
586 			reg.regspec_bustype = 1;
587 			break;
588 
589 		case PCI_ADDR_CONFIG:
590 			/*
591 			 * If this is an unmap/unlock of a standard config
592 			 * space mapping (memory-mapped config space mappings
593 			 * would have the DDI_ACCATTR_CPU_VADDR bit set in the
594 			 * acc_attr), undo that setup here.
595 			 */
596 			if (NPE_IS_HANDLE_FOR_STDCFG_ACC(mp->map_handlep)) {
597 
598 				if (DDI_FM_ACC_ERR_CAP(ddi_fm_capable(rdip)) &&
599 				    mp->map_handlep->ah_acc.devacc_attr_access
600 				    != DDI_DEFAULT_ACC) {
601 					ndi_fmc_remove(rdip, ACC_HANDLE,
602 					    (void *)mp->map_handlep);
603 				}
604 				return (DDI_SUCCESS);
605 			}
606 
607 			pci_rp->pci_size_low = PCIE_CONF_HDR_SIZE;
608 
609 			/* FALLTHROUGH */
610 		case PCI_ADDR_MEM64:
611 		case PCI_ADDR_MEM32:
612 			reg.regspec_bustype = 0;
613 			break;
614 
615 		default:
616 			return (DDI_FAILURE);
617 		}
618 
619 		reg.regspec_addr = (uint64_t)pci_rp->pci_phys_mid << 32 |
620 		    (uint64_t)pci_rp->pci_phys_low;
621 		reg.regspec_size = (uint64_t)pci_rp->pci_size_hi << 32 |
622 		    (uint64_t)pci_rp->pci_size_low;
623 
624 		/*
625 		 * Adjust offset and length
626 		 * A non-zero length means override the one in the regspec.
627 		 */
628 		if (reg.regspec_addr + offset < MAX(reg.regspec_addr, offset))
629 			return (DDI_FAILURE);
630 		reg.regspec_addr += offset;
631 		if (len != 0)
632 			reg.regspec_size = len;
633 
634 		mp->map_obj.rp = (struct regspec *)&reg;
635 		mp->map_flags |= DDI_MF_EXT_REGSPEC;
636 		retval = ddi_map(dip, mp, (off_t)0, (off_t)0, vaddrp);
637 		if (DDI_FM_ACC_ERR_CAP(ddi_fm_capable(rdip)) &&
638 		    mp->map_handlep->ah_acc.devacc_attr_access !=
639 		    DDI_DEFAULT_ACC) {
640 			ndi_fmc_remove(rdip, ACC_HANDLE,
641 			    (void *)mp->map_handlep);
642 		}
643 		return (retval);
644 
645 	}
646 
647 	/* check for user mapping request - not legal for Config */
648 	if (mp->map_op == DDI_MO_MAP_HANDLE && space == PCI_ADDR_CONFIG) {
649 		cmn_err(CE_NOTE, "npe: Config mapping request from user\n");
650 		return (DDI_FAILURE);
651 	}
652 
653 
654 	/*
655 	 * Note that pci_fm_acc_setup() is called to serve two purposes
656 	 * i) enable legacy PCI I/O style config space access
657 	 * ii) register with FMA
658 	 */
659 	if (space == PCI_ADDR_CONFIG) {
660 
661 		/* Can't map config space without a handle */
662 		hp = (ddi_acc_hdl_t *)mp->map_handlep;
663 		if (hp == NULL)
664 			return (DDI_FAILURE);
665 
666 		/* record the device address for future reference */
667 		cfp = (pci_acc_cfblk_t *)&hp->ah_bus_private;
668 		cfp->c_busnum = PCI_REG_BUS_G(pci_rp->pci_phys_hi);
669 		cfp->c_devnum = PCI_REG_DEV_G(pci_rp->pci_phys_hi);
670 		cfp->c_funcnum = PCI_REG_FUNC_G(pci_rp->pci_phys_hi);
671 
672 		*vaddrp = (caddr_t)offset;
673 
674 		/* Check if MMCFG is supported */
675 		if (!npe_is_mmcfg_supported(rdip)) {
676 			return (npe_setup_std_pcicfg_acc(rdip, mp, hp,
677 			    offset, len));
678 		}
679 
680 
681 		if (ddi_prop_lookup_int64_array(DDI_DEV_T_ANY, rdip, 0,
682 		    "ecfg", &ecfginfo, &nelem) == DDI_PROP_SUCCESS) {
683 
684 			if (nelem != 4 ||
685 			    cfp->c_busnum < ecfginfo[2] ||
686 			    cfp->c_busnum > ecfginfo[3]) {
687 				/*
688 				 * Invalid property or Doesn't contain the
689 				 * requested bus; fall back to standard
690 				 * (I/O-based) config access.
691 				 */
692 				ddi_prop_free(ecfginfo);
693 				return (npe_setup_std_pcicfg_acc(rdip, mp, hp,
694 				    offset, len));
695 			} else {
696 				pci_rp->pci_phys_low = ecfginfo[0];
697 
698 				ddi_prop_free(ecfginfo);
699 
700 				pci_rp->pci_phys_low += ((cfp->c_busnum << 20) |
701 				    (cfp->c_devnum) << 15 |
702 				    (cfp->c_funcnum << 12));
703 
704 				pci_rp->pci_size_low = PCIE_CONF_HDR_SIZE;
705 			}
706 		} else {
707 			/*
708 			 * Couldn't find the MMCFG property -- fall back to
709 			 * standard config access
710 			 */
711 			return (npe_setup_std_pcicfg_acc(rdip, mp, hp,
712 			    offset, len));
713 		}
714 	}
715 
716 	/*
717 	 * range check
718 	 */
719 	pci_rlength = (uint64_t)pci_rp->pci_size_low |
720 	    (uint64_t)pci_rp->pci_size_hi << 32;
721 	if ((offset >= pci_rlength) || (len > pci_rlength) ||
722 	    (offset + len > pci_rlength) || (offset + len < MAX(offset, len))) {
723 		return (DDI_FAILURE);
724 	}
725 
726 	/*
727 	 * convert the pci regsec into the generic regspec used by the
728 	 * parent root nexus driver.
729 	 */
730 	switch (space) {
731 	case PCI_ADDR_IO:
732 		reg.regspec_bustype = 1;
733 		break;
734 	case PCI_ADDR_CONFIG:
735 	case PCI_ADDR_MEM64:
736 	case PCI_ADDR_MEM32:
737 		reg.regspec_bustype = 0;
738 		break;
739 	default:
740 		return (DDI_FAILURE);
741 	}
742 
743 	reg.regspec_addr = (uint64_t)pci_rp->pci_phys_mid << 32 |
744 	    (uint64_t)pci_rp->pci_phys_low;
745 	reg.regspec_size = pci_rlength;
746 
747 	/*
748 	 * Adjust offset and length
749 	 * A non-zero length means override the one in the regspec.
750 	 */
751 	if (reg.regspec_addr + offset < MAX(reg.regspec_addr, offset))
752 		return (DDI_FAILURE);
753 	reg.regspec_addr += offset;
754 	if (len != 0)
755 		reg.regspec_size = len;
756 
757 
758 	mp->map_obj.rp = (struct regspec *)&reg;
759 	mp->map_flags |= DDI_MF_EXT_REGSPEC;
760 	retval = ddi_map(dip, mp, (off_t)0, (off_t)0, vaddrp);
761 	if (retval == DDI_SUCCESS) {
762 		/*
763 		 * For config space gets force use of cautious access routines.
764 		 * These will handle default and protected mode accesses too.
765 		 */
766 		if (space == PCI_ADDR_CONFIG) {
767 			ap = (ddi_acc_impl_t *)mp->map_handlep;
768 			ap->ahi_acc_attr &= ~DDI_ACCATTR_DIRECT;
769 			ap->ahi_acc_attr |= DDI_ACCATTR_CONFIG_SPACE;
770 			ap->ahi_get8 = i_ddi_caut_get8;
771 			ap->ahi_get16 = i_ddi_caut_get16;
772 			ap->ahi_get32 = i_ddi_caut_get32;
773 			ap->ahi_get64 = i_ddi_caut_get64;
774 			ap->ahi_rep_get8 = i_ddi_caut_rep_get8;
775 			ap->ahi_rep_get16 = i_ddi_caut_rep_get16;
776 			ap->ahi_rep_get32 = i_ddi_caut_rep_get32;
777 			ap->ahi_rep_get64 = i_ddi_caut_rep_get64;
778 		}
779 		if (DDI_FM_ACC_ERR_CAP(ddi_fm_capable(rdip)) &&
780 		    mp->map_handlep->ah_acc.devacc_attr_access !=
781 		    DDI_DEFAULT_ACC) {
782 			ndi_fmc_insert(rdip, ACC_HANDLE,
783 			    (void *)mp->map_handlep, NULL);
784 		}
785 	}
786 	return (retval);
787 }
788 
789 
790 
791 /*ARGSUSED*/
792 static int
npe_ctlops(dev_info_t * dip,dev_info_t * rdip,ddi_ctl_enum_t ctlop,void * arg,void * result)793 npe_ctlops(dev_info_t *dip, dev_info_t *rdip,
794     ddi_ctl_enum_t ctlop, void *arg, void *result)
795 {
796 	int		totreg;
797 	uint_t		reglen;
798 	pci_regspec_t	*drv_regp;
799 	struct attachspec *asp;
800 	struct detachspec *dsp;
801 	pci_state_t	*pci_p = ddi_get_soft_state(npe_statep,
802 	    ddi_get_instance(dip));
803 
804 	switch (ctlop) {
805 	case DDI_CTLOPS_REPORTDEV:
806 		if (rdip == (dev_info_t *)0)
807 			return (DDI_FAILURE);
808 		cmn_err(CE_CONT, "?PCI Express-device: %s@%s, %s%d\n",
809 		    ddi_node_name(rdip), ddi_get_name_addr(rdip),
810 		    ddi_driver_name(rdip), ddi_get_instance(rdip));
811 		return (DDI_SUCCESS);
812 
813 	case DDI_CTLOPS_INITCHILD:
814 		return (npe_initchild((dev_info_t *)arg));
815 
816 	case DDI_CTLOPS_UNINITCHILD:
817 		return (npe_removechild((dev_info_t *)arg));
818 
819 	case DDI_CTLOPS_SIDDEV:
820 		return (DDI_SUCCESS);
821 
822 	case DDI_CTLOPS_REGSIZE:
823 	case DDI_CTLOPS_NREGS:
824 		if (rdip == (dev_info_t *)0)
825 			return (DDI_FAILURE);
826 
827 		*(int *)result = 0;
828 		if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, rdip,
829 		    DDI_PROP_DONTPASS, "reg", (int **)&drv_regp,
830 		    &reglen) != DDI_PROP_SUCCESS) {
831 			return (DDI_FAILURE);
832 		}
833 
834 		totreg = (reglen * sizeof (int)) / sizeof (pci_regspec_t);
835 		if (ctlop == DDI_CTLOPS_NREGS)
836 			*(int *)result = totreg;
837 		else if (ctlop == DDI_CTLOPS_REGSIZE) {
838 			uint64_t val;
839 			int rn;
840 
841 			rn = *(int *)arg;
842 			if (rn >= totreg) {
843 				ddi_prop_free(drv_regp);
844 				return (DDI_FAILURE);
845 			}
846 			val = drv_regp[rn].pci_size_low |
847 			    (uint64_t)drv_regp[rn].pci_size_hi << 32;
848 			if (val > OFF_MAX) {
849 				int ce = CE_NOTE;
850 #ifdef DEBUG
851 				ce = CE_WARN;
852 #endif
853 				dev_err(rdip, ce, "failed to get register "
854 				    "size, value larger than OFF_MAX: 0x%"
855 				    PRIx64 "\n", val);
856 				return (DDI_FAILURE);
857 			}
858 			*(off_t *)result = (off_t)val;
859 		}
860 		ddi_prop_free(drv_regp);
861 
862 		return (DDI_SUCCESS);
863 
864 	case DDI_CTLOPS_POWER:
865 	{
866 		power_req_t	*reqp = (power_req_t *)arg;
867 		/*
868 		 * We currently understand reporting of PCI_PM_IDLESPEED
869 		 * capability. Everything else is passed up.
870 		 */
871 		if ((reqp->request_type == PMR_REPORT_PMCAP) &&
872 		    (reqp->req.report_pmcap_req.cap ==  PCI_PM_IDLESPEED))
873 			return (DDI_SUCCESS);
874 
875 		break;
876 	}
877 
878 	case DDI_CTLOPS_PEEK:
879 	case DDI_CTLOPS_POKE:
880 		return (pci_common_peekpoke(dip, rdip, ctlop, arg, result));
881 
882 	/* X86 systems support PME wakeup from suspended state */
883 	case DDI_CTLOPS_ATTACH:
884 		if (!pcie_is_child(dip, rdip))
885 			return (DDI_SUCCESS);
886 
887 		asp = (struct attachspec *)arg;
888 		if ((asp->when == DDI_POST) && (asp->result == DDI_SUCCESS)) {
889 			pf_init(rdip, (void *)pci_p->pci_fm_ibc, asp->cmd);
890 			(void) pcie_postattach_child(rdip);
891 		}
892 
893 		/* only do this for immediate children */
894 		if (asp->cmd == DDI_RESUME && asp->when == DDI_PRE &&
895 		    ddi_get_parent(rdip) == dip)
896 			if (pci_pre_resume(rdip) != DDI_SUCCESS) {
897 				/* Not good, better stop now. */
898 				cmn_err(CE_PANIC,
899 				    "Couldn't pre-resume device %p",
900 				    (void *) dip);
901 				/* NOTREACHED */
902 			}
903 
904 		return (DDI_SUCCESS);
905 
906 	case DDI_CTLOPS_DETACH:
907 		if (!pcie_is_child(dip, rdip))
908 			return (DDI_SUCCESS);
909 
910 		dsp = (struct detachspec *)arg;
911 
912 		if (dsp->when == DDI_PRE)
913 			pf_fini(rdip, dsp->cmd);
914 
915 		/* only do this for immediate children */
916 		if (dsp->cmd == DDI_SUSPEND && dsp->when == DDI_POST &&
917 		    ddi_get_parent(rdip) == dip)
918 			if (pci_post_suspend(rdip) != DDI_SUCCESS)
919 				return (DDI_FAILURE);
920 
921 		return (DDI_SUCCESS);
922 
923 	default:
924 		break;
925 	}
926 
927 	return (ddi_ctlops(dip, rdip, ctlop, arg, result));
928 
929 }
930 
931 
932 /*
933  * npe_intr_ops
934  */
935 static int
npe_intr_ops(dev_info_t * pdip,dev_info_t * rdip,ddi_intr_op_t intr_op,ddi_intr_handle_impl_t * hdlp,void * result)936 npe_intr_ops(dev_info_t *pdip, dev_info_t *rdip, ddi_intr_op_t intr_op,
937     ddi_intr_handle_impl_t *hdlp, void *result)
938 {
939 	return (pci_common_intr_ops(pdip, rdip, intr_op, hdlp, result));
940 }
941 
942 
943 static int
npe_initchild(dev_info_t * child)944 npe_initchild(dev_info_t *child)
945 {
946 	char		name[80];
947 	pcie_bus_t	*bus_p;
948 	uint32_t	regs;
949 	ddi_acc_handle_t	cfg_hdl;
950 
951 	/*
952 	 * Do not bind drivers to empty bridges.
953 	 * Fail above, if the bridge is found to be hotplug capable
954 	 */
955 	if (npe_disable_empty_bridges_workaround(child) == 1)
956 		return (DDI_FAILURE);
957 
958 	if (pci_common_name_child(child, name, 80) != DDI_SUCCESS)
959 		return (DDI_FAILURE);
960 
961 	ddi_set_name_addr(child, name);
962 
963 	/*
964 	 * Pseudo nodes indicate a prototype node with per-instance
965 	 * properties to be merged into the real h/w device node.
966 	 * The interpretation of the unit-address is DD[,F]
967 	 * where DD is the device id and F is the function.
968 	 */
969 	if (ndi_dev_is_persistent_node(child) == 0) {
970 		extern int pci_allow_pseudo_children;
971 
972 		ddi_set_parent_data(child, NULL);
973 
974 		/*
975 		 * Try to merge the properties from this prototype
976 		 * node into real h/w nodes.
977 		 */
978 		if (ndi_merge_node(child, pci_common_name_child) ==
979 		    DDI_SUCCESS) {
980 			/*
981 			 * Merged ok - return failure to remove the node.
982 			 */
983 			ddi_set_name_addr(child, NULL);
984 			return (DDI_FAILURE);
985 		}
986 
987 		/* workaround for DDIVS to run under PCI Express */
988 		if (pci_allow_pseudo_children) {
989 			/*
990 			 * If the "interrupts" property doesn't exist,
991 			 * this must be the ddivs no-intr case, and it returns
992 			 * DDI_SUCCESS instead of DDI_FAILURE.
993 			 */
994 			if (ddi_prop_get_int(DDI_DEV_T_ANY, child,
995 			    DDI_PROP_DONTPASS, "interrupts", -1) == -1)
996 				return (DDI_SUCCESS);
997 			/*
998 			 * Create the ddi_parent_private_data for a pseudo
999 			 * child.
1000 			 */
1001 			pci_common_set_parent_private_data(child);
1002 			return (DDI_SUCCESS);
1003 		}
1004 
1005 		/*
1006 		 * The child was not merged into a h/w node,
1007 		 * but there's not much we can do with it other
1008 		 * than return failure to cause the node to be removed.
1009 		 */
1010 		cmn_err(CE_WARN, "!%s@%s: %s.conf properties not merged",
1011 		    ddi_get_name(child), ddi_get_name_addr(child),
1012 		    ddi_get_name(child));
1013 		ddi_set_name_addr(child, NULL);
1014 		return (DDI_NOT_WELL_FORMED);
1015 	}
1016 
1017 	if (ddi_prop_get_int(DDI_DEV_T_ANY, child, DDI_PROP_DONTPASS,
1018 	    "interrupts", -1) != -1)
1019 		pci_common_set_parent_private_data(child);
1020 	else
1021 		ddi_set_parent_data(child, NULL);
1022 
1023 	/* Disable certain errors on PCIe drivers for x86 platforms */
1024 	regs = pcie_get_aer_uce_mask() | npe_aer_uce_mask;
1025 	pcie_set_aer_uce_mask(regs);
1026 	regs = pcie_get_aer_ce_mask() | npe_aer_ce_mask;
1027 	pcie_set_aer_ce_mask(regs);
1028 	regs = pcie_get_aer_suce_mask() | npe_aer_suce_mask;
1029 	pcie_set_aer_suce_mask(regs);
1030 
1031 	/*
1032 	 * If URs are disabled, mask SERRs as well, otherwise the system will
1033 	 * still be notified of URs
1034 	 */
1035 	if (npe_aer_uce_mask & PCIE_AER_UCE_UR)
1036 		pcie_set_serr_mask(1);
1037 
1038 	if (pci_config_setup(child, &cfg_hdl) == DDI_SUCCESS) {
1039 		npe_ck804_fix_aer_ptr(cfg_hdl);
1040 		npe_nvidia_error_workaround(cfg_hdl);
1041 		npe_intel_error_workaround(cfg_hdl);
1042 		pci_config_teardown(&cfg_hdl);
1043 	}
1044 
1045 	bus_p = PCIE_DIP2BUS(child);
1046 	if (bus_p) {
1047 		uint16_t device_id = (uint16_t)(bus_p->bus_dev_ven_id >> 16);
1048 		uint16_t vendor_id = (uint16_t)(bus_p->bus_dev_ven_id & 0xFFFF);
1049 		uint16_t rev_id = bus_p->bus_rev_id;
1050 
1051 		/* Disable AER for certain NVIDIA Chipsets */
1052 		if ((vendor_id == NVIDIA_VENDOR_ID) &&
1053 		    (device_id == NVIDIA_CK804_DEVICE_ID) &&
1054 		    (rev_id < NVIDIA_CK804_AER_VALID_REVID))
1055 			bus_p->bus_aer_off = 0;
1056 
1057 		pcie_init_dom(child);
1058 		(void) pcie_initchild(child);
1059 	}
1060 
1061 	return (DDI_SUCCESS);
1062 }
1063 
1064 
1065 static int
npe_removechild(dev_info_t * dip)1066 npe_removechild(dev_info_t *dip)
1067 {
1068 	pcie_uninitchild(dip);
1069 
1070 	ddi_set_name_addr(dip, NULL);
1071 
1072 	/*
1073 	 * Strip the node to properly convert it back to prototype form
1074 	 */
1075 	ddi_remove_minor_node(dip, NULL);
1076 
1077 	ddi_prop_remove_all(dip);
1078 
1079 	return (DDI_SUCCESS);
1080 }
1081 
1082 static int
npe_open(dev_t * devp,int flags,int otyp,cred_t * credp)1083 npe_open(dev_t *devp, int flags, int otyp, cred_t *credp)
1084 {
1085 	minor_t		minor = getminor(*devp);
1086 	int		instance = PCI_MINOR_NUM_TO_INSTANCE(minor);
1087 	pci_state_t	*pci_p = ddi_get_soft_state(npe_statep, instance);
1088 	int	rv;
1089 
1090 	/*
1091 	 * Make sure the open is for the right file type.
1092 	 */
1093 	if (otyp != OTYP_CHR)
1094 		return (EINVAL);
1095 
1096 	if (pci_p == NULL)
1097 		return (ENXIO);
1098 
1099 	mutex_enter(&pci_p->pci_mutex);
1100 	switch (PCI_MINOR_NUM_TO_PCI_DEVNUM(minor)) {
1101 	case PCI_TOOL_REG_MINOR_NUM:
1102 	case PCI_TOOL_INTR_MINOR_NUM:
1103 		break;
1104 	default:
1105 		/* Handle devctl ioctls */
1106 		rv = pcie_open(pci_p->pci_dip, devp, flags, otyp, credp);
1107 		mutex_exit(&pci_p->pci_mutex);
1108 		return (rv);
1109 	}
1110 
1111 	/* Handle pcitool ioctls */
1112 	if (flags & FEXCL) {
1113 		if (pci_p->pci_soft_state != PCI_SOFT_STATE_CLOSED) {
1114 			mutex_exit(&pci_p->pci_mutex);
1115 			cmn_err(CE_NOTE, "npe_open: busy");
1116 			return (EBUSY);
1117 		}
1118 		pci_p->pci_soft_state = PCI_SOFT_STATE_OPEN_EXCL;
1119 	} else {
1120 		if (pci_p->pci_soft_state == PCI_SOFT_STATE_OPEN_EXCL) {
1121 			mutex_exit(&pci_p->pci_mutex);
1122 			cmn_err(CE_NOTE, "npe_open: busy");
1123 			return (EBUSY);
1124 		}
1125 		pci_p->pci_soft_state = PCI_SOFT_STATE_OPEN;
1126 	}
1127 	mutex_exit(&pci_p->pci_mutex);
1128 
1129 	return (0);
1130 }
1131 
1132 static int
npe_close(dev_t dev,int flags,int otyp,cred_t * credp)1133 npe_close(dev_t dev, int flags, int otyp, cred_t *credp)
1134 {
1135 	minor_t		minor = getminor(dev);
1136 	int		instance = PCI_MINOR_NUM_TO_INSTANCE(minor);
1137 	pci_state_t	*pci_p = ddi_get_soft_state(npe_statep, instance);
1138 	int	rv;
1139 
1140 	if (pci_p == NULL)
1141 		return (ENXIO);
1142 
1143 	mutex_enter(&pci_p->pci_mutex);
1144 
1145 	switch (PCI_MINOR_NUM_TO_PCI_DEVNUM(minor)) {
1146 	case PCI_TOOL_REG_MINOR_NUM:
1147 	case PCI_TOOL_INTR_MINOR_NUM:
1148 		break;
1149 	default:
1150 		/* Handle devctl ioctls */
1151 		rv = pcie_close(pci_p->pci_dip, dev, flags, otyp, credp);
1152 		mutex_exit(&pci_p->pci_mutex);
1153 		return (rv);
1154 	}
1155 
1156 	/* Handle pcitool ioctls */
1157 	pci_p->pci_soft_state = PCI_SOFT_STATE_CLOSED;
1158 	mutex_exit(&pci_p->pci_mutex);
1159 	return (0);
1160 }
1161 
1162 static int
npe_ioctl(dev_t dev,int cmd,intptr_t arg,int mode,cred_t * credp,int * rvalp)1163 npe_ioctl(dev_t dev, int cmd, intptr_t arg, int mode, cred_t *credp, int *rvalp)
1164 {
1165 	minor_t		minor = getminor(dev);
1166 	int		instance = PCI_MINOR_NUM_TO_INSTANCE(minor);
1167 	pci_state_t	*pci_p = ddi_get_soft_state(npe_statep, instance);
1168 	int		ret = ENOTTY;
1169 
1170 	if (pci_p == NULL)
1171 		return (ENXIO);
1172 
1173 	switch (PCI_MINOR_NUM_TO_PCI_DEVNUM(minor)) {
1174 	case PCI_TOOL_REG_MINOR_NUM:
1175 	case PCI_TOOL_INTR_MINOR_NUM:
1176 		/* To handle pcitool related ioctls */
1177 		ret =  pci_common_ioctl(pci_p->pci_dip, dev, cmd, arg, mode,
1178 		    credp, rvalp);
1179 		break;
1180 	default:
1181 		/* To handle devctl and hotplug related ioctls */
1182 		ret = pcie_ioctl(pci_p->pci_dip, dev, cmd, arg, mode, credp,
1183 		    rvalp);
1184 		break;
1185 	}
1186 
1187 	return (ret);
1188 }
1189 
1190 /*ARGSUSED*/
1191 static int
npe_fm_init(dev_info_t * dip,dev_info_t * tdip,int cap,ddi_iblock_cookie_t * ibc)1192 npe_fm_init(dev_info_t *dip, dev_info_t *tdip, int cap,
1193     ddi_iblock_cookie_t *ibc)
1194 {
1195 	pci_state_t  *pcip = ddi_get_soft_state(npe_statep,
1196 	    ddi_get_instance(dip));
1197 
1198 	ASSERT(ibc != NULL);
1199 	*ibc = pcip->pci_fm_ibc;
1200 
1201 	return (pcip->pci_fmcap);
1202 }
1203 
1204 static int
npe_bus_get_eventcookie(dev_info_t * dip,dev_info_t * rdip,char * eventname,ddi_eventcookie_t * cookiep)1205 npe_bus_get_eventcookie(dev_info_t *dip, dev_info_t *rdip, char *eventname,
1206     ddi_eventcookie_t *cookiep)
1207 {
1208 	pci_state_t *pcip = ddi_get_soft_state(npe_statep,
1209 	    ddi_get_instance(dip));
1210 
1211 	return (ndi_event_retrieve_cookie(pcip->pci_ndi_event_hdl, rdip,
1212 	    eventname, cookiep, NDI_EVENT_NOPASS));
1213 }
1214 
1215 static int
npe_bus_add_eventcall(dev_info_t * dip,dev_info_t * rdip,ddi_eventcookie_t cookie,void (* callback)(dev_info_t * dip,ddi_eventcookie_t cookie,void * arg,void * bus_impldata),void * arg,ddi_callback_id_t * cb_id)1216 npe_bus_add_eventcall(dev_info_t *dip, dev_info_t *rdip,
1217     ddi_eventcookie_t cookie, void (*callback)(dev_info_t *dip,
1218     ddi_eventcookie_t cookie, void *arg, void *bus_impldata),
1219     void *arg, ddi_callback_id_t *cb_id)
1220 {
1221 	pci_state_t *pcip = ddi_get_soft_state(npe_statep,
1222 	    ddi_get_instance(dip));
1223 
1224 	return (ndi_event_add_callback(pcip->pci_ndi_event_hdl, rdip, cookie,
1225 	    callback, arg, NDI_SLEEP, cb_id));
1226 }
1227 
1228 static int
npe_bus_remove_eventcall(dev_info_t * dip,ddi_callback_id_t cb_id)1229 npe_bus_remove_eventcall(dev_info_t *dip, ddi_callback_id_t cb_id)
1230 {
1231 	pci_state_t *pcip = ddi_get_soft_state(npe_statep,
1232 	    ddi_get_instance(dip));
1233 	return (ndi_event_remove_callback(pcip->pci_ndi_event_hdl, cb_id));
1234 }
1235 
1236 static int
npe_bus_post_event(dev_info_t * dip,dev_info_t * rdip,ddi_eventcookie_t cookie,void * impl_data)1237 npe_bus_post_event(dev_info_t *dip, dev_info_t *rdip,
1238     ddi_eventcookie_t cookie, void *impl_data)
1239 {
1240 	pci_state_t *pcip = ddi_get_soft_state(npe_statep,
1241 	    ddi_get_instance(dip));
1242 	return (ndi_event_do_callback(pcip->pci_ndi_event_hdl, rdip, cookie,
1243 	    impl_data));
1244 
1245 }
1246 
1247 /*ARGSUSED*/
1248 static int
npe_fm_callback(dev_info_t * dip,ddi_fm_error_t * derr,const void * no_used)1249 npe_fm_callback(dev_info_t *dip, ddi_fm_error_t *derr, const void *no_used)
1250 {
1251 	/*
1252 	 * On current x86 systems, npe's callback does not get called for failed
1253 	 * loads.  If in the future this feature is used, the fault PA should be
1254 	 * logged in the derr->fme_bus_specific field.  The appropriate PCIe
1255 	 * error handling code should be called and needs to be coordinated with
1256 	 * safe access handling.
1257 	 */
1258 
1259 	return (DDI_FM_OK);
1260 }
1261