1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 */
26
27 /*
28 * Copyright 2012 Garrett D'Amore <garrett@damore.org>. All rights reserved.
29 * Copyright 2019 Joyent, Inc.
30 */
31
32 /*
33 * npe (Nexus PCIe driver): Host to PCI-Express local bus driver
34 *
35 * npe serves as the driver for PCIe Root Complexes and as the nexus driver
36 * for PCIe devices. See also: npe(7D). For more information about hotplug,
37 * see the big theory statement at uts/common/os/ddi_hp_impl.c.
38 *
39 *
40 * NDI EVENT HANDLING SUPPORT
41 *
42 * npe supports NDI event handling. The only available event is surprise
43 * removal of a device. Child drivers can register surprise removal event
44 * callbacks by requesting an event cookie using ddi_get_eventcookie for
45 * the DDI_DEVI_REMOVE_EVENT and add their callback using
46 * ddi_add_event_handler. For an example, see the nvme driver in
47 * uts/common/io/nvme/nvme.c.
48 *
49 * The NDI events in npe are retrieved using NDI_EVENT_NOPASS, which
50 * prevent them from being propagated up the tree once they reach the npe's
51 * bus_get_eventcookie operations. This is important because npe maintains
52 * the state of PCIe devices and their receptacles, via the PCIe hotplug
53 * controller driver (pciehpc).
54 *
55 * Hot removal events are ultimately posted by the PCIe hotplug controller
56 * interrupt handler for hotplug events. Events are posted using the
57 * ndi_post_event interface.
58 */
59
60 #include <sys/conf.h>
61 #include <sys/modctl.h>
62 #include <sys/file.h>
63 #include <sys/pci_impl.h>
64 #include <sys/pcie_impl.h>
65 #include <sys/sysmacros.h>
66 #include <sys/ddi_intr.h>
67 #include <sys/sunndi.h>
68 #include <sys/sunddi.h>
69 #include <sys/ddifm.h>
70 #include <sys/ndifm.h>
71 #include <sys/fm/util.h>
72 #include <sys/hotplug/pci/pcie_hp.h>
73 #include <io/pci/pci_tools_ext.h>
74 #include <io/pci/pci_common.h>
75 #include <io/pciex/pcie_nvidia.h>
76
77 /*
78 * Helper Macros
79 */
80 #define NPE_IS_HANDLE_FOR_STDCFG_ACC(hp) \
81 ((hp) != NULL && \
82 ((ddi_acc_hdl_t *)(hp))->ah_platform_private != NULL && \
83 (((ddi_acc_impl_t *)((ddi_acc_hdl_t *)(hp))-> \
84 ah_platform_private)-> \
85 ahi_acc_attr &(DDI_ACCATTR_CPU_VADDR|DDI_ACCATTR_CONFIG_SPACE)) \
86 == DDI_ACCATTR_CONFIG_SPACE)
87
88 /*
89 * Bus Operation functions
90 */
91 static int npe_bus_map(dev_info_t *, dev_info_t *, ddi_map_req_t *,
92 off_t, off_t, caddr_t *);
93 static int npe_ctlops(dev_info_t *, dev_info_t *, ddi_ctl_enum_t,
94 void *, void *);
95 static int npe_intr_ops(dev_info_t *, dev_info_t *, ddi_intr_op_t,
96 ddi_intr_handle_impl_t *, void *);
97 static int npe_fm_init(dev_info_t *, dev_info_t *, int,
98 ddi_iblock_cookie_t *);
99 static int npe_bus_get_eventcookie(dev_info_t *, dev_info_t *, char *,
100 ddi_eventcookie_t *);
101 static int npe_bus_add_eventcall(dev_info_t *, dev_info_t *,
102 ddi_eventcookie_t, void (*)(dev_info_t *,
103 ddi_eventcookie_t, void *, void *),
104 void *, ddi_callback_id_t *);
105 static int npe_bus_remove_eventcall(dev_info_t *, ddi_callback_id_t);
106 static int npe_bus_post_event(dev_info_t *, dev_info_t *,
107 ddi_eventcookie_t, void *);
108
109 static int npe_fm_callback(dev_info_t *, ddi_fm_error_t *, const void *);
110
111 /*
112 * Disable URs and Received MA for all PCIe devices. Until x86 SW is changed so
113 * that random drivers do not do PIO accesses on devices that it does not own,
114 * these error bits must be disabled. SERR must also be disabled if URs have
115 * been masked.
116 */
117 uint32_t npe_aer_uce_mask = PCIE_AER_UCE_UR;
118 uint32_t npe_aer_ce_mask = 0;
119 uint32_t npe_aer_suce_mask = PCIE_AER_SUCE_RCVD_MA;
120
121 struct bus_ops npe_bus_ops = {
122 BUSO_REV,
123 npe_bus_map,
124 NULL,
125 NULL,
126 NULL,
127 i_ddi_map_fault,
128 NULL,
129 ddi_dma_allochdl,
130 ddi_dma_freehdl,
131 ddi_dma_bindhdl,
132 ddi_dma_unbindhdl,
133 ddi_dma_flush,
134 ddi_dma_win,
135 ddi_dma_mctl,
136 npe_ctlops,
137 ddi_bus_prop_op,
138 npe_bus_get_eventcookie,
139 npe_bus_add_eventcall,
140 npe_bus_remove_eventcall,
141 npe_bus_post_event,
142 0, /* (*bus_intr_ctl)(); */
143 0, /* (*bus_config)(); */
144 0, /* (*bus_unconfig)(); */
145 npe_fm_init, /* (*bus_fm_init)(); */
146 NULL, /* (*bus_fm_fini)(); */
147 NULL, /* (*bus_fm_access_enter)(); */
148 NULL, /* (*bus_fm_access_exit)(); */
149 NULL, /* (*bus_power)(); */
150 npe_intr_ops, /* (*bus_intr_op)(); */
151 pcie_hp_common_ops /* (*bus_hp_op)(); */
152 };
153
154 static int npe_open(dev_t *, int, int, cred_t *);
155 static int npe_close(dev_t, int, int, cred_t *);
156 static int npe_ioctl(dev_t, int, intptr_t, int, cred_t *, int *);
157
158 struct cb_ops npe_cb_ops = {
159 npe_open, /* open */
160 npe_close, /* close */
161 nodev, /* strategy */
162 nodev, /* print */
163 nodev, /* dump */
164 nodev, /* read */
165 nodev, /* write */
166 npe_ioctl, /* ioctl */
167 nodev, /* devmap */
168 nodev, /* mmap */
169 nodev, /* segmap */
170 nochpoll, /* poll */
171 pcie_prop_op, /* cb_prop_op */
172 NULL, /* streamtab */
173 D_NEW | D_MP | D_HOTPLUG, /* Driver compatibility flag */
174 CB_REV, /* rev */
175 nodev, /* int (*cb_aread)() */
176 nodev /* int (*cb_awrite)() */
177 };
178
179
180 /*
181 * Device Node Operation functions
182 */
183 static int npe_attach(dev_info_t *devi, ddi_attach_cmd_t cmd);
184 static int npe_detach(dev_info_t *devi, ddi_detach_cmd_t cmd);
185 static int npe_info(dev_info_t *, ddi_info_cmd_t, void *, void **);
186
187 struct dev_ops npe_ops = {
188 DEVO_REV, /* devo_rev */
189 0, /* refcnt */
190 npe_info, /* info */
191 nulldev, /* identify */
192 nulldev, /* probe */
193 npe_attach, /* attach */
194 npe_detach, /* detach */
195 nulldev, /* reset */
196 &npe_cb_ops, /* driver operations */
197 &npe_bus_ops, /* bus operations */
198 NULL, /* power */
199 ddi_quiesce_not_needed, /* quiesce */
200 };
201
202 /*
203 * Internal routines in support of particular npe_ctlops.
204 */
205 static int npe_removechild(dev_info_t *child);
206 static int npe_initchild(dev_info_t *child);
207
208 /*
209 * External support routine
210 */
211 extern void npe_query_acpi_mcfg(dev_info_t *dip);
212 extern void npe_ck804_fix_aer_ptr(ddi_acc_handle_t cfg_hdl);
213 extern int npe_disable_empty_bridges_workaround(dev_info_t *child);
214 extern void npe_nvidia_error_workaround(ddi_acc_handle_t cfg_hdl);
215 extern void npe_intel_error_workaround(ddi_acc_handle_t cfg_hdl);
216 extern boolean_t npe_is_mmcfg_supported(dev_info_t *dip);
217 extern void npe_enable_htmsi_children(dev_info_t *dip);
218 extern int npe_save_htconfig_children(dev_info_t *dip);
219 extern int npe_restore_htconfig_children(dev_info_t *dip);
220
221 /*
222 * Module linkage information for the kernel.
223 */
224 static struct modldrv modldrv = {
225 &mod_driverops, /* Type of module */
226 "Host to PCIe nexus driver", /* Name of module */
227 &npe_ops, /* driver ops */
228 };
229
230 static struct modlinkage modlinkage = {
231 MODREV_1,
232 (void *)&modldrv,
233 NULL
234 };
235
236 /* Save minimal state. */
237 void *npe_statep;
238
239 int
_init(void)240 _init(void)
241 {
242 int e;
243
244 /*
245 * Initialize per-pci bus soft state pointer.
246 */
247 e = ddi_soft_state_init(&npe_statep, sizeof (pci_state_t), 1);
248 if (e != 0)
249 return (e);
250
251 if ((e = mod_install(&modlinkage)) != 0)
252 ddi_soft_state_fini(&npe_statep);
253
254 return (e);
255 }
256
257
258 int
_fini(void)259 _fini(void)
260 {
261 int rc;
262
263 rc = mod_remove(&modlinkage);
264 if (rc != 0)
265 return (rc);
266
267 ddi_soft_state_fini(&npe_statep);
268 return (rc);
269 }
270
271
272 int
_info(struct modinfo * modinfop)273 _info(struct modinfo *modinfop)
274 {
275 return (mod_info(&modlinkage, modinfop));
276 }
277
278 /*ARGSUSED*/
279 static int
npe_info(dev_info_t * dip,ddi_info_cmd_t cmd,void * arg,void ** result)280 npe_info(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, void **result)
281 {
282 minor_t minor = getminor((dev_t)arg);
283 int instance = PCI_MINOR_NUM_TO_INSTANCE(minor);
284 pci_state_t *pcip = ddi_get_soft_state(npe_statep, instance);
285 int ret = DDI_SUCCESS;
286
287 switch (cmd) {
288 case DDI_INFO_DEVT2INSTANCE:
289 *result = (void *)(intptr_t)instance;
290 break;
291 case DDI_INFO_DEVT2DEVINFO:
292 if (pcip == NULL) {
293 ret = DDI_FAILURE;
294 break;
295 }
296
297 *result = (void *)pcip->pci_dip;
298 break;
299 default:
300 ret = DDI_FAILURE;
301 break;
302 }
303
304 return (ret);
305 }
306
307 /*
308 * See big theory statement at the top of this file for more information about
309 * surprise removal events.
310 */
311 #define NPE_EVENT_TAG_HOT_REMOVAL 0
312 static ndi_event_definition_t npe_ndi_event_defs[1] = {
313 {NPE_EVENT_TAG_HOT_REMOVAL, DDI_DEVI_REMOVE_EVENT, EPL_KERNEL,
314 NDI_EVENT_POST_TO_ALL}
315 };
316
317 static ndi_event_set_t npe_ndi_events = {
318 NDI_EVENTS_REV1, ARRAY_SIZE(npe_ndi_event_defs), npe_ndi_event_defs
319 };
320
321 /*ARGSUSED*/
322 static int
npe_attach(dev_info_t * devi,ddi_attach_cmd_t cmd)323 npe_attach(dev_info_t *devi, ddi_attach_cmd_t cmd)
324 {
325 int instance = ddi_get_instance(devi);
326 pci_state_t *pcip = NULL;
327 int ret;
328
329 if (cmd == DDI_RESUME) {
330 /*
331 * the system might still be able to resume even if this fails
332 */
333 (void) npe_restore_htconfig_children(devi);
334 return (DDI_SUCCESS);
335 }
336
337 /*
338 * We must do this here in order to ensure that all top level devices
339 * get their HyperTransport MSI mapping regs programmed first.
340 * "Memory controller" and "hostbridge" class devices are leaf devices
341 * that may affect MSI translation functionality for devices
342 * connected to the same link/bus.
343 *
344 * This will also program HT MSI mapping registers on root buses
345 * devices (basically sitting on an HT bus) that are not dependent
346 * on the aforementioned HT devices for MSI translation.
347 */
348 npe_enable_htmsi_children(devi);
349
350 if (ddi_prop_update_string(DDI_DEV_T_NONE, devi, "device_type",
351 "pciex") != DDI_PROP_SUCCESS) {
352 cmn_err(CE_WARN, "npe: 'device_type' prop create failed");
353 }
354
355 if (ddi_soft_state_zalloc(npe_statep, instance) == DDI_SUCCESS)
356 pcip = ddi_get_soft_state(npe_statep, instance);
357
358 if (pcip == NULL)
359 return (DDI_FAILURE);
360
361 pcip->pci_dip = devi;
362 pcip->pci_soft_state = PCI_SOFT_STATE_CLOSED;
363
364 if (pcie_init(devi, NULL) != DDI_SUCCESS)
365 goto fail1;
366
367 ret = ndi_event_alloc_hdl(pcip->pci_dip, NULL, &pcip->pci_ndi_event_hdl,
368 NDI_SLEEP);
369 if (ret == NDI_SUCCESS) {
370 ret = ndi_event_bind_set(pcip->pci_ndi_event_hdl,
371 &npe_ndi_events, NDI_SLEEP);
372 if (ret != NDI_SUCCESS) {
373 dev_err(pcip->pci_dip, CE_WARN, "npe: failed to bind "
374 "NDI event set (error=%d)", ret);
375 goto fail1;
376 }
377 } else {
378 dev_err(pcip->pci_dip, CE_WARN, "npe: failed to allocate "
379 "event handle (error=%d)", ret);
380 goto fail1;
381 }
382
383 /* Second arg: initialize for pci_express root nexus */
384 if (pcitool_init(devi, B_TRUE) != DDI_SUCCESS)
385 goto fail2;
386
387 pcip->pci_fmcap = DDI_FM_EREPORT_CAPABLE | DDI_FM_ERRCB_CAPABLE |
388 DDI_FM_ACCCHK_CAPABLE | DDI_FM_DMACHK_CAPABLE;
389 ddi_fm_init(devi, &pcip->pci_fmcap, &pcip->pci_fm_ibc);
390
391 if (pcip->pci_fmcap & DDI_FM_ERRCB_CAPABLE) {
392 ddi_fm_handler_register(devi, npe_fm_callback, NULL);
393 }
394
395 PCIE_DIP2PFD(devi) = kmem_zalloc(sizeof (pf_data_t), KM_SLEEP);
396 pcie_rc_init_pfd(devi, PCIE_DIP2PFD(devi));
397
398 npe_query_acpi_mcfg(devi);
399 ddi_report_dev(devi);
400 pcie_fab_init_bus(devi, PCIE_BUS_FINAL);
401
402 return (DDI_SUCCESS);
403
404 fail2:
405 (void) pcie_uninit(devi);
406 fail1:
407 pcie_rc_fini_bus(devi);
408 ddi_soft_state_free(npe_statep, instance);
409
410 return (DDI_FAILURE);
411 }
412
413 /*ARGSUSED*/
414 static int
npe_detach(dev_info_t * devi,ddi_detach_cmd_t cmd)415 npe_detach(dev_info_t *devi, ddi_detach_cmd_t cmd)
416 {
417 int instance = ddi_get_instance(devi);
418 pci_state_t *pcip;
419 int ret;
420
421 pcip = ddi_get_soft_state(npe_statep, ddi_get_instance(devi));
422
423 switch (cmd) {
424 case DDI_DETACH:
425
426 /*
427 * Clean up event handling first, to ensure there are no
428 * oustanding callbacks registered.
429 */
430 ret = ndi_event_unbind_set(pcip->pci_ndi_event_hdl,
431 &npe_ndi_events, NDI_SLEEP);
432 if (ret == NDI_SUCCESS) {
433 /* ndi_event_free_hdl always succeeds. */
434 (void) ndi_event_free_hdl(pcip->pci_ndi_event_hdl);
435 } else {
436 /*
437 * The event set will only fail to unbind if there are
438 * outstanding callbacks registered for it, which
439 * probably means a child driver still has one
440 * registered and thus was not cleaned up properly
441 * before npe's detach routine was called. Consequently,
442 * we should fail the detach here.
443 */
444 dev_err(pcip->pci_dip, CE_WARN, "npe: failed to "
445 "unbind NDI event set (error=%d)", ret);
446 return (DDI_FAILURE);
447 }
448
449 pcie_fab_fini_bus(devi, PCIE_BUS_INITIAL);
450
451 /* Uninitialize pcitool support. */
452 pcitool_uninit(devi);
453
454 if (pcie_uninit(devi) != DDI_SUCCESS)
455 return (DDI_FAILURE);
456
457 if (pcip->pci_fmcap & DDI_FM_ERRCB_CAPABLE)
458 ddi_fm_handler_unregister(devi);
459
460 pcie_rc_fini_pfd(PCIE_DIP2PFD(devi));
461 kmem_free(PCIE_DIP2PFD(devi), sizeof (pf_data_t));
462
463 ddi_fm_fini(devi);
464 ddi_soft_state_free(npe_statep, instance);
465
466 return (DDI_SUCCESS);
467
468 case DDI_SUSPEND:
469 /*
470 * the system might still be able to suspend/resume even if
471 * this fails
472 */
473 (void) npe_save_htconfig_children(devi);
474 return (DDI_SUCCESS);
475 default:
476 return (DDI_FAILURE);
477 }
478 }
479
480 /*
481 * Configure the access handle for standard configuration space
482 * access (see pci_fm_acc_setup for code that initializes the
483 * access-function pointers).
484 */
485 static int
npe_setup_std_pcicfg_acc(dev_info_t * rdip,ddi_map_req_t * mp,ddi_acc_hdl_t * hp,off_t offset,off_t len)486 npe_setup_std_pcicfg_acc(dev_info_t *rdip, ddi_map_req_t *mp,
487 ddi_acc_hdl_t *hp, off_t offset, off_t len)
488 {
489 int ret;
490
491 if ((ret = pci_fm_acc_setup(hp, offset, len)) ==
492 DDI_SUCCESS) {
493 if (DDI_FM_ACC_ERR_CAP(ddi_fm_capable(rdip)) &&
494 mp->map_handlep->ah_acc.devacc_attr_access
495 != DDI_DEFAULT_ACC) {
496 ndi_fmc_insert(rdip, ACC_HANDLE,
497 (void *)mp->map_handlep, NULL);
498 }
499 }
500 return (ret);
501 }
502
503 static int
npe_bus_map(dev_info_t * dip,dev_info_t * rdip,ddi_map_req_t * mp,off_t offset,off_t len,caddr_t * vaddrp)504 npe_bus_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp,
505 off_t offset, off_t len, caddr_t *vaddrp)
506 {
507 int rnumber;
508 int space;
509 ddi_acc_impl_t *ap;
510 ddi_acc_hdl_t *hp;
511 ddi_map_req_t mr;
512 pci_regspec_t pci_reg;
513 pci_regspec_t *pci_rp;
514 struct regspec64 reg;
515 pci_acc_cfblk_t *cfp;
516 int retval;
517 int64_t *ecfginfo;
518 uint_t nelem;
519 uint64_t pci_rlength;
520
521 mr = *mp; /* Get private copy of request */
522 mp = &mr;
523
524 /*
525 * check for register number
526 */
527 switch (mp->map_type) {
528 case DDI_MT_REGSPEC:
529 pci_reg = *(pci_regspec_t *)(mp->map_obj.rp);
530 pci_rp = &pci_reg;
531 if (pci_common_get_reg_prop(rdip, pci_rp) != DDI_SUCCESS)
532 return (DDI_FAILURE);
533 break;
534 case DDI_MT_RNUMBER:
535 rnumber = mp->map_obj.rnumber;
536 /*
537 * get ALL "reg" properties for dip, select the one of
538 * of interest. In x86, "assigned-addresses" property
539 * is identical to the "reg" property, so there is no
540 * need to cross check the two to determine the physical
541 * address of the registers.
542 * This routine still performs some validity checks to
543 * make sure that everything is okay.
544 */
545 if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, rdip,
546 DDI_PROP_DONTPASS, "reg", (int **)&pci_rp, &nelem) !=
547 DDI_PROP_SUCCESS)
548 return (DDI_FAILURE);
549
550 /*
551 * validate the register number.
552 */
553 nelem /= (sizeof (pci_regspec_t) / sizeof (int));
554 if (rnumber >= nelem) {
555 ddi_prop_free(pci_rp);
556 return (DDI_FAILURE);
557 }
558
559 /*
560 * copy the required entry.
561 */
562 pci_reg = pci_rp[rnumber];
563
564 /*
565 * free the memory allocated by ddi_prop_lookup_int_array
566 */
567 ddi_prop_free(pci_rp);
568
569 pci_rp = &pci_reg;
570 if (pci_common_get_reg_prop(rdip, pci_rp) != DDI_SUCCESS)
571 return (DDI_FAILURE);
572 mp->map_type = DDI_MT_REGSPEC;
573 break;
574 default:
575 return (DDI_ME_INVAL);
576 }
577
578 space = pci_rp->pci_phys_hi & PCI_REG_ADDR_M;
579
580 /*
581 * check for unmap and unlock of address space
582 */
583 if ((mp->map_op == DDI_MO_UNMAP) || (mp->map_op == DDI_MO_UNLOCK)) {
584 switch (space) {
585 case PCI_ADDR_IO:
586 reg.regspec_bustype = 1;
587 break;
588
589 case PCI_ADDR_CONFIG:
590 /*
591 * If this is an unmap/unlock of a standard config
592 * space mapping (memory-mapped config space mappings
593 * would have the DDI_ACCATTR_CPU_VADDR bit set in the
594 * acc_attr), undo that setup here.
595 */
596 if (NPE_IS_HANDLE_FOR_STDCFG_ACC(mp->map_handlep)) {
597
598 if (DDI_FM_ACC_ERR_CAP(ddi_fm_capable(rdip)) &&
599 mp->map_handlep->ah_acc.devacc_attr_access
600 != DDI_DEFAULT_ACC) {
601 ndi_fmc_remove(rdip, ACC_HANDLE,
602 (void *)mp->map_handlep);
603 }
604 return (DDI_SUCCESS);
605 }
606
607 pci_rp->pci_size_low = PCIE_CONF_HDR_SIZE;
608
609 /* FALLTHROUGH */
610 case PCI_ADDR_MEM64:
611 case PCI_ADDR_MEM32:
612 reg.regspec_bustype = 0;
613 break;
614
615 default:
616 return (DDI_FAILURE);
617 }
618
619 reg.regspec_addr = (uint64_t)pci_rp->pci_phys_mid << 32 |
620 (uint64_t)pci_rp->pci_phys_low;
621 reg.regspec_size = (uint64_t)pci_rp->pci_size_hi << 32 |
622 (uint64_t)pci_rp->pci_size_low;
623
624 /*
625 * Adjust offset and length
626 * A non-zero length means override the one in the regspec.
627 */
628 if (reg.regspec_addr + offset < MAX(reg.regspec_addr, offset))
629 return (DDI_FAILURE);
630 reg.regspec_addr += offset;
631 if (len != 0)
632 reg.regspec_size = len;
633
634 mp->map_obj.rp = (struct regspec *)®
635 mp->map_flags |= DDI_MF_EXT_REGSPEC;
636 retval = ddi_map(dip, mp, (off_t)0, (off_t)0, vaddrp);
637 if (DDI_FM_ACC_ERR_CAP(ddi_fm_capable(rdip)) &&
638 mp->map_handlep->ah_acc.devacc_attr_access !=
639 DDI_DEFAULT_ACC) {
640 ndi_fmc_remove(rdip, ACC_HANDLE,
641 (void *)mp->map_handlep);
642 }
643 return (retval);
644
645 }
646
647 /* check for user mapping request - not legal for Config */
648 if (mp->map_op == DDI_MO_MAP_HANDLE && space == PCI_ADDR_CONFIG) {
649 cmn_err(CE_NOTE, "npe: Config mapping request from user\n");
650 return (DDI_FAILURE);
651 }
652
653
654 /*
655 * Note that pci_fm_acc_setup() is called to serve two purposes
656 * i) enable legacy PCI I/O style config space access
657 * ii) register with FMA
658 */
659 if (space == PCI_ADDR_CONFIG) {
660
661 /* Can't map config space without a handle */
662 hp = (ddi_acc_hdl_t *)mp->map_handlep;
663 if (hp == NULL)
664 return (DDI_FAILURE);
665
666 /* record the device address for future reference */
667 cfp = (pci_acc_cfblk_t *)&hp->ah_bus_private;
668 cfp->c_busnum = PCI_REG_BUS_G(pci_rp->pci_phys_hi);
669 cfp->c_devnum = PCI_REG_DEV_G(pci_rp->pci_phys_hi);
670 cfp->c_funcnum = PCI_REG_FUNC_G(pci_rp->pci_phys_hi);
671
672 *vaddrp = (caddr_t)offset;
673
674 /* Check if MMCFG is supported */
675 if (!npe_is_mmcfg_supported(rdip)) {
676 return (npe_setup_std_pcicfg_acc(rdip, mp, hp,
677 offset, len));
678 }
679
680
681 if (ddi_prop_lookup_int64_array(DDI_DEV_T_ANY, rdip, 0,
682 "ecfg", &ecfginfo, &nelem) == DDI_PROP_SUCCESS) {
683
684 if (nelem != 4 ||
685 cfp->c_busnum < ecfginfo[2] ||
686 cfp->c_busnum > ecfginfo[3]) {
687 /*
688 * Invalid property or Doesn't contain the
689 * requested bus; fall back to standard
690 * (I/O-based) config access.
691 */
692 ddi_prop_free(ecfginfo);
693 return (npe_setup_std_pcicfg_acc(rdip, mp, hp,
694 offset, len));
695 } else {
696 pci_rp->pci_phys_low = ecfginfo[0];
697
698 ddi_prop_free(ecfginfo);
699
700 pci_rp->pci_phys_low += ((cfp->c_busnum << 20) |
701 (cfp->c_devnum) << 15 |
702 (cfp->c_funcnum << 12));
703
704 pci_rp->pci_size_low = PCIE_CONF_HDR_SIZE;
705 }
706 } else {
707 /*
708 * Couldn't find the MMCFG property -- fall back to
709 * standard config access
710 */
711 return (npe_setup_std_pcicfg_acc(rdip, mp, hp,
712 offset, len));
713 }
714 }
715
716 /*
717 * range check
718 */
719 pci_rlength = (uint64_t)pci_rp->pci_size_low |
720 (uint64_t)pci_rp->pci_size_hi << 32;
721 if ((offset >= pci_rlength) || (len > pci_rlength) ||
722 (offset + len > pci_rlength) || (offset + len < MAX(offset, len))) {
723 return (DDI_FAILURE);
724 }
725
726 /*
727 * convert the pci regsec into the generic regspec used by the
728 * parent root nexus driver.
729 */
730 switch (space) {
731 case PCI_ADDR_IO:
732 reg.regspec_bustype = 1;
733 break;
734 case PCI_ADDR_CONFIG:
735 case PCI_ADDR_MEM64:
736 case PCI_ADDR_MEM32:
737 reg.regspec_bustype = 0;
738 break;
739 default:
740 return (DDI_FAILURE);
741 }
742
743 reg.regspec_addr = (uint64_t)pci_rp->pci_phys_mid << 32 |
744 (uint64_t)pci_rp->pci_phys_low;
745 reg.regspec_size = pci_rlength;
746
747 /*
748 * Adjust offset and length
749 * A non-zero length means override the one in the regspec.
750 */
751 if (reg.regspec_addr + offset < MAX(reg.regspec_addr, offset))
752 return (DDI_FAILURE);
753 reg.regspec_addr += offset;
754 if (len != 0)
755 reg.regspec_size = len;
756
757
758 mp->map_obj.rp = (struct regspec *)®
759 mp->map_flags |= DDI_MF_EXT_REGSPEC;
760 retval = ddi_map(dip, mp, (off_t)0, (off_t)0, vaddrp);
761 if (retval == DDI_SUCCESS) {
762 /*
763 * For config space gets force use of cautious access routines.
764 * These will handle default and protected mode accesses too.
765 */
766 if (space == PCI_ADDR_CONFIG) {
767 ap = (ddi_acc_impl_t *)mp->map_handlep;
768 ap->ahi_acc_attr &= ~DDI_ACCATTR_DIRECT;
769 ap->ahi_acc_attr |= DDI_ACCATTR_CONFIG_SPACE;
770 ap->ahi_get8 = i_ddi_caut_get8;
771 ap->ahi_get16 = i_ddi_caut_get16;
772 ap->ahi_get32 = i_ddi_caut_get32;
773 ap->ahi_get64 = i_ddi_caut_get64;
774 ap->ahi_rep_get8 = i_ddi_caut_rep_get8;
775 ap->ahi_rep_get16 = i_ddi_caut_rep_get16;
776 ap->ahi_rep_get32 = i_ddi_caut_rep_get32;
777 ap->ahi_rep_get64 = i_ddi_caut_rep_get64;
778 }
779 if (DDI_FM_ACC_ERR_CAP(ddi_fm_capable(rdip)) &&
780 mp->map_handlep->ah_acc.devacc_attr_access !=
781 DDI_DEFAULT_ACC) {
782 ndi_fmc_insert(rdip, ACC_HANDLE,
783 (void *)mp->map_handlep, NULL);
784 }
785 }
786 return (retval);
787 }
788
789
790
791 /*ARGSUSED*/
792 static int
npe_ctlops(dev_info_t * dip,dev_info_t * rdip,ddi_ctl_enum_t ctlop,void * arg,void * result)793 npe_ctlops(dev_info_t *dip, dev_info_t *rdip,
794 ddi_ctl_enum_t ctlop, void *arg, void *result)
795 {
796 int totreg;
797 uint_t reglen;
798 pci_regspec_t *drv_regp;
799 struct attachspec *asp;
800 struct detachspec *dsp;
801 pci_state_t *pci_p = ddi_get_soft_state(npe_statep,
802 ddi_get_instance(dip));
803
804 switch (ctlop) {
805 case DDI_CTLOPS_REPORTDEV:
806 if (rdip == (dev_info_t *)0)
807 return (DDI_FAILURE);
808 cmn_err(CE_CONT, "?PCI Express-device: %s@%s, %s%d\n",
809 ddi_node_name(rdip), ddi_get_name_addr(rdip),
810 ddi_driver_name(rdip), ddi_get_instance(rdip));
811 return (DDI_SUCCESS);
812
813 case DDI_CTLOPS_INITCHILD:
814 return (npe_initchild((dev_info_t *)arg));
815
816 case DDI_CTLOPS_UNINITCHILD:
817 return (npe_removechild((dev_info_t *)arg));
818
819 case DDI_CTLOPS_SIDDEV:
820 return (DDI_SUCCESS);
821
822 case DDI_CTLOPS_REGSIZE:
823 case DDI_CTLOPS_NREGS:
824 if (rdip == (dev_info_t *)0)
825 return (DDI_FAILURE);
826
827 *(int *)result = 0;
828 if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, rdip,
829 DDI_PROP_DONTPASS, "reg", (int **)&drv_regp,
830 ®len) != DDI_PROP_SUCCESS) {
831 return (DDI_FAILURE);
832 }
833
834 totreg = (reglen * sizeof (int)) / sizeof (pci_regspec_t);
835 if (ctlop == DDI_CTLOPS_NREGS)
836 *(int *)result = totreg;
837 else if (ctlop == DDI_CTLOPS_REGSIZE) {
838 uint64_t val;
839 int rn;
840
841 rn = *(int *)arg;
842 if (rn >= totreg) {
843 ddi_prop_free(drv_regp);
844 return (DDI_FAILURE);
845 }
846 val = drv_regp[rn].pci_size_low |
847 (uint64_t)drv_regp[rn].pci_size_hi << 32;
848 if (val > OFF_MAX) {
849 int ce = CE_NOTE;
850 #ifdef DEBUG
851 ce = CE_WARN;
852 #endif
853 dev_err(rdip, ce, "failed to get register "
854 "size, value larger than OFF_MAX: 0x%"
855 PRIx64 "\n", val);
856 return (DDI_FAILURE);
857 }
858 *(off_t *)result = (off_t)val;
859 }
860 ddi_prop_free(drv_regp);
861
862 return (DDI_SUCCESS);
863
864 case DDI_CTLOPS_POWER:
865 {
866 power_req_t *reqp = (power_req_t *)arg;
867 /*
868 * We currently understand reporting of PCI_PM_IDLESPEED
869 * capability. Everything else is passed up.
870 */
871 if ((reqp->request_type == PMR_REPORT_PMCAP) &&
872 (reqp->req.report_pmcap_req.cap == PCI_PM_IDLESPEED))
873 return (DDI_SUCCESS);
874
875 break;
876 }
877
878 case DDI_CTLOPS_PEEK:
879 case DDI_CTLOPS_POKE:
880 return (pci_common_peekpoke(dip, rdip, ctlop, arg, result));
881
882 /* X86 systems support PME wakeup from suspended state */
883 case DDI_CTLOPS_ATTACH:
884 if (!pcie_is_child(dip, rdip))
885 return (DDI_SUCCESS);
886
887 asp = (struct attachspec *)arg;
888 if ((asp->when == DDI_POST) && (asp->result == DDI_SUCCESS)) {
889 pf_init(rdip, (void *)pci_p->pci_fm_ibc, asp->cmd);
890 (void) pcie_postattach_child(rdip);
891 }
892
893 /* only do this for immediate children */
894 if (asp->cmd == DDI_RESUME && asp->when == DDI_PRE &&
895 ddi_get_parent(rdip) == dip)
896 if (pci_pre_resume(rdip) != DDI_SUCCESS) {
897 /* Not good, better stop now. */
898 cmn_err(CE_PANIC,
899 "Couldn't pre-resume device %p",
900 (void *) dip);
901 /* NOTREACHED */
902 }
903
904 return (DDI_SUCCESS);
905
906 case DDI_CTLOPS_DETACH:
907 if (!pcie_is_child(dip, rdip))
908 return (DDI_SUCCESS);
909
910 dsp = (struct detachspec *)arg;
911
912 if (dsp->when == DDI_PRE)
913 pf_fini(rdip, dsp->cmd);
914
915 /* only do this for immediate children */
916 if (dsp->cmd == DDI_SUSPEND && dsp->when == DDI_POST &&
917 ddi_get_parent(rdip) == dip)
918 if (pci_post_suspend(rdip) != DDI_SUCCESS)
919 return (DDI_FAILURE);
920
921 return (DDI_SUCCESS);
922
923 default:
924 break;
925 }
926
927 return (ddi_ctlops(dip, rdip, ctlop, arg, result));
928
929 }
930
931
932 /*
933 * npe_intr_ops
934 */
935 static int
npe_intr_ops(dev_info_t * pdip,dev_info_t * rdip,ddi_intr_op_t intr_op,ddi_intr_handle_impl_t * hdlp,void * result)936 npe_intr_ops(dev_info_t *pdip, dev_info_t *rdip, ddi_intr_op_t intr_op,
937 ddi_intr_handle_impl_t *hdlp, void *result)
938 {
939 return (pci_common_intr_ops(pdip, rdip, intr_op, hdlp, result));
940 }
941
942
943 static int
npe_initchild(dev_info_t * child)944 npe_initchild(dev_info_t *child)
945 {
946 char name[80];
947 pcie_bus_t *bus_p;
948 uint32_t regs;
949 ddi_acc_handle_t cfg_hdl;
950
951 /*
952 * Do not bind drivers to empty bridges.
953 * Fail above, if the bridge is found to be hotplug capable
954 */
955 if (npe_disable_empty_bridges_workaround(child) == 1)
956 return (DDI_FAILURE);
957
958 if (pci_common_name_child(child, name, 80) != DDI_SUCCESS)
959 return (DDI_FAILURE);
960
961 ddi_set_name_addr(child, name);
962
963 /*
964 * Pseudo nodes indicate a prototype node with per-instance
965 * properties to be merged into the real h/w device node.
966 * The interpretation of the unit-address is DD[,F]
967 * where DD is the device id and F is the function.
968 */
969 if (ndi_dev_is_persistent_node(child) == 0) {
970 extern int pci_allow_pseudo_children;
971
972 ddi_set_parent_data(child, NULL);
973
974 /*
975 * Try to merge the properties from this prototype
976 * node into real h/w nodes.
977 */
978 if (ndi_merge_node(child, pci_common_name_child) ==
979 DDI_SUCCESS) {
980 /*
981 * Merged ok - return failure to remove the node.
982 */
983 ddi_set_name_addr(child, NULL);
984 return (DDI_FAILURE);
985 }
986
987 /* workaround for DDIVS to run under PCI Express */
988 if (pci_allow_pseudo_children) {
989 /*
990 * If the "interrupts" property doesn't exist,
991 * this must be the ddivs no-intr case, and it returns
992 * DDI_SUCCESS instead of DDI_FAILURE.
993 */
994 if (ddi_prop_get_int(DDI_DEV_T_ANY, child,
995 DDI_PROP_DONTPASS, "interrupts", -1) == -1)
996 return (DDI_SUCCESS);
997 /*
998 * Create the ddi_parent_private_data for a pseudo
999 * child.
1000 */
1001 pci_common_set_parent_private_data(child);
1002 return (DDI_SUCCESS);
1003 }
1004
1005 /*
1006 * The child was not merged into a h/w node,
1007 * but there's not much we can do with it other
1008 * than return failure to cause the node to be removed.
1009 */
1010 cmn_err(CE_WARN, "!%s@%s: %s.conf properties not merged",
1011 ddi_get_name(child), ddi_get_name_addr(child),
1012 ddi_get_name(child));
1013 ddi_set_name_addr(child, NULL);
1014 return (DDI_NOT_WELL_FORMED);
1015 }
1016
1017 if (ddi_prop_get_int(DDI_DEV_T_ANY, child, DDI_PROP_DONTPASS,
1018 "interrupts", -1) != -1)
1019 pci_common_set_parent_private_data(child);
1020 else
1021 ddi_set_parent_data(child, NULL);
1022
1023 /* Disable certain errors on PCIe drivers for x86 platforms */
1024 regs = pcie_get_aer_uce_mask() | npe_aer_uce_mask;
1025 pcie_set_aer_uce_mask(regs);
1026 regs = pcie_get_aer_ce_mask() | npe_aer_ce_mask;
1027 pcie_set_aer_ce_mask(regs);
1028 regs = pcie_get_aer_suce_mask() | npe_aer_suce_mask;
1029 pcie_set_aer_suce_mask(regs);
1030
1031 /*
1032 * If URs are disabled, mask SERRs as well, otherwise the system will
1033 * still be notified of URs
1034 */
1035 if (npe_aer_uce_mask & PCIE_AER_UCE_UR)
1036 pcie_set_serr_mask(1);
1037
1038 if (pci_config_setup(child, &cfg_hdl) == DDI_SUCCESS) {
1039 npe_ck804_fix_aer_ptr(cfg_hdl);
1040 npe_nvidia_error_workaround(cfg_hdl);
1041 npe_intel_error_workaround(cfg_hdl);
1042 pci_config_teardown(&cfg_hdl);
1043 }
1044
1045 bus_p = PCIE_DIP2BUS(child);
1046 if (bus_p) {
1047 uint16_t device_id = (uint16_t)(bus_p->bus_dev_ven_id >> 16);
1048 uint16_t vendor_id = (uint16_t)(bus_p->bus_dev_ven_id & 0xFFFF);
1049 uint16_t rev_id = bus_p->bus_rev_id;
1050
1051 /* Disable AER for certain NVIDIA Chipsets */
1052 if ((vendor_id == NVIDIA_VENDOR_ID) &&
1053 (device_id == NVIDIA_CK804_DEVICE_ID) &&
1054 (rev_id < NVIDIA_CK804_AER_VALID_REVID))
1055 bus_p->bus_aer_off = 0;
1056
1057 pcie_init_dom(child);
1058 (void) pcie_initchild(child);
1059 }
1060
1061 return (DDI_SUCCESS);
1062 }
1063
1064
1065 static int
npe_removechild(dev_info_t * dip)1066 npe_removechild(dev_info_t *dip)
1067 {
1068 pcie_uninitchild(dip);
1069
1070 ddi_set_name_addr(dip, NULL);
1071
1072 /*
1073 * Strip the node to properly convert it back to prototype form
1074 */
1075 ddi_remove_minor_node(dip, NULL);
1076
1077 ddi_prop_remove_all(dip);
1078
1079 return (DDI_SUCCESS);
1080 }
1081
1082 static int
npe_open(dev_t * devp,int flags,int otyp,cred_t * credp)1083 npe_open(dev_t *devp, int flags, int otyp, cred_t *credp)
1084 {
1085 minor_t minor = getminor(*devp);
1086 int instance = PCI_MINOR_NUM_TO_INSTANCE(minor);
1087 pci_state_t *pci_p = ddi_get_soft_state(npe_statep, instance);
1088 int rv;
1089
1090 /*
1091 * Make sure the open is for the right file type.
1092 */
1093 if (otyp != OTYP_CHR)
1094 return (EINVAL);
1095
1096 if (pci_p == NULL)
1097 return (ENXIO);
1098
1099 mutex_enter(&pci_p->pci_mutex);
1100 switch (PCI_MINOR_NUM_TO_PCI_DEVNUM(minor)) {
1101 case PCI_TOOL_REG_MINOR_NUM:
1102 case PCI_TOOL_INTR_MINOR_NUM:
1103 break;
1104 default:
1105 /* Handle devctl ioctls */
1106 rv = pcie_open(pci_p->pci_dip, devp, flags, otyp, credp);
1107 mutex_exit(&pci_p->pci_mutex);
1108 return (rv);
1109 }
1110
1111 /* Handle pcitool ioctls */
1112 if (flags & FEXCL) {
1113 if (pci_p->pci_soft_state != PCI_SOFT_STATE_CLOSED) {
1114 mutex_exit(&pci_p->pci_mutex);
1115 cmn_err(CE_NOTE, "npe_open: busy");
1116 return (EBUSY);
1117 }
1118 pci_p->pci_soft_state = PCI_SOFT_STATE_OPEN_EXCL;
1119 } else {
1120 if (pci_p->pci_soft_state == PCI_SOFT_STATE_OPEN_EXCL) {
1121 mutex_exit(&pci_p->pci_mutex);
1122 cmn_err(CE_NOTE, "npe_open: busy");
1123 return (EBUSY);
1124 }
1125 pci_p->pci_soft_state = PCI_SOFT_STATE_OPEN;
1126 }
1127 mutex_exit(&pci_p->pci_mutex);
1128
1129 return (0);
1130 }
1131
1132 static int
npe_close(dev_t dev,int flags,int otyp,cred_t * credp)1133 npe_close(dev_t dev, int flags, int otyp, cred_t *credp)
1134 {
1135 minor_t minor = getminor(dev);
1136 int instance = PCI_MINOR_NUM_TO_INSTANCE(minor);
1137 pci_state_t *pci_p = ddi_get_soft_state(npe_statep, instance);
1138 int rv;
1139
1140 if (pci_p == NULL)
1141 return (ENXIO);
1142
1143 mutex_enter(&pci_p->pci_mutex);
1144
1145 switch (PCI_MINOR_NUM_TO_PCI_DEVNUM(minor)) {
1146 case PCI_TOOL_REG_MINOR_NUM:
1147 case PCI_TOOL_INTR_MINOR_NUM:
1148 break;
1149 default:
1150 /* Handle devctl ioctls */
1151 rv = pcie_close(pci_p->pci_dip, dev, flags, otyp, credp);
1152 mutex_exit(&pci_p->pci_mutex);
1153 return (rv);
1154 }
1155
1156 /* Handle pcitool ioctls */
1157 pci_p->pci_soft_state = PCI_SOFT_STATE_CLOSED;
1158 mutex_exit(&pci_p->pci_mutex);
1159 return (0);
1160 }
1161
1162 static int
npe_ioctl(dev_t dev,int cmd,intptr_t arg,int mode,cred_t * credp,int * rvalp)1163 npe_ioctl(dev_t dev, int cmd, intptr_t arg, int mode, cred_t *credp, int *rvalp)
1164 {
1165 minor_t minor = getminor(dev);
1166 int instance = PCI_MINOR_NUM_TO_INSTANCE(minor);
1167 pci_state_t *pci_p = ddi_get_soft_state(npe_statep, instance);
1168 int ret = ENOTTY;
1169
1170 if (pci_p == NULL)
1171 return (ENXIO);
1172
1173 switch (PCI_MINOR_NUM_TO_PCI_DEVNUM(minor)) {
1174 case PCI_TOOL_REG_MINOR_NUM:
1175 case PCI_TOOL_INTR_MINOR_NUM:
1176 /* To handle pcitool related ioctls */
1177 ret = pci_common_ioctl(pci_p->pci_dip, dev, cmd, arg, mode,
1178 credp, rvalp);
1179 break;
1180 default:
1181 /* To handle devctl and hotplug related ioctls */
1182 ret = pcie_ioctl(pci_p->pci_dip, dev, cmd, arg, mode, credp,
1183 rvalp);
1184 break;
1185 }
1186
1187 return (ret);
1188 }
1189
1190 /*ARGSUSED*/
1191 static int
npe_fm_init(dev_info_t * dip,dev_info_t * tdip,int cap,ddi_iblock_cookie_t * ibc)1192 npe_fm_init(dev_info_t *dip, dev_info_t *tdip, int cap,
1193 ddi_iblock_cookie_t *ibc)
1194 {
1195 pci_state_t *pcip = ddi_get_soft_state(npe_statep,
1196 ddi_get_instance(dip));
1197
1198 ASSERT(ibc != NULL);
1199 *ibc = pcip->pci_fm_ibc;
1200
1201 return (pcip->pci_fmcap);
1202 }
1203
1204 static int
npe_bus_get_eventcookie(dev_info_t * dip,dev_info_t * rdip,char * eventname,ddi_eventcookie_t * cookiep)1205 npe_bus_get_eventcookie(dev_info_t *dip, dev_info_t *rdip, char *eventname,
1206 ddi_eventcookie_t *cookiep)
1207 {
1208 pci_state_t *pcip = ddi_get_soft_state(npe_statep,
1209 ddi_get_instance(dip));
1210
1211 return (ndi_event_retrieve_cookie(pcip->pci_ndi_event_hdl, rdip,
1212 eventname, cookiep, NDI_EVENT_NOPASS));
1213 }
1214
1215 static int
npe_bus_add_eventcall(dev_info_t * dip,dev_info_t * rdip,ddi_eventcookie_t cookie,void (* callback)(dev_info_t * dip,ddi_eventcookie_t cookie,void * arg,void * bus_impldata),void * arg,ddi_callback_id_t * cb_id)1216 npe_bus_add_eventcall(dev_info_t *dip, dev_info_t *rdip,
1217 ddi_eventcookie_t cookie, void (*callback)(dev_info_t *dip,
1218 ddi_eventcookie_t cookie, void *arg, void *bus_impldata),
1219 void *arg, ddi_callback_id_t *cb_id)
1220 {
1221 pci_state_t *pcip = ddi_get_soft_state(npe_statep,
1222 ddi_get_instance(dip));
1223
1224 return (ndi_event_add_callback(pcip->pci_ndi_event_hdl, rdip, cookie,
1225 callback, arg, NDI_SLEEP, cb_id));
1226 }
1227
1228 static int
npe_bus_remove_eventcall(dev_info_t * dip,ddi_callback_id_t cb_id)1229 npe_bus_remove_eventcall(dev_info_t *dip, ddi_callback_id_t cb_id)
1230 {
1231 pci_state_t *pcip = ddi_get_soft_state(npe_statep,
1232 ddi_get_instance(dip));
1233 return (ndi_event_remove_callback(pcip->pci_ndi_event_hdl, cb_id));
1234 }
1235
1236 static int
npe_bus_post_event(dev_info_t * dip,dev_info_t * rdip,ddi_eventcookie_t cookie,void * impl_data)1237 npe_bus_post_event(dev_info_t *dip, dev_info_t *rdip,
1238 ddi_eventcookie_t cookie, void *impl_data)
1239 {
1240 pci_state_t *pcip = ddi_get_soft_state(npe_statep,
1241 ddi_get_instance(dip));
1242 return (ndi_event_do_callback(pcip->pci_ndi_event_hdl, rdip, cookie,
1243 impl_data));
1244
1245 }
1246
1247 /*ARGSUSED*/
1248 static int
npe_fm_callback(dev_info_t * dip,ddi_fm_error_t * derr,const void * no_used)1249 npe_fm_callback(dev_info_t *dip, ddi_fm_error_t *derr, const void *no_used)
1250 {
1251 /*
1252 * On current x86 systems, npe's callback does not get called for failed
1253 * loads. If in the future this feature is used, the fault PA should be
1254 * logged in the derr->fme_bus_specific field. The appropriate PCIe
1255 * error handling code should be called and needs to be coordinated with
1256 * safe access handling.
1257 */
1258
1259 return (DDI_FM_OK);
1260 }
1261