1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22/*
23 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright 2019 Joyent, Inc.
25 */
26
27#include <sys/sysmacros.h>
28#include <sys/types.h>
29#include <sys/kmem.h>
30#include <sys/modctl.h>
31#include <sys/ddi.h>
32#include <sys/sunddi.h>
33#include <sys/sunndi.h>
34#include <sys/fm/protocol.h>
35#include <sys/fm/util.h>
36#include <sys/promif.h>
37#include <sys/disp.h>
38#include <sys/stat.h>
39#include <sys/file.h>
40#include <sys/pci_cap.h>
41#include <sys/pci_impl.h>
42#include <sys/pcie_impl.h>
43#include <sys/hotplug/pci/pcie_hp.h>
44#include <sys/hotplug/pci/pciehpc.h>
45#include <sys/hotplug/pci/pcishpc.h>
46#include <sys/hotplug/pci/pcicfg.h>
47#include <sys/pci_cfgacc.h>
48#include <sys/sysevent.h>
49#include <sys/sysevent/eventdefs.h>
50#include <sys/sysevent/pcie.h>
51
52/* Local functions prototypes */
53static void pcie_init_pfd(dev_info_t *);
54static void pcie_fini_pfd(dev_info_t *);
55
56#if defined(__i386) || defined(__amd64)
57static void pcie_check_io_mem_range(ddi_acc_handle_t, boolean_t *, boolean_t *);
58#endif /* defined(__i386) || defined(__amd64) */
59
60#ifdef DEBUG
61uint_t pcie_debug_flags = 0;
62static void pcie_print_bus(pcie_bus_t *bus_p);
63void pcie_dbg(char *fmt, ...);
64#endif /* DEBUG */
65
66/* Variable to control default PCI-Express config settings */
67ushort_t pcie_command_default =
68    PCI_COMM_SERR_ENABLE |
69    PCI_COMM_WAIT_CYC_ENAB |
70    PCI_COMM_PARITY_DETECT |
71    PCI_COMM_ME |
72    PCI_COMM_MAE |
73    PCI_COMM_IO;
74
75/* xxx_fw are bits that are controlled by FW and should not be modified */
76ushort_t pcie_command_default_fw =
77    PCI_COMM_SPEC_CYC |
78    PCI_COMM_MEMWR_INVAL |
79    PCI_COMM_PALETTE_SNOOP |
80    PCI_COMM_WAIT_CYC_ENAB |
81    0xF800; /* Reserved Bits */
82
83ushort_t pcie_bdg_command_default_fw =
84    PCI_BCNF_BCNTRL_ISA_ENABLE |
85    PCI_BCNF_BCNTRL_VGA_ENABLE |
86    0xF000; /* Reserved Bits */
87
88/* PCI-Express Base error defaults */
89ushort_t pcie_base_err_default =
90    PCIE_DEVCTL_CE_REPORTING_EN |
91    PCIE_DEVCTL_NFE_REPORTING_EN |
92    PCIE_DEVCTL_FE_REPORTING_EN |
93    PCIE_DEVCTL_UR_REPORTING_EN;
94
95/* PCI-Express Device Control Register */
96uint16_t pcie_devctl_default = PCIE_DEVCTL_RO_EN |
97    PCIE_DEVCTL_MAX_READ_REQ_512;
98
99/* PCI-Express AER Root Control Register */
100#define	PCIE_ROOT_SYS_ERR	(PCIE_ROOTCTL_SYS_ERR_ON_CE_EN | \
101				PCIE_ROOTCTL_SYS_ERR_ON_NFE_EN | \
102				PCIE_ROOTCTL_SYS_ERR_ON_FE_EN)
103
104ushort_t pcie_root_ctrl_default =
105    PCIE_ROOTCTL_SYS_ERR_ON_CE_EN |
106    PCIE_ROOTCTL_SYS_ERR_ON_NFE_EN |
107    PCIE_ROOTCTL_SYS_ERR_ON_FE_EN;
108
109/* PCI-Express Root Error Command Register */
110ushort_t pcie_root_error_cmd_default =
111    PCIE_AER_RE_CMD_CE_REP_EN |
112    PCIE_AER_RE_CMD_NFE_REP_EN |
113    PCIE_AER_RE_CMD_FE_REP_EN;
114
115/* ECRC settings in the PCIe AER Control Register */
116uint32_t pcie_ecrc_value =
117    PCIE_AER_CTL_ECRC_GEN_ENA |
118    PCIE_AER_CTL_ECRC_CHECK_ENA;
119
120/*
121 * If a particular platform wants to disable certain errors such as UR/MA,
122 * instead of using #defines have the platform's PCIe Root Complex driver set
123 * these masks using the pcie_get_XXX_mask and pcie_set_XXX_mask functions.  For
124 * x86 the closest thing to a PCIe root complex driver is NPE.	For SPARC the
125 * closest PCIe root complex driver is PX.
126 *
127 * pcie_serr_disable_flag : disable SERR only (in RCR and command reg) x86
128 * systems may want to disable SERR in general.  For root ports, enabling SERR
129 * causes NMIs which are not handled and results in a watchdog timeout error.
130 */
131uint32_t pcie_aer_uce_mask = 0;		/* AER UE Mask */
132uint32_t pcie_aer_ce_mask = 0;		/* AER CE Mask */
133uint32_t pcie_aer_suce_mask = 0;	/* AER Secondary UE Mask */
134uint32_t pcie_serr_disable_flag = 0;	/* Disable SERR */
135
136/* Default severities needed for eversholt.  Error handling doesn't care */
137uint32_t pcie_aer_uce_severity = PCIE_AER_UCE_MTLP | PCIE_AER_UCE_RO | \
138    PCIE_AER_UCE_FCP | PCIE_AER_UCE_SD | PCIE_AER_UCE_DLP | \
139    PCIE_AER_UCE_TRAINING;
140uint32_t pcie_aer_suce_severity = PCIE_AER_SUCE_SERR_ASSERT | \
141    PCIE_AER_SUCE_UC_ADDR_ERR | PCIE_AER_SUCE_UC_ATTR_ERR | \
142    PCIE_AER_SUCE_USC_MSG_DATA_ERR;
143
144int pcie_max_mps = PCIE_DEVCTL_MAX_PAYLOAD_4096 >> 5;
145int pcie_disable_ari = 0;
146
147/*
148 * On some platforms, such as the AMD B450 chipset, we've seen an odd
149 * relationship between enabling link bandwidth notifications and AERs about
150 * ECRC errors. This provides a mechanism to disable it.
151 */
152int pcie_disable_lbw = 0;
153
154/*
155 * Amount of time to wait for an in-progress retraining. The default is to try
156 * 500 times in 10ms chunks, thus a total of 5s.
157 */
158uint32_t pcie_link_retrain_count = 500;
159uint32_t pcie_link_retrain_delay_ms = 10;
160
161taskq_t *pcie_link_tq;
162kmutex_t pcie_link_tq_mutex;
163
164static void pcie_scan_mps(dev_info_t *rc_dip, dev_info_t *dip,
165	int *max_supported);
166static int pcie_get_max_supported(dev_info_t *dip, void *arg);
167static int pcie_map_phys(dev_info_t *dip, pci_regspec_t *phys_spec,
168    caddr_t *addrp, ddi_acc_handle_t *handlep);
169static void pcie_unmap_phys(ddi_acc_handle_t *handlep,	pci_regspec_t *ph);
170static int pcie_link_bw_intr(dev_info_t *);
171static void pcie_capture_speeds(dev_info_t *);
172
173dev_info_t *pcie_get_rc_dip(dev_info_t *dip);
174
175/*
176 * modload support
177 */
178
179static struct modlmisc modlmisc	= {
180	&mod_miscops,	/* Type	of module */
181	"PCI Express Framework Module"
182};
183
184static struct modlinkage modlinkage = {
185	MODREV_1,
186	(void	*)&modlmisc,
187	NULL
188};
189
190/*
191 * Global Variables needed for a non-atomic version of ddi_fm_ereport_post.
192 * Currently used to send the pci.fabric ereports whose payload depends on the
193 * type of PCI device it is being sent for.
194 */
195char		*pcie_nv_buf;
196nv_alloc_t	*pcie_nvap;
197nvlist_t	*pcie_nvl;
198
199int
200_init(void)
201{
202	int rval;
203
204	pcie_nv_buf = kmem_alloc(ERPT_DATA_SZ, KM_SLEEP);
205	pcie_nvap = fm_nva_xcreate(pcie_nv_buf, ERPT_DATA_SZ);
206	pcie_nvl = fm_nvlist_create(pcie_nvap);
207	mutex_init(&pcie_link_tq_mutex, NULL, MUTEX_DRIVER, NULL);
208
209	if ((rval = mod_install(&modlinkage)) != 0) {
210		mutex_destroy(&pcie_link_tq_mutex);
211		fm_nvlist_destroy(pcie_nvl, FM_NVA_RETAIN);
212		fm_nva_xdestroy(pcie_nvap);
213		kmem_free(pcie_nv_buf, ERPT_DATA_SZ);
214	}
215	return (rval);
216}
217
218int
219_fini()
220{
221	int		rval;
222
223	if ((rval = mod_remove(&modlinkage)) == 0) {
224		if (pcie_link_tq != NULL) {
225			taskq_destroy(pcie_link_tq);
226		}
227		mutex_destroy(&pcie_link_tq_mutex);
228		fm_nvlist_destroy(pcie_nvl, FM_NVA_RETAIN);
229		fm_nva_xdestroy(pcie_nvap);
230		kmem_free(pcie_nv_buf, ERPT_DATA_SZ);
231	}
232	return (rval);
233}
234
235int
236_info(struct modinfo *modinfop)
237{
238	return (mod_info(&modlinkage, modinfop));
239}
240
241/* ARGSUSED */
242int
243pcie_init(dev_info_t *dip, caddr_t arg)
244{
245	int	ret = DDI_SUCCESS;
246
247	/*
248	 * Our _init function is too early to create a taskq. Create the pcie
249	 * link management taskq here now instead.
250	 */
251	mutex_enter(&pcie_link_tq_mutex);
252	if (pcie_link_tq == NULL) {
253		pcie_link_tq = taskq_create("pcie_link", 1, minclsyspri, 0, 0,
254		    0);
255	}
256	mutex_exit(&pcie_link_tq_mutex);
257
258
259	/*
260	 * Create a "devctl" minor node to support DEVCTL_DEVICE_*
261	 * and DEVCTL_BUS_* ioctls to this bus.
262	 */
263	if ((ret = ddi_create_minor_node(dip, "devctl", S_IFCHR,
264	    PCI_MINOR_NUM(ddi_get_instance(dip), PCI_DEVCTL_MINOR),
265	    DDI_NT_NEXUS, 0)) != DDI_SUCCESS) {
266		PCIE_DBG("Failed to create devctl minor node for %s%d\n",
267		    ddi_driver_name(dip), ddi_get_instance(dip));
268
269		return (ret);
270	}
271
272	if ((ret = pcie_hp_init(dip, arg)) != DDI_SUCCESS) {
273		/*
274		 * On some x86 platforms, we observed unexpected hotplug
275		 * initialization failures in recent years. The known cause
276		 * is a hardware issue: while the problem PCI bridges have
277		 * the Hotplug Capable registers set, the machine actually
278		 * does not implement the expected ACPI object.
279		 *
280		 * We don't want to stop PCI driver attach and system boot
281		 * just because of this hotplug initialization failure.
282		 * Continue with a debug message printed.
283		 */
284		PCIE_DBG("%s%d: Failed setting hotplug framework\n",
285		    ddi_driver_name(dip), ddi_get_instance(dip));
286
287#if defined(__sparc)
288		ddi_remove_minor_node(dip, "devctl");
289
290		return (ret);
291#endif /* defined(__sparc) */
292	}
293
294	return (DDI_SUCCESS);
295}
296
297/* ARGSUSED */
298int
299pcie_uninit(dev_info_t *dip)
300{
301	int	ret = DDI_SUCCESS;
302
303	if (pcie_ari_is_enabled(dip) == PCIE_ARI_FORW_ENABLED)
304		(void) pcie_ari_disable(dip);
305
306	if ((ret = pcie_hp_uninit(dip)) != DDI_SUCCESS) {
307		PCIE_DBG("Failed to uninitialize hotplug for %s%d\n",
308		    ddi_driver_name(dip), ddi_get_instance(dip));
309
310		return (ret);
311	}
312
313	if (pcie_link_bw_supported(dip)) {
314		(void) pcie_link_bw_disable(dip);
315	}
316
317	ddi_remove_minor_node(dip, "devctl");
318
319	return (ret);
320}
321
322/*
323 * PCIe module interface for enabling hotplug interrupt.
324 *
325 * It should be called after pcie_init() is done and bus driver's
326 * interrupt handlers have being attached.
327 */
328int
329pcie_hpintr_enable(dev_info_t *dip)
330{
331	pcie_bus_t	*bus_p = PCIE_DIP2BUS(dip);
332	pcie_hp_ctrl_t	*ctrl_p = PCIE_GET_HP_CTRL(dip);
333
334	if (PCIE_IS_PCIE_HOTPLUG_ENABLED(bus_p)) {
335		(void) (ctrl_p->hc_ops.enable_hpc_intr)(ctrl_p);
336	} else if (PCIE_IS_PCI_HOTPLUG_ENABLED(bus_p)) {
337		(void) pcishpc_enable_irqs(ctrl_p);
338	}
339	return (DDI_SUCCESS);
340}
341
342/*
343 * PCIe module interface for disabling hotplug interrupt.
344 *
345 * It should be called before pcie_uninit() is called and bus driver's
346 * interrupt handlers is dettached.
347 */
348int
349pcie_hpintr_disable(dev_info_t *dip)
350{
351	pcie_bus_t	*bus_p = PCIE_DIP2BUS(dip);
352	pcie_hp_ctrl_t	*ctrl_p = PCIE_GET_HP_CTRL(dip);
353
354	if (PCIE_IS_PCIE_HOTPLUG_ENABLED(bus_p)) {
355		(void) (ctrl_p->hc_ops.disable_hpc_intr)(ctrl_p);
356	} else if (PCIE_IS_PCI_HOTPLUG_ENABLED(bus_p)) {
357		(void) pcishpc_disable_irqs(ctrl_p);
358	}
359	return (DDI_SUCCESS);
360}
361
362/* ARGSUSED */
363int
364pcie_intr(dev_info_t *dip)
365{
366	int hp, lbw;
367
368	hp = pcie_hp_intr(dip);
369	lbw = pcie_link_bw_intr(dip);
370
371	if (hp == DDI_INTR_CLAIMED || lbw == DDI_INTR_CLAIMED) {
372		return (DDI_INTR_CLAIMED);
373	}
374
375	return (DDI_INTR_UNCLAIMED);
376}
377
378/* ARGSUSED */
379int
380pcie_open(dev_info_t *dip, dev_t *devp, int flags, int otyp, cred_t *credp)
381{
382	pcie_bus_t	*bus_p = PCIE_DIP2BUS(dip);
383
384	/*
385	 * Make sure the open is for the right file type.
386	 */
387	if (otyp != OTYP_CHR)
388		return (EINVAL);
389
390	/*
391	 * Handle the open by tracking the device state.
392	 */
393	if ((bus_p->bus_soft_state == PCI_SOFT_STATE_OPEN_EXCL) ||
394	    ((flags & FEXCL) &&
395	    (bus_p->bus_soft_state != PCI_SOFT_STATE_CLOSED))) {
396		return (EBUSY);
397	}
398
399	if (flags & FEXCL)
400		bus_p->bus_soft_state = PCI_SOFT_STATE_OPEN_EXCL;
401	else
402		bus_p->bus_soft_state = PCI_SOFT_STATE_OPEN;
403
404	return (0);
405}
406
407/* ARGSUSED */
408int
409pcie_close(dev_info_t *dip, dev_t dev, int flags, int otyp, cred_t *credp)
410{
411	pcie_bus_t	*bus_p = PCIE_DIP2BUS(dip);
412
413	if (otyp != OTYP_CHR)
414		return (EINVAL);
415
416	bus_p->bus_soft_state = PCI_SOFT_STATE_CLOSED;
417
418	return (0);
419}
420
421/* ARGSUSED */
422int
423pcie_ioctl(dev_info_t *dip, dev_t dev, int cmd, intptr_t arg, int mode,
424    cred_t *credp, int *rvalp)
425{
426	struct devctl_iocdata	*dcp;
427	uint_t			bus_state;
428	int			rv = DDI_SUCCESS;
429
430	/*
431	 * We can use the generic implementation for devctl ioctl
432	 */
433	switch (cmd) {
434	case DEVCTL_DEVICE_GETSTATE:
435	case DEVCTL_DEVICE_ONLINE:
436	case DEVCTL_DEVICE_OFFLINE:
437	case DEVCTL_BUS_GETSTATE:
438		return (ndi_devctl_ioctl(dip, cmd, arg, mode, 0));
439	default:
440		break;
441	}
442
443	/*
444	 * read devctl ioctl data
445	 */
446	if (ndi_dc_allochdl((void *)arg, &dcp) != NDI_SUCCESS)
447		return (EFAULT);
448
449	switch (cmd) {
450	case DEVCTL_BUS_QUIESCE:
451		if (ndi_get_bus_state(dip, &bus_state) == NDI_SUCCESS)
452			if (bus_state == BUS_QUIESCED)
453				break;
454		(void) ndi_set_bus_state(dip, BUS_QUIESCED);
455		break;
456	case DEVCTL_BUS_UNQUIESCE:
457		if (ndi_get_bus_state(dip, &bus_state) == NDI_SUCCESS)
458			if (bus_state == BUS_ACTIVE)
459				break;
460		(void) ndi_set_bus_state(dip, BUS_ACTIVE);
461		break;
462	case DEVCTL_BUS_RESET:
463	case DEVCTL_BUS_RESETALL:
464	case DEVCTL_DEVICE_RESET:
465		rv = ENOTSUP;
466		break;
467	default:
468		rv = ENOTTY;
469	}
470
471	ndi_dc_freehdl(dcp);
472	return (rv);
473}
474
475/* ARGSUSED */
476int
477pcie_prop_op(dev_t dev, dev_info_t *dip, ddi_prop_op_t prop_op,
478    int flags, char *name, caddr_t valuep, int *lengthp)
479{
480	if (dev == DDI_DEV_T_ANY)
481		goto skip;
482
483	if (PCIE_IS_HOTPLUG_CAPABLE(dip) &&
484	    strcmp(name, "pci-occupant") == 0) {
485		int	pci_dev = PCI_MINOR_NUM_TO_PCI_DEVNUM(getminor(dev));
486
487		pcie_hp_create_occupant_props(dip, dev, pci_dev);
488	}
489
490skip:
491	return (ddi_prop_op(dev, dip, prop_op, flags, name, valuep, lengthp));
492}
493
494int
495pcie_init_cfghdl(dev_info_t *cdip)
496{
497	pcie_bus_t		*bus_p;
498	ddi_acc_handle_t	eh = NULL;
499
500	bus_p = PCIE_DIP2BUS(cdip);
501	if (bus_p == NULL)
502		return (DDI_FAILURE);
503
504	/* Create an config access special to error handling */
505	if (pci_config_setup(cdip, &eh) != DDI_SUCCESS) {
506		cmn_err(CE_WARN, "Cannot setup config access"
507		    " for BDF 0x%x\n", bus_p->bus_bdf);
508		return (DDI_FAILURE);
509	}
510
511	bus_p->bus_cfg_hdl = eh;
512	return (DDI_SUCCESS);
513}
514
515void
516pcie_fini_cfghdl(dev_info_t *cdip)
517{
518	pcie_bus_t	*bus_p = PCIE_DIP2BUS(cdip);
519
520	pci_config_teardown(&bus_p->bus_cfg_hdl);
521}
522
523void
524pcie_determine_serial(dev_info_t *dip)
525{
526	pcie_bus_t		*bus_p = PCIE_DIP2BUS(dip);
527	ddi_acc_handle_t	h;
528	uint16_t		cap;
529	uchar_t			serial[8];
530	uint32_t		low, high;
531
532	if (!PCIE_IS_PCIE(bus_p))
533		return;
534
535	h = bus_p->bus_cfg_hdl;
536
537	if ((PCI_CAP_LOCATE(h, PCI_CAP_XCFG_SPC(PCIE_EXT_CAP_ID_SER), &cap)) ==
538	    DDI_FAILURE)
539		return;
540
541	high = PCI_XCAP_GET32(h, 0, cap, PCIE_SER_SID_UPPER_DW);
542	low = PCI_XCAP_GET32(h, 0, cap, PCIE_SER_SID_LOWER_DW);
543
544	/*
545	 * Here, we're trying to figure out if we had an invalid PCIe read. From
546	 * looking at the contents of the value, it can be hard to tell the
547	 * difference between a value that has all 1s correctly versus if we had
548	 * an error. In this case, we only assume it's invalid if both register
549	 * reads are invalid. We also only use 32-bit reads as we're not sure if
550	 * all devices will support these as 64-bit reads, while we know that
551	 * they'll support these as 32-bit reads.
552	 */
553	if (high == PCI_EINVAL32 && low == PCI_EINVAL32)
554		return;
555
556	serial[0] = low & 0xff;
557	serial[1] = (low >> 8) & 0xff;
558	serial[2] = (low >> 16) & 0xff;
559	serial[3] = (low >> 24) & 0xff;
560	serial[4] = high & 0xff;
561	serial[5] = (high >> 8) & 0xff;
562	serial[6] = (high >> 16) & 0xff;
563	serial[7] = (high >> 24) & 0xff;
564
565	(void) ndi_prop_update_byte_array(DDI_DEV_T_NONE, dip, "pcie-serial",
566	    serial, sizeof (serial));
567}
568
569static void
570pcie_determine_aspm(dev_info_t *dip)
571{
572	pcie_bus_t	*bus_p = PCIE_DIP2BUS(dip);
573	uint32_t	linkcap;
574	uint16_t	linkctl;
575
576	if (!PCIE_IS_PCIE(bus_p))
577		return;
578
579	linkcap = PCIE_CAP_GET(32, bus_p, PCIE_LINKCAP);
580	linkctl = PCIE_CAP_GET(16, bus_p, PCIE_LINKCTL);
581
582	switch (linkcap & PCIE_LINKCAP_ASPM_SUP_MASK) {
583	case PCIE_LINKCAP_ASPM_SUP_L0S:
584		(void) ndi_prop_update_string(DDI_DEV_T_NONE, dip,
585		    "pcie-aspm-support", "l0s");
586		break;
587	case PCIE_LINKCAP_ASPM_SUP_L1:
588		(void) ndi_prop_update_string(DDI_DEV_T_NONE, dip,
589		    "pcie-aspm-support", "l1");
590		break;
591	case PCIE_LINKCAP_ASPM_SUP_L0S_L1:
592		(void) ndi_prop_update_string(DDI_DEV_T_NONE, dip,
593		    "pcie-aspm-support", "l0s,l1");
594		break;
595	default:
596		return;
597	}
598
599	switch (linkctl & PCIE_LINKCTL_ASPM_CTL_MASK) {
600	case PCIE_LINKCTL_ASPM_CTL_DIS:
601		(void) ndi_prop_update_string(DDI_DEV_T_NONE, dip,
602		    "pcie-aspm-state", "disabled");
603		break;
604	case PCIE_LINKCTL_ASPM_CTL_L0S:
605		(void) ndi_prop_update_string(DDI_DEV_T_NONE, dip,
606		    "pcie-aspm-state", "l0s");
607		break;
608	case PCIE_LINKCTL_ASPM_CTL_L1:
609		(void) ndi_prop_update_string(DDI_DEV_T_NONE, dip,
610		    "pcie-aspm-state", "l1");
611		break;
612	case PCIE_LINKCTL_ASPM_CTL_L0S_L1:
613		(void) ndi_prop_update_string(DDI_DEV_T_NONE, dip,
614		    "pcie-aspm-state", "l0s,l1");
615		break;
616	}
617}
618
619/*
620 * PCI-Express child device initialization.
621 * This function enables generic pci-express interrupts and error
622 * handling.
623 *
624 * @param pdip		root dip (root nexus's dip)
625 * @param cdip		child's dip (device's dip)
626 * @return		DDI_SUCCESS or DDI_FAILURE
627 */
628/* ARGSUSED */
629int
630pcie_initchild(dev_info_t *cdip)
631{
632	uint16_t		tmp16, reg16;
633	pcie_bus_t		*bus_p;
634	uint32_t		devid, venid;
635
636	bus_p = PCIE_DIP2BUS(cdip);
637	if (bus_p == NULL) {
638		PCIE_DBG("%s: BUS not found.\n",
639		    ddi_driver_name(cdip));
640
641		return (DDI_FAILURE);
642	}
643
644	if (pcie_init_cfghdl(cdip) != DDI_SUCCESS)
645		return (DDI_FAILURE);
646
647	/*
648	 * Update pcie_bus_t with real Vendor Id Device Id.
649	 *
650	 * For assigned devices in IOV environment, the OBP will return
651	 * faked device id/vendor id on configration read and for both
652	 * properties in root domain. translate_devid() function will
653	 * update the properties with real device-id/vendor-id on such
654	 * platforms, so that we can utilize the properties here to get
655	 * real device-id/vendor-id and overwrite the faked ids.
656	 *
657	 * For unassigned devices or devices in non-IOV environment, the
658	 * operation below won't make a difference.
659	 *
660	 * The IOV implementation only supports assignment of PCIE
661	 * endpoint devices. Devices under pci-pci bridges don't need
662	 * operation like this.
663	 */
664	devid = ddi_prop_get_int(DDI_DEV_T_ANY, cdip, DDI_PROP_DONTPASS,
665	    "device-id", -1);
666	venid = ddi_prop_get_int(DDI_DEV_T_ANY, cdip, DDI_PROP_DONTPASS,
667	    "vendor-id", -1);
668	bus_p->bus_dev_ven_id = (devid << 16) | (venid & 0xffff);
669
670	/* Clear the device's status register */
671	reg16 = PCIE_GET(16, bus_p, PCI_CONF_STAT);
672	PCIE_PUT(16, bus_p, PCI_CONF_STAT, reg16);
673
674	/* Setup the device's command register */
675	reg16 = PCIE_GET(16, bus_p, PCI_CONF_COMM);
676	tmp16 = (reg16 & pcie_command_default_fw) | pcie_command_default;
677
678#if defined(__i386) || defined(__amd64)
679	boolean_t empty_io_range = B_FALSE;
680	boolean_t empty_mem_range = B_FALSE;
681	/*
682	 * Check for empty IO and Mem ranges on bridges. If so disable IO/Mem
683	 * access as it can cause a hang if enabled.
684	 */
685	pcie_check_io_mem_range(bus_p->bus_cfg_hdl, &empty_io_range,
686	    &empty_mem_range);
687	if ((empty_io_range == B_TRUE) &&
688	    (pcie_command_default & PCI_COMM_IO)) {
689		tmp16 &= ~PCI_COMM_IO;
690		PCIE_DBG("No I/O range found for %s, bdf 0x%x\n",
691		    ddi_driver_name(cdip), bus_p->bus_bdf);
692	}
693	if ((empty_mem_range == B_TRUE) &&
694	    (pcie_command_default & PCI_COMM_MAE)) {
695		tmp16 &= ~PCI_COMM_MAE;
696		PCIE_DBG("No Mem range found for %s, bdf 0x%x\n",
697		    ddi_driver_name(cdip), bus_p->bus_bdf);
698	}
699#endif /* defined(__i386) || defined(__amd64) */
700
701	if (pcie_serr_disable_flag && PCIE_IS_PCIE(bus_p))
702		tmp16 &= ~PCI_COMM_SERR_ENABLE;
703
704	PCIE_PUT(16, bus_p, PCI_CONF_COMM, tmp16);
705	PCIE_DBG_CFG(cdip, bus_p, "COMMAND", 16, PCI_CONF_COMM, reg16);
706
707	/*
708	 * If the device has a bus control register then program it
709	 * based on the settings in the command register.
710	 */
711	if (PCIE_IS_BDG(bus_p)) {
712		/* Clear the device's secondary status register */
713		reg16 = PCIE_GET(16, bus_p, PCI_BCNF_SEC_STATUS);
714		PCIE_PUT(16, bus_p, PCI_BCNF_SEC_STATUS, reg16);
715
716		/* Setup the device's secondary command register */
717		reg16 = PCIE_GET(16, bus_p, PCI_BCNF_BCNTRL);
718		tmp16 = (reg16 & pcie_bdg_command_default_fw);
719
720		tmp16 |= PCI_BCNF_BCNTRL_SERR_ENABLE;
721		/*
722		 * Workaround for this Nvidia bridge. Don't enable the SERR
723		 * enable bit in the bridge control register as it could lead to
724		 * bogus NMIs.
725		 */
726		if (bus_p->bus_dev_ven_id == 0x037010DE)
727			tmp16 &= ~PCI_BCNF_BCNTRL_SERR_ENABLE;
728
729		if (pcie_command_default & PCI_COMM_PARITY_DETECT)
730			tmp16 |= PCI_BCNF_BCNTRL_PARITY_ENABLE;
731
732		/*
733		 * Enable Master Abort Mode only if URs have not been masked.
734		 * For PCI and PCIe-PCI bridges, enabling this bit causes a
735		 * Master Aborts/UR to be forwarded as a UR/TA or SERR.  If this
736		 * bit is masked, posted requests are dropped and non-posted
737		 * requests are returned with -1.
738		 */
739		if (pcie_aer_uce_mask & PCIE_AER_UCE_UR)
740			tmp16 &= ~PCI_BCNF_BCNTRL_MAST_AB_MODE;
741		else
742			tmp16 |= PCI_BCNF_BCNTRL_MAST_AB_MODE;
743		PCIE_PUT(16, bus_p, PCI_BCNF_BCNTRL, tmp16);
744		PCIE_DBG_CFG(cdip, bus_p, "SEC CMD", 16, PCI_BCNF_BCNTRL,
745		    reg16);
746	}
747
748	if (PCIE_IS_PCIE(bus_p)) {
749		/* Setup PCIe device control register */
750		reg16 = PCIE_CAP_GET(16, bus_p, PCIE_DEVCTL);
751		/* note: MPS/MRRS are initialized in pcie_initchild_mps() */
752		tmp16 = (reg16 & (PCIE_DEVCTL_MAX_READ_REQ_MASK |
753		    PCIE_DEVCTL_MAX_PAYLOAD_MASK)) |
754		    (pcie_devctl_default & ~(PCIE_DEVCTL_MAX_READ_REQ_MASK |
755		    PCIE_DEVCTL_MAX_PAYLOAD_MASK));
756		PCIE_CAP_PUT(16, bus_p, PCIE_DEVCTL, tmp16);
757		PCIE_DBG_CAP(cdip, bus_p, "DEVCTL", 16, PCIE_DEVCTL, reg16);
758
759		/* Enable PCIe errors */
760		pcie_enable_errors(cdip);
761
762		pcie_determine_serial(cdip);
763
764		pcie_determine_aspm(cdip);
765
766		pcie_capture_speeds(cdip);
767	}
768
769	bus_p->bus_ari = B_FALSE;
770	if ((pcie_ari_is_enabled(ddi_get_parent(cdip))
771	    == PCIE_ARI_FORW_ENABLED) && (pcie_ari_device(cdip)
772	    == PCIE_ARI_DEVICE)) {
773		bus_p->bus_ari = B_TRUE;
774	}
775
776	if (pcie_initchild_mps(cdip) == DDI_FAILURE) {
777		pcie_fini_cfghdl(cdip);
778		return (DDI_FAILURE);
779	}
780
781	return (DDI_SUCCESS);
782}
783
784static void
785pcie_init_pfd(dev_info_t *dip)
786{
787	pf_data_t	*pfd_p = PCIE_ZALLOC(pf_data_t);
788	pcie_bus_t	*bus_p = PCIE_DIP2BUS(dip);
789
790	PCIE_DIP2PFD(dip) = pfd_p;
791
792	pfd_p->pe_bus_p = bus_p;
793	pfd_p->pe_severity_flags = 0;
794	pfd_p->pe_severity_mask = 0;
795	pfd_p->pe_orig_severity_flags = 0;
796	pfd_p->pe_lock = B_FALSE;
797	pfd_p->pe_valid = B_FALSE;
798
799	/* Allocate the root fault struct for both RC and RP */
800	if (PCIE_IS_ROOT(bus_p)) {
801		PCIE_ROOT_FAULT(pfd_p) = PCIE_ZALLOC(pf_root_fault_t);
802		PCIE_ROOT_FAULT(pfd_p)->scan_bdf = PCIE_INVALID_BDF;
803		PCIE_ROOT_EH_SRC(pfd_p) = PCIE_ZALLOC(pf_root_eh_src_t);
804	}
805
806	PCI_ERR_REG(pfd_p) = PCIE_ZALLOC(pf_pci_err_regs_t);
807	PFD_AFFECTED_DEV(pfd_p) = PCIE_ZALLOC(pf_affected_dev_t);
808	PFD_AFFECTED_DEV(pfd_p)->pe_affected_bdf = PCIE_INVALID_BDF;
809
810	if (PCIE_IS_BDG(bus_p))
811		PCI_BDG_ERR_REG(pfd_p) = PCIE_ZALLOC(pf_pci_bdg_err_regs_t);
812
813	if (PCIE_IS_PCIE(bus_p)) {
814		PCIE_ERR_REG(pfd_p) = PCIE_ZALLOC(pf_pcie_err_regs_t);
815
816		if (PCIE_IS_RP(bus_p))
817			PCIE_RP_REG(pfd_p) =
818			    PCIE_ZALLOC(pf_pcie_rp_err_regs_t);
819
820		PCIE_ADV_REG(pfd_p) = PCIE_ZALLOC(pf_pcie_adv_err_regs_t);
821		PCIE_ADV_REG(pfd_p)->pcie_ue_tgt_bdf = PCIE_INVALID_BDF;
822
823		if (PCIE_IS_RP(bus_p)) {
824			PCIE_ADV_RP_REG(pfd_p) =
825			    PCIE_ZALLOC(pf_pcie_adv_rp_err_regs_t);
826			PCIE_ADV_RP_REG(pfd_p)->pcie_rp_ce_src_id =
827			    PCIE_INVALID_BDF;
828			PCIE_ADV_RP_REG(pfd_p)->pcie_rp_ue_src_id =
829			    PCIE_INVALID_BDF;
830		} else if (PCIE_IS_PCIE_BDG(bus_p)) {
831			PCIE_ADV_BDG_REG(pfd_p) =
832			    PCIE_ZALLOC(pf_pcie_adv_bdg_err_regs_t);
833			PCIE_ADV_BDG_REG(pfd_p)->pcie_sue_tgt_bdf =
834			    PCIE_INVALID_BDF;
835		}
836
837		if (PCIE_IS_PCIE_BDG(bus_p) && PCIE_IS_PCIX(bus_p)) {
838			PCIX_BDG_ERR_REG(pfd_p) =
839			    PCIE_ZALLOC(pf_pcix_bdg_err_regs_t);
840
841			if (PCIX_ECC_VERSION_CHECK(bus_p)) {
842				PCIX_BDG_ECC_REG(pfd_p, 0) =
843				    PCIE_ZALLOC(pf_pcix_ecc_regs_t);
844				PCIX_BDG_ECC_REG(pfd_p, 1) =
845				    PCIE_ZALLOC(pf_pcix_ecc_regs_t);
846			}
847		}
848	} else if (PCIE_IS_PCIX(bus_p)) {
849		if (PCIE_IS_BDG(bus_p)) {
850			PCIX_BDG_ERR_REG(pfd_p) =
851			    PCIE_ZALLOC(pf_pcix_bdg_err_regs_t);
852
853			if (PCIX_ECC_VERSION_CHECK(bus_p)) {
854				PCIX_BDG_ECC_REG(pfd_p, 0) =
855				    PCIE_ZALLOC(pf_pcix_ecc_regs_t);
856				PCIX_BDG_ECC_REG(pfd_p, 1) =
857				    PCIE_ZALLOC(pf_pcix_ecc_regs_t);
858			}
859		} else {
860			PCIX_ERR_REG(pfd_p) = PCIE_ZALLOC(pf_pcix_err_regs_t);
861
862			if (PCIX_ECC_VERSION_CHECK(bus_p))
863				PCIX_ECC_REG(pfd_p) =
864				    PCIE_ZALLOC(pf_pcix_ecc_regs_t);
865		}
866	}
867}
868
869static void
870pcie_fini_pfd(dev_info_t *dip)
871{
872	pf_data_t	*pfd_p = PCIE_DIP2PFD(dip);
873	pcie_bus_t	*bus_p = PCIE_DIP2BUS(dip);
874
875	if (PCIE_IS_PCIE(bus_p)) {
876		if (PCIE_IS_PCIE_BDG(bus_p) && PCIE_IS_PCIX(bus_p)) {
877			if (PCIX_ECC_VERSION_CHECK(bus_p)) {
878				kmem_free(PCIX_BDG_ECC_REG(pfd_p, 0),
879				    sizeof (pf_pcix_ecc_regs_t));
880				kmem_free(PCIX_BDG_ECC_REG(pfd_p, 1),
881				    sizeof (pf_pcix_ecc_regs_t));
882			}
883
884			kmem_free(PCIX_BDG_ERR_REG(pfd_p),
885			    sizeof (pf_pcix_bdg_err_regs_t));
886		}
887
888		if (PCIE_IS_RP(bus_p))
889			kmem_free(PCIE_ADV_RP_REG(pfd_p),
890			    sizeof (pf_pcie_adv_rp_err_regs_t));
891		else if (PCIE_IS_PCIE_BDG(bus_p))
892			kmem_free(PCIE_ADV_BDG_REG(pfd_p),
893			    sizeof (pf_pcie_adv_bdg_err_regs_t));
894
895		kmem_free(PCIE_ADV_REG(pfd_p),
896		    sizeof (pf_pcie_adv_err_regs_t));
897
898		if (PCIE_IS_RP(bus_p))
899			kmem_free(PCIE_RP_REG(pfd_p),
900			    sizeof (pf_pcie_rp_err_regs_t));
901
902		kmem_free(PCIE_ERR_REG(pfd_p), sizeof (pf_pcie_err_regs_t));
903	} else if (PCIE_IS_PCIX(bus_p)) {
904		if (PCIE_IS_BDG(bus_p)) {
905			if (PCIX_ECC_VERSION_CHECK(bus_p)) {
906				kmem_free(PCIX_BDG_ECC_REG(pfd_p, 0),
907				    sizeof (pf_pcix_ecc_regs_t));
908				kmem_free(PCIX_BDG_ECC_REG(pfd_p, 1),
909				    sizeof (pf_pcix_ecc_regs_t));
910			}
911
912			kmem_free(PCIX_BDG_ERR_REG(pfd_p),
913			    sizeof (pf_pcix_bdg_err_regs_t));
914		} else {
915			if (PCIX_ECC_VERSION_CHECK(bus_p))
916				kmem_free(PCIX_ECC_REG(pfd_p),
917				    sizeof (pf_pcix_ecc_regs_t));
918
919			kmem_free(PCIX_ERR_REG(pfd_p),
920			    sizeof (pf_pcix_err_regs_t));
921		}
922	}
923
924	if (PCIE_IS_BDG(bus_p))
925		kmem_free(PCI_BDG_ERR_REG(pfd_p),
926		    sizeof (pf_pci_bdg_err_regs_t));
927
928	kmem_free(PFD_AFFECTED_DEV(pfd_p), sizeof (pf_affected_dev_t));
929	kmem_free(PCI_ERR_REG(pfd_p), sizeof (pf_pci_err_regs_t));
930
931	if (PCIE_IS_ROOT(bus_p)) {
932		kmem_free(PCIE_ROOT_FAULT(pfd_p), sizeof (pf_root_fault_t));
933		kmem_free(PCIE_ROOT_EH_SRC(pfd_p), sizeof (pf_root_eh_src_t));
934	}
935
936	kmem_free(PCIE_DIP2PFD(dip), sizeof (pf_data_t));
937
938	PCIE_DIP2PFD(dip) = NULL;
939}
940
941
942/*
943 * Special functions to allocate pf_data_t's for PCIe root complexes.
944 * Note: Root Complex not Root Port
945 */
946void
947pcie_rc_init_pfd(dev_info_t *dip, pf_data_t *pfd_p)
948{
949	pfd_p->pe_bus_p = PCIE_DIP2DOWNBUS(dip);
950	pfd_p->pe_severity_flags = 0;
951	pfd_p->pe_severity_mask = 0;
952	pfd_p->pe_orig_severity_flags = 0;
953	pfd_p->pe_lock = B_FALSE;
954	pfd_p->pe_valid = B_FALSE;
955
956	PCIE_ROOT_FAULT(pfd_p) = PCIE_ZALLOC(pf_root_fault_t);
957	PCIE_ROOT_FAULT(pfd_p)->scan_bdf = PCIE_INVALID_BDF;
958	PCIE_ROOT_EH_SRC(pfd_p) = PCIE_ZALLOC(pf_root_eh_src_t);
959	PCI_ERR_REG(pfd_p) = PCIE_ZALLOC(pf_pci_err_regs_t);
960	PFD_AFFECTED_DEV(pfd_p) = PCIE_ZALLOC(pf_affected_dev_t);
961	PFD_AFFECTED_DEV(pfd_p)->pe_affected_bdf = PCIE_INVALID_BDF;
962	PCI_BDG_ERR_REG(pfd_p) = PCIE_ZALLOC(pf_pci_bdg_err_regs_t);
963	PCIE_ERR_REG(pfd_p) = PCIE_ZALLOC(pf_pcie_err_regs_t);
964	PCIE_RP_REG(pfd_p) = PCIE_ZALLOC(pf_pcie_rp_err_regs_t);
965	PCIE_ADV_REG(pfd_p) = PCIE_ZALLOC(pf_pcie_adv_err_regs_t);
966	PCIE_ADV_RP_REG(pfd_p) = PCIE_ZALLOC(pf_pcie_adv_rp_err_regs_t);
967	PCIE_ADV_RP_REG(pfd_p)->pcie_rp_ce_src_id = PCIE_INVALID_BDF;
968	PCIE_ADV_RP_REG(pfd_p)->pcie_rp_ue_src_id = PCIE_INVALID_BDF;
969
970	PCIE_ADV_REG(pfd_p)->pcie_ue_sev = pcie_aer_uce_severity;
971}
972
973void
974pcie_rc_fini_pfd(pf_data_t *pfd_p)
975{
976	kmem_free(PCIE_ADV_RP_REG(pfd_p), sizeof (pf_pcie_adv_rp_err_regs_t));
977	kmem_free(PCIE_ADV_REG(pfd_p), sizeof (pf_pcie_adv_err_regs_t));
978	kmem_free(PCIE_RP_REG(pfd_p), sizeof (pf_pcie_rp_err_regs_t));
979	kmem_free(PCIE_ERR_REG(pfd_p), sizeof (pf_pcie_err_regs_t));
980	kmem_free(PCI_BDG_ERR_REG(pfd_p), sizeof (pf_pci_bdg_err_regs_t));
981	kmem_free(PFD_AFFECTED_DEV(pfd_p), sizeof (pf_affected_dev_t));
982	kmem_free(PCI_ERR_REG(pfd_p), sizeof (pf_pci_err_regs_t));
983	kmem_free(PCIE_ROOT_FAULT(pfd_p), sizeof (pf_root_fault_t));
984	kmem_free(PCIE_ROOT_EH_SRC(pfd_p), sizeof (pf_root_eh_src_t));
985}
986
987/*
988 * init pcie_bus_t for root complex
989 *
990 * Only a few of the fields in bus_t is valid for root complex.
991 * The fields that are bracketed are initialized in this routine:
992 *
993 * dev_info_t *		<bus_dip>
994 * dev_info_t *		bus_rp_dip
995 * ddi_acc_handle_t	bus_cfg_hdl
996 * uint_t		<bus_fm_flags>
997 * pcie_req_id_t	bus_bdf
998 * pcie_req_id_t	bus_rp_bdf
999 * uint32_t		bus_dev_ven_id
1000 * uint8_t		bus_rev_id
1001 * uint8_t		<bus_hdr_type>
1002 * uint16_t		<bus_dev_type>
1003 * uint8_t		bus_bdg_secbus
1004 * uint16_t		bus_pcie_off
1005 * uint16_t		<bus_aer_off>
1006 * uint16_t		bus_pcix_off
1007 * uint16_t		bus_ecc_ver
1008 * pci_bus_range_t	bus_bus_range
1009 * ppb_ranges_t	*	bus_addr_ranges
1010 * int			bus_addr_entries
1011 * pci_regspec_t *	bus_assigned_addr
1012 * int			bus_assigned_entries
1013 * pf_data_t *		bus_pfd
1014 * pcie_domain_t *	<bus_dom>
1015 * int			bus_mps
1016 * uint64_t		bus_cfgacc_base
1017 * void	*		bus_plat_private
1018 */
1019void
1020pcie_rc_init_bus(dev_info_t *dip)
1021{
1022	pcie_bus_t *bus_p;
1023
1024	bus_p = (pcie_bus_t *)kmem_zalloc(sizeof (pcie_bus_t), KM_SLEEP);
1025	bus_p->bus_dip = dip;
1026	bus_p->bus_dev_type = PCIE_PCIECAP_DEV_TYPE_RC_PSEUDO;
1027	bus_p->bus_hdr_type = PCI_HEADER_ONE;
1028
1029	/* Fake that there are AER logs */
1030	bus_p->bus_aer_off = (uint16_t)-1;
1031
1032	/* Needed only for handle lookup */
1033	atomic_or_uint(&bus_p->bus_fm_flags, PF_FM_READY);
1034
1035	ndi_set_bus_private(dip, B_FALSE, DEVI_PORT_TYPE_PCI, bus_p);
1036
1037	PCIE_BUS2DOM(bus_p) = PCIE_ZALLOC(pcie_domain_t);
1038}
1039
1040void
1041pcie_rc_fini_bus(dev_info_t *dip)
1042{
1043	pcie_bus_t *bus_p = PCIE_DIP2DOWNBUS(dip);
1044	ndi_set_bus_private(dip, B_FALSE, 0, NULL);
1045	kmem_free(PCIE_BUS2DOM(bus_p), sizeof (pcie_domain_t));
1046	kmem_free(bus_p, sizeof (pcie_bus_t));
1047}
1048
1049static int
1050pcie_width_to_int(pcie_link_width_t width)
1051{
1052	switch (width) {
1053	case PCIE_LINK_WIDTH_X1:
1054		return (1);
1055	case PCIE_LINK_WIDTH_X2:
1056		return (2);
1057	case PCIE_LINK_WIDTH_X4:
1058		return (4);
1059	case PCIE_LINK_WIDTH_X8:
1060		return (8);
1061	case PCIE_LINK_WIDTH_X12:
1062		return (12);
1063	case PCIE_LINK_WIDTH_X16:
1064		return (16);
1065	case PCIE_LINK_WIDTH_X32:
1066		return (32);
1067	default:
1068		return (0);
1069	}
1070}
1071
1072/*
1073 * Return the speed in Transfers / second. This is a signed quantity to match
1074 * the ndi/ddi property interfaces.
1075 */
1076static int64_t
1077pcie_speed_to_int(pcie_link_speed_t speed)
1078{
1079	switch (speed) {
1080	case PCIE_LINK_SPEED_2_5:
1081		return (2500000000LL);
1082	case PCIE_LINK_SPEED_5:
1083		return (5000000000LL);
1084	case PCIE_LINK_SPEED_8:
1085		return (8000000000LL);
1086	case PCIE_LINK_SPEED_16:
1087		return (16000000000LL);
1088	default:
1089		return (0);
1090	}
1091}
1092
1093/*
1094 * Translate the recorded speed information into devinfo properties.
1095 */
1096static void
1097pcie_speeds_to_devinfo(dev_info_t *dip, pcie_bus_t *bus_p)
1098{
1099	if (bus_p->bus_max_width != PCIE_LINK_WIDTH_UNKNOWN) {
1100		(void) ndi_prop_update_int(DDI_DEV_T_NONE, dip,
1101		    "pcie-link-maximum-width",
1102		    pcie_width_to_int(bus_p->bus_max_width));
1103	}
1104
1105	if (bus_p->bus_cur_width != PCIE_LINK_WIDTH_UNKNOWN) {
1106		(void) ndi_prop_update_int(DDI_DEV_T_NONE, dip,
1107		    "pcie-link-current-width",
1108		    pcie_width_to_int(bus_p->bus_cur_width));
1109	}
1110
1111	if (bus_p->bus_cur_speed != PCIE_LINK_SPEED_UNKNOWN) {
1112		(void) ndi_prop_update_int64(DDI_DEV_T_NONE, dip,
1113		    "pcie-link-current-speed",
1114		    pcie_speed_to_int(bus_p->bus_cur_speed));
1115	}
1116
1117	if (bus_p->bus_max_speed != PCIE_LINK_SPEED_UNKNOWN) {
1118		(void) ndi_prop_update_int64(DDI_DEV_T_NONE, dip,
1119		    "pcie-link-maximum-speed",
1120		    pcie_speed_to_int(bus_p->bus_max_speed));
1121	}
1122
1123	if (bus_p->bus_target_speed != PCIE_LINK_SPEED_UNKNOWN) {
1124		(void) ndi_prop_update_int64(DDI_DEV_T_NONE, dip,
1125		    "pcie-link-target-speed",
1126		    pcie_speed_to_int(bus_p->bus_target_speed));
1127	}
1128
1129	if ((bus_p->bus_speed_flags & PCIE_LINK_F_ADMIN_TARGET) != 0) {
1130		(void) ndi_prop_create_boolean(DDI_DEV_T_NONE, dip,
1131		    "pcie-link-admin-target-speed");
1132	}
1133
1134	if (bus_p->bus_sup_speed != PCIE_LINK_SPEED_UNKNOWN) {
1135		int64_t speeds[4];
1136		uint_t nspeeds = 0;
1137
1138		if (bus_p->bus_sup_speed & PCIE_LINK_SPEED_2_5) {
1139			speeds[nspeeds++] =
1140			    pcie_speed_to_int(PCIE_LINK_SPEED_2_5);
1141		}
1142
1143		if (bus_p->bus_sup_speed & PCIE_LINK_SPEED_5) {
1144			speeds[nspeeds++] =
1145			    pcie_speed_to_int(PCIE_LINK_SPEED_5);
1146		}
1147
1148		if (bus_p->bus_sup_speed & PCIE_LINK_SPEED_8) {
1149			speeds[nspeeds++] =
1150			    pcie_speed_to_int(PCIE_LINK_SPEED_8);
1151		}
1152
1153		if (bus_p->bus_sup_speed & PCIE_LINK_SPEED_16) {
1154			speeds[nspeeds++] =
1155			    pcie_speed_to_int(PCIE_LINK_SPEED_16);
1156		}
1157
1158		(void) ndi_prop_update_int64_array(DDI_DEV_T_NONE, dip,
1159		    "pcie-link-supported-speeds", speeds, nspeeds);
1160	}
1161}
1162
1163/*
1164 * We need to capture the supported, maximum, and current device speed and
1165 * width. The way that this has been done has changed over time.
1166 *
1167 * Prior to PCIe Gen 3, there were only current and supported speed fields.
1168 * These were found in the link status and link capabilities registers of the
1169 * PCI express capability. With the change to PCIe Gen 3, the information in the
1170 * link capabilities changed to the maximum value. The supported speeds vector
1171 * was moved to the link capabilities 2 register.
1172 *
1173 * Now, a device may not implement some of these registers. To determine whether
1174 * or not it's here, we have to do the following. First, we need to check the
1175 * revision of the PCI express capability. The link capabilities 2 register did
1176 * not exist prior to version 2 of this capability. If a modern device does not
1177 * implement it, it is supposed to return zero for the register.
1178 */
1179static void
1180pcie_capture_speeds(dev_info_t *dip)
1181{
1182	uint16_t	vers, status;
1183	uint32_t	cap, cap2, ctl2;
1184	pcie_bus_t	*bus_p = PCIE_DIP2BUS(dip);
1185
1186	if (!PCIE_IS_PCIE(bus_p))
1187		return;
1188
1189	vers = PCIE_CAP_GET(16, bus_p, PCIE_PCIECAP);
1190	if (vers == PCI_EINVAL16)
1191		return;
1192	vers &= PCIE_PCIECAP_VER_MASK;
1193
1194	/*
1195	 * Verify the capability's version.
1196	 */
1197	switch (vers) {
1198	case PCIE_PCIECAP_VER_1_0:
1199		cap2 = 0;
1200		ctl2 = 0;
1201		break;
1202	case PCIE_PCIECAP_VER_2_0:
1203		cap2 = PCIE_CAP_GET(32, bus_p, PCIE_LINKCAP2);
1204		if (cap2 == PCI_EINVAL32)
1205			cap2 = 0;
1206		ctl2 = PCIE_CAP_GET(16, bus_p, PCIE_LINKCTL2);
1207		if (ctl2 == PCI_EINVAL16)
1208			ctl2 = 0;
1209		break;
1210	default:
1211		/* Don't try and handle an unknown version */
1212		return;
1213	}
1214
1215	status = PCIE_CAP_GET(16, bus_p, PCIE_LINKSTS);
1216	cap = PCIE_CAP_GET(32, bus_p, PCIE_LINKCAP);
1217	if (status == PCI_EINVAL16 || cap == PCI_EINVAL32)
1218		return;
1219
1220	mutex_enter(&bus_p->bus_speed_mutex);
1221
1222	switch (status & PCIE_LINKSTS_SPEED_MASK) {
1223	case PCIE_LINKSTS_SPEED_2_5:
1224		bus_p->bus_cur_speed = PCIE_LINK_SPEED_2_5;
1225		break;
1226	case PCIE_LINKSTS_SPEED_5:
1227		bus_p->bus_cur_speed = PCIE_LINK_SPEED_5;
1228		break;
1229	case PCIE_LINKSTS_SPEED_8:
1230		bus_p->bus_cur_speed = PCIE_LINK_SPEED_8;
1231		break;
1232	case PCIE_LINKSTS_SPEED_16:
1233		bus_p->bus_cur_speed = PCIE_LINK_SPEED_16;
1234		break;
1235	default:
1236		bus_p->bus_cur_speed = PCIE_LINK_SPEED_UNKNOWN;
1237		break;
1238	}
1239
1240	switch (status & PCIE_LINKSTS_NEG_WIDTH_MASK) {
1241	case PCIE_LINKSTS_NEG_WIDTH_X1:
1242		bus_p->bus_cur_width = PCIE_LINK_WIDTH_X1;
1243		break;
1244	case PCIE_LINKSTS_NEG_WIDTH_X2:
1245		bus_p->bus_cur_width = PCIE_LINK_WIDTH_X2;
1246		break;
1247	case PCIE_LINKSTS_NEG_WIDTH_X4:
1248		bus_p->bus_cur_width = PCIE_LINK_WIDTH_X4;
1249		break;
1250	case PCIE_LINKSTS_NEG_WIDTH_X8:
1251		bus_p->bus_cur_width = PCIE_LINK_WIDTH_X8;
1252		break;
1253	case PCIE_LINKSTS_NEG_WIDTH_X12:
1254		bus_p->bus_cur_width = PCIE_LINK_WIDTH_X12;
1255		break;
1256	case PCIE_LINKSTS_NEG_WIDTH_X16:
1257		bus_p->bus_cur_width = PCIE_LINK_WIDTH_X16;
1258		break;
1259	case PCIE_LINKSTS_NEG_WIDTH_X32:
1260		bus_p->bus_cur_width = PCIE_LINK_WIDTH_X32;
1261		break;
1262	default:
1263		bus_p->bus_cur_width = PCIE_LINK_WIDTH_UNKNOWN;
1264		break;
1265	}
1266
1267	switch (cap & PCIE_LINKCAP_MAX_WIDTH_MASK) {
1268	case PCIE_LINKCAP_MAX_WIDTH_X1:
1269		bus_p->bus_max_width = PCIE_LINK_WIDTH_X1;
1270		break;
1271	case PCIE_LINKCAP_MAX_WIDTH_X2:
1272		bus_p->bus_max_width = PCIE_LINK_WIDTH_X2;
1273		break;
1274	case PCIE_LINKCAP_MAX_WIDTH_X4:
1275		bus_p->bus_max_width = PCIE_LINK_WIDTH_X4;
1276		break;
1277	case PCIE_LINKCAP_MAX_WIDTH_X8:
1278		bus_p->bus_max_width = PCIE_LINK_WIDTH_X8;
1279		break;
1280	case PCIE_LINKCAP_MAX_WIDTH_X12:
1281		bus_p->bus_max_width = PCIE_LINK_WIDTH_X12;
1282		break;
1283	case PCIE_LINKCAP_MAX_WIDTH_X16:
1284		bus_p->bus_max_width = PCIE_LINK_WIDTH_X16;
1285		break;
1286	case PCIE_LINKCAP_MAX_WIDTH_X32:
1287		bus_p->bus_max_width = PCIE_LINK_WIDTH_X32;
1288		break;
1289	default:
1290		bus_p->bus_max_width = PCIE_LINK_WIDTH_UNKNOWN;
1291		break;
1292	}
1293
1294	/*
1295	 * If we have the Link Capabilities 2, then we can get the supported
1296	 * speeds from it and treat the bits in Link Capabilities 1 as the
1297	 * maximum. If we don't, then we need to follow the Implementation Note
1298	 * in the standard under Link Capabilities 2. Effectively, this means
1299	 * that if the value of 10b is set in Link Capabilities register, that
1300	 * it supports both 2.5 and 5 GT/s speeds.
1301	 */
1302	if (cap2 != 0) {
1303		if (cap2 & PCIE_LINKCAP2_SPEED_2_5)
1304			bus_p->bus_sup_speed |= PCIE_LINK_SPEED_2_5;
1305		if (cap2 & PCIE_LINKCAP2_SPEED_5)
1306			bus_p->bus_sup_speed |= PCIE_LINK_SPEED_5;
1307		if (cap2 & PCIE_LINKCAP2_SPEED_8)
1308			bus_p->bus_sup_speed |= PCIE_LINK_SPEED_8;
1309		if (cap2 & PCIE_LINKCAP2_SPEED_16)
1310			bus_p->bus_sup_speed |= PCIE_LINK_SPEED_16;
1311
1312		switch (cap & PCIE_LINKCAP_MAX_SPEED_MASK) {
1313		case PCIE_LINKCAP_MAX_SPEED_2_5:
1314			bus_p->bus_max_speed = PCIE_LINK_SPEED_2_5;
1315			break;
1316		case PCIE_LINKCAP_MAX_SPEED_5:
1317			bus_p->bus_max_speed = PCIE_LINK_SPEED_5;
1318			break;
1319		case PCIE_LINKCAP_MAX_SPEED_8:
1320			bus_p->bus_max_speed = PCIE_LINK_SPEED_8;
1321			break;
1322		case PCIE_LINKCAP_MAX_SPEED_16:
1323			bus_p->bus_max_speed = PCIE_LINK_SPEED_16;
1324			break;
1325		default:
1326			bus_p->bus_max_speed = PCIE_LINK_SPEED_UNKNOWN;
1327			break;
1328		}
1329	} else {
1330		if (cap & PCIE_LINKCAP_MAX_SPEED_5) {
1331			bus_p->bus_max_speed = PCIE_LINK_SPEED_5;
1332			bus_p->bus_sup_speed = PCIE_LINK_SPEED_2_5 |
1333			    PCIE_LINK_SPEED_5;
1334		} else if (cap & PCIE_LINKCAP_MAX_SPEED_2_5) {
1335			bus_p->bus_max_speed = PCIE_LINK_SPEED_2_5;
1336			bus_p->bus_sup_speed = PCIE_LINK_SPEED_2_5;
1337		}
1338	}
1339
1340	switch (ctl2 & PCIE_LINKCTL2_TARGET_SPEED_MASK) {
1341	case PCIE_LINKCTL2_TARGET_SPEED_2_5:
1342		bus_p->bus_target_speed = PCIE_LINK_SPEED_2_5;
1343		break;
1344	case PCIE_LINKCTL2_TARGET_SPEED_5:
1345		bus_p->bus_target_speed = PCIE_LINK_SPEED_5;
1346		break;
1347	case PCIE_LINKCTL2_TARGET_SPEED_8:
1348		bus_p->bus_target_speed = PCIE_LINK_SPEED_8;
1349		break;
1350	case PCIE_LINKCTL2_TARGET_SPEED_16:
1351		bus_p->bus_target_speed = PCIE_LINK_SPEED_16;
1352		break;
1353	default:
1354		bus_p->bus_target_speed = PCIE_LINK_SPEED_UNKNOWN;
1355		break;
1356	}
1357
1358	pcie_speeds_to_devinfo(dip, bus_p);
1359	mutex_exit(&bus_p->bus_speed_mutex);
1360}
1361
1362/*
1363 * partially init pcie_bus_t for device (dip,bdf) for accessing pci
1364 * config space
1365 *
1366 * This routine is invoked during boot, either after creating a devinfo node
1367 * (x86 case) or during px driver attach (sparc case); it is also invoked
1368 * in hotplug context after a devinfo node is created.
1369 *
1370 * The fields that are bracketed are initialized if flag PCIE_BUS_INITIAL
1371 * is set:
1372 *
1373 * dev_info_t *		<bus_dip>
1374 * dev_info_t *		<bus_rp_dip>
1375 * ddi_acc_handle_t	bus_cfg_hdl
1376 * uint_t		bus_fm_flags
1377 * pcie_req_id_t	<bus_bdf>
1378 * pcie_req_id_t	<bus_rp_bdf>
1379 * uint32_t		<bus_dev_ven_id>
1380 * uint8_t		<bus_rev_id>
1381 * uint8_t		<bus_hdr_type>
1382 * uint16_t		<bus_dev_type>
1383 * uint8_t		<bus_bdg_secbus
1384 * uint16_t		<bus_pcie_off>
1385 * uint16_t		<bus_aer_off>
1386 * uint16_t		<bus_pcix_off>
1387 * uint16_t		<bus_ecc_ver>
1388 * pci_bus_range_t	bus_bus_range
1389 * ppb_ranges_t	*	bus_addr_ranges
1390 * int			bus_addr_entries
1391 * pci_regspec_t *	bus_assigned_addr
1392 * int			bus_assigned_entries
1393 * pf_data_t *		bus_pfd
1394 * pcie_domain_t *	bus_dom
1395 * int			bus_mps
1396 * uint64_t		bus_cfgacc_base
1397 * void	*		bus_plat_private
1398 *
1399 * The fields that are bracketed are initialized if flag PCIE_BUS_FINAL
1400 * is set:
1401 *
1402 * dev_info_t *		bus_dip
1403 * dev_info_t *		bus_rp_dip
1404 * ddi_acc_handle_t	bus_cfg_hdl
1405 * uint_t		bus_fm_flags
1406 * pcie_req_id_t	bus_bdf
1407 * pcie_req_id_t	bus_rp_bdf
1408 * uint32_t		bus_dev_ven_id
1409 * uint8_t		bus_rev_id
1410 * uint8_t		bus_hdr_type
1411 * uint16_t		bus_dev_type
1412 * uint8_t		<bus_bdg_secbus>
1413 * uint16_t		bus_pcie_off
1414 * uint16_t		bus_aer_off
1415 * uint16_t		bus_pcix_off
1416 * uint16_t		bus_ecc_ver
1417 * pci_bus_range_t	<bus_bus_range>
1418 * ppb_ranges_t	*	<bus_addr_ranges>
1419 * int			<bus_addr_entries>
1420 * pci_regspec_t *	<bus_assigned_addr>
1421 * int			<bus_assigned_entries>
1422 * pf_data_t *		<bus_pfd>
1423 * pcie_domain_t *	bus_dom
1424 * int			bus_mps
1425 * uint64_t		bus_cfgacc_base
1426 * void	*		<bus_plat_private>
1427 */
1428
1429pcie_bus_t *
1430pcie_init_bus(dev_info_t *dip, pcie_req_id_t bdf, uint8_t flags)
1431{
1432	uint16_t	status, base, baseptr, num_cap;
1433	uint32_t	capid;
1434	int		range_size;
1435	pcie_bus_t	*bus_p = NULL;
1436	dev_info_t	*rcdip;
1437	dev_info_t	*pdip;
1438	const char	*errstr = NULL;
1439
1440	if (!(flags & PCIE_BUS_INITIAL))
1441		goto initial_done;
1442
1443	bus_p = kmem_zalloc(sizeof (pcie_bus_t), KM_SLEEP);
1444
1445	bus_p->bus_dip = dip;
1446	bus_p->bus_bdf = bdf;
1447
1448	rcdip = pcie_get_rc_dip(dip);
1449	ASSERT(rcdip != NULL);
1450
1451	/* Save the Vendor ID, Device ID and revision ID */
1452	bus_p->bus_dev_ven_id = pci_cfgacc_get32(rcdip, bdf, PCI_CONF_VENID);
1453	bus_p->bus_rev_id = pci_cfgacc_get8(rcdip, bdf, PCI_CONF_REVID);
1454	/* Save the Header Type */
1455	bus_p->bus_hdr_type = pci_cfgacc_get8(rcdip, bdf, PCI_CONF_HEADER);
1456	bus_p->bus_hdr_type &= PCI_HEADER_TYPE_M;
1457
1458	/*
1459	 * Figure out the device type and all the relavant capability offsets
1460	 */
1461	/* set default value */
1462	bus_p->bus_dev_type = PCIE_PCIECAP_DEV_TYPE_PCI_PSEUDO;
1463
1464	status = pci_cfgacc_get16(rcdip, bdf, PCI_CONF_STAT);
1465	if (status == PCI_CAP_EINVAL16 || !(status & PCI_STAT_CAP))
1466		goto caps_done; /* capability not supported */
1467
1468	/* Relevant conventional capabilities first */
1469
1470	/* Conventional caps: PCI_CAP_ID_PCI_E, PCI_CAP_ID_PCIX */
1471	num_cap = 2;
1472
1473	switch (bus_p->bus_hdr_type) {
1474	case PCI_HEADER_ZERO:
1475		baseptr = PCI_CONF_CAP_PTR;
1476		break;
1477	case PCI_HEADER_PPB:
1478		baseptr = PCI_BCNF_CAP_PTR;
1479		break;
1480	case PCI_HEADER_CARDBUS:
1481		baseptr = PCI_CBUS_CAP_PTR;
1482		break;
1483	default:
1484		cmn_err(CE_WARN, "%s: unexpected pci header type:%x",
1485		    __func__, bus_p->bus_hdr_type);
1486		goto caps_done;
1487	}
1488
1489	base = baseptr;
1490	for (base = pci_cfgacc_get8(rcdip, bdf, base); base && num_cap;
1491	    base = pci_cfgacc_get8(rcdip, bdf, base + PCI_CAP_NEXT_PTR)) {
1492		capid = pci_cfgacc_get8(rcdip, bdf, base);
1493		switch (capid) {
1494		case PCI_CAP_ID_PCI_E:
1495			bus_p->bus_pcie_off = base;
1496			bus_p->bus_dev_type = pci_cfgacc_get16(rcdip, bdf,
1497			    base + PCIE_PCIECAP) & PCIE_PCIECAP_DEV_TYPE_MASK;
1498
1499			/* Check and save PCIe hotplug capability information */
1500			if ((PCIE_IS_RP(bus_p) || PCIE_IS_SWD(bus_p)) &&
1501			    (pci_cfgacc_get16(rcdip, bdf, base + PCIE_PCIECAP)
1502			    & PCIE_PCIECAP_SLOT_IMPL) &&
1503			    (pci_cfgacc_get32(rcdip, bdf, base + PCIE_SLOTCAP)
1504			    & PCIE_SLOTCAP_HP_CAPABLE))
1505				bus_p->bus_hp_sup_modes |= PCIE_NATIVE_HP_MODE;
1506
1507			num_cap--;
1508			break;
1509		case PCI_CAP_ID_PCIX:
1510			bus_p->bus_pcix_off = base;
1511			if (PCIE_IS_BDG(bus_p))
1512				bus_p->bus_ecc_ver =
1513				    pci_cfgacc_get16(rcdip, bdf, base +
1514				    PCI_PCIX_SEC_STATUS) & PCI_PCIX_VER_MASK;
1515			else
1516				bus_p->bus_ecc_ver =
1517				    pci_cfgacc_get16(rcdip, bdf, base +
1518				    PCI_PCIX_COMMAND) & PCI_PCIX_VER_MASK;
1519			num_cap--;
1520			break;
1521		default:
1522			break;
1523		}
1524	}
1525
1526	/* Check and save PCI hotplug (SHPC) capability information */
1527	if (PCIE_IS_BDG(bus_p)) {
1528		base = baseptr;
1529		for (base = pci_cfgacc_get8(rcdip, bdf, base);
1530		    base; base = pci_cfgacc_get8(rcdip, bdf,
1531		    base + PCI_CAP_NEXT_PTR)) {
1532			capid = pci_cfgacc_get8(rcdip, bdf, base);
1533			if (capid == PCI_CAP_ID_PCI_HOTPLUG) {
1534				bus_p->bus_pci_hp_off = base;
1535				bus_p->bus_hp_sup_modes |= PCIE_PCI_HP_MODE;
1536				break;
1537			}
1538		}
1539	}
1540
1541	/* Then, relevant extended capabilities */
1542
1543	if (!PCIE_IS_PCIE(bus_p))
1544		goto caps_done;
1545
1546	/* Extended caps: PCIE_EXT_CAP_ID_AER */
1547	for (base = PCIE_EXT_CAP; base; base = (capid >>
1548	    PCIE_EXT_CAP_NEXT_PTR_SHIFT) & PCIE_EXT_CAP_NEXT_PTR_MASK) {
1549		capid = pci_cfgacc_get32(rcdip, bdf, base);
1550		if (capid == PCI_CAP_EINVAL32)
1551			break;
1552		if (((capid >> PCIE_EXT_CAP_ID_SHIFT) & PCIE_EXT_CAP_ID_MASK)
1553		    == PCIE_EXT_CAP_ID_AER) {
1554			bus_p->bus_aer_off = base;
1555			break;
1556		}
1557	}
1558
1559	/*
1560	 * Save and record speed information about the device.
1561	 */
1562
1563caps_done:
1564	/* save RP dip and RP bdf */
1565	if (PCIE_IS_RP(bus_p)) {
1566		bus_p->bus_rp_dip = dip;
1567		bus_p->bus_rp_bdf = bus_p->bus_bdf;
1568	} else {
1569		for (pdip = ddi_get_parent(dip); pdip;
1570		    pdip = ddi_get_parent(pdip)) {
1571			pcie_bus_t *parent_bus_p = PCIE_DIP2BUS(pdip);
1572
1573			/*
1574			 * If RP dip and RP bdf in parent's bus_t have
1575			 * been initialized, simply use these instead of
1576			 * continuing up to the RC.
1577			 */
1578			if (parent_bus_p->bus_rp_dip != NULL) {
1579				bus_p->bus_rp_dip = parent_bus_p->bus_rp_dip;
1580				bus_p->bus_rp_bdf = parent_bus_p->bus_rp_bdf;
1581				break;
1582			}
1583
1584			/*
1585			 * When debugging be aware that some NVIDIA x86
1586			 * architectures have 2 nodes for each RP, One at Bus
1587			 * 0x0 and one at Bus 0x80.  The requester is from Bus
1588			 * 0x80
1589			 */
1590			if (PCIE_IS_ROOT(parent_bus_p)) {
1591				bus_p->bus_rp_dip = pdip;
1592				bus_p->bus_rp_bdf = parent_bus_p->bus_bdf;
1593				break;
1594			}
1595		}
1596	}
1597
1598	bus_p->bus_soft_state = PCI_SOFT_STATE_CLOSED;
1599	(void) atomic_swap_uint(&bus_p->bus_fm_flags, 0);
1600	bus_p->bus_mps = 0;
1601
1602	ndi_set_bus_private(dip, B_TRUE, DEVI_PORT_TYPE_PCI, (void *)bus_p);
1603
1604	if (PCIE_IS_HOTPLUG_CAPABLE(dip))
1605		(void) ndi_prop_create_boolean(DDI_DEV_T_NONE, dip,
1606		    "hotplug-capable");
1607
1608initial_done:
1609	if (!(flags & PCIE_BUS_FINAL))
1610		goto final_done;
1611
1612	/* already initialized? */
1613	bus_p = PCIE_DIP2BUS(dip);
1614
1615	/* Save the Range information if device is a switch/bridge */
1616	if (PCIE_IS_BDG(bus_p)) {
1617		/* get "bus_range" property */
1618		range_size = sizeof (pci_bus_range_t);
1619		if (ddi_getlongprop_buf(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
1620		    "bus-range", (caddr_t)&bus_p->bus_bus_range, &range_size)
1621		    != DDI_PROP_SUCCESS) {
1622			errstr = "Cannot find \"bus-range\" property";
1623			cmn_err(CE_WARN,
1624			    "PCIE init err info failed BDF 0x%x:%s\n",
1625			    bus_p->bus_bdf, errstr);
1626		}
1627
1628		/* get secondary bus number */
1629		rcdip = pcie_get_rc_dip(dip);
1630		ASSERT(rcdip != NULL);
1631
1632		bus_p->bus_bdg_secbus = pci_cfgacc_get8(rcdip,
1633		    bus_p->bus_bdf, PCI_BCNF_SECBUS);
1634
1635		/* Get "ranges" property */
1636		if (ddi_getlongprop(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
1637		    "ranges", (caddr_t)&bus_p->bus_addr_ranges,
1638		    &bus_p->bus_addr_entries) != DDI_PROP_SUCCESS)
1639			bus_p->bus_addr_entries = 0;
1640		bus_p->bus_addr_entries /= sizeof (ppb_ranges_t);
1641	}
1642
1643	/* save "assigned-addresses" property array, ignore failues */
1644	if (ddi_getlongprop(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
1645	    "assigned-addresses", (caddr_t)&bus_p->bus_assigned_addr,
1646	    &bus_p->bus_assigned_entries) == DDI_PROP_SUCCESS)
1647		bus_p->bus_assigned_entries /= sizeof (pci_regspec_t);
1648	else
1649		bus_p->bus_assigned_entries = 0;
1650
1651	pcie_init_pfd(dip);
1652
1653	pcie_init_plat(dip);
1654
1655final_done:
1656
1657	PCIE_DBG("Add %s(dip 0x%p, bdf 0x%x, secbus 0x%x)\n",
1658	    ddi_driver_name(dip), (void *)dip, bus_p->bus_bdf,
1659	    bus_p->bus_bdg_secbus);
1660#ifdef DEBUG
1661	if (bus_p != NULL) {
1662		pcie_print_bus(bus_p);
1663	}
1664#endif
1665
1666	return (bus_p);
1667}
1668
1669/*
1670 * Invoked before destroying devinfo node, mostly during hotplug
1671 * operation to free pcie_bus_t data structure
1672 */
1673/* ARGSUSED */
1674void
1675pcie_fini_bus(dev_info_t *dip, uint8_t flags)
1676{
1677	pcie_bus_t *bus_p = PCIE_DIP2UPBUS(dip);
1678	ASSERT(bus_p);
1679
1680	if (flags & PCIE_BUS_INITIAL) {
1681		pcie_fini_plat(dip);
1682		pcie_fini_pfd(dip);
1683
1684		kmem_free(bus_p->bus_assigned_addr,
1685		    (sizeof (pci_regspec_t) * bus_p->bus_assigned_entries));
1686		kmem_free(bus_p->bus_addr_ranges,
1687		    (sizeof (ppb_ranges_t) * bus_p->bus_addr_entries));
1688		/* zero out the fields that have been destroyed */
1689		bus_p->bus_assigned_addr = NULL;
1690		bus_p->bus_addr_ranges = NULL;
1691		bus_p->bus_assigned_entries = 0;
1692		bus_p->bus_addr_entries = 0;
1693	}
1694
1695	if (flags & PCIE_BUS_FINAL) {
1696		if (PCIE_IS_HOTPLUG_CAPABLE(dip)) {
1697			(void) ndi_prop_remove(DDI_DEV_T_NONE, dip,
1698			    "hotplug-capable");
1699		}
1700
1701		ndi_set_bus_private(dip, B_TRUE, 0, NULL);
1702		kmem_free(bus_p, sizeof (pcie_bus_t));
1703	}
1704}
1705
1706int
1707pcie_postattach_child(dev_info_t *cdip)
1708{
1709	pcie_bus_t *bus_p = PCIE_DIP2BUS(cdip);
1710
1711	if (!bus_p)
1712		return (DDI_FAILURE);
1713
1714	return (pcie_enable_ce(cdip));
1715}
1716
1717/*
1718 * PCI-Express child device de-initialization.
1719 * This function disables generic pci-express interrupts and error
1720 * handling.
1721 */
1722void
1723pcie_uninitchild(dev_info_t *cdip)
1724{
1725	pcie_disable_errors(cdip);
1726	pcie_fini_cfghdl(cdip);
1727	pcie_fini_dom(cdip);
1728}
1729
1730/*
1731 * find the root complex dip
1732 */
1733dev_info_t *
1734pcie_get_rc_dip(dev_info_t *dip)
1735{
1736	dev_info_t *rcdip;
1737	pcie_bus_t *rc_bus_p;
1738
1739	for (rcdip = ddi_get_parent(dip); rcdip;
1740	    rcdip = ddi_get_parent(rcdip)) {
1741		rc_bus_p = PCIE_DIP2BUS(rcdip);
1742		if (rc_bus_p && PCIE_IS_RC(rc_bus_p))
1743			break;
1744	}
1745
1746	return (rcdip);
1747}
1748
1749boolean_t
1750pcie_is_pci_device(dev_info_t *dip)
1751{
1752	dev_info_t	*pdip;
1753	char		*device_type;
1754
1755	pdip = ddi_get_parent(dip);
1756	if (pdip == NULL)
1757		return (B_FALSE);
1758
1759	if (ddi_prop_lookup_string(DDI_DEV_T_ANY, pdip, DDI_PROP_DONTPASS,
1760	    "device_type", &device_type) != DDI_PROP_SUCCESS)
1761		return (B_FALSE);
1762
1763	if (strcmp(device_type, "pciex") != 0 &&
1764	    strcmp(device_type, "pci") != 0) {
1765		ddi_prop_free(device_type);
1766		return (B_FALSE);
1767	}
1768
1769	ddi_prop_free(device_type);
1770	return (B_TRUE);
1771}
1772
1773typedef struct {
1774	boolean_t	init;
1775	uint8_t		flags;
1776} pcie_bus_arg_t;
1777
1778/*ARGSUSED*/
1779static int
1780pcie_fab_do_init_fini(dev_info_t *dip, void *arg)
1781{
1782	pcie_req_id_t	bdf;
1783	pcie_bus_arg_t	*bus_arg = (pcie_bus_arg_t *)arg;
1784
1785	if (!pcie_is_pci_device(dip))
1786		goto out;
1787
1788	if (bus_arg->init) {
1789		if (pcie_get_bdf_from_dip(dip, &bdf) != DDI_SUCCESS)
1790			goto out;
1791
1792		(void) pcie_init_bus(dip, bdf, bus_arg->flags);
1793	} else {
1794		(void) pcie_fini_bus(dip, bus_arg->flags);
1795	}
1796
1797	return (DDI_WALK_CONTINUE);
1798
1799out:
1800	return (DDI_WALK_PRUNECHILD);
1801}
1802
1803void
1804pcie_fab_init_bus(dev_info_t *rcdip, uint8_t flags)
1805{
1806	int		circular_count;
1807	dev_info_t	*dip = ddi_get_child(rcdip);
1808	pcie_bus_arg_t	arg;
1809
1810	arg.init = B_TRUE;
1811	arg.flags = flags;
1812
1813	ndi_devi_enter(rcdip, &circular_count);
1814	ddi_walk_devs(dip, pcie_fab_do_init_fini, &arg);
1815	ndi_devi_exit(rcdip, circular_count);
1816}
1817
1818void
1819pcie_fab_fini_bus(dev_info_t *rcdip, uint8_t flags)
1820{
1821	int		circular_count;
1822	dev_info_t	*dip = ddi_get_child(rcdip);
1823	pcie_bus_arg_t	arg;
1824
1825	arg.init = B_FALSE;
1826	arg.flags = flags;
1827
1828	ndi_devi_enter(rcdip, &circular_count);
1829	ddi_walk_devs(dip, pcie_fab_do_init_fini, &arg);
1830	ndi_devi_exit(rcdip, circular_count);
1831}
1832
1833void
1834pcie_enable_errors(dev_info_t *dip)
1835{
1836	pcie_bus_t	*bus_p = PCIE_DIP2BUS(dip);
1837	uint16_t	reg16, tmp16;
1838	uint32_t	reg32, tmp32;
1839
1840	ASSERT(bus_p);
1841
1842	/*
1843	 * Clear any pending errors
1844	 */
1845	pcie_clear_errors(dip);
1846
1847	if (!PCIE_IS_PCIE(bus_p))
1848		return;
1849
1850	/*
1851	 * Enable Baseline Error Handling but leave CE reporting off (poweron
1852	 * default).
1853	 */
1854	if ((reg16 = PCIE_CAP_GET(16, bus_p, PCIE_DEVCTL)) !=
1855	    PCI_CAP_EINVAL16) {
1856		tmp16 = (reg16 & (PCIE_DEVCTL_MAX_READ_REQ_MASK |
1857		    PCIE_DEVCTL_MAX_PAYLOAD_MASK)) |
1858		    (pcie_devctl_default & ~(PCIE_DEVCTL_MAX_READ_REQ_MASK |
1859		    PCIE_DEVCTL_MAX_PAYLOAD_MASK)) |
1860		    (pcie_base_err_default & (~PCIE_DEVCTL_CE_REPORTING_EN));
1861
1862		PCIE_CAP_PUT(16, bus_p, PCIE_DEVCTL, tmp16);
1863		PCIE_DBG_CAP(dip, bus_p, "DEVCTL", 16, PCIE_DEVCTL, reg16);
1864	}
1865
1866	/* Enable Root Port Baseline Error Receiving */
1867	if (PCIE_IS_ROOT(bus_p) &&
1868	    (reg16 = PCIE_CAP_GET(16, bus_p, PCIE_ROOTCTL)) !=
1869	    PCI_CAP_EINVAL16) {
1870
1871		tmp16 = pcie_serr_disable_flag ?
1872		    (pcie_root_ctrl_default & ~PCIE_ROOT_SYS_ERR) :
1873		    pcie_root_ctrl_default;
1874		PCIE_CAP_PUT(16, bus_p, PCIE_ROOTCTL, tmp16);
1875		PCIE_DBG_CAP(dip, bus_p, "ROOT DEVCTL", 16, PCIE_ROOTCTL,
1876		    reg16);
1877	}
1878
1879	/*
1880	 * Enable PCI-Express Advanced Error Handling if Exists
1881	 */
1882	if (!PCIE_HAS_AER(bus_p))
1883		return;
1884
1885	/* Set Uncorrectable Severity */
1886	if ((reg32 = PCIE_AER_GET(32, bus_p, PCIE_AER_UCE_SERV)) !=
1887	    PCI_CAP_EINVAL32) {
1888		tmp32 = pcie_aer_uce_severity;
1889
1890		PCIE_AER_PUT(32, bus_p, PCIE_AER_UCE_SERV, tmp32);
1891		PCIE_DBG_AER(dip, bus_p, "AER UCE SEV", 32, PCIE_AER_UCE_SERV,
1892		    reg32);
1893	}
1894
1895	/* Enable Uncorrectable errors */
1896	if ((reg32 = PCIE_AER_GET(32, bus_p, PCIE_AER_UCE_MASK)) !=
1897	    PCI_CAP_EINVAL32) {
1898		tmp32 = pcie_aer_uce_mask;
1899
1900		PCIE_AER_PUT(32, bus_p, PCIE_AER_UCE_MASK, tmp32);
1901		PCIE_DBG_AER(dip, bus_p, "AER UCE MASK", 32, PCIE_AER_UCE_MASK,
1902		    reg32);
1903	}
1904
1905	/* Enable ECRC generation and checking */
1906	if ((reg32 = PCIE_AER_GET(32, bus_p, PCIE_AER_CTL)) !=
1907	    PCI_CAP_EINVAL32) {
1908		tmp32 = reg32 | pcie_ecrc_value;
1909		PCIE_AER_PUT(32, bus_p, PCIE_AER_CTL, tmp32);
1910		PCIE_DBG_AER(dip, bus_p, "AER CTL", 32, PCIE_AER_CTL, reg32);
1911	}
1912
1913	/* Enable Secondary Uncorrectable errors if this is a bridge */
1914	if (!PCIE_IS_PCIE_BDG(bus_p))
1915		goto root;
1916
1917	/* Set Uncorrectable Severity */
1918	if ((reg32 = PCIE_AER_GET(32, bus_p, PCIE_AER_SUCE_SERV)) !=
1919	    PCI_CAP_EINVAL32) {
1920		tmp32 = pcie_aer_suce_severity;
1921
1922		PCIE_AER_PUT(32, bus_p, PCIE_AER_SUCE_SERV, tmp32);
1923		PCIE_DBG_AER(dip, bus_p, "AER SUCE SEV", 32, PCIE_AER_SUCE_SERV,
1924		    reg32);
1925	}
1926
1927	if ((reg32 = PCIE_AER_GET(32, bus_p, PCIE_AER_SUCE_MASK)) !=
1928	    PCI_CAP_EINVAL32) {
1929		PCIE_AER_PUT(32, bus_p, PCIE_AER_SUCE_MASK, pcie_aer_suce_mask);
1930		PCIE_DBG_AER(dip, bus_p, "AER SUCE MASK", 32,
1931		    PCIE_AER_SUCE_MASK, reg32);
1932	}
1933
1934root:
1935	/*
1936	 * Enable Root Control this is a Root device
1937	 */
1938	if (!PCIE_IS_ROOT(bus_p))
1939		return;
1940
1941	if ((reg16 = PCIE_AER_GET(16, bus_p, PCIE_AER_RE_CMD)) !=
1942	    PCI_CAP_EINVAL16) {
1943		PCIE_AER_PUT(16, bus_p, PCIE_AER_RE_CMD,
1944		    pcie_root_error_cmd_default);
1945		PCIE_DBG_AER(dip, bus_p, "AER Root Err Cmd", 16,
1946		    PCIE_AER_RE_CMD, reg16);
1947	}
1948}
1949
1950/*
1951 * This function is used for enabling CE reporting and setting the AER CE mask.
1952 * When called from outside the pcie module it should always be preceded by
1953 * a call to pcie_enable_errors.
1954 */
1955int
1956pcie_enable_ce(dev_info_t *dip)
1957{
1958	pcie_bus_t	*bus_p = PCIE_DIP2BUS(dip);
1959	uint16_t	device_sts, device_ctl;
1960	uint32_t	tmp_pcie_aer_ce_mask;
1961
1962	if (!PCIE_IS_PCIE(bus_p))
1963		return (DDI_SUCCESS);
1964
1965	/*
1966	 * The "pcie_ce_mask" property is used to control both the CE reporting
1967	 * enable field in the device control register and the AER CE mask. We
1968	 * leave CE reporting disabled if pcie_ce_mask is set to -1.
1969	 */
1970
1971	tmp_pcie_aer_ce_mask = (uint32_t)ddi_prop_get_int(DDI_DEV_T_ANY, dip,
1972	    DDI_PROP_DONTPASS, "pcie_ce_mask", pcie_aer_ce_mask);
1973
1974	if (tmp_pcie_aer_ce_mask == (uint32_t)-1) {
1975		/*
1976		 * Nothing to do since CE reporting has already been disabled.
1977		 */
1978		return (DDI_SUCCESS);
1979	}
1980
1981	if (PCIE_HAS_AER(bus_p)) {
1982		/* Enable AER CE */
1983		PCIE_AER_PUT(32, bus_p, PCIE_AER_CE_MASK, tmp_pcie_aer_ce_mask);
1984		PCIE_DBG_AER(dip, bus_p, "AER CE MASK", 32, PCIE_AER_CE_MASK,
1985		    0);
1986
1987		/* Clear any pending AER CE errors */
1988		PCIE_AER_PUT(32, bus_p, PCIE_AER_CE_STS, -1);
1989	}
1990
1991	/* clear any pending CE errors */
1992	if ((device_sts = PCIE_CAP_GET(16, bus_p, PCIE_DEVSTS)) !=
1993	    PCI_CAP_EINVAL16)
1994		PCIE_CAP_PUT(16, bus_p, PCIE_DEVSTS,
1995		    device_sts & (~PCIE_DEVSTS_CE_DETECTED));
1996
1997	/* Enable CE reporting */
1998	device_ctl = PCIE_CAP_GET(16, bus_p, PCIE_DEVCTL);
1999	PCIE_CAP_PUT(16, bus_p, PCIE_DEVCTL,
2000	    (device_ctl & (~PCIE_DEVCTL_ERR_MASK)) | pcie_base_err_default);
2001	PCIE_DBG_CAP(dip, bus_p, "DEVCTL", 16, PCIE_DEVCTL, device_ctl);
2002
2003	return (DDI_SUCCESS);
2004}
2005
2006/* ARGSUSED */
2007void
2008pcie_disable_errors(dev_info_t *dip)
2009{
2010	pcie_bus_t	*bus_p = PCIE_DIP2BUS(dip);
2011	uint16_t	device_ctl;
2012	uint32_t	aer_reg;
2013
2014	if (!PCIE_IS_PCIE(bus_p))
2015		return;
2016
2017	/*
2018	 * Disable PCI-Express Baseline Error Handling
2019	 */
2020	device_ctl = PCIE_CAP_GET(16, bus_p, PCIE_DEVCTL);
2021	device_ctl &= ~PCIE_DEVCTL_ERR_MASK;
2022	PCIE_CAP_PUT(16, bus_p, PCIE_DEVCTL, device_ctl);
2023
2024	/*
2025	 * Disable PCI-Express Advanced Error Handling if Exists
2026	 */
2027	if (!PCIE_HAS_AER(bus_p))
2028		goto root;
2029
2030	/* Disable Uncorrectable errors */
2031	PCIE_AER_PUT(32, bus_p, PCIE_AER_UCE_MASK, PCIE_AER_UCE_BITS);
2032
2033	/* Disable Correctable errors */
2034	PCIE_AER_PUT(32, bus_p, PCIE_AER_CE_MASK, PCIE_AER_CE_BITS);
2035
2036	/* Disable ECRC generation and checking */
2037	if ((aer_reg = PCIE_AER_GET(32, bus_p, PCIE_AER_CTL)) !=
2038	    PCI_CAP_EINVAL32) {
2039		aer_reg &= ~(PCIE_AER_CTL_ECRC_GEN_ENA |
2040		    PCIE_AER_CTL_ECRC_CHECK_ENA);
2041
2042		PCIE_AER_PUT(32, bus_p, PCIE_AER_CTL, aer_reg);
2043	}
2044	/*
2045	 * Disable Secondary Uncorrectable errors if this is a bridge
2046	 */
2047	if (!PCIE_IS_PCIE_BDG(bus_p))
2048		goto root;
2049
2050	PCIE_AER_PUT(32, bus_p, PCIE_AER_SUCE_MASK, PCIE_AER_SUCE_BITS);
2051
2052root:
2053	/*
2054	 * disable Root Control this is a Root device
2055	 */
2056	if (!PCIE_IS_ROOT(bus_p))
2057		return;
2058
2059	if (!pcie_serr_disable_flag) {
2060		device_ctl = PCIE_CAP_GET(16, bus_p, PCIE_ROOTCTL);
2061		device_ctl &= ~PCIE_ROOT_SYS_ERR;
2062		PCIE_CAP_PUT(16, bus_p, PCIE_ROOTCTL, device_ctl);
2063	}
2064
2065	if (!PCIE_HAS_AER(bus_p))
2066		return;
2067
2068	if ((device_ctl = PCIE_CAP_GET(16, bus_p, PCIE_AER_RE_CMD)) !=
2069	    PCI_CAP_EINVAL16) {
2070		device_ctl &= ~pcie_root_error_cmd_default;
2071		PCIE_CAP_PUT(16, bus_p, PCIE_AER_RE_CMD, device_ctl);
2072	}
2073}
2074
2075/*
2076 * Extract bdf from "reg" property.
2077 */
2078int
2079pcie_get_bdf_from_dip(dev_info_t *dip, pcie_req_id_t *bdf)
2080{
2081	pci_regspec_t	*regspec;
2082	int		reglen;
2083
2084	if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
2085	    "reg", (int **)&regspec, (uint_t *)&reglen) != DDI_SUCCESS)
2086		return (DDI_FAILURE);
2087
2088	if (reglen < (sizeof (pci_regspec_t) / sizeof (int))) {
2089		ddi_prop_free(regspec);
2090		return (DDI_FAILURE);
2091	}
2092
2093	/* Get phys_hi from first element.  All have same bdf. */
2094	*bdf = (regspec->pci_phys_hi & (PCI_REG_BDFR_M ^ PCI_REG_REG_M)) >> 8;
2095
2096	ddi_prop_free(regspec);
2097	return (DDI_SUCCESS);
2098}
2099
2100dev_info_t *
2101pcie_get_my_childs_dip(dev_info_t *dip, dev_info_t *rdip)
2102{
2103	dev_info_t *cdip = rdip;
2104
2105	for (; ddi_get_parent(cdip) != dip; cdip = ddi_get_parent(cdip))
2106		;
2107
2108	return (cdip);
2109}
2110
2111uint32_t
2112pcie_get_bdf_for_dma_xfer(dev_info_t *dip, dev_info_t *rdip)
2113{
2114	dev_info_t *cdip;
2115
2116	/*
2117	 * As part of the probing, the PCI fcode interpreter may setup a DMA
2118	 * request if a given card has a fcode on it using dip and rdip of the
2119	 * hotplug connector i.e, dip and rdip of px/pcieb driver. In this
2120	 * case, return a invalid value for the bdf since we cannot get to the
2121	 * bdf value of the actual device which will be initiating this DMA.
2122	 */
2123	if (rdip == dip)
2124		return (PCIE_INVALID_BDF);
2125
2126	cdip = pcie_get_my_childs_dip(dip, rdip);
2127
2128	/*
2129	 * For a given rdip, return the bdf value of dip's (px or pcieb)
2130	 * immediate child or secondary bus-id if dip is a PCIe2PCI bridge.
2131	 *
2132	 * XXX - For now, return a invalid bdf value for all PCI and PCI-X
2133	 * devices since this needs more work.
2134	 */
2135	return (PCI_GET_PCIE2PCI_SECBUS(cdip) ?
2136	    PCIE_INVALID_BDF : PCI_GET_BDF(cdip));
2137}
2138
2139uint32_t
2140pcie_get_aer_uce_mask()
2141{
2142	return (pcie_aer_uce_mask);
2143}
2144uint32_t
2145pcie_get_aer_ce_mask()
2146{
2147	return (pcie_aer_ce_mask);
2148}
2149uint32_t
2150pcie_get_aer_suce_mask()
2151{
2152	return (pcie_aer_suce_mask);
2153}
2154uint32_t
2155pcie_get_serr_mask()
2156{
2157	return (pcie_serr_disable_flag);
2158}
2159
2160void
2161pcie_set_aer_uce_mask(uint32_t mask)
2162{
2163	pcie_aer_uce_mask = mask;
2164	if (mask & PCIE_AER_UCE_UR)
2165		pcie_base_err_default &= ~PCIE_DEVCTL_UR_REPORTING_EN;
2166	else
2167		pcie_base_err_default |= PCIE_DEVCTL_UR_REPORTING_EN;
2168
2169	if (mask & PCIE_AER_UCE_ECRC)
2170		pcie_ecrc_value = 0;
2171}
2172
2173void
2174pcie_set_aer_ce_mask(uint32_t mask)
2175{
2176	pcie_aer_ce_mask = mask;
2177}
2178void
2179pcie_set_aer_suce_mask(uint32_t mask)
2180{
2181	pcie_aer_suce_mask = mask;
2182}
2183void
2184pcie_set_serr_mask(uint32_t mask)
2185{
2186	pcie_serr_disable_flag = mask;
2187}
2188
2189/*
2190 * Is the rdip a child of dip.	Used for checking certain CTLOPS from bubbling
2191 * up erronously.  Ex.	ISA ctlops to a PCI-PCI Bridge.
2192 */
2193boolean_t
2194pcie_is_child(dev_info_t *dip, dev_info_t *rdip)
2195{
2196	dev_info_t	*cdip = ddi_get_child(dip);
2197	for (; cdip; cdip = ddi_get_next_sibling(cdip))
2198		if (cdip == rdip)
2199			break;
2200	return (cdip != NULL);
2201}
2202
2203boolean_t
2204pcie_is_link_disabled(dev_info_t *dip)
2205{
2206	pcie_bus_t *bus_p = PCIE_DIP2BUS(dip);
2207
2208	if (PCIE_IS_PCIE(bus_p)) {
2209		if (PCIE_CAP_GET(16, bus_p, PCIE_LINKCTL) &
2210		    PCIE_LINKCTL_LINK_DISABLE)
2211			return (B_TRUE);
2212	}
2213	return (B_FALSE);
2214}
2215
2216/*
2217 * Initialize the MPS for a root port.
2218 *
2219 * dip - dip of root port device.
2220 */
2221void
2222pcie_init_root_port_mps(dev_info_t *dip)
2223{
2224	pcie_bus_t	*bus_p = PCIE_DIP2BUS(dip);
2225	int rp_cap, max_supported = pcie_max_mps;
2226
2227	(void) pcie_get_fabric_mps(ddi_get_parent(dip),
2228	    ddi_get_child(dip), &max_supported);
2229
2230	rp_cap = PCI_CAP_GET16(bus_p->bus_cfg_hdl, 0,
2231	    bus_p->bus_pcie_off, PCIE_DEVCAP) &
2232	    PCIE_DEVCAP_MAX_PAYLOAD_MASK;
2233
2234	if (rp_cap < max_supported)
2235		max_supported = rp_cap;
2236
2237	bus_p->bus_mps = max_supported;
2238	(void) pcie_initchild_mps(dip);
2239}
2240
2241/*
2242 * Initialize the Maximum Payload Size of a device.
2243 *
2244 * cdip - dip of device.
2245 *
2246 * returns - DDI_SUCCESS or DDI_FAILURE
2247 */
2248int
2249pcie_initchild_mps(dev_info_t *cdip)
2250{
2251	pcie_bus_t	*bus_p;
2252	dev_info_t	*pdip = ddi_get_parent(cdip);
2253	uint8_t		dev_type;
2254
2255	bus_p = PCIE_DIP2BUS(cdip);
2256	if (bus_p == NULL) {
2257		PCIE_DBG("%s: BUS not found.\n",
2258		    ddi_driver_name(cdip));
2259		return (DDI_FAILURE);
2260	}
2261
2262	dev_type = bus_p->bus_dev_type;
2263
2264	/*
2265	 * For ARI Devices, only function zero's MPS needs to be set.
2266	 */
2267	if ((dev_type == PCIE_PCIECAP_DEV_TYPE_PCIE_DEV) &&
2268	    (pcie_ari_is_enabled(pdip) == PCIE_ARI_FORW_ENABLED)) {
2269		pcie_req_id_t child_bdf;
2270
2271		if (pcie_get_bdf_from_dip(cdip, &child_bdf) == DDI_FAILURE)
2272			return (DDI_FAILURE);
2273		if ((child_bdf & PCIE_REQ_ID_ARI_FUNC_MASK) != 0)
2274			return (DDI_SUCCESS);
2275	}
2276
2277	if (PCIE_IS_PCIE(bus_p)) {
2278		int suggested_mrrs, fabric_mps;
2279		uint16_t device_mps, device_mps_cap, device_mrrs, dev_ctrl;
2280
2281		dev_ctrl = PCIE_CAP_GET(16, bus_p, PCIE_DEVCTL);
2282		if ((fabric_mps = (PCIE_IS_RP(bus_p) ? bus_p :
2283		    PCIE_DIP2BUS(pdip))->bus_mps) < 0) {
2284			dev_ctrl = (dev_ctrl & ~(PCIE_DEVCTL_MAX_READ_REQ_MASK |
2285			    PCIE_DEVCTL_MAX_PAYLOAD_MASK)) |
2286			    (pcie_devctl_default &
2287			    (PCIE_DEVCTL_MAX_READ_REQ_MASK |
2288			    PCIE_DEVCTL_MAX_PAYLOAD_MASK));
2289
2290			PCIE_CAP_PUT(16, bus_p, PCIE_DEVCTL, dev_ctrl);
2291			return (DDI_SUCCESS);
2292		}
2293
2294		device_mps_cap = PCIE_CAP_GET(16, bus_p, PCIE_DEVCAP) &
2295		    PCIE_DEVCAP_MAX_PAYLOAD_MASK;
2296
2297		device_mrrs = (dev_ctrl & PCIE_DEVCTL_MAX_READ_REQ_MASK) >>
2298		    PCIE_DEVCTL_MAX_READ_REQ_SHIFT;
2299
2300		if (device_mps_cap < fabric_mps)
2301			device_mrrs = device_mps = device_mps_cap;
2302		else
2303			device_mps = (uint16_t)fabric_mps;
2304
2305		suggested_mrrs = (uint32_t)ddi_prop_get_int(DDI_DEV_T_ANY,
2306		    cdip, DDI_PROP_DONTPASS, "suggested-mrrs", device_mrrs);
2307
2308		if ((device_mps == fabric_mps) ||
2309		    (suggested_mrrs < device_mrrs))
2310			device_mrrs = (uint16_t)suggested_mrrs;
2311
2312		/*
2313		 * Replace MPS and MRRS settings.
2314		 */
2315		dev_ctrl &= ~(PCIE_DEVCTL_MAX_READ_REQ_MASK |
2316		    PCIE_DEVCTL_MAX_PAYLOAD_MASK);
2317
2318		dev_ctrl |= ((device_mrrs << PCIE_DEVCTL_MAX_READ_REQ_SHIFT) |
2319		    device_mps << PCIE_DEVCTL_MAX_PAYLOAD_SHIFT);
2320
2321		PCIE_CAP_PUT(16, bus_p, PCIE_DEVCTL, dev_ctrl);
2322
2323		bus_p->bus_mps = device_mps;
2324	}
2325
2326	return (DDI_SUCCESS);
2327}
2328
2329/*
2330 * Scans a device tree/branch for a maximum payload size capabilities.
2331 *
2332 * rc_dip - dip of Root Complex.
2333 * dip - dip of device where scan will begin.
2334 * max_supported (IN) - maximum allowable MPS.
2335 * max_supported (OUT) - maximum payload size capability of fabric.
2336 */
2337void
2338pcie_get_fabric_mps(dev_info_t *rc_dip, dev_info_t *dip, int *max_supported)
2339{
2340	if (dip == NULL)
2341		return;
2342
2343	/*
2344	 * Perform a fabric scan to obtain Maximum Payload Capabilities
2345	 */
2346	(void) pcie_scan_mps(rc_dip, dip, max_supported);
2347
2348	PCIE_DBG("MPS: Highest Common MPS= %x\n", max_supported);
2349}
2350
2351/*
2352 * Scans fabric and determines Maximum Payload Size based on
2353 * highest common denominator alogorithm
2354 */
2355static void
2356pcie_scan_mps(dev_info_t *rc_dip, dev_info_t *dip, int *max_supported)
2357{
2358	int circular_count;
2359	pcie_max_supported_t max_pay_load_supported;
2360
2361	max_pay_load_supported.dip = rc_dip;
2362	max_pay_load_supported.highest_common_mps = *max_supported;
2363
2364	ndi_devi_enter(ddi_get_parent(dip), &circular_count);
2365	ddi_walk_devs(dip, pcie_get_max_supported,
2366	    (void *)&max_pay_load_supported);
2367	ndi_devi_exit(ddi_get_parent(dip), circular_count);
2368
2369	*max_supported = max_pay_load_supported.highest_common_mps;
2370}
2371
2372/*
2373 * Called as part of the Maximum Payload Size scan.
2374 */
2375static int
2376pcie_get_max_supported(dev_info_t *dip, void *arg)
2377{
2378	uint32_t max_supported;
2379	uint16_t cap_ptr;
2380	pcie_max_supported_t *current = (pcie_max_supported_t *)arg;
2381	pci_regspec_t *reg;
2382	int rlen;
2383	caddr_t virt;
2384	ddi_acc_handle_t config_handle;
2385
2386	if (ddi_get_child(current->dip) == NULL) {
2387		goto fail1;
2388	}
2389
2390	if (pcie_dev(dip) == DDI_FAILURE) {
2391		PCIE_DBG("MPS: pcie_get_max_supported: %s:  "
2392		    "Not a PCIe dev\n", ddi_driver_name(dip));
2393		goto fail1;
2394	}
2395
2396	/*
2397	 * If the suggested-mrrs property exists, then don't include this
2398	 * device in the MPS capabilities scan.
2399	 */
2400	if (ddi_prop_exists(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
2401	    "suggested-mrrs") != 0)
2402		goto fail1;
2403
2404	if (ddi_getlongprop(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, "reg",
2405	    (caddr_t)&reg, &rlen) != DDI_PROP_SUCCESS) {
2406		PCIE_DBG("MPS: pcie_get_max_supported: %s:  "
2407		    "Can not read reg\n", ddi_driver_name(dip));
2408		goto fail1;
2409	}
2410
2411	if (pcie_map_phys(ddi_get_child(current->dip), reg, &virt,
2412	    &config_handle) != DDI_SUCCESS) {
2413		PCIE_DBG("MPS: pcie_get_max_supported: %s:  pcie_map_phys "
2414		    "failed\n", ddi_driver_name(dip));
2415		goto fail2;
2416	}
2417
2418	if ((PCI_CAP_LOCATE(config_handle, PCI_CAP_ID_PCI_E, &cap_ptr)) ==
2419	    DDI_FAILURE) {
2420		goto fail3;
2421	}
2422
2423	max_supported = PCI_CAP_GET16(config_handle, 0, cap_ptr,
2424	    PCIE_DEVCAP) & PCIE_DEVCAP_MAX_PAYLOAD_MASK;
2425
2426	PCIE_DBG("PCIE MPS: %s: MPS Capabilities %x\n", ddi_driver_name(dip),
2427	    max_supported);
2428
2429	if (max_supported < current->highest_common_mps)
2430		current->highest_common_mps = max_supported;
2431
2432fail3:
2433	pcie_unmap_phys(&config_handle, reg);
2434fail2:
2435	kmem_free(reg, rlen);
2436fail1:
2437	return (DDI_WALK_CONTINUE);
2438}
2439
2440/*
2441 * Determines if there are any root ports attached to a root complex.
2442 *
2443 * dip - dip of root complex
2444 *
2445 * Returns - DDI_SUCCESS if there is at least one root port otherwise
2446 *	     DDI_FAILURE.
2447 */
2448int
2449pcie_root_port(dev_info_t *dip)
2450{
2451	int port_type;
2452	uint16_t cap_ptr;
2453	ddi_acc_handle_t config_handle;
2454	dev_info_t *cdip = ddi_get_child(dip);
2455
2456	/*
2457	 * Determine if any of the children of the passed in dip
2458	 * are root ports.
2459	 */
2460	for (; cdip; cdip = ddi_get_next_sibling(cdip)) {
2461
2462		if (pci_config_setup(cdip, &config_handle) != DDI_SUCCESS)
2463			continue;
2464
2465		if ((PCI_CAP_LOCATE(config_handle, PCI_CAP_ID_PCI_E,
2466		    &cap_ptr)) == DDI_FAILURE) {
2467			pci_config_teardown(&config_handle);
2468			continue;
2469		}
2470
2471		port_type = PCI_CAP_GET16(config_handle, 0, cap_ptr,
2472		    PCIE_PCIECAP) & PCIE_PCIECAP_DEV_TYPE_MASK;
2473
2474		pci_config_teardown(&config_handle);
2475
2476		if (port_type == PCIE_PCIECAP_DEV_TYPE_ROOT)
2477			return (DDI_SUCCESS);
2478	}
2479
2480	/* No root ports were found */
2481
2482	return (DDI_FAILURE);
2483}
2484
2485/*
2486 * Function that determines if a device a PCIe device.
2487 *
2488 * dip - dip of device.
2489 *
2490 * returns - DDI_SUCCESS if device is a PCIe device, otherwise DDI_FAILURE.
2491 */
2492int
2493pcie_dev(dev_info_t *dip)
2494{
2495	/* get parent device's device_type property */
2496	char *device_type;
2497	int rc = DDI_FAILURE;
2498	dev_info_t *pdip = ddi_get_parent(dip);
2499
2500	if (ddi_prop_lookup_string(DDI_DEV_T_ANY, pdip,
2501	    DDI_PROP_DONTPASS, "device_type", &device_type)
2502	    != DDI_PROP_SUCCESS) {
2503		return (DDI_FAILURE);
2504	}
2505
2506	if (strcmp(device_type, "pciex") == 0)
2507		rc = DDI_SUCCESS;
2508	else
2509		rc = DDI_FAILURE;
2510
2511	ddi_prop_free(device_type);
2512	return (rc);
2513}
2514
2515/*
2516 * Function to map in a device's memory space.
2517 */
2518static int
2519pcie_map_phys(dev_info_t *dip, pci_regspec_t *phys_spec,
2520    caddr_t *addrp, ddi_acc_handle_t *handlep)
2521{
2522	ddi_map_req_t mr;
2523	ddi_acc_hdl_t *hp;
2524	int result;
2525	ddi_device_acc_attr_t attr;
2526
2527	attr.devacc_attr_version = DDI_DEVICE_ATTR_V0;
2528	attr.devacc_attr_endian_flags = DDI_STRUCTURE_LE_ACC;
2529	attr.devacc_attr_dataorder = DDI_STRICTORDER_ACC;
2530	attr.devacc_attr_access = DDI_CAUTIOUS_ACC;
2531
2532	*handlep = impl_acc_hdl_alloc(KM_SLEEP, NULL);
2533	hp = impl_acc_hdl_get(*handlep);
2534	hp->ah_vers = VERS_ACCHDL;
2535	hp->ah_dip = dip;
2536	hp->ah_rnumber = 0;
2537	hp->ah_offset = 0;
2538	hp->ah_len = 0;
2539	hp->ah_acc = attr;
2540
2541	mr.map_op = DDI_MO_MAP_LOCKED;
2542	mr.map_type = DDI_MT_REGSPEC;
2543	mr.map_obj.rp = (struct regspec *)phys_spec;
2544	mr.map_prot = PROT_READ | PROT_WRITE;
2545	mr.map_flags = DDI_MF_KERNEL_MAPPING;
2546	mr.map_handlep = hp;
2547	mr.map_vers = DDI_MAP_VERSION;
2548
2549	result = ddi_map(dip, &mr, 0, 0, addrp);
2550
2551	if (result != DDI_SUCCESS) {
2552		impl_acc_hdl_free(*handlep);
2553		*handlep = (ddi_acc_handle_t)NULL;
2554	} else {
2555		hp->ah_addr = *addrp;
2556	}
2557
2558	return (result);
2559}
2560
2561/*
2562 * Map out memory that was mapped in with pcie_map_phys();
2563 */
2564static void
2565pcie_unmap_phys(ddi_acc_handle_t *handlep,  pci_regspec_t *ph)
2566{
2567	ddi_map_req_t mr;
2568	ddi_acc_hdl_t *hp;
2569
2570	hp = impl_acc_hdl_get(*handlep);
2571	ASSERT(hp);
2572
2573	mr.map_op = DDI_MO_UNMAP;
2574	mr.map_type = DDI_MT_REGSPEC;
2575	mr.map_obj.rp = (struct regspec *)ph;
2576	mr.map_prot = PROT_READ | PROT_WRITE;
2577	mr.map_flags = DDI_MF_KERNEL_MAPPING;
2578	mr.map_handlep = hp;
2579	mr.map_vers = DDI_MAP_VERSION;
2580
2581	(void) ddi_map(hp->ah_dip, &mr, hp->ah_offset,
2582	    hp->ah_len, &hp->ah_addr);
2583
2584	impl_acc_hdl_free(*handlep);
2585	*handlep = (ddi_acc_handle_t)NULL;
2586}
2587
2588void
2589pcie_set_rber_fatal(dev_info_t *dip, boolean_t val)
2590{
2591	pcie_bus_t *bus_p = PCIE_DIP2UPBUS(dip);
2592	bus_p->bus_pfd->pe_rber_fatal = val;
2593}
2594
2595/*
2596 * Return parent Root Port's pe_rber_fatal value.
2597 */
2598boolean_t
2599pcie_get_rber_fatal(dev_info_t *dip)
2600{
2601	pcie_bus_t *bus_p = PCIE_DIP2UPBUS(dip);
2602	pcie_bus_t *rp_bus_p = PCIE_DIP2UPBUS(bus_p->bus_rp_dip);
2603	return (rp_bus_p->bus_pfd->pe_rber_fatal);
2604}
2605
2606int
2607pcie_ari_supported(dev_info_t *dip)
2608{
2609	uint32_t devcap2;
2610	uint16_t pciecap;
2611	pcie_bus_t *bus_p = PCIE_DIP2BUS(dip);
2612	uint8_t dev_type;
2613
2614	PCIE_DBG("pcie_ari_supported: dip=%p\n", dip);
2615
2616	if (bus_p == NULL)
2617		return (PCIE_ARI_FORW_NOT_SUPPORTED);
2618
2619	dev_type = bus_p->bus_dev_type;
2620
2621	if ((dev_type != PCIE_PCIECAP_DEV_TYPE_DOWN) &&
2622	    (dev_type != PCIE_PCIECAP_DEV_TYPE_ROOT))
2623		return (PCIE_ARI_FORW_NOT_SUPPORTED);
2624
2625	if (pcie_disable_ari) {
2626		PCIE_DBG("pcie_ari_supported: dip=%p: ARI Disabled\n", dip);
2627		return (PCIE_ARI_FORW_NOT_SUPPORTED);
2628	}
2629
2630	pciecap = PCIE_CAP_GET(16, bus_p, PCIE_PCIECAP);
2631
2632	if ((pciecap & PCIE_PCIECAP_VER_MASK) < PCIE_PCIECAP_VER_2_0) {
2633		PCIE_DBG("pcie_ari_supported: dip=%p: Not 2.0\n", dip);
2634		return (PCIE_ARI_FORW_NOT_SUPPORTED);
2635	}
2636
2637	devcap2 = PCIE_CAP_GET(32, bus_p, PCIE_DEVCAP2);
2638
2639	PCIE_DBG("pcie_ari_supported: dip=%p: DevCap2=0x%x\n",
2640	    dip, devcap2);
2641
2642	if (devcap2 & PCIE_DEVCAP2_ARI_FORWARD) {
2643		PCIE_DBG("pcie_ari_supported: "
2644		    "dip=%p: ARI Forwarding is supported\n", dip);
2645		return (PCIE_ARI_FORW_SUPPORTED);
2646	}
2647	return (PCIE_ARI_FORW_NOT_SUPPORTED);
2648}
2649
2650int
2651pcie_ari_enable(dev_info_t *dip)
2652{
2653	uint16_t devctl2;
2654	pcie_bus_t *bus_p = PCIE_DIP2BUS(dip);
2655
2656	PCIE_DBG("pcie_ari_enable: dip=%p\n", dip);
2657
2658	if (pcie_ari_supported(dip) == PCIE_ARI_FORW_NOT_SUPPORTED)
2659		return (DDI_FAILURE);
2660
2661	devctl2 = PCIE_CAP_GET(16, bus_p, PCIE_DEVCTL2);
2662	devctl2 |= PCIE_DEVCTL2_ARI_FORWARD_EN;
2663	PCIE_CAP_PUT(16, bus_p, PCIE_DEVCTL2, devctl2);
2664
2665	PCIE_DBG("pcie_ari_enable: dip=%p: writing 0x%x to DevCtl2\n",
2666	    dip, devctl2);
2667
2668	return (DDI_SUCCESS);
2669}
2670
2671int
2672pcie_ari_disable(dev_info_t *dip)
2673{
2674	uint16_t devctl2;
2675	pcie_bus_t *bus_p = PCIE_DIP2BUS(dip);
2676
2677	PCIE_DBG("pcie_ari_disable: dip=%p\n", dip);
2678
2679	if (pcie_ari_supported(dip) == PCIE_ARI_FORW_NOT_SUPPORTED)
2680		return (DDI_FAILURE);
2681
2682	devctl2 = PCIE_CAP_GET(16, bus_p, PCIE_DEVCTL2);
2683	devctl2 &= ~PCIE_DEVCTL2_ARI_FORWARD_EN;
2684	PCIE_CAP_PUT(16, bus_p, PCIE_DEVCTL2, devctl2);
2685
2686	PCIE_DBG("pcie_ari_disable: dip=%p: writing 0x%x to DevCtl2\n",
2687	    dip, devctl2);
2688
2689	return (DDI_SUCCESS);
2690}
2691
2692int
2693pcie_ari_is_enabled(dev_info_t *dip)
2694{
2695	uint16_t devctl2;
2696	pcie_bus_t *bus_p = PCIE_DIP2BUS(dip);
2697
2698	PCIE_DBG("pcie_ari_is_enabled: dip=%p\n", dip);
2699
2700	if (pcie_ari_supported(dip) == PCIE_ARI_FORW_NOT_SUPPORTED)
2701		return (PCIE_ARI_FORW_DISABLED);
2702
2703	devctl2 = PCIE_CAP_GET(32, bus_p, PCIE_DEVCTL2);
2704
2705	PCIE_DBG("pcie_ari_is_enabled: dip=%p: DevCtl2=0x%x\n",
2706	    dip, devctl2);
2707
2708	if (devctl2 & PCIE_DEVCTL2_ARI_FORWARD_EN) {
2709		PCIE_DBG("pcie_ari_is_enabled: "
2710		    "dip=%p: ARI Forwarding is enabled\n", dip);
2711		return (PCIE_ARI_FORW_ENABLED);
2712	}
2713
2714	return (PCIE_ARI_FORW_DISABLED);
2715}
2716
2717int
2718pcie_ari_device(dev_info_t *dip)
2719{
2720	ddi_acc_handle_t handle;
2721	uint16_t cap_ptr;
2722
2723	PCIE_DBG("pcie_ari_device: dip=%p\n", dip);
2724
2725	/*
2726	 * XXX - This function may be called before the bus_p structure
2727	 * has been populated.  This code can be changed to remove
2728	 * pci_config_setup()/pci_config_teardown() when the RFE
2729	 * to populate the bus_p structures early in boot is putback.
2730	 */
2731
2732	/* First make sure it is a PCIe device */
2733
2734	if (pci_config_setup(dip, &handle) != DDI_SUCCESS)
2735		return (PCIE_NOT_ARI_DEVICE);
2736
2737	if ((PCI_CAP_LOCATE(handle, PCI_CAP_ID_PCI_E, &cap_ptr))
2738	    != DDI_SUCCESS) {
2739		pci_config_teardown(&handle);
2740		return (PCIE_NOT_ARI_DEVICE);
2741	}
2742
2743	/* Locate the ARI Capability */
2744
2745	if ((PCI_CAP_LOCATE(handle, PCI_CAP_XCFG_SPC(PCIE_EXT_CAP_ID_ARI),
2746	    &cap_ptr)) == DDI_FAILURE) {
2747		pci_config_teardown(&handle);
2748		return (PCIE_NOT_ARI_DEVICE);
2749	}
2750
2751	/* ARI Capability was found so it must be a ARI device */
2752	PCIE_DBG("pcie_ari_device: ARI Device dip=%p\n", dip);
2753
2754	pci_config_teardown(&handle);
2755	return (PCIE_ARI_DEVICE);
2756}
2757
2758int
2759pcie_ari_get_next_function(dev_info_t *dip, int *func)
2760{
2761	uint32_t val;
2762	uint16_t cap_ptr, next_function;
2763	ddi_acc_handle_t handle;
2764
2765	/*
2766	 * XXX - This function may be called before the bus_p structure
2767	 * has been populated.  This code can be changed to remove
2768	 * pci_config_setup()/pci_config_teardown() when the RFE
2769	 * to populate the bus_p structures early in boot is putback.
2770	 */
2771
2772	if (pci_config_setup(dip, &handle) != DDI_SUCCESS)
2773		return (DDI_FAILURE);
2774
2775	if ((PCI_CAP_LOCATE(handle,
2776	    PCI_CAP_XCFG_SPC(PCIE_EXT_CAP_ID_ARI), &cap_ptr)) == DDI_FAILURE) {
2777		pci_config_teardown(&handle);
2778		return (DDI_FAILURE);
2779	}
2780
2781	val = PCI_CAP_GET32(handle, 0, cap_ptr, PCIE_ARI_CAP);
2782
2783	next_function = (val >> PCIE_ARI_CAP_NEXT_FUNC_SHIFT) &
2784	    PCIE_ARI_CAP_NEXT_FUNC_MASK;
2785
2786	pci_config_teardown(&handle);
2787
2788	*func = next_function;
2789
2790	return (DDI_SUCCESS);
2791}
2792
2793dev_info_t *
2794pcie_func_to_dip(dev_info_t *dip, pcie_req_id_t function)
2795{
2796	pcie_req_id_t child_bdf;
2797	dev_info_t *cdip;
2798
2799	for (cdip = ddi_get_child(dip); cdip;
2800	    cdip = ddi_get_next_sibling(cdip)) {
2801
2802		if (pcie_get_bdf_from_dip(cdip, &child_bdf) == DDI_FAILURE)
2803			return (NULL);
2804
2805		if ((child_bdf & PCIE_REQ_ID_ARI_FUNC_MASK) == function)
2806			return (cdip);
2807	}
2808	return (NULL);
2809}
2810
2811#ifdef	DEBUG
2812
2813static void
2814pcie_print_bus(pcie_bus_t *bus_p)
2815{
2816	pcie_dbg("\tbus_dip = 0x%p\n", bus_p->bus_dip);
2817	pcie_dbg("\tbus_fm_flags = 0x%x\n", bus_p->bus_fm_flags);
2818
2819	pcie_dbg("\tbus_bdf = 0x%x\n", bus_p->bus_bdf);
2820	pcie_dbg("\tbus_dev_ven_id = 0x%x\n", bus_p->bus_dev_ven_id);
2821	pcie_dbg("\tbus_rev_id = 0x%x\n", bus_p->bus_rev_id);
2822	pcie_dbg("\tbus_hdr_type = 0x%x\n", bus_p->bus_hdr_type);
2823	pcie_dbg("\tbus_dev_type = 0x%x\n", bus_p->bus_dev_type);
2824	pcie_dbg("\tbus_bdg_secbus = 0x%x\n", bus_p->bus_bdg_secbus);
2825	pcie_dbg("\tbus_pcie_off = 0x%x\n", bus_p->bus_pcie_off);
2826	pcie_dbg("\tbus_aer_off = 0x%x\n", bus_p->bus_aer_off);
2827	pcie_dbg("\tbus_pcix_off = 0x%x\n", bus_p->bus_pcix_off);
2828	pcie_dbg("\tbus_ecc_ver = 0x%x\n", bus_p->bus_ecc_ver);
2829}
2830
2831/*
2832 * For debugging purposes set pcie_dbg_print != 0 to see printf messages
2833 * during interrupt.
2834 *
2835 * When a proper solution is in place this code will disappear.
2836 * Potential solutions are:
2837 * o circular buffers
2838 * o taskq to print at lower pil
2839 */
2840int pcie_dbg_print = 0;
2841void
2842pcie_dbg(char *fmt, ...)
2843{
2844	va_list ap;
2845
2846	if (!pcie_debug_flags) {
2847		return;
2848	}
2849	va_start(ap, fmt);
2850	if (servicing_interrupt()) {
2851		if (pcie_dbg_print) {
2852			prom_vprintf(fmt, ap);
2853		}
2854	} else {
2855		prom_vprintf(fmt, ap);
2856	}
2857	va_end(ap);
2858}
2859#endif	/* DEBUG */
2860
2861#if defined(__i386) || defined(__amd64)
2862static void
2863pcie_check_io_mem_range(ddi_acc_handle_t cfg_hdl, boolean_t *empty_io_range,
2864    boolean_t *empty_mem_range)
2865{
2866	uint8_t	class, subclass;
2867	uint_t	val;
2868
2869	class = pci_config_get8(cfg_hdl, PCI_CONF_BASCLASS);
2870	subclass = pci_config_get8(cfg_hdl, PCI_CONF_SUBCLASS);
2871
2872	if ((class == PCI_CLASS_BRIDGE) && (subclass == PCI_BRIDGE_PCI)) {
2873		val = (((uint_t)pci_config_get8(cfg_hdl, PCI_BCNF_IO_BASE_LOW) &
2874		    PCI_BCNF_IO_MASK) << 8);
2875		/*
2876		 * Assuming that a zero based io_range[0] implies an
2877		 * invalid I/O range.  Likewise for mem_range[0].
2878		 */
2879		if (val == 0)
2880			*empty_io_range = B_TRUE;
2881		val = (((uint_t)pci_config_get16(cfg_hdl, PCI_BCNF_MEM_BASE) &
2882		    PCI_BCNF_MEM_MASK) << 16);
2883		if (val == 0)
2884			*empty_mem_range = B_TRUE;
2885	}
2886}
2887
2888#endif /* defined(__i386) || defined(__amd64) */
2889
2890boolean_t
2891pcie_link_bw_supported(dev_info_t *dip)
2892{
2893	uint32_t linkcap;
2894	pcie_bus_t *bus_p = PCIE_DIP2BUS(dip);
2895
2896	if (!PCIE_IS_PCIE(bus_p)) {
2897		return (B_FALSE);
2898	}
2899
2900	if (!PCIE_IS_RP(bus_p) && !PCIE_IS_SWD(bus_p)) {
2901		return (B_FALSE);
2902	}
2903
2904	linkcap = PCIE_CAP_GET(32, bus_p, PCIE_LINKCAP);
2905	return ((linkcap & PCIE_LINKCAP_LINK_BW_NOTIFY_CAP) != 0);
2906}
2907
2908int
2909pcie_link_bw_enable(dev_info_t *dip)
2910{
2911	uint16_t linkctl;
2912	pcie_bus_t *bus_p = PCIE_DIP2BUS(dip);
2913
2914	if (pcie_disable_lbw != 0) {
2915		return (DDI_FAILURE);
2916	}
2917
2918	if (!pcie_link_bw_supported(dip)) {
2919		return (DDI_FAILURE);
2920	}
2921
2922	mutex_init(&bus_p->bus_lbw_mutex, NULL, MUTEX_DRIVER, NULL);
2923	cv_init(&bus_p->bus_lbw_cv, NULL, CV_DRIVER, NULL);
2924	linkctl = PCIE_CAP_GET(16, bus_p, PCIE_LINKCTL);
2925	linkctl |= PCIE_LINKCTL_LINK_BW_INTR_EN;
2926	linkctl |= PCIE_LINKCTL_LINK_AUTO_BW_INTR_EN;
2927	PCIE_CAP_PUT(16, bus_p, PCIE_LINKCTL, linkctl);
2928
2929	bus_p->bus_lbw_pbuf = kmem_zalloc(MAXPATHLEN, KM_SLEEP);
2930	bus_p->bus_lbw_cbuf = kmem_zalloc(MAXPATHLEN, KM_SLEEP);
2931	bus_p->bus_lbw_state |= PCIE_LBW_S_ENABLED;
2932
2933	return (DDI_SUCCESS);
2934}
2935
2936int
2937pcie_link_bw_disable(dev_info_t *dip)
2938{
2939	uint16_t linkctl;
2940	pcie_bus_t *bus_p = PCIE_DIP2BUS(dip);
2941
2942	if ((bus_p->bus_lbw_state & PCIE_LBW_S_ENABLED) == 0) {
2943		return (DDI_FAILURE);
2944	}
2945
2946	mutex_enter(&bus_p->bus_lbw_mutex);
2947	while ((bus_p->bus_lbw_state &
2948	    (PCIE_LBW_S_DISPATCHED | PCIE_LBW_S_RUNNING)) != 0) {
2949		cv_wait(&bus_p->bus_lbw_cv, &bus_p->bus_lbw_mutex);
2950	}
2951	mutex_exit(&bus_p->bus_lbw_mutex);
2952
2953	linkctl = PCIE_CAP_GET(16, bus_p, PCIE_LINKCTL);
2954	linkctl &= ~PCIE_LINKCTL_LINK_BW_INTR_EN;
2955	linkctl &= ~PCIE_LINKCTL_LINK_AUTO_BW_INTR_EN;
2956	PCIE_CAP_PUT(16, bus_p, PCIE_LINKCTL, linkctl);
2957
2958	bus_p->bus_lbw_state &= ~PCIE_LBW_S_ENABLED;
2959	kmem_free(bus_p->bus_lbw_pbuf, MAXPATHLEN);
2960	kmem_free(bus_p->bus_lbw_cbuf, MAXPATHLEN);
2961	bus_p->bus_lbw_pbuf = NULL;
2962	bus_p->bus_lbw_cbuf = NULL;
2963
2964	mutex_destroy(&bus_p->bus_lbw_mutex);
2965	cv_destroy(&bus_p->bus_lbw_cv);
2966
2967	return (DDI_SUCCESS);
2968}
2969
2970void
2971pcie_link_bw_taskq(void *arg)
2972{
2973	dev_info_t *dip = arg;
2974	pcie_bus_t *bus_p = PCIE_DIP2BUS(dip);
2975	dev_info_t *cdip;
2976	boolean_t again;
2977	sysevent_t *se;
2978	sysevent_value_t se_val;
2979	sysevent_id_t eid;
2980	sysevent_attr_list_t *ev_attr_list;
2981	int circular;
2982
2983top:
2984	ndi_devi_enter(dip, &circular);
2985	se = NULL;
2986	ev_attr_list = NULL;
2987	mutex_enter(&bus_p->bus_lbw_mutex);
2988	bus_p->bus_lbw_state &= ~PCIE_LBW_S_DISPATCHED;
2989	bus_p->bus_lbw_state |= PCIE_LBW_S_RUNNING;
2990	mutex_exit(&bus_p->bus_lbw_mutex);
2991
2992	/*
2993	 * Update our own speeds as we've likely changed something.
2994	 */
2995	pcie_capture_speeds(dip);
2996
2997	/*
2998	 * Walk our children. We only care about updating this on function 0
2999	 * because the PCIe specification requires that these all be the same
3000	 * otherwise.
3001	 */
3002	for (cdip = ddi_get_child(dip); cdip != NULL;
3003	    cdip = ddi_get_next_sibling(cdip)) {
3004		pcie_bus_t *cbus_p = PCIE_DIP2BUS(cdip);
3005
3006		if (cbus_p == NULL) {
3007			continue;
3008		}
3009
3010		if ((cbus_p->bus_bdf & PCIE_REQ_ID_FUNC_MASK) != 0) {
3011			continue;
3012		}
3013
3014		/*
3015		 * It's possible that this can fire while a child is otherwise
3016		 * only partially constructed. Therefore, if we don't have the
3017		 * config handle, don't bother updating the child.
3018		 */
3019		if (cbus_p->bus_cfg_hdl == NULL) {
3020			continue;
3021		}
3022
3023		pcie_capture_speeds(cdip);
3024		break;
3025	}
3026
3027	se = sysevent_alloc(EC_PCIE, ESC_PCIE_LINK_STATE,
3028	    ILLUMOS_KERN_PUB "pcie", SE_SLEEP);
3029
3030	(void) ddi_pathname(dip, bus_p->bus_lbw_pbuf);
3031	se_val.value_type = SE_DATA_TYPE_STRING;
3032	se_val.value.sv_string = bus_p->bus_lbw_pbuf;
3033	if (sysevent_add_attr(&ev_attr_list, PCIE_EV_DETECTOR_PATH, &se_val,
3034	    SE_SLEEP) != 0) {
3035		ndi_devi_exit(dip, circular);
3036		goto err;
3037	}
3038
3039	if (cdip != NULL) {
3040		(void) ddi_pathname(cdip, bus_p->bus_lbw_cbuf);
3041
3042		se_val.value_type = SE_DATA_TYPE_STRING;
3043		se_val.value.sv_string = bus_p->bus_lbw_cbuf;
3044
3045		/*
3046		 * If this fails, that's OK. We'd rather get the event off and
3047		 * there's a chance that there may not be anything there for us.
3048		 */
3049		(void) sysevent_add_attr(&ev_attr_list, PCIE_EV_CHILD_PATH,
3050		    &se_val, SE_SLEEP);
3051	}
3052
3053	ndi_devi_exit(dip, circular);
3054
3055	/*
3056	 * Before we generate and send down a sysevent, we need to tell the
3057	 * system that parts of the devinfo cache need to be invalidated. While
3058	 * the function below takes several args, it ignores them all. Because
3059	 * this is a global invalidation, we don't bother trying to do much more
3060	 * than requesting a global invalidation, lest we accidentally kick off
3061	 * several in a row.
3062	 */
3063	ddi_prop_cache_invalidate(DDI_DEV_T_NONE, NULL, NULL, 0);
3064
3065	if (sysevent_attach_attributes(se, ev_attr_list) != 0) {
3066		goto err;
3067	}
3068	ev_attr_list = NULL;
3069
3070	if (log_sysevent(se, SE_SLEEP, &eid) != 0) {
3071		goto err;
3072	}
3073
3074err:
3075	sysevent_free_attr(ev_attr_list);
3076	sysevent_free(se);
3077
3078	mutex_enter(&bus_p->bus_lbw_mutex);
3079	bus_p->bus_lbw_state &= ~PCIE_LBW_S_RUNNING;
3080	cv_broadcast(&bus_p->bus_lbw_cv);
3081	again = (bus_p->bus_lbw_state & PCIE_LBW_S_DISPATCHED) != 0;
3082	mutex_exit(&bus_p->bus_lbw_mutex);
3083
3084	if (again) {
3085		goto top;
3086	}
3087}
3088
3089int
3090pcie_link_bw_intr(dev_info_t *dip)
3091{
3092	pcie_bus_t *bus_p = PCIE_DIP2BUS(dip);
3093	uint16_t linksts;
3094	uint16_t flags = PCIE_LINKSTS_LINK_BW_MGMT | PCIE_LINKSTS_AUTO_BW;
3095
3096	if ((bus_p->bus_lbw_state & PCIE_LBW_S_ENABLED) == 0) {
3097		return (DDI_INTR_UNCLAIMED);
3098	}
3099
3100	linksts = PCIE_CAP_GET(16, bus_p, PCIE_LINKSTS);
3101	if ((linksts & flags) == 0) {
3102		return (DDI_INTR_UNCLAIMED);
3103	}
3104
3105	/*
3106	 * Check if we've already dispatched this event. If we have already
3107	 * dispatched it, then there's nothing else to do, we coalesce multiple
3108	 * events.
3109	 */
3110	mutex_enter(&bus_p->bus_lbw_mutex);
3111	bus_p->bus_lbw_nevents++;
3112	if ((bus_p->bus_lbw_state & PCIE_LBW_S_DISPATCHED) == 0) {
3113		if ((bus_p->bus_lbw_state & PCIE_LBW_S_RUNNING) == 0) {
3114			taskq_dispatch_ent(pcie_link_tq, pcie_link_bw_taskq,
3115			    dip, 0, &bus_p->bus_lbw_ent);
3116		}
3117
3118		bus_p->bus_lbw_state |= PCIE_LBW_S_DISPATCHED;
3119	}
3120	mutex_exit(&bus_p->bus_lbw_mutex);
3121
3122	PCIE_CAP_PUT(16, bus_p, PCIE_LINKSTS, flags);
3123	return (DDI_INTR_CLAIMED);
3124}
3125
3126int
3127pcie_link_set_target(dev_info_t *dip, pcie_link_speed_t speed)
3128{
3129	uint16_t ctl2, rval;
3130	pcie_bus_t *bus_p = PCIE_DIP2BUS(dip);
3131
3132	if (!PCIE_IS_PCIE(bus_p)) {
3133		return (ENOTSUP);
3134	}
3135
3136	if (!PCIE_IS_RP(bus_p) && !PCIE_IS_SWD(bus_p)) {
3137		return (ENOTSUP);
3138	}
3139
3140	switch (speed) {
3141	case PCIE_LINK_SPEED_2_5:
3142		rval = PCIE_LINKCTL2_TARGET_SPEED_2_5;
3143		break;
3144	case PCIE_LINK_SPEED_5:
3145		rval = PCIE_LINKCTL2_TARGET_SPEED_5;
3146		break;
3147	case PCIE_LINK_SPEED_8:
3148		rval = PCIE_LINKCTL2_TARGET_SPEED_8;
3149		break;
3150	case PCIE_LINK_SPEED_16:
3151		rval = PCIE_LINKCTL2_TARGET_SPEED_16;
3152		break;
3153	default:
3154		return (EINVAL);
3155	}
3156
3157	mutex_enter(&bus_p->bus_speed_mutex);
3158	bus_p->bus_target_speed = speed;
3159	bus_p->bus_speed_flags |= PCIE_LINK_F_ADMIN_TARGET;
3160
3161	ctl2 = PCIE_CAP_GET(16, bus_p, PCIE_LINKCTL2);
3162	ctl2 &= ~PCIE_LINKCTL2_TARGET_SPEED_MASK;
3163	ctl2 |= rval;
3164	PCIE_CAP_PUT(16, bus_p, PCIE_LINKCTL2, ctl2);
3165	mutex_exit(&bus_p->bus_speed_mutex);
3166
3167	/*
3168	 * Make sure our updates have been reflected in devinfo.
3169	 */
3170	pcie_capture_speeds(dip);
3171
3172	return (0);
3173}
3174
3175int
3176pcie_link_retrain(dev_info_t *dip)
3177{
3178	uint16_t ctl;
3179	pcie_bus_t *bus_p = PCIE_DIP2BUS(dip);
3180
3181	if (!PCIE_IS_PCIE(bus_p)) {
3182		return (ENOTSUP);
3183	}
3184
3185	if (!PCIE_IS_RP(bus_p) && !PCIE_IS_SWD(bus_p)) {
3186		return (ENOTSUP);
3187	}
3188
3189	/*
3190	 * The PCIe specification suggests that we make sure that the link isn't
3191	 * in training before issuing this command in case there was a state
3192	 * machine transition prior to when we got here. We wait and then go
3193	 * ahead and issue the command anyways.
3194	 */
3195	for (uint32_t i = 0; i < pcie_link_retrain_count; i++) {
3196		uint16_t sts;
3197
3198		sts = PCIE_CAP_GET(16, bus_p, PCIE_LINKSTS);
3199		if ((sts & PCIE_LINKSTS_LINK_TRAINING) == 0)
3200			break;
3201		delay(drv_usectohz(pcie_link_retrain_delay_ms * 1000));
3202	}
3203
3204	ctl = PCIE_CAP_GET(16, bus_p, PCIE_LINKCTL);
3205	ctl |= PCIE_LINKCTL_RETRAIN_LINK;
3206	PCIE_CAP_PUT(16, bus_p, PCIE_LINKCTL, ctl);
3207
3208	/*
3209	 * Wait again to see if it clears before returning to the user.
3210	 */
3211	for (uint32_t i = 0; i < pcie_link_retrain_count; i++) {
3212		uint16_t sts;
3213
3214		sts = PCIE_CAP_GET(16, bus_p, PCIE_LINKSTS);
3215		if ((sts & PCIE_LINKSTS_LINK_TRAINING) == 0)
3216			break;
3217		delay(drv_usectohz(pcie_link_retrain_delay_ms * 1000));
3218	}
3219
3220	return (0);
3221}
3222