12fcbc377Syt /*
22fcbc377Syt  * CDDL HEADER START
32fcbc377Syt  *
42fcbc377Syt  * The contents of this file are subject to the terms of the
52fcbc377Syt  * Common Development and Distribution License (the "License").
62fcbc377Syt  * You may not use this file except in compliance with the License.
72fcbc377Syt  *
82fcbc377Syt  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
92fcbc377Syt  * or http://www.opensolaris.org/os/licensing.
102fcbc377Syt  * See the License for the specific language governing permissions
112fcbc377Syt  * and limitations under the License.
122fcbc377Syt  *
132fcbc377Syt  * When distributing Covered Code, include this CDDL HEADER in each
142fcbc377Syt  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
152fcbc377Syt  * If applicable, add the following below this CDDL HEADER, with the
162fcbc377Syt  * fields enclosed by brackets "[]" replaced with your own identifying
172fcbc377Syt  * information: Portions Copyright [yyyy] [name of copyright owner]
182fcbc377Syt  *
192fcbc377Syt  * CDDL HEADER END
202fcbc377Syt  */
212fcbc377Syt 
222fcbc377Syt /*
2309121340Syt  * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
242fcbc377Syt  * Use is subject to license terms.
252fcbc377Syt  */
262fcbc377Syt 
272fcbc377Syt #pragma ident	"%Z%%M%	%I%	%E% SMI"
282fcbc377Syt 
292fcbc377Syt /*
302fcbc377Syt  * AHCI (Advanced Host Controller Interface) SATA HBA Driver
312fcbc377Syt  */
322fcbc377Syt 
332fcbc377Syt #include <sys/scsi/scsi.h>
342fcbc377Syt #include <sys/pci.h>
352fcbc377Syt #include <sys/sata/sata_hba.h>
362fcbc377Syt #include <sys/sata/adapters/ahci/ahcireg.h>
372fcbc377Syt #include <sys/sata/adapters/ahci/ahcivar.h>
382fcbc377Syt 
392fcbc377Syt /*
402fcbc377Syt  * Function prototypes for driver entry points
412fcbc377Syt  */
422fcbc377Syt static	int ahci_attach(dev_info_t *, ddi_attach_cmd_t);
432fcbc377Syt static	int ahci_detach(dev_info_t *, ddi_detach_cmd_t);
442fcbc377Syt static	int ahci_getinfo(dev_info_t *, ddi_info_cmd_t, void *, void **);
452fcbc377Syt 
462fcbc377Syt /*
472fcbc377Syt  * Function prototypes for SATA Framework interfaces
482fcbc377Syt  */
492fcbc377Syt static	int ahci_register_sata_hba_tran(ahci_ctl_t *, uint32_t);
502fcbc377Syt static	int ahci_unregister_sata_hba_tran(ahci_ctl_t *);
512fcbc377Syt 
522fcbc377Syt static	int ahci_tran_probe_port(dev_info_t *, sata_device_t *);
532fcbc377Syt static	int ahci_tran_start(dev_info_t *, sata_pkt_t *spkt);
542fcbc377Syt static	int ahci_tran_abort(dev_info_t *, sata_pkt_t *, int);
552fcbc377Syt static	int ahci_tran_reset_dport(dev_info_t *, sata_device_t *);
562fcbc377Syt static	int ahci_tran_hotplug_port_activate(dev_info_t *, sata_device_t *);
572fcbc377Syt static	int ahci_tran_hotplug_port_deactivate(dev_info_t *, sata_device_t *);
582fcbc377Syt #if defined(__lock_lint)
592fcbc377Syt static	int ahci_selftest(dev_info_t *, sata_device_t *);
602fcbc377Syt #endif
612fcbc377Syt 
622fcbc377Syt /*
632fcbc377Syt  * Local function prototypes
642fcbc377Syt  */
65*68d33a25Syt static	int ahci_alloc_ports_state(ahci_ctl_t *);
66*68d33a25Syt static	void ahci_dealloc_ports_state(ahci_ctl_t *);
672fcbc377Syt static	int ahci_alloc_port_state(ahci_ctl_t *, uint8_t);
682fcbc377Syt static	void ahci_dealloc_port_state(ahci_ctl_t *, uint8_t);
692fcbc377Syt static	int ahci_alloc_rcvd_fis(ahci_ctl_t *, ahci_port_t *, uint8_t);
702fcbc377Syt static	void ahci_dealloc_rcvd_fis(ahci_ctl_t *, ahci_port_t *);
712fcbc377Syt static	int ahci_alloc_cmd_list(ahci_ctl_t *, ahci_port_t *, uint8_t);
722fcbc377Syt static	void ahci_dealloc_cmd_list(ahci_ctl_t *, ahci_port_t *);
732fcbc377Syt static  int ahci_alloc_cmd_tables(ahci_ctl_t *, ahci_port_t *);
742fcbc377Syt static  void ahci_dealloc_cmd_tables(ahci_ctl_t *, ahci_port_t *);
752fcbc377Syt 
76*68d33a25Syt static	int ahci_initialize_controller(ahci_ctl_t *);
77*68d33a25Syt static	void ahci_uninitialize_controller(ahci_ctl_t *);
782fcbc377Syt static	int ahci_initialize_port(ahci_ctl_t *, ahci_port_t *, uint8_t);
79*68d33a25Syt 
80*68d33a25Syt static	int ahci_start_port(ahci_ctl_t *, ahci_port_t *, uint8_t);
81*68d33a25Syt static	void ahci_find_dev_signature(ahci_ctl_t *, ahci_port_t *, uint8_t);
822fcbc377Syt static	void ahci_update_sata_registers(ahci_ctl_t *, uint8_t, sata_device_t *);
832fcbc377Syt static	int ahci_deliver_satapkt(ahci_ctl_t *, ahci_port_t *,
842fcbc377Syt     uint8_t, sata_pkt_t *);
85*68d33a25Syt static	int ahci_do_sync_start(ahci_ctl_t *, ahci_port_t *,
86*68d33a25Syt     uint8_t, sata_pkt_t *);
872fcbc377Syt static	int ahci_claim_free_slot(ahci_ctl_t *, ahci_port_t *);
88*68d33a25Syt static  void ahci_copy_err_cnxt(sata_cmd_t *, ahci_fis_d2h_register_t *);
892fcbc377Syt static	void ahci_copy_out_regs(sata_cmd_t *, ahci_fis_d2h_register_t *);
902fcbc377Syt 
912fcbc377Syt static	int ahci_software_reset(ahci_ctl_t *, ahci_port_t *, uint8_t);
922fcbc377Syt static	int ahci_hba_reset(ahci_ctl_t *);
932fcbc377Syt static	int ahci_port_reset(ahci_ctl_t *, ahci_port_t *, uint8_t);
942fcbc377Syt static	void ahci_reject_all_abort_pkts(ahci_ctl_t *, ahci_port_t *, uint8_t);
952fcbc377Syt static	int ahci_reset_device_reject_pkts(ahci_ctl_t *, ahci_port_t *, uint8_t);
962fcbc377Syt static	int ahci_reset_port_reject_pkts(ahci_ctl_t *, ahci_port_t *, uint8_t);
972fcbc377Syt static	int ahci_reset_hba_reject_pkts(ahci_ctl_t *);
98*68d33a25Syt static	int ahci_put_port_into_notrunning_state(ahci_ctl_t *, ahci_port_t *,
992fcbc377Syt     uint8_t);
1002fcbc377Syt static	int ahci_restart_port_wait_till_ready(ahci_ctl_t *, ahci_port_t *,
101*68d33a25Syt     uint8_t, int, int *);
1022fcbc377Syt static	void ahci_mop_commands(ahci_ctl_t *, ahci_port_t *, uint8_t,
1032fcbc377Syt     uint32_t, uint32_t, uint32_t, uint32_t, uint32_t);
104*68d33a25Syt static void ahci_get_rqsense_data(ahci_ctl_t *, ahci_port_t *,
105*68d33a25Syt     uint8_t, sata_pkt_t *);
106*68d33a25Syt static	void ahci_fatal_error_recovery_handler(ahci_ctl_t *, ahci_port_t *,
107*68d33a25Syt     uint8_t, uint32_t, uint32_t);
108*68d33a25Syt static	void ahci_timeout_pkts(ahci_ctl_t *, ahci_port_t *,
109*68d33a25Syt     uint8_t, uint32_t, uint32_t);
110*68d33a25Syt static	void ahci_events_handler(void *);
1112fcbc377Syt static	void ahci_watchdog_handler(ahci_ctl_t *);
1122fcbc377Syt 
1132fcbc377Syt static	uint_t ahci_intr(caddr_t, caddr_t);
114*68d33a25Syt static	void ahci_port_intr(ahci_ctl_t *, ahci_port_t *, uint8_t, uint32_t);
1152fcbc377Syt static	int ahci_add_legacy_intrs(ahci_ctl_t *);
1162fcbc377Syt static	int ahci_add_msi_intrs(ahci_ctl_t *);
1172fcbc377Syt static	void ahci_rem_intrs(ahci_ctl_t *);
1182fcbc377Syt static	void ahci_enable_all_intrs(ahci_ctl_t *);
1192fcbc377Syt static	void ahci_disable_all_intrs(ahci_ctl_t *);
1202fcbc377Syt static	void ahci_enable_port_intrs(ahci_ctl_t *, ahci_port_t *, uint8_t);
1212fcbc377Syt static	void ahci_disable_port_intrs(ahci_ctl_t *, ahci_port_t *, uint8_t);
1222fcbc377Syt 
123*68d33a25Syt static  int ahci_intr_cmd_cmplt(ahci_ctl_t *, ahci_port_t *, uint8_t,
124*68d33a25Syt     uint32_t, uint32_t);
125*68d33a25Syt static	int ahci_intr_set_device_bits(ahci_ctl_t *, ahci_port_t *, uint8_t);
1262fcbc377Syt static	int ahci_intr_port_connect_change(ahci_ctl_t *, ahci_port_t *, uint8_t);
1272fcbc377Syt static	int ahci_intr_device_mechanical_presence_status(ahci_ctl_t *,
1282fcbc377Syt     ahci_port_t *, uint8_t);
1292fcbc377Syt static	int ahci_intr_phyrdy_change(ahci_ctl_t *, ahci_port_t *, uint8_t);
130*68d33a25Syt static	int ahci_intr_non_fatal_error(ahci_ctl_t *, ahci_port_t *,
131*68d33a25Syt     uint8_t, uint32_t);
132*68d33a25Syt static  int ahci_intr_fatal_error(ahci_ctl_t *, ahci_port_t *,
133*68d33a25Syt     uint8_t, uint32_t, uint32_t);
1342fcbc377Syt static	int ahci_intr_cold_port_detect(ahci_ctl_t *, ahci_port_t *, uint8_t);
1352fcbc377Syt 
1362fcbc377Syt static	int ahci_get_num_implemented_ports(uint32_t);
137*68d33a25Syt static  void ahci_log_fatal_error_message(ahci_ctl_t *, uint8_t port,
138*68d33a25Syt     uint32_t);
139*68d33a25Syt static	void ahci_log_serror_message(ahci_ctl_t *, uint8_t, uint32_t);
1402fcbc377Syt static	void ahci_log(ahci_ctl_t *, uint_t, char *, ...);
1412fcbc377Syt 
1422fcbc377Syt 
1432fcbc377Syt /*
1442fcbc377Syt  * DMA attributes for the data buffer
1452fcbc377Syt  *
1462fcbc377Syt  * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA
1472fcbc377Syt  * does not support 64-bit addressing
1482fcbc377Syt  */
1492fcbc377Syt static ddi_dma_attr_t buffer_dma_attr = {
1502fcbc377Syt 	DMA_ATTR_V0,		/* dma_attr_version */
151*68d33a25Syt 	0x0ull,			/* dma_attr_addr_lo: lowest bus address */
1522fcbc377Syt 	0xffffffffffffffffull,	/* dma_attr_addr_hi: highest bus address */
1532fcbc377Syt 	0x3fffffull,		/* dma_attr_count_max i.e. for one cookie */
154*68d33a25Syt 	0x2ull,			/* dma_attr_align: word aligned */
1552fcbc377Syt 	1,			/* dma_attr_burstsizes */
1562fcbc377Syt 	1,			/* dma_attr_minxfer */
1572fcbc377Syt 	0xffffffffull,		/* dma_attr_maxxfer i.e. includes all cookies */
1582fcbc377Syt 	0xffffffffull,		/* dma_attr_seg */
1592fcbc377Syt 	AHCI_PRDT_NUMBER,	/* dma_attr_sgllen */
1602fcbc377Syt 	512,			/* dma_attr_granular */
1612fcbc377Syt 	0,			/* dma_attr_flags */
1622fcbc377Syt };
1632fcbc377Syt 
1642fcbc377Syt /*
1652fcbc377Syt  * DMA attributes for the rcvd FIS
1662fcbc377Syt  *
1672fcbc377Syt  * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA
1682fcbc377Syt  * does not support 64-bit addressing
1692fcbc377Syt  */
1702fcbc377Syt static ddi_dma_attr_t rcvd_fis_dma_attr = {
1712fcbc377Syt 	DMA_ATTR_V0,		/* dma_attr_version */
172*68d33a25Syt 	0x0ull,			/* dma_attr_addr_lo: lowest bus address */
1732fcbc377Syt 	0xffffffffffffffffull,	/* dma_attr_addr_hi: highest bus address */
1742fcbc377Syt 	0xffffffffull,		/* dma_attr_count_max i.e. for one cookie */
175*68d33a25Syt 	0x100ull,		/* dma_attr_align: 256-byte aligned */
1762fcbc377Syt 	1,			/* dma_attr_burstsizes */
1772fcbc377Syt 	1,			/* dma_attr_minxfer */
1782fcbc377Syt 	0xffffffffull,		/* dma_attr_maxxfer i.e. includes all cookies */
1792fcbc377Syt 	0xffffffffull,		/* dma_attr_seg */
1802fcbc377Syt 	1,			/* dma_attr_sgllen */
1812fcbc377Syt 	1,			/* dma_attr_granular */
1822fcbc377Syt 	0,			/* dma_attr_flags */
1832fcbc377Syt };
1842fcbc377Syt 
1852fcbc377Syt /*
1862fcbc377Syt  * DMA attributes for the command list
1872fcbc377Syt  *
1882fcbc377Syt  * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA
1892fcbc377Syt  * does not support 64-bit addressing
1902fcbc377Syt  */
1912fcbc377Syt static ddi_dma_attr_t cmd_list_dma_attr = {
1922fcbc377Syt 	DMA_ATTR_V0,		/* dma_attr_version */
193*68d33a25Syt 	0x0ull,			/* dma_attr_addr_lo: lowest bus address */
1942fcbc377Syt 	0xffffffffffffffffull,	/* dma_attr_addr_hi: highest bus address */
1952fcbc377Syt 	0xffffffffull,		/* dma_attr_count_max i.e. for one cookie */
196*68d33a25Syt 	0x400ull,		/* dma_attr_align: 1K-byte aligned */
1972fcbc377Syt 	1,			/* dma_attr_burstsizes */
1982fcbc377Syt 	1,			/* dma_attr_minxfer */
1992fcbc377Syt 	0xffffffffull,		/* dma_attr_maxxfer i.e. includes all cookies */
2002fcbc377Syt 	0xffffffffull,		/* dma_attr_seg */
2012fcbc377Syt 	1,			/* dma_attr_sgllen */
2022fcbc377Syt 	1,			/* dma_attr_granular */
2032fcbc377Syt 	0,			/* dma_attr_flags */
2042fcbc377Syt };
2052fcbc377Syt 
2062fcbc377Syt /*
2072fcbc377Syt  * DMA attributes for cmd tables
2082fcbc377Syt  *
2092fcbc377Syt  * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA
2102fcbc377Syt  * does not support 64-bit addressing
2112fcbc377Syt  */
2122fcbc377Syt static ddi_dma_attr_t cmd_table_dma_attr = {
2132fcbc377Syt 	DMA_ATTR_V0,		/* dma_attr_version */
214*68d33a25Syt 	0x0ull,			/* dma_attr_addr_lo: lowest bus address */
2152fcbc377Syt 	0xffffffffffffffffull,	/* dma_attr_addr_hi: highest bus address */
2162fcbc377Syt 	0xffffffffull,		/* dma_attr_count_max i.e. for one cookie */
217*68d33a25Syt 	0x80ull,		/* dma_attr_align: 128-byte aligned */
2182fcbc377Syt 	1,			/* dma_attr_burstsizes */
2192fcbc377Syt 	1,			/* dma_attr_minxfer */
2202fcbc377Syt 	0xffffffffull,		/* dma_attr_maxxfer i.e. includes all cookies */
2212fcbc377Syt 	0xffffffffull,		/* dma_attr_seg */
2222fcbc377Syt 	1,			/* dma_attr_sgllen */
2232fcbc377Syt 	1,			/* dma_attr_granular */
2242fcbc377Syt 	0,			/* dma_attr_flags */
2252fcbc377Syt };
2262fcbc377Syt 
2272fcbc377Syt 
2282fcbc377Syt /* Device access attributes */
2292fcbc377Syt static ddi_device_acc_attr_t accattr = {
2302fcbc377Syt 	DDI_DEVICE_ATTR_V0,
2312fcbc377Syt 	DDI_STRUCTURE_LE_ACC,
2322fcbc377Syt 	DDI_STRICTORDER_ACC
2332fcbc377Syt };
2342fcbc377Syt 
2352fcbc377Syt 
2362fcbc377Syt static struct dev_ops ahcictl_dev_ops = {
2372fcbc377Syt 	DEVO_REV,		/* devo_rev */
2382fcbc377Syt 	0,			/* refcnt  */
2392fcbc377Syt 	ahci_getinfo,		/* info */
2402fcbc377Syt 	nulldev,		/* identify */
2412fcbc377Syt 	nulldev,		/* probe */
2422fcbc377Syt 	ahci_attach,		/* attach */
2432fcbc377Syt 	ahci_detach,		/* detach */
2442fcbc377Syt 	nodev,			/* no reset */
2452fcbc377Syt 	(struct cb_ops *)0,	/* driver operations */
2462fcbc377Syt 	NULL,			/* bus operations */
2472fcbc377Syt 	NULL			/* power */
2482fcbc377Syt };
2492fcbc377Syt 
2502fcbc377Syt static sata_tran_hotplug_ops_t ahci_tran_hotplug_ops = {
2512fcbc377Syt 	SATA_TRAN_HOTPLUG_OPS_REV_1,
2522fcbc377Syt 	ahci_tran_hotplug_port_activate,
2532fcbc377Syt 	ahci_tran_hotplug_port_deactivate
2542fcbc377Syt };
2552fcbc377Syt 
2562fcbc377Syt extern struct mod_ops mod_driverops;
2572fcbc377Syt 
2582fcbc377Syt static  struct modldrv modldrv = {
2592fcbc377Syt 	&mod_driverops,		/* driverops */
2602fcbc377Syt 	"ahci driver %I%",
2612fcbc377Syt 	&ahcictl_dev_ops,	/* driver ops */
2622fcbc377Syt };
2632fcbc377Syt 
2642fcbc377Syt static  struct modlinkage modlinkage = {
2652fcbc377Syt 	MODREV_1,
2662fcbc377Syt 	&modldrv,
2672fcbc377Syt 	NULL
2682fcbc377Syt };
2692fcbc377Syt 
2702fcbc377Syt static int ahci_watchdog_timeout = 5; /* 5 seconds */
2712fcbc377Syt static int ahci_watchdog_tick;
2722fcbc377Syt 
2732fcbc377Syt /* The following is needed for ahci_log() */
2742fcbc377Syt static kmutex_t ahci_log_mutex;
2752fcbc377Syt static char ahci_log_buf[512];
2762fcbc377Syt 
2772fcbc377Syt static size_t ahci_cmd_table_size;
2782fcbc377Syt 
2792fcbc377Syt /* The number of Physical Region Descriptor Table(PRDT) in Command Table */
2802fcbc377Syt int ahci_dma_prdt_number = AHCI_PRDT_NUMBER;
2812fcbc377Syt 
2822fcbc377Syt /*
2832fcbc377Syt  * AHCI MSI tunable:
2842fcbc377Syt  *
2852fcbc377Syt  * MSI will be enabled in phase 2.
2862fcbc377Syt  */
2872fcbc377Syt boolean_t ahci_msi_enabled = B_FALSE;
2882fcbc377Syt 
2892fcbc377Syt #if AHCI_DEBUG
2902fcbc377Syt uint32_t ahci_debug_flags = 0;
2912fcbc377Syt #endif
2922fcbc377Syt 
2932fcbc377Syt /* Opaque state pointer initialized by ddi_soft_state_init() */
2942fcbc377Syt static void *ahci_statep = NULL;
2952fcbc377Syt 
2962fcbc377Syt /*
2972fcbc377Syt  *  ahci module initialization.
2982fcbc377Syt  */
2992fcbc377Syt int
3002fcbc377Syt _init(void)
3012fcbc377Syt {
3022fcbc377Syt 	int	ret;
3032fcbc377Syt 
3042fcbc377Syt 	ret = ddi_soft_state_init(&ahci_statep, sizeof (ahci_ctl_t), 0);
3052fcbc377Syt 	if (ret != 0) {
3062fcbc377Syt 		goto err_out;
3072fcbc377Syt 	}
3082fcbc377Syt 
3092fcbc377Syt 	mutex_init(&ahci_log_mutex, NULL, MUTEX_DRIVER, NULL);
3102fcbc377Syt 
3112fcbc377Syt 	if ((ret = sata_hba_init(&modlinkage)) != 0) {
3122fcbc377Syt 		mutex_destroy(&ahci_log_mutex);
3132fcbc377Syt 		ddi_soft_state_fini(&ahci_statep);
3142fcbc377Syt 		goto err_out;
3152fcbc377Syt 	}
3162fcbc377Syt 
3172fcbc377Syt 	ret = mod_install(&modlinkage);
3182fcbc377Syt 	if (ret != 0) {
3192fcbc377Syt 		sata_hba_fini(&modlinkage);
3202fcbc377Syt 		mutex_destroy(&ahci_log_mutex);
3212fcbc377Syt 		ddi_soft_state_fini(&ahci_statep);
3222fcbc377Syt 		goto err_out;
3232fcbc377Syt 	}
3242fcbc377Syt 
3252fcbc377Syt 	/* watchdog tick */
3262fcbc377Syt 	ahci_watchdog_tick = drv_usectohz(
3272fcbc377Syt 	    (clock_t)ahci_watchdog_timeout * 1000000);
3282fcbc377Syt 	return (ret);
3292fcbc377Syt 
3302fcbc377Syt err_out:
3312fcbc377Syt 	cmn_err(CE_WARN, "!Module init failed");
3322fcbc377Syt 	return (ret);
3332fcbc377Syt }
3342fcbc377Syt 
3352fcbc377Syt /*
3362fcbc377Syt  * ahci module uninitialize.
3372fcbc377Syt  */
3382fcbc377Syt int
3392fcbc377Syt _fini(void)
3402fcbc377Syt {
3412fcbc377Syt 	int	ret;
3422fcbc377Syt 
3432fcbc377Syt 	ret = mod_remove(&modlinkage);
3442fcbc377Syt 	if (ret != 0) {
3452fcbc377Syt 		return (ret);
3462fcbc377Syt 	}
3472fcbc377Syt 
3482fcbc377Syt 	/* Remove the resources allocated in _init(). */
3492fcbc377Syt 	sata_hba_fini(&modlinkage);
3502fcbc377Syt 	mutex_destroy(&ahci_log_mutex);
3512fcbc377Syt 	ddi_soft_state_fini(&ahci_statep);
3522fcbc377Syt 
3532fcbc377Syt 	return (ret);
3542fcbc377Syt }
3552fcbc377Syt 
3562fcbc377Syt /*
3572fcbc377Syt  * _info entry point
3582fcbc377Syt  */
3592fcbc377Syt int
3602fcbc377Syt _info(struct modinfo *modinfop)
3612fcbc377Syt {
3622fcbc377Syt 	return (mod_info(&modlinkage, modinfop));
3632fcbc377Syt }
3642fcbc377Syt 
3652fcbc377Syt /*
3662fcbc377Syt  * The attach entry point for dev_ops.
3672fcbc377Syt  */
3682fcbc377Syt static int
3692fcbc377Syt ahci_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
3702fcbc377Syt {
3712fcbc377Syt 	ahci_ctl_t *ahci_ctlp;
3722fcbc377Syt 	int instance = ddi_get_instance(dip);
3732fcbc377Syt 	int status;
3742fcbc377Syt 	int attach_state;
3752fcbc377Syt 	uint32_t cap_status, ahci_version;
3762fcbc377Syt 	int intr_types;
3772fcbc377Syt 	ushort_t venid;
3782fcbc377Syt 	uint8_t revision;
379*68d33a25Syt 	int i;
380*68d33a25Syt #if AHCI_DEBUG
381*68d33a25Syt 	int speed;
382*68d33a25Syt #endif
3832fcbc377Syt 
3842fcbc377Syt 	AHCIDBG0(AHCIDBG_INIT|AHCIDBG_ENTRY, NULL, "ahci_attach enter");
3852fcbc377Syt 
3862fcbc377Syt 	switch (cmd) {
3872fcbc377Syt 	case DDI_ATTACH:
3882fcbc377Syt 		break;
3892fcbc377Syt 
3902fcbc377Syt 	case DDI_RESUME:
3912fcbc377Syt 		/* It will be implemented in Phase 2 */
3922fcbc377Syt 		return (DDI_FAILURE);
3932fcbc377Syt 
3942fcbc377Syt 	default:
3952fcbc377Syt 		return (DDI_FAILURE);
3962fcbc377Syt 	}
3972fcbc377Syt 
3982fcbc377Syt 	attach_state = AHCI_ATTACH_STATE_NONE;
3992fcbc377Syt 
4002fcbc377Syt 	/* Allocate soft state */
4012fcbc377Syt 	status = ddi_soft_state_zalloc(ahci_statep, instance);
4022fcbc377Syt 	if (status != DDI_SUCCESS) {
403*68d33a25Syt 		cmn_err(CE_WARN, "!Cannot allocate soft state");
4042fcbc377Syt 		goto err_out;
4052fcbc377Syt 	}
4062fcbc377Syt 
4072fcbc377Syt 	ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
4082fcbc377Syt 	ahci_ctlp->ahcictl_dip = dip;
4092fcbc377Syt 
410*68d33a25Syt 	/* Initialize the cport/port mapping */
411*68d33a25Syt 	for (i = 0; i < AHCI_MAX_PORTS; i++) {
412*68d33a25Syt 		ahci_ctlp->ahcictl_port_to_cport[i] = 0xff;
413*68d33a25Syt 		ahci_ctlp->ahcictl_cport_to_port[i] = 0xff;
414*68d33a25Syt 	}
415*68d33a25Syt 
4162fcbc377Syt 	attach_state |= AHCI_ATTACH_STATE_STATEP_ALLOC;
4172fcbc377Syt 
4182fcbc377Syt 	/*
4192fcbc377Syt 	 * Now map the AHCI base address; which includes global
4202fcbc377Syt 	 * registers and port control registers
4212fcbc377Syt 	 */
4222fcbc377Syt 	status = ddi_regs_map_setup(dip,
4232fcbc377Syt 	    AHCI_BASE_REG,
4242fcbc377Syt 	    (caddr_t *)&ahci_ctlp->ahcictl_ahci_addr,
4252fcbc377Syt 	    0,
4262fcbc377Syt 	    0,
4272fcbc377Syt 	    &accattr,
4282fcbc377Syt 	    &ahci_ctlp->ahcictl_ahci_acc_handle);
4292fcbc377Syt 	if (status != DDI_SUCCESS) {
430*68d33a25Syt 		cmn_err(CE_WARN, "!Cannot map register space");
4312fcbc377Syt 		goto err_out;
4322fcbc377Syt 	}
4332fcbc377Syt 
4342fcbc377Syt 	attach_state |= AHCI_ATTACH_STATE_REG_MAP;
4352fcbc377Syt 
4362fcbc377Syt 	/* Get the AHCI version information */
4372fcbc377Syt 	ahci_version = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4382fcbc377Syt 	    (uint32_t *)AHCI_GLOBAL_VS(ahci_ctlp));
4392fcbc377Syt 
440*68d33a25Syt 	cmn_err(CE_NOTE, "!hba AHCI version = %x.%x",
4412fcbc377Syt 	    (ahci_version & 0xffff0000) >> 16,
4422fcbc377Syt 	    ((ahci_version & 0x0000ff00) >> 4 |
4432fcbc377Syt 	    (ahci_version & 0x000000ff)));
4442fcbc377Syt 
4452fcbc377Syt 	/* We don't support controllers whose versions are lower than 1.0 */
4462fcbc377Syt 	if (!(ahci_version & 0xffff0000)) {
447*68d33a25Syt 		cmn_err(CE_WARN, "Don't support AHCI HBA with lower than "
448*68d33a25Syt 		    "version 1.0");
4492fcbc377Syt 		goto err_out;
4502fcbc377Syt 	}
4512fcbc377Syt 
4522fcbc377Syt 	/* Get the HBA capabilities information */
4532fcbc377Syt 	cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4542fcbc377Syt 	    (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp));
4552fcbc377Syt 
4562fcbc377Syt 	AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, "hba capabilites = 0x%x",
4572fcbc377Syt 	    cap_status);
4582fcbc377Syt 
459*68d33a25Syt #if AHCI_DEBUG
460*68d33a25Syt 	/* Get the interface speed supported by the HBA */
461*68d33a25Syt 	speed = (cap_status & AHCI_HBA_CAP_ISS) >> AHCI_HBA_CAP_ISS_SHIFT;
462*68d33a25Syt 	if (speed == 0x01) {
463*68d33a25Syt 		AHCIDBG0(AHCIDBG_INIT, ahci_ctlp,
464*68d33a25Syt 		    "hba interface speed support: Gen 1 (1.5Gbps)");
465*68d33a25Syt 	} else if (speed == 0x10) {
466*68d33a25Syt 		AHCIDBG0(AHCIDBG_INIT, ahci_ctlp,
467*68d33a25Syt 		    "hba interface speed support: Gen 2 (3 Gbps)");
468*68d33a25Syt 	} else if (speed == 0x11) {
469*68d33a25Syt 		AHCIDBG0(AHCIDBG_INIT, ahci_ctlp,
470*68d33a25Syt 		    "hba interface speed support: Gen 3 (6 Gbps)");
471*68d33a25Syt 	}
472*68d33a25Syt #endif
473*68d33a25Syt 
4742fcbc377Syt 	/* Get the number of command slots supported by the HBA */
4752fcbc377Syt 	ahci_ctlp->ahcictl_num_cmd_slots =
4762fcbc377Syt 	    ((cap_status & AHCI_HBA_CAP_NCS) >>
4772fcbc377Syt 	    AHCI_HBA_CAP_NCS_SHIFT) + 1;
4782fcbc377Syt 
479*68d33a25Syt 	AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, "hba number of cmd slots: %d",
4802fcbc377Syt 	    ahci_ctlp->ahcictl_num_cmd_slots);
4812fcbc377Syt 
4822fcbc377Syt 	/* Get the bit map which indicates ports implemented by the HBA */
4832fcbc377Syt 	ahci_ctlp->ahcictl_ports_implemented =
4842fcbc377Syt 	    ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4852fcbc377Syt 	    (uint32_t *)AHCI_GLOBAL_PI(ahci_ctlp));
4862fcbc377Syt 
4872fcbc377Syt 	AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, "hba implementation of ports: 0x%x",
4882fcbc377Syt 	    ahci_ctlp->ahcictl_ports_implemented);
4892fcbc377Syt 
49009121340Syt 	/*
49109121340Syt 	 * According to the AHCI spec, CAP.NP should indicate the maximum
49209121340Syt 	 * number of ports supported by the HBA silicon, but we found
49309121340Syt 	 * this value of ICH8 chipset only indicates the number of ports
49409121340Syt 	 * implemented (exposed) by it. Therefore, the driver should calculate
49509121340Syt 	 * the potential maximum value by checking PI register, and use
49609121340Syt 	 * the maximum of this value and CAP.NP.
49709121340Syt 	 */
49809121340Syt 	ahci_ctlp->ahcictl_num_ports = max(
49909121340Syt 	    (cap_status & AHCI_HBA_CAP_NP) + 1,
50009121340Syt 	    ddi_fls(ahci_ctlp->ahcictl_ports_implemented));
50109121340Syt 
50209121340Syt 	AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, "hba number of ports: %d",
50309121340Syt 	    ahci_ctlp->ahcictl_num_ports);
50409121340Syt 
5052fcbc377Syt 	/* Get the number of implemented ports by the HBA */
5062fcbc377Syt 	ahci_ctlp->ahcictl_num_implemented_ports =
5072fcbc377Syt 	    ahci_get_num_implemented_ports(
5082fcbc377Syt 	    ahci_ctlp->ahcictl_ports_implemented);
5092fcbc377Syt 
5102fcbc377Syt 	AHCIDBG1(AHCIDBG_INIT, ahci_ctlp,
511*68d33a25Syt 	    "hba number of implemented ports: %d",
5122fcbc377Syt 	    ahci_ctlp->ahcictl_num_implemented_ports);
5132fcbc377Syt 
5142fcbc377Syt 	ahci_ctlp->ahcictl_buffer_dma_attr = buffer_dma_attr;
5152fcbc377Syt 
5162fcbc377Syt 	if (!(cap_status & AHCI_HBA_CAP_S64A)) {
5172fcbc377Syt 		AHCIDBG0(AHCIDBG_INIT, ahci_ctlp,
5182fcbc377Syt 		    "hba does not support 64-bit addressing");
5192fcbc377Syt 		ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_addr_hi =
5202fcbc377Syt 		    0xffffffffull;
5212fcbc377Syt 	}
5222fcbc377Syt 
5232fcbc377Syt 	if (pci_config_setup(dip, &ahci_ctlp->ahcictl_pci_conf_handle)
5242fcbc377Syt 	    != DDI_SUCCESS) {
525*68d33a25Syt 		cmn_err(CE_WARN, "!Cannot set up pci configure space");
5262fcbc377Syt 		goto err_out;
5272fcbc377Syt 	}
5282fcbc377Syt 
529*68d33a25Syt 	attach_state |= AHCI_ATTACH_STATE_PCICFG_SETUP;
530*68d33a25Syt 
531*68d33a25Syt 	/*
532*68d33a25Syt 	 * Modify dma_attr_align of ahcictl_buffer_dma_attr. For VT8251, those
533*68d33a25Syt 	 * controllers with 0x00 revision id work on 4-byte aligned buffer,
534*68d33a25Syt 	 * which is a bug and was fixed after 0x00 revision id controllers.
535*68d33a25Syt 	 *
536*68d33a25Syt 	 * And VT8251 cannot use multiple command lists for non-queued commands
537*68d33a25Syt 	 * because the Command Issue register bit can be cleared by software
538*68d33a25Syt 	 * set.
539*68d33a25Syt 	 */
5402fcbc377Syt 	venid = pci_config_get16(ahci_ctlp->ahcictl_pci_conf_handle,
5412fcbc377Syt 	    PCI_CONF_VENID);
5422fcbc377Syt 
5432fcbc377Syt 	if (venid == VIA_VENID) {
5442fcbc377Syt 		revision = pci_config_get8(ahci_ctlp->ahcictl_pci_conf_handle,
5452fcbc377Syt 		    PCI_CONF_REVID);
5462fcbc377Syt 		AHCIDBG1(AHCIDBG_INIT, ahci_ctlp,
5472fcbc377Syt 		    "revision id = 0x%x", revision);
5482fcbc377Syt 		if (revision == 0x00) {
5492fcbc377Syt 			ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_align = 0x4;
5502fcbc377Syt 			AHCIDBG0(AHCIDBG_INIT, ahci_ctlp,
5512fcbc377Syt 			    "change ddi_attr_align to 0x4");
5522fcbc377Syt 		}
5532fcbc377Syt 
554*68d33a25Syt 		ahci_ctlp->ahcictl_cap = AHCI_CAP_NO_MCMDLIST_NONQUEUE;
555*68d33a25Syt 		AHCIDBG0(AHCIDBG_INIT, ahci_ctlp,
556*68d33a25Syt 		    "VT8251 cannot use multiple command lists for "
557*68d33a25Syt 		    "non-queued commands");
558*68d33a25Syt 	}
5592fcbc377Syt 
5602fcbc377Syt 	/*
5612fcbc377Syt 	 * Disable the whole controller interrupts before adding
5622fcbc377Syt 	 * interrupt handlers(s).
5632fcbc377Syt 	 */
5642fcbc377Syt 	ahci_disable_all_intrs(ahci_ctlp);
5652fcbc377Syt 
5662fcbc377Syt 	/* Get supported interrupt types */
5672fcbc377Syt 	if (ddi_intr_get_supported_types(dip, &intr_types) != DDI_SUCCESS) {
568*68d33a25Syt 		cmn_err(CE_WARN, "!ddi_intr_get_supported_types failed");
5692fcbc377Syt 		goto err_out;
5702fcbc377Syt 	}
5712fcbc377Syt 
5722fcbc377Syt 	AHCIDBG1(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
5732fcbc377Syt 	    "ddi_intr_get_supported_types() returned: 0x%x",
5742fcbc377Syt 	    intr_types);
5752fcbc377Syt 
5762fcbc377Syt 	if (ahci_msi_enabled && (intr_types & DDI_INTR_TYPE_MSI)) {
5772fcbc377Syt 		/*
5782fcbc377Syt 		 * Try MSI first, but fall back to FIXED if failed
5792fcbc377Syt 		 */
5802fcbc377Syt 		if (ahci_add_msi_intrs(ahci_ctlp) == DDI_SUCCESS) {
5812fcbc377Syt 			ahci_ctlp->ahcictl_intr_type = DDI_INTR_TYPE_MSI;
5822fcbc377Syt 			AHCIDBG0(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
5832fcbc377Syt 			    "Using MSI interrupt type");
5842fcbc377Syt 			goto intr_done;
5852fcbc377Syt 		}
5862fcbc377Syt 
5872fcbc377Syt 		AHCIDBG0(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
5882fcbc377Syt 		    "MSI registration failed, "
5892fcbc377Syt 		    "trying FIXED interrupts");
5902fcbc377Syt 	}
5912fcbc377Syt 
5922fcbc377Syt 	if (intr_types & DDI_INTR_TYPE_FIXED) {
5932fcbc377Syt 		if (ahci_add_legacy_intrs(ahci_ctlp) == DDI_SUCCESS) {
5942fcbc377Syt 			ahci_ctlp->ahcictl_intr_type = DDI_INTR_TYPE_FIXED;
5952fcbc377Syt 			AHCIDBG0(AHCIDBG_INIT|AHCIDBG_INTR, NULL,
5962fcbc377Syt 			    "Using FIXED interrupt type");
5972fcbc377Syt 			goto intr_done;
5982fcbc377Syt 		}
5992fcbc377Syt 
6002fcbc377Syt 		AHCIDBG0(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
6012fcbc377Syt 		    "FIXED interrupt registration failed");
6022fcbc377Syt 	}
6032fcbc377Syt 
6042fcbc377Syt 	cmn_err(CE_WARN, "!Interrupt registration failed");
6052fcbc377Syt 
6062fcbc377Syt 	goto err_out;
6072fcbc377Syt 
6082fcbc377Syt intr_done:
6092fcbc377Syt 
6102fcbc377Syt 	attach_state |= AHCI_ATTACH_STATE_INTR_ADDED;
6112fcbc377Syt 
6122fcbc377Syt 	/* Initialize the controller mutex */
6132fcbc377Syt 	mutex_init(&ahci_ctlp->ahcictl_mutex, NULL, MUTEX_DRIVER,
6142fcbc377Syt 	    (void *)(uintptr_t)ahci_ctlp->ahcictl_intr_pri);
6152fcbc377Syt 
6162fcbc377Syt 	attach_state |= AHCI_ATTACH_STATE_MUTEX_INIT;
6172fcbc377Syt 
6182fcbc377Syt 	if (ahci_dma_prdt_number < AHCI_MIN_PRDT_NUMBER) {
6192fcbc377Syt 		ahci_dma_prdt_number = AHCI_MIN_PRDT_NUMBER;
6202fcbc377Syt 	} else if (ahci_dma_prdt_number > AHCI_MAX_PRDT_NUMBER) {
6212fcbc377Syt 		ahci_dma_prdt_number = AHCI_MAX_PRDT_NUMBER;
6222fcbc377Syt 	}
6232fcbc377Syt 
6242fcbc377Syt 	ahci_cmd_table_size = (sizeof (ahci_cmd_table_t) +
6252fcbc377Syt 	    (ahci_dma_prdt_number - AHCI_PRDT_NUMBER) *
6262fcbc377Syt 	    sizeof (ahci_prdt_item_t));
6272fcbc377Syt 
6282fcbc377Syt 	AHCIDBG2(AHCIDBG_INIT, ahci_ctlp,
6292fcbc377Syt 	    "ahci_attach: ahci_dma_prdt_number set by user is 0x%x,"
6302fcbc377Syt 	    " ahci_cmd_table_size is 0x%x",
6312fcbc377Syt 	    ahci_dma_prdt_number, ahci_cmd_table_size);
6322fcbc377Syt 
6332fcbc377Syt 	if (ahci_dma_prdt_number != AHCI_PRDT_NUMBER)
6342fcbc377Syt 		ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_sgllen =
6352fcbc377Syt 		    ahci_dma_prdt_number;
6362fcbc377Syt 
637*68d33a25Syt 	/* Allocate the ports structure */
638*68d33a25Syt 	status = ahci_alloc_ports_state(ahci_ctlp);
639*68d33a25Syt 	if (status != AHCI_SUCCESS) {
640*68d33a25Syt 		cmn_err(CE_WARN, "!Cannot allocate ports structure");
641*68d33a25Syt 		goto err_out;
642*68d33a25Syt 	}
643*68d33a25Syt 
644*68d33a25Syt 	attach_state |= AHCI_ATTACH_STATE_PORT_ALLOC;
645*68d33a25Syt 
646*68d33a25Syt 	/*
647*68d33a25Syt 	 * A taskq is created for dealing with events
648*68d33a25Syt 	 */
649*68d33a25Syt 	if ((ahci_ctlp->ahcictl_event_taskq = ddi_taskq_create(dip,
650*68d33a25Syt 	    "ahci_event_handle_taskq", 1, TASKQ_DEFAULTPRI, 0)) == NULL) {
651*68d33a25Syt 		cmn_err(CE_WARN, "!ddi_taskq_create failed for event handle");
652*68d33a25Syt 		goto err_out;
653*68d33a25Syt 	}
654*68d33a25Syt 
655*68d33a25Syt 	attach_state |= AHCI_ATTACH_STATE_ERR_RECV_TASKQ;
656*68d33a25Syt 
6572fcbc377Syt 	/*
658*68d33a25Syt 	 * Initialize the controller and ports.
6592fcbc377Syt 	 */
6602fcbc377Syt 	ahci_ctlp->ahcictl_flags |= AHCI_ATTACH;
6612fcbc377Syt 	status = ahci_initialize_controller(ahci_ctlp);
6622fcbc377Syt 	ahci_ctlp->ahcictl_flags &= ~AHCI_ATTACH;
6632fcbc377Syt 	if (status != AHCI_SUCCESS) {
664*68d33a25Syt 		cmn_err(CE_WARN, "!HBA initialization failed");
6652fcbc377Syt 		goto err_out;
6662fcbc377Syt 	}
6672fcbc377Syt 
6682fcbc377Syt 	attach_state |= AHCI_ATTACH_STATE_HW_INIT;
6692fcbc377Syt 
6702fcbc377Syt 	/* Start one thread to check packet timeouts */
6712fcbc377Syt 	ahci_ctlp->ahcictl_timeout_id = timeout(
6722fcbc377Syt 	    (void (*)(void *))ahci_watchdog_handler,
6732fcbc377Syt 	    (caddr_t)ahci_ctlp, ahci_watchdog_tick);
6742fcbc377Syt 
6752fcbc377Syt 	attach_state |= AHCI_ATTACH_STATE_TIMEOUT_ENABLED;
6762fcbc377Syt 
6772fcbc377Syt 	if (ahci_register_sata_hba_tran(ahci_ctlp, cap_status)) {
678*68d33a25Syt 		cmn_err(CE_WARN, "!sata hba tran registration failed");
6792fcbc377Syt 		goto err_out;
6802fcbc377Syt 	}
6812fcbc377Syt 
6822fcbc377Syt 	AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, "ahci_attach success!");
6832fcbc377Syt 
6842fcbc377Syt 	return (DDI_SUCCESS);
6852fcbc377Syt 
6862fcbc377Syt err_out:
6872fcbc377Syt 	if (attach_state & AHCI_ATTACH_STATE_TIMEOUT_ENABLED) {
6882fcbc377Syt 		mutex_enter(&ahci_ctlp->ahcictl_mutex);
6892fcbc377Syt 		(void) untimeout(ahci_ctlp->ahcictl_timeout_id);
6902fcbc377Syt 		ahci_ctlp->ahcictl_timeout_id = 0;
6912fcbc377Syt 		mutex_exit(&ahci_ctlp->ahcictl_mutex);
6922fcbc377Syt 	}
6932fcbc377Syt 
6942fcbc377Syt 	if (attach_state & AHCI_ATTACH_STATE_HW_INIT) {
6952fcbc377Syt 		mutex_enter(&ahci_ctlp->ahcictl_mutex);
696*68d33a25Syt 		ahci_uninitialize_controller(ahci_ctlp);
6972fcbc377Syt 		mutex_exit(&ahci_ctlp->ahcictl_mutex);
6982fcbc377Syt 	}
6992fcbc377Syt 
700*68d33a25Syt 	if (attach_state & AHCI_ATTACH_STATE_ERR_RECV_TASKQ) {
701*68d33a25Syt 		ddi_taskq_destroy(ahci_ctlp->ahcictl_event_taskq);
702*68d33a25Syt 	}
703*68d33a25Syt 
704*68d33a25Syt 	if (attach_state & AHCI_ATTACH_STATE_PORT_ALLOC) {
705*68d33a25Syt 		ahci_dealloc_ports_state(ahci_ctlp);
706*68d33a25Syt 	}
707*68d33a25Syt 
7082fcbc377Syt 	if (attach_state & AHCI_ATTACH_STATE_MUTEX_INIT) {
7092fcbc377Syt 		mutex_destroy(&ahci_ctlp->ahcictl_mutex);
7102fcbc377Syt 	}
7112fcbc377Syt 
7122fcbc377Syt 	if (attach_state & AHCI_ATTACH_STATE_INTR_ADDED) {
7132fcbc377Syt 		ahci_rem_intrs(ahci_ctlp);
7142fcbc377Syt 	}
7152fcbc377Syt 
716*68d33a25Syt 	if (attach_state & AHCI_ATTACH_STATE_PCICFG_SETUP) {
717*68d33a25Syt 		pci_config_teardown(&ahci_ctlp->ahcictl_pci_conf_handle);
718*68d33a25Syt 	}
719*68d33a25Syt 
7202fcbc377Syt 	if (attach_state & AHCI_ATTACH_STATE_REG_MAP) {
7212fcbc377Syt 		ddi_regs_map_free(&ahci_ctlp->ahcictl_ahci_acc_handle);
7222fcbc377Syt 	}
7232fcbc377Syt 
7242fcbc377Syt 	if (attach_state & AHCI_ATTACH_STATE_STATEP_ALLOC) {
7252fcbc377Syt 		ddi_soft_state_free(ahci_statep, instance);
7262fcbc377Syt 	}
7272fcbc377Syt 
7282fcbc377Syt 	return (DDI_FAILURE);
7292fcbc377Syt }
7302fcbc377Syt 
7312fcbc377Syt /*
7322fcbc377Syt  * The detach entry point for dev_ops.
7332fcbc377Syt  */
7342fcbc377Syt static int
7352fcbc377Syt ahci_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
7362fcbc377Syt {
7372fcbc377Syt 	ahci_ctl_t *ahci_ctlp;
7382fcbc377Syt 	int instance;
7392fcbc377Syt 	int ret;
7402fcbc377Syt 
7412fcbc377Syt 	instance = ddi_get_instance(dip);
7422fcbc377Syt 	ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
7432fcbc377Syt 
7442fcbc377Syt 	AHCIDBG0(AHCIDBG_ENTRY, ahci_ctlp, "ahci_detach enter");
7452fcbc377Syt 
7462fcbc377Syt 	switch (cmd) {
7472fcbc377Syt 	case DDI_DETACH:
7482fcbc377Syt 		/* disable the interrupts for an uninterrupted detach */
7492fcbc377Syt 		mutex_enter(&ahci_ctlp->ahcictl_mutex);
7502fcbc377Syt 		ahci_disable_all_intrs(ahci_ctlp);
7512fcbc377Syt 		mutex_exit(&ahci_ctlp->ahcictl_mutex);
7522fcbc377Syt 
7532fcbc377Syt 		/* unregister from the sata framework. */
7542fcbc377Syt 		ret = ahci_unregister_sata_hba_tran(ahci_ctlp);
7552fcbc377Syt 		if (ret != AHCI_SUCCESS) {
7562fcbc377Syt 			mutex_enter(&ahci_ctlp->ahcictl_mutex);
7572fcbc377Syt 			ahci_enable_all_intrs(ahci_ctlp);
7582fcbc377Syt 			mutex_exit(&ahci_ctlp->ahcictl_mutex);
7592fcbc377Syt 			return (DDI_FAILURE);
7602fcbc377Syt 		}
7612fcbc377Syt 
7622fcbc377Syt 		mutex_enter(&ahci_ctlp->ahcictl_mutex);
7632fcbc377Syt 
7642fcbc377Syt 		/* stop the watchdog handler */
7652fcbc377Syt 		(void) untimeout(ahci_ctlp->ahcictl_timeout_id);
7662fcbc377Syt 		ahci_ctlp->ahcictl_timeout_id = 0;
7672fcbc377Syt 
768*68d33a25Syt 		mutex_exit(&ahci_ctlp->ahcictl_mutex);
769*68d33a25Syt 
770*68d33a25Syt 		/* uninitialize the controller */
7712fcbc377Syt 		ahci_ctlp->ahcictl_flags |= AHCI_DETACH;
772*68d33a25Syt 		ahci_uninitialize_controller(ahci_ctlp);
7732fcbc377Syt 		ahci_ctlp->ahcictl_flags &= ~AHCI_DETACH;
7742fcbc377Syt 
775*68d33a25Syt 		/* remove the interrupts */
776*68d33a25Syt 		ahci_rem_intrs(ahci_ctlp);
7772fcbc377Syt 
778*68d33a25Syt 		/* destroy the taskq */
779*68d33a25Syt 		ddi_taskq_destroy(ahci_ctlp->ahcictl_event_taskq);
7802fcbc377Syt 
781*68d33a25Syt 		/* deallocate the ports structures */
782*68d33a25Syt 		ahci_dealloc_ports_state(ahci_ctlp);
783*68d33a25Syt 
784*68d33a25Syt 		/* destroy mutex */
7852fcbc377Syt 		mutex_destroy(&ahci_ctlp->ahcictl_mutex);
7862fcbc377Syt 
787*68d33a25Syt 		/* teardown the pci config */
788*68d33a25Syt 		pci_config_teardown(&ahci_ctlp->ahcictl_pci_conf_handle);
7892fcbc377Syt 
7902fcbc377Syt 		/* remove the reg maps. */
7912fcbc377Syt 		ddi_regs_map_free(&ahci_ctlp->ahcictl_ahci_acc_handle);
7922fcbc377Syt 
7932fcbc377Syt 		/* free the soft state. */
7942fcbc377Syt 		ddi_soft_state_free(ahci_statep, instance);
7952fcbc377Syt 
7962fcbc377Syt 		return (DDI_SUCCESS);
7972fcbc377Syt 
7982fcbc377Syt 	case DDI_SUSPEND:
7992fcbc377Syt 		/* It will be implemented in Phase 2 */
8002fcbc377Syt 		return (DDI_FAILURE);
8012fcbc377Syt 
8022fcbc377Syt 	default:
8032fcbc377Syt 		return (DDI_FAILURE);
8042fcbc377Syt 	}
8052fcbc377Syt }
8062fcbc377Syt 
8072fcbc377Syt /*
8082fcbc377Syt  * The info entry point for dev_ops.
8092fcbc377Syt  *
8102fcbc377Syt  */
8112fcbc377Syt static int
8122fcbc377Syt ahci_getinfo(dev_info_t *dip, ddi_info_cmd_t infocmd,
8132fcbc377Syt 		    void *arg, void **result)
8142fcbc377Syt {
8152fcbc377Syt #ifndef __lock_lint
8162fcbc377Syt 	_NOTE(ARGUNUSED(dip))
8172fcbc377Syt #endif /* __lock_lint */
8182fcbc377Syt 
8192fcbc377Syt 	ahci_ctl_t *ahci_ctlp;
8202fcbc377Syt 	int instance;
8212fcbc377Syt 	dev_t dev;
8222fcbc377Syt 
8232fcbc377Syt 	dev = (dev_t)arg;
8242fcbc377Syt 	instance = getminor(dev);
8252fcbc377Syt 
8262fcbc377Syt 	switch (infocmd) {
8272fcbc377Syt 		case DDI_INFO_DEVT2DEVINFO:
8282fcbc377Syt 			ahci_ctlp = ddi_get_soft_state(ahci_statep,  instance);
8292fcbc377Syt 			if (ahci_ctlp != NULL) {
8302fcbc377Syt 				*result = ahci_ctlp->ahcictl_dip;
8312fcbc377Syt 				return (DDI_SUCCESS);
8322fcbc377Syt 			} else {
8332fcbc377Syt 				*result = NULL;
8342fcbc377Syt 				return (DDI_FAILURE);
8352fcbc377Syt 			}
8362fcbc377Syt 		case DDI_INFO_DEVT2INSTANCE:
8372fcbc377Syt 			*(int *)result = instance;
8382fcbc377Syt 			break;
8392fcbc377Syt 		default:
8402fcbc377Syt 			break;
8412fcbc377Syt 	}
8422fcbc377Syt 
8432fcbc377Syt 	return (DDI_SUCCESS);
8442fcbc377Syt }
8452fcbc377Syt 
8462fcbc377Syt /*
8472fcbc377Syt  * Registers the ahci with sata framework.
8482fcbc377Syt  */
8492fcbc377Syt static int
8502fcbc377Syt ahci_register_sata_hba_tran(ahci_ctl_t *ahci_ctlp, uint32_t cap_status)
8512fcbc377Syt {
8522fcbc377Syt 	struct 	sata_hba_tran	*sata_hba_tran;
8532fcbc377Syt 
8542fcbc377Syt 	AHCIDBG0(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
8552fcbc377Syt 	    "ahci_register_sata_hba_tran enter");
8562fcbc377Syt 
8572fcbc377Syt 	mutex_enter(&ahci_ctlp->ahcictl_mutex);
8582fcbc377Syt 
8592fcbc377Syt 	/* Allocate memory for the sata_hba_tran  */
8602fcbc377Syt 	sata_hba_tran = kmem_zalloc(sizeof (sata_hba_tran_t), KM_SLEEP);
8612fcbc377Syt 
862*68d33a25Syt 	sata_hba_tran->sata_tran_hba_rev = SATA_TRAN_HBA_REV_2;
8632fcbc377Syt 	sata_hba_tran->sata_tran_hba_dip = ahci_ctlp->ahcictl_dip;
8642fcbc377Syt 	sata_hba_tran->sata_tran_hba_dma_attr =
8652fcbc377Syt 	    &ahci_ctlp->ahcictl_buffer_dma_attr;
8662fcbc377Syt 
8672fcbc377Syt 	/* Report the number of implemented ports */
8682fcbc377Syt 	sata_hba_tran->sata_tran_hba_num_cports =
8692fcbc377Syt 	    ahci_ctlp->ahcictl_num_implemented_ports;
8702fcbc377Syt 
871*68d33a25Syt 	/* Support ATAPI device */
872*68d33a25Syt 	sata_hba_tran->sata_tran_hba_features_support = SATA_CTLF_ATAPI;
8732fcbc377Syt 
8742fcbc377Syt 	/* Get the data transfer capability for PIO command by the HBA */
8752fcbc377Syt 	if (cap_status & AHCI_HBA_CAP_PMD) {
876*68d33a25Syt 		ahci_ctlp->ahcictl_flags |= AHCI_CAP_PIO_MDRQ;
8772fcbc377Syt 		AHCIDBG0(AHCIDBG_INFO, ahci_ctlp, "HBA supports multiple "
8782fcbc377Syt 		    "DRQ block data transfer for PIO command protocol");
8792fcbc377Syt 	} else {
8802fcbc377Syt 		AHCIDBG0(AHCIDBG_INFO, ahci_ctlp, "HBA only supports single "
8812fcbc377Syt 		    "DRQ block data transfer for PIO command protocol");
8822fcbc377Syt 	}
8832fcbc377Syt 
8842fcbc377Syt 	/* Report the number of command slots */
8852fcbc377Syt 	sata_hba_tran->sata_tran_hba_qdepth = ahci_ctlp->ahcictl_num_cmd_slots;
8862fcbc377Syt 
8872fcbc377Syt 	sata_hba_tran->sata_tran_probe_port = ahci_tran_probe_port;
8882fcbc377Syt 	sata_hba_tran->sata_tran_start = ahci_tran_start;
8892fcbc377Syt 	sata_hba_tran->sata_tran_abort = ahci_tran_abort;
8902fcbc377Syt 	sata_hba_tran->sata_tran_reset_dport = ahci_tran_reset_dport;
8912fcbc377Syt 	sata_hba_tran->sata_tran_hotplug_ops = &ahci_tran_hotplug_ops;
8922fcbc377Syt #ifdef __lock_lint
8932fcbc377Syt 	sata_hba_tran->sata_tran_selftest = ahci_selftest;
8942fcbc377Syt #endif
8952fcbc377Syt 	/*
8962fcbc377Syt 	 * When SATA framework adds support for pwrmgt the
8972fcbc377Syt 	 * pwrmgt_ops needs to be updated
8982fcbc377Syt 	 */
8992fcbc377Syt 	sata_hba_tran->sata_tran_pwrmgt_ops = NULL;
9002fcbc377Syt 	sata_hba_tran->sata_tran_ioctl = NULL;
9012fcbc377Syt 
9022fcbc377Syt 	ahci_ctlp->ahcictl_sata_hba_tran = sata_hba_tran;
9032fcbc377Syt 
9042fcbc377Syt 	mutex_exit(&ahci_ctlp->ahcictl_mutex);
9052fcbc377Syt 
9062fcbc377Syt 	/* Attach it to SATA framework */
9072fcbc377Syt 	if (sata_hba_attach(ahci_ctlp->ahcictl_dip, sata_hba_tran, DDI_ATTACH)
9082fcbc377Syt 	    != DDI_SUCCESS) {
9092fcbc377Syt 		kmem_free((void *)sata_hba_tran, sizeof (sata_hba_tran_t));
9102fcbc377Syt 		mutex_enter(&ahci_ctlp->ahcictl_mutex);
9112fcbc377Syt 		ahci_ctlp->ahcictl_sata_hba_tran = NULL;
9122fcbc377Syt 		mutex_exit(&ahci_ctlp->ahcictl_mutex);
9132fcbc377Syt 		return (AHCI_FAILURE);
9142fcbc377Syt 	}
9152fcbc377Syt 
9162fcbc377Syt 	return (AHCI_SUCCESS);
9172fcbc377Syt }
9182fcbc377Syt 
9192fcbc377Syt /*
9202fcbc377Syt  * Unregisters the ahci with sata framework.
9212fcbc377Syt  */
9222fcbc377Syt static int
9232fcbc377Syt ahci_unregister_sata_hba_tran(ahci_ctl_t *ahci_ctlp)
9242fcbc377Syt {
9252fcbc377Syt 	AHCIDBG0(AHCIDBG_ENTRY, ahci_ctlp,
9262fcbc377Syt 	    "ahci_unregister_sata_hba_tran enter");
9272fcbc377Syt 
9282fcbc377Syt 	/* Detach from the SATA framework. */
9292fcbc377Syt 	if (sata_hba_detach(ahci_ctlp->ahcictl_dip, DDI_DETACH) !=
9302fcbc377Syt 	    DDI_SUCCESS) {
9312fcbc377Syt 		return (AHCI_FAILURE);
9322fcbc377Syt 	}
9332fcbc377Syt 
9342fcbc377Syt 	/* Deallocate sata_hba_tran. */
9352fcbc377Syt 	kmem_free((void *)ahci_ctlp->ahcictl_sata_hba_tran,
9362fcbc377Syt 	    sizeof (sata_hba_tran_t));
9372fcbc377Syt 
9382fcbc377Syt 	mutex_enter(&ahci_ctlp->ahcictl_mutex);
9392fcbc377Syt 	ahci_ctlp->ahcictl_sata_hba_tran = NULL;
9402fcbc377Syt 	mutex_exit(&ahci_ctlp->ahcictl_mutex);
9412fcbc377Syt 
9422fcbc377Syt 	return (AHCI_SUCCESS);
9432fcbc377Syt }
9442fcbc377Syt 
9452fcbc377Syt /*
9462fcbc377Syt  * ahci_tran_probe_port is called by SATA framework. It returns port state,
9472fcbc377Syt  * port status registers and an attached device type via sata_device
9482fcbc377Syt  * structure.
9492fcbc377Syt  *
9502fcbc377Syt  * We return the cached information from a previous hardware probe. The
9512fcbc377Syt  * actual hardware probing itself was done either from within
9522fcbc377Syt  * ahci_initialize_controller() during the driver attach or from a phy
9532fcbc377Syt  * ready change interrupt handler.
9542fcbc377Syt  */
9552fcbc377Syt static int
9562fcbc377Syt ahci_tran_probe_port(dev_info_t *dip, sata_device_t *sd)
9572fcbc377Syt {
9582fcbc377Syt 	ahci_ctl_t *ahci_ctlp;
9592fcbc377Syt 	ahci_port_t *ahci_portp;
9602fcbc377Syt 	uint8_t	cport = sd->satadev_addr.cport;
9612fcbc377Syt 	uint8_t pmport = sd->satadev_addr.pmport;
9622fcbc377Syt 	uint8_t qual = sd->satadev_addr.qual;
9632fcbc377Syt 	uint8_t	device_type;
9642fcbc377Syt 	uint32_t port_state;
9652fcbc377Syt 	uint8_t port;
9662fcbc377Syt 
9672fcbc377Syt 	ahci_ctlp = ddi_get_soft_state(ahci_statep, ddi_get_instance(dip));
9682fcbc377Syt 	port = ahci_ctlp->ahcictl_cport_to_port[cport];
9692fcbc377Syt 
9702fcbc377Syt 	AHCIDBG3(AHCIDBG_ENTRY, ahci_ctlp,
971*68d33a25Syt 	    "ahci_tran_probe_port enter: cport: %d, "
972*68d33a25Syt 	    "pmport: %d, qual: %d", cport, pmport, qual);
9732fcbc377Syt 
9742fcbc377Syt 	ahci_portp = ahci_ctlp->ahcictl_ports[port];
9752fcbc377Syt 
9762fcbc377Syt 	mutex_enter(&ahci_portp->ahciport_mutex);
9772fcbc377Syt 
9782fcbc377Syt 	port_state = ahci_portp->ahciport_port_state;
9792fcbc377Syt 	switch (port_state) {
9802fcbc377Syt 
9812fcbc377Syt 	case SATA_PSTATE_FAILED:
9822fcbc377Syt 		sd->satadev_state = SATA_PSTATE_FAILED;
9832fcbc377Syt 		AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp,
984*68d33a25Syt 		    "ahci_tran_probe_port: port %d PORT FAILED", port);
9852fcbc377Syt 		goto out;
9862fcbc377Syt 
9872fcbc377Syt 	case SATA_PSTATE_SHUTDOWN:
9882fcbc377Syt 		sd->satadev_state = SATA_PSTATE_SHUTDOWN;
9892fcbc377Syt 		AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp,
990*68d33a25Syt 		    "ahci_tran_probe_port: port %d PORT SHUTDOWN", port);
9912fcbc377Syt 		goto out;
9922fcbc377Syt 
9932fcbc377Syt 	case SATA_PSTATE_PWROFF:
9942fcbc377Syt 		sd->satadev_state = SATA_PSTATE_PWROFF;
9952fcbc377Syt 		AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp,
996*68d33a25Syt 		    "ahci_tran_probe_port: port %d PORT PWROFF", port);
9972fcbc377Syt 		goto out;
9982fcbc377Syt 
9992fcbc377Syt 	case SATA_PSTATE_PWRON:
10002fcbc377Syt 		sd->satadev_state = SATA_PSTATE_PWRON;
10012fcbc377Syt 		AHCIDBG1(AHCIDBG_INFO, ahci_ctlp,
1002*68d33a25Syt 		    "ahci_tran_probe_port: port %d PORT PWRON", port);
10032fcbc377Syt 		break;
10042fcbc377Syt 
10052fcbc377Syt 	default:
10062fcbc377Syt 		sd->satadev_state = port_state;
10072fcbc377Syt 		AHCIDBG2(AHCIDBG_INFO, ahci_ctlp,
1008*68d33a25Syt 		    "ahci_tran_probe_port: port %d PORT NORMAL %x",
10092fcbc377Syt 		    port, port_state);
10102fcbc377Syt 		break;
10112fcbc377Syt 	}
10122fcbc377Syt 
10132fcbc377Syt 	device_type = ahci_portp->ahciport_device_type;
10142fcbc377Syt 
10152fcbc377Syt 	switch (device_type) {
10162fcbc377Syt 
10172fcbc377Syt 	case SATA_DTYPE_ATADISK:
10182fcbc377Syt 		sd->satadev_type = SATA_DTYPE_ATADISK;
10192fcbc377Syt 		AHCIDBG1(AHCIDBG_INFO, ahci_ctlp,
1020*68d33a25Syt 		    "ahci_tran_probe_port: port %d DISK found", port);
10212fcbc377Syt 		break;
10222fcbc377Syt 
10232fcbc377Syt 	case SATA_DTYPE_ATAPICD:
10242fcbc377Syt 		sd->satadev_type = SATA_DTYPE_ATAPICD;
10252fcbc377Syt 		AHCIDBG1(AHCIDBG_INFO, ahci_ctlp,
1026*68d33a25Syt 		    "ahci_tran_probe_port: port %d ATAPI found", port);
10272fcbc377Syt 		break;
10282fcbc377Syt 
10292fcbc377Syt 	case SATA_DTYPE_PMULT:
10302fcbc377Syt 		sd->satadev_type = SATA_DTYPE_PMULT;
10312fcbc377Syt 		AHCIDBG1(AHCIDBG_INFO, ahci_ctlp,
1032*68d33a25Syt 		    "ahci_tran_probe_port: port %d Port Multiplier found",
10332fcbc377Syt 		    port);
10342fcbc377Syt 		break;
10352fcbc377Syt 
10362fcbc377Syt 	case SATA_DTYPE_UNKNOWN:
10372fcbc377Syt 		sd->satadev_type = SATA_DTYPE_UNKNOWN;
10382fcbc377Syt 		AHCIDBG1(AHCIDBG_INFO, ahci_ctlp,
1039*68d33a25Syt 		    "ahci_tran_probe_port: port %d Unknown device found", port);
10402fcbc377Syt 		break;
10412fcbc377Syt 
10422fcbc377Syt 	default:
10432fcbc377Syt 		/* we don't support any other device types */
10442fcbc377Syt 		sd->satadev_type = SATA_DTYPE_NONE;
10452fcbc377Syt 		AHCIDBG1(AHCIDBG_INFO, ahci_ctlp,
1046*68d33a25Syt 		    "ahci_tran_probe_port: port %d No device found", port);
10472fcbc377Syt 		break;
10482fcbc377Syt 	}
10492fcbc377Syt 
10502fcbc377Syt out:
10512fcbc377Syt 	ahci_update_sata_registers(ahci_ctlp, port, sd);
10522fcbc377Syt 	mutex_exit(&ahci_portp->ahciport_mutex);
10532fcbc377Syt 
10542fcbc377Syt 	return (SATA_SUCCESS);
10552fcbc377Syt }
10562fcbc377Syt 
1057*68d33a25Syt /*
1058*68d33a25Syt  * There are four operation modes in sata framework:
1059*68d33a25Syt  * SATA_OPMODE_INTERRUPTS
1060*68d33a25Syt  * SATA_OPMODE_POLLING
1061*68d33a25Syt  * SATA_OPMODE_ASYNCH
1062*68d33a25Syt  * SATA_OPMODE_SYNCH
1063*68d33a25Syt  *
1064*68d33a25Syt  * Their combined meanings as following:
1065*68d33a25Syt  *
1066*68d33a25Syt  * SATA_OPMODE_SYNCH
1067*68d33a25Syt  * The command has to be completed before sata_tran_start functions returns.
1068*68d33a25Syt  * Either interrupts or polling could be used - it's up to the driver.
1069*68d33a25Syt  * Mode used currently for internal, sata-module initiated operations.
1070*68d33a25Syt  *
1071*68d33a25Syt  * SATA_OPMODE_SYNCH | SATA_OPMODE_INTERRUPTS
1072*68d33a25Syt  * It is the same as the one above.
1073*68d33a25Syt  *
1074*68d33a25Syt  * SATA_OPMODE_SYNCH | SATA_OPMODE_POLLING
1075*68d33a25Syt  * The command has to be completed before sata_tran_start function returns.
1076*68d33a25Syt  * No interrupt used, polling only. This should be the mode used for scsi
1077*68d33a25Syt  * packets with FLAG_NOINTR.
1078*68d33a25Syt  *
1079*68d33a25Syt  * SATA_OPMODE_ASYNCH | SATA_OPMODE_INTERRUPTS
1080*68d33a25Syt  * The command may be queued (callback function specified). Interrupts could
1081*68d33a25Syt  * be used. It's normal operation mode.
1082*68d33a25Syt  */
10832fcbc377Syt /*
10842fcbc377Syt  * Called by sata framework to transport a sata packet down stream.
10852fcbc377Syt  */
10862fcbc377Syt static int
10872fcbc377Syt ahci_tran_start(dev_info_t *dip, sata_pkt_t *spkt)
10882fcbc377Syt {
10892fcbc377Syt 	ahci_ctl_t *ahci_ctlp;
10902fcbc377Syt 	ahci_port_t *ahci_portp;
10912fcbc377Syt 	uint8_t	cport = spkt->satapkt_device.satadev_addr.cport;
10922fcbc377Syt 	uint8_t port;
10932fcbc377Syt 
10942fcbc377Syt 	ahci_ctlp = ddi_get_soft_state(ahci_statep, ddi_get_instance(dip));
10952fcbc377Syt 	port = ahci_ctlp->ahcictl_cport_to_port[cport];
10962fcbc377Syt 
10972fcbc377Syt 	AHCIDBG2(AHCIDBG_ENTRY, ahci_ctlp,
10982fcbc377Syt 	    "ahci_tran_start enter: cport %d satapkt 0x%p",
10992fcbc377Syt 	    cport, (void *)spkt);
11002fcbc377Syt 
11012fcbc377Syt 	ahci_portp = ahci_ctlp->ahcictl_ports[port];
11022fcbc377Syt 
11032fcbc377Syt 	mutex_enter(&ahci_portp->ahciport_mutex);
11042fcbc377Syt 
11052fcbc377Syt 	if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED |
11062fcbc377Syt 	    ahci_portp->ahciport_port_state & SATA_PSTATE_SHUTDOWN |
11072fcbc377Syt 	    ahci_portp->ahciport_port_state & SATA_PSTATE_PWROFF) {
11082fcbc377Syt 		/*
11092fcbc377Syt 		 * In case the targer driver would send the packet before
11102fcbc377Syt 		 * sata framework can have the opportunity to process those
11112fcbc377Syt 		 * event reports.
11122fcbc377Syt 		 */
11132fcbc377Syt 		spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
11142fcbc377Syt 		spkt->satapkt_device.satadev_state =
11152fcbc377Syt 		    ahci_portp->ahciport_port_state;
11162fcbc377Syt 		ahci_update_sata_registers(ahci_ctlp, port,
11172fcbc377Syt 		    &spkt->satapkt_device);
11182fcbc377Syt 		AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp,
11192fcbc377Syt 		    "ahci_tran_start returning PORT_ERROR while "
11202fcbc377Syt 		    "port in FAILED/SHUTDOWN/PWROFF state: "
1121*68d33a25Syt 		    "port: %d", port);
11222fcbc377Syt 		mutex_exit(&ahci_portp->ahciport_mutex);
11232fcbc377Syt 		return (SATA_TRAN_PORT_ERROR);
11242fcbc377Syt 	}
11252fcbc377Syt 
11262fcbc377Syt 	if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
11272fcbc377Syt 		/*
11282fcbc377Syt 		 * ahci_intr_phyrdy_change() may have rendered it to
11292fcbc377Syt 		 * SATA_DTYPE_NONE.
11302fcbc377Syt 		 */
11312fcbc377Syt 		spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
11322fcbc377Syt 		spkt->satapkt_device.satadev_type = SATA_DTYPE_NONE;
11332fcbc377Syt 		spkt->satapkt_device.satadev_state =
11342fcbc377Syt 		    ahci_portp->ahciport_port_state;
11352fcbc377Syt 		ahci_update_sata_registers(ahci_ctlp, port,
11362fcbc377Syt 		    &spkt->satapkt_device);
11372fcbc377Syt 		AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp,
11382fcbc377Syt 		    "ahci_tran_start returning PORT_ERROR while "
1139*68d33a25Syt 		    "no device attached: port: %d", port);
11402fcbc377Syt 		mutex_exit(&ahci_portp->ahciport_mutex);
11412fcbc377Syt 		return (SATA_TRAN_PORT_ERROR);
11422fcbc377Syt 	}
11432fcbc377Syt 
11442fcbc377Syt 	/*
11452fcbc377Syt 	 * SATA HBA driver should remember that a device was reset and it
11462fcbc377Syt 	 * is supposed to reject any packets which do not specify either
11472fcbc377Syt 	 * SATA_IGNORE_DEV_RESET_STATE or SATA_CLEAR_DEV_RESET_STATE.
11482fcbc377Syt 	 *
11492fcbc377Syt 	 * This is to prevent a race condition when a device was arbitrarily
11502fcbc377Syt 	 * reset by the HBA driver (and lost it's setting) and a target
11512fcbc377Syt 	 * driver sending some commands to a device before the sata framework
11522fcbc377Syt 	 * has a chance to restore the device setting (such as cache enable/
11532fcbc377Syt 	 * disable or other resettable stuff).
11542fcbc377Syt 	 */
11552fcbc377Syt 	if (spkt->satapkt_cmd.satacmd_flags.sata_clear_dev_reset) {
11562fcbc377Syt 		ahci_portp->ahciport_reset_in_progress = 0;
11572fcbc377Syt 		AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp,
11582fcbc377Syt 		    "ahci_tran_start clearing the "
1159*68d33a25Syt 		    "reset_in_progress for port: %d", port);
11602fcbc377Syt 	}
11612fcbc377Syt 
11622fcbc377Syt 	if (ahci_portp->ahciport_reset_in_progress &&
11632fcbc377Syt 	    ! spkt->satapkt_cmd.satacmd_flags.sata_ignore_dev_reset &&
11642fcbc377Syt 	    ! ddi_in_panic()) {
11652fcbc377Syt 		spkt->satapkt_reason = SATA_PKT_BUSY;
11662fcbc377Syt 		AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp,
11672fcbc377Syt 		    "ahci_tran_start returning BUSY while "
1168*68d33a25Syt 		    "reset in progress: port: %d", port);
11692fcbc377Syt 		mutex_exit(&ahci_portp->ahciport_mutex);
11702fcbc377Syt 		return (SATA_TRAN_BUSY);
11712fcbc377Syt 	}
11722fcbc377Syt 
1173*68d33a25Syt 	if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
11742fcbc377Syt 		spkt->satapkt_reason = SATA_PKT_BUSY;
11752fcbc377Syt 		AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp,
11762fcbc377Syt 		    "ahci_tran_start returning BUSY while "
1177*68d33a25Syt 		    "mopping in progress: port: %d", port);
11782fcbc377Syt 		mutex_exit(&ahci_portp->ahciport_mutex);
11792fcbc377Syt 		return (SATA_TRAN_BUSY);
11802fcbc377Syt 	}
11812fcbc377Syt 
1182*68d33a25Syt 	if ((ahci_ctlp->ahcictl_cap & AHCI_CAP_NO_MCMDLIST_NONQUEUE) &&
1183*68d33a25Syt 	    (ahci_portp->ahciport_pending_tags != 0)) {
1184*68d33a25Syt 		AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, "ahci_tran_start "
1185*68d33a25Syt 		    "return QUEUE_FULL: port %d because HBA cannot "
1186*68d33a25Syt 		    "use multiple command lists for non-queued commands",
1187*68d33a25Syt 		    port);
11882fcbc377Syt 		mutex_exit(&ahci_portp->ahciport_mutex);
11892fcbc377Syt 		return (SATA_TRAN_QUEUE_FULL);
11902fcbc377Syt 	}
11912fcbc377Syt 
11922fcbc377Syt 	if (spkt->satapkt_op_mode &
1193*68d33a25Syt 	    (SATA_OPMODE_SYNCH | SATA_OPMODE_POLLING)) {
1194*68d33a25Syt 		/* We need to do the sync start now */
1195*68d33a25Syt 		if (ahci_do_sync_start(ahci_ctlp, ahci_portp, port,
1196*68d33a25Syt 		    spkt) == AHCI_FAILURE) {
1197*68d33a25Syt 			AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "ahci_tran_start "
1198*68d33a25Syt 			    "return QUEUE_FULL: port %d", port);
1199*68d33a25Syt 			mutex_exit(&ahci_portp->ahciport_mutex);
1200*68d33a25Syt 			return (SATA_TRAN_QUEUE_FULL);
1201*68d33a25Syt 		}
1202*68d33a25Syt 	} else {
1203*68d33a25Syt 		/* Async start, using interrupt */
1204*68d33a25Syt 		if (ahci_deliver_satapkt(ahci_ctlp, ahci_portp, port, spkt)
1205*68d33a25Syt 		    == AHCI_FAILURE) {
1206*68d33a25Syt 			spkt->satapkt_reason = SATA_PKT_QUEUE_FULL;
1207*68d33a25Syt 			AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "ahci_tran_start "
1208*68d33a25Syt 			    "returning QUEUE_FULL: port %d", port);
1209*68d33a25Syt 			mutex_exit(&ahci_portp->ahciport_mutex);
1210*68d33a25Syt 			return (SATA_TRAN_QUEUE_FULL);
1211*68d33a25Syt 		}
12122fcbc377Syt 	}
12132fcbc377Syt 
12142fcbc377Syt 	AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, "ahci_tran_start "
12152fcbc377Syt 	    "sata tran accepted: port %d", port);
12162fcbc377Syt 
1217*68d33a25Syt 	mutex_exit(&ahci_portp->ahciport_mutex);
12182fcbc377Syt 	return (SATA_TRAN_ACCEPTED);
12192fcbc377Syt }
12202fcbc377Syt 
1221*68d33a25Syt /*
1222*68d33a25Syt  * SATA_OPMODE_SYNCH flag is set
1223*68d33a25Syt  *
1224*68d33a25Syt  * If SATA_OPMODE_POLLING flag is set, then we must poll the command
1225*68d33a25Syt  * without interrupt, otherwise we can still use the interrupt.
1226*68d33a25Syt  *
1227*68d33a25Syt  * WARNING!!! ahciport_mutex should be acquired before the function
1228*68d33a25Syt  * is called.
1229*68d33a25Syt  */
1230*68d33a25Syt static int
1231*68d33a25Syt ahci_do_sync_start(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
1232*68d33a25Syt     uint8_t port, sata_pkt_t *spkt)
1233*68d33a25Syt {
1234*68d33a25Syt 	int pkt_timeout_ticks;
1235*68d33a25Syt 	uint32_t timeout_tags;
1236*68d33a25Syt 	uint32_t retrieve_errinfo_slot_status = 0;
1237*68d33a25Syt 	int rval;
1238*68d33a25Syt 
1239*68d33a25Syt 	AHCIDBG2(AHCIDBG_ENTRY, ahci_ctlp, "ahci_do_sync_start enter: "
1240*68d33a25Syt 	    "port %d spkt 0x%p", port, spkt);
1241*68d33a25Syt 
1242*68d33a25Syt 	if (spkt->satapkt_op_mode & SATA_OPMODE_POLLING) {
1243*68d33a25Syt 		/* Disable the port interrupt */
1244*68d33a25Syt 		ahci_disable_port_intrs(ahci_ctlp, ahci_portp, port);
1245*68d33a25Syt 		ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_POLLING;
1246*68d33a25Syt 
1247*68d33a25Syt 		if ((rval = ahci_deliver_satapkt(ahci_ctlp, ahci_portp,
1248*68d33a25Syt 		    port, spkt)) == AHCI_FAILURE) {
1249*68d33a25Syt 			ahci_portp->ahciport_flags &= ~ AHCI_PORT_FLAG_POLLING;
1250*68d33a25Syt 			ahci_enable_port_intrs(ahci_ctlp, ahci_portp, port);
1251*68d33a25Syt 			return (rval);
1252*68d33a25Syt 		}
1253*68d33a25Syt 
1254*68d33a25Syt 		pkt_timeout_ticks =
1255*68d33a25Syt 		    drv_usectohz((clock_t)spkt->satapkt_time * 1000000);
1256*68d33a25Syt 
1257*68d33a25Syt 		/*
1258*68d33a25Syt 		 * AHCI_PORT_FLAG_RQSENSE means the command is the REQUEST
1259*68d33a25Syt 		 * SENSE which is sent down to retrieve sense data during
1260*68d33a25Syt 		 * error recovery, so we need to keep the slot number of it
1261*68d33a25Syt 		 * in order to handle it's completion.
1262*68d33a25Syt 		 */
1263*68d33a25Syt 		if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_RQSENSE) {
1264*68d33a25Syt 			/* rval is the allocated command slot */
1265*68d33a25Syt 			retrieve_errinfo_slot_status = 0x1 << rval;
1266*68d33a25Syt 		}
1267*68d33a25Syt 
1268*68d33a25Syt 		while (spkt->satapkt_reason == SATA_PKT_BUSY) {
1269*68d33a25Syt 			mutex_exit(&ahci_portp->ahciport_mutex);
1270*68d33a25Syt 
1271*68d33a25Syt 			/* Simulate the interrupt */
1272*68d33a25Syt 			ahci_port_intr(ahci_ctlp, ahci_portp, port,
1273*68d33a25Syt 			    retrieve_errinfo_slot_status);
1274*68d33a25Syt 
1275*68d33a25Syt 			drv_usecwait(AHCI_1MS_USECS);
1276*68d33a25Syt 
1277*68d33a25Syt 			mutex_enter(&ahci_portp->ahciport_mutex);
1278*68d33a25Syt 			pkt_timeout_ticks -= AHCI_1MS_TICKS;
1279*68d33a25Syt 			if (pkt_timeout_ticks < 0) {
1280*68d33a25Syt 				cmn_err(CE_NOTE, "!ahci_do_sync_start: "
1281*68d33a25Syt 				    "port %d satapkt 0x%p timed out\n",
1282*68d33a25Syt 				    port, (void *)spkt);
1283*68d33a25Syt 				timeout_tags = (0x1 << rval);
1284*68d33a25Syt 				mutex_exit(&ahci_portp->ahciport_mutex);
1285*68d33a25Syt 				ahci_timeout_pkts(ahci_ctlp, ahci_portp,
1286*68d33a25Syt 				    port, timeout_tags,
1287*68d33a25Syt 				    retrieve_errinfo_slot_status);
1288*68d33a25Syt 				mutex_enter(&ahci_portp->ahciport_mutex);
1289*68d33a25Syt 			}
1290*68d33a25Syt 		}
1291*68d33a25Syt 		ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_POLLING;
1292*68d33a25Syt 		/* Enable the port interrupt */
1293*68d33a25Syt 		ahci_enable_port_intrs(ahci_ctlp, ahci_portp, port);
1294*68d33a25Syt 		return (AHCI_SUCCESS);
1295*68d33a25Syt 
1296*68d33a25Syt 	} else {
1297*68d33a25Syt 		if ((rval = ahci_deliver_satapkt(ahci_ctlp, ahci_portp,
1298*68d33a25Syt 		    port, spkt)) == AHCI_FAILURE)
1299*68d33a25Syt 			return (rval);
1300*68d33a25Syt 
1301*68d33a25Syt 		while (spkt->satapkt_reason == SATA_PKT_BUSY)
1302*68d33a25Syt 			cv_wait(&ahci_portp->ahciport_cv,
1303*68d33a25Syt 			    &ahci_portp->ahciport_mutex);
1304*68d33a25Syt 
1305*68d33a25Syt 		return (AHCI_SUCCESS);
1306*68d33a25Syt 	}
1307*68d33a25Syt }
1308*68d33a25Syt 
13092fcbc377Syt #define	SENDUP_PACKET(ahci_portp, satapkt, reason)			\
13102fcbc377Syt 	if (satapkt) {							\
13112fcbc377Syt 		satapkt->satapkt_reason = reason;			\
13122fcbc377Syt 		/*							\
13132fcbc377Syt 		 * We set the satapkt_reason in both sync and		\
13142fcbc377Syt 		 * non-sync cases.					\
13152fcbc377Syt 		 */							\
13162fcbc377Syt 	}								\
13172fcbc377Syt 	if (satapkt &&							\
1318*68d33a25Syt 	    ! (satapkt->satapkt_op_mode & SATA_OPMODE_SYNCH) &&		\
13192fcbc377Syt 	    satapkt->satapkt_comp) {					\
13202fcbc377Syt 		mutex_exit(&ahci_portp->ahciport_mutex);		\
13212fcbc377Syt 		(*satapkt->satapkt_comp)(satapkt);			\
13222fcbc377Syt 		mutex_enter(&ahci_portp->ahciport_mutex);		\
1323*68d33a25Syt 	} else {							\
1324*68d33a25Syt 		if (satapkt &&						\
1325*68d33a25Syt 		    (satapkt->satapkt_op_mode & SATA_OPMODE_SYNCH) &&	\
1326*68d33a25Syt 		    ! (satapkt->satapkt_op_mode & SATA_OPMODE_POLLING))	\
1327*68d33a25Syt 			cv_signal(&ahci_portp->ahciport_cv);		\
13282fcbc377Syt 	}
13292fcbc377Syt 
13302fcbc377Syt /*
13312fcbc377Syt  * Searches for and claims a free slot.
13322fcbc377Syt  *
13332fcbc377Syt  * Returns:     AHCI_FAILURE if no slots found
13342fcbc377Syt  *              claimed slot number if successful
13352fcbc377Syt  *
13362fcbc377Syt  * WARNING!!! ahciport_mutex should be acquired before the function
13372fcbc377Syt  * is called.
13382fcbc377Syt  */
13392fcbc377Syt /*ARGSUSED*/
13402fcbc377Syt static int
13412fcbc377Syt ahci_claim_free_slot(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
13422fcbc377Syt {
13432fcbc377Syt 	uint32_t free_slots;
13442fcbc377Syt 	int slot;
13452fcbc377Syt 
13462fcbc377Syt 	AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp, "ahci_claim_free_slot enter "
13472fcbc377Syt 	    "ahciport_pending_tags = 0x%x",
13482fcbc377Syt 	    ahci_portp->ahciport_pending_tags);
13492fcbc377Syt 
13502fcbc377Syt 	free_slots = (~ahci_portp->ahciport_pending_tags)
13512fcbc377Syt 	    & AHCI_SLOT_MASK(ahci_ctlp);
13522fcbc377Syt 
13532fcbc377Syt 	slot = ddi_ffs(free_slots) - 1;
13542fcbc377Syt 	if (slot == -1) {
13552fcbc377Syt 		AHCIDBG0(AHCIDBG_VERBOSE, ahci_ctlp,
13562fcbc377Syt 		    "ahci_claim_free_slot: no empty slots");
13572fcbc377Syt 		return (AHCI_FAILURE);
13582fcbc377Syt 	}
13592fcbc377Syt 
13602fcbc377Syt 	ahci_portp->ahciport_pending_tags |= (0x1 << slot);
13612fcbc377Syt 
13622fcbc377Syt 	AHCIDBG1(AHCIDBG_VERBOSE, ahci_ctlp,
13632fcbc377Syt 	    "ahci_claim_free_slot: found slot: 0x%x", slot);
13642fcbc377Syt 
13652fcbc377Syt 	return (slot);
13662fcbc377Syt }
13672fcbc377Syt 
13682fcbc377Syt /*
13692fcbc377Syt  * Builds the Command Table for the sata packet and delivers it to controller.
13702fcbc377Syt  *
13712fcbc377Syt  * Returns:
13722fcbc377Syt  * 	slot number if we can obtain a slot successfully
13732fcbc377Syt  *	otherwise, return AHCI_FAILURE
13742fcbc377Syt  *
13752fcbc377Syt  * WARNING!!! ahciport_mutex should be acquired before the function is called.
13762fcbc377Syt  */
13772fcbc377Syt static int
13782fcbc377Syt ahci_deliver_satapkt(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
13792fcbc377Syt     uint8_t port, sata_pkt_t *spkt)
13802fcbc377Syt {
1381*68d33a25Syt 	int cmd_slot;
1382*68d33a25Syt 	sata_cmd_t *scmd;
13832fcbc377Syt 	ahci_fis_h2d_register_t *h2d_register_fisp;
13842fcbc377Syt 	ahci_cmd_table_t *cmd_table;
13852fcbc377Syt 	ahci_cmd_header_t *cmd_header;
13862fcbc377Syt 	int ncookies;
13872fcbc377Syt 	int i;
1388*68d33a25Syt #if AHCI_DEBUG
1389*68d33a25Syt 	uint32_t *ptr;
1390*68d33a25Syt 	uint8_t *ptr2;
1391*68d33a25Syt #endif
13922fcbc377Syt 
13932fcbc377Syt 	spkt->satapkt_reason = SATA_PKT_BUSY;
13942fcbc377Syt 
1395*68d33a25Syt 	scmd = &spkt->satapkt_cmd;
13962fcbc377Syt 
1397*68d33a25Syt 	/* check if there is an empty command slot */
1398*68d33a25Syt 	cmd_slot = ahci_claim_free_slot(ahci_ctlp, ahci_portp);
1399*68d33a25Syt 	if (cmd_slot == AHCI_FAILURE) {
14002fcbc377Syt 		AHCIDBG0(AHCIDBG_INFO, ahci_ctlp, "no free slot");
14012fcbc377Syt 		return (AHCI_FAILURE);
14022fcbc377Syt 	}
14032fcbc377Syt 
14042fcbc377Syt 	AHCIDBG4(AHCIDBG_ENTRY|AHCIDBG_INFO, ahci_ctlp,
1405*68d33a25Syt 	    "ahci_deliver_satapkt enter: cmd_reg: 0x%x, cmd_slot: 0x%x, "
1406*68d33a25Syt 	    "port: %d, satapkt: 0x%p", scmd->satacmd_cmd_reg,
1407*68d33a25Syt 	    cmd_slot, port, (void *)spkt);
14082fcbc377Syt 
1409*68d33a25Syt 	cmd_table = ahci_portp->ahciport_cmd_tables[cmd_slot];
14102fcbc377Syt 	bzero((void *)cmd_table, ahci_cmd_table_size);
14112fcbc377Syt 
1412*68d33a25Syt 	/* for data transfer operations, this is the h2d register fis */
14132fcbc377Syt 	h2d_register_fisp =
14142fcbc377Syt 	    &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register);
14152fcbc377Syt 
14162fcbc377Syt 	SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE);
14172fcbc377Syt 	if ((spkt->satapkt_device.satadev_addr.qual == SATA_ADDR_PMPORT) ||
14182fcbc377Syt 	    (spkt->satapkt_device.satadev_addr.qual == SATA_ADDR_DPMPORT)) {
14192fcbc377Syt 		SET_FIS_PMP(h2d_register_fisp,
14202fcbc377Syt 		    spkt->satapkt_device.satadev_addr.pmport);
14212fcbc377Syt 	}
14222fcbc377Syt 
14232fcbc377Syt 	SET_FIS_CDMDEVCTL(h2d_register_fisp, 1);
1424*68d33a25Syt 	SET_FIS_COMMAND(h2d_register_fisp, scmd->satacmd_cmd_reg);
1425*68d33a25Syt 	SET_FIS_FEATURES(h2d_register_fisp, scmd->satacmd_features_reg);
1426*68d33a25Syt 	SET_FIS_SECTOR_COUNT(h2d_register_fisp, scmd->satacmd_sec_count_lsb);
14272fcbc377Syt 
1428*68d33a25Syt 	switch (scmd->satacmd_addr_type) {
14292fcbc377Syt 
14302fcbc377Syt 	case ATA_ADDR_LBA:
14312fcbc377Syt 		/* fallthrough */
14322fcbc377Syt 
14332fcbc377Syt 	case ATA_ADDR_LBA28:
14342fcbc377Syt 		/* LBA[7:0] */
1435*68d33a25Syt 		SET_FIS_SECTOR(h2d_register_fisp, scmd->satacmd_lba_low_lsb);
14362fcbc377Syt 
14372fcbc377Syt 		/* LBA[15:8] */
1438*68d33a25Syt 		SET_FIS_CYL_LOW(h2d_register_fisp, scmd->satacmd_lba_mid_lsb);
14392fcbc377Syt 
14402fcbc377Syt 		/* LBA[23:16] */
1441*68d33a25Syt 		SET_FIS_CYL_HI(h2d_register_fisp, scmd->satacmd_lba_high_lsb);
14422fcbc377Syt 
14432fcbc377Syt 		/* LBA [27:24] (also called dev_head) */
1444*68d33a25Syt 		SET_FIS_DEV_HEAD(h2d_register_fisp, scmd->satacmd_device_reg);
14452fcbc377Syt 
14462fcbc377Syt 		break;
14472fcbc377Syt 
14482fcbc377Syt 	case ATA_ADDR_LBA48:
14492fcbc377Syt 		/* LBA[7:0] */
1450*68d33a25Syt 		SET_FIS_SECTOR(h2d_register_fisp, scmd->satacmd_lba_low_lsb);
14512fcbc377Syt 
14522fcbc377Syt 		/* LBA[15:8] */
1453*68d33a25Syt 		SET_FIS_CYL_LOW(h2d_register_fisp, scmd->satacmd_lba_mid_lsb);
14542fcbc377Syt 
14552fcbc377Syt 		/* LBA[23:16] */
1456*68d33a25Syt 		SET_FIS_CYL_HI(h2d_register_fisp, scmd->satacmd_lba_high_lsb);
14572fcbc377Syt 
14582fcbc377Syt 		/* LBA [31:24] */
14592fcbc377Syt 		SET_FIS_SECTOR_EXP(h2d_register_fisp,
1460*68d33a25Syt 		    scmd->satacmd_lba_low_msb);
14612fcbc377Syt 
14622fcbc377Syt 		/* LBA [39:32] */
14632fcbc377Syt 		SET_FIS_CYL_LOW_EXP(h2d_register_fisp,
1464*68d33a25Syt 		    scmd->satacmd_lba_mid_msb);
14652fcbc377Syt 
14662fcbc377Syt 		/* LBA [47:40] */
14672fcbc377Syt 		SET_FIS_CYL_HI_EXP(h2d_register_fisp,
1468*68d33a25Syt 		    scmd->satacmd_lba_high_msb);
14692fcbc377Syt 
14702fcbc377Syt 		/* Set dev_head */
14712fcbc377Syt 		SET_FIS_DEV_HEAD(h2d_register_fisp,
1472*68d33a25Syt 		    scmd->satacmd_device_reg);
14732fcbc377Syt 
14742fcbc377Syt 		/* Set the extended sector count and features */
14752fcbc377Syt 		SET_FIS_SECTOR_COUNT_EXP(h2d_register_fisp,
1476*68d33a25Syt 		    scmd->satacmd_sec_count_msb);
14772fcbc377Syt 		SET_FIS_FEATURES_EXP(h2d_register_fisp,
1478*68d33a25Syt 		    scmd->satacmd_features_reg_ext);
14792fcbc377Syt 		break;
14802fcbc377Syt 	}
14812fcbc377Syt 
1482*68d33a25Syt 	ncookies = scmd->satacmd_num_dma_cookies;
14832fcbc377Syt 	AHCIDBG2(AHCIDBG_INFO, ahci_ctlp,
14842fcbc377Syt 	    "ncookies = 0x%x, ahci_dma_prdt_number = 0x%x",
14852fcbc377Syt 	    ncookies, ahci_dma_prdt_number);
14862fcbc377Syt 
14872fcbc377Syt 	ASSERT(ncookies <= ahci_dma_prdt_number);
14882fcbc377Syt 
14892fcbc377Syt 	/* *** now fill the scatter gather list ******* */
14902fcbc377Syt 	for (i = 0; i < ncookies; i++) {
14912fcbc377Syt 		cmd_table->ahcict_prdt[i].ahcipi_data_base_addr =
1492*68d33a25Syt 		    scmd->satacmd_dma_cookie_list[i]._dmu._dmac_la[0];
14932fcbc377Syt 		cmd_table->ahcict_prdt[i].ahcipi_data_base_addr_upper =
1494*68d33a25Syt 		    scmd->satacmd_dma_cookie_list[i]._dmu._dmac_la[1];
14952fcbc377Syt 		cmd_table->ahcict_prdt[i].ahcipi_descr_info =
1496*68d33a25Syt 		    scmd->satacmd_dma_cookie_list[i].dmac_size - 1;
1497*68d33a25Syt 	}
1498*68d33a25Syt 
1499*68d33a25Syt 	/* The ACMD field is filled in for ATAPI command */
1500*68d33a25Syt 	if (scmd->satacmd_cmd_reg == SATAC_PACKET) {
1501*68d33a25Syt 		bcopy(scmd->satacmd_acdb, cmd_table->ahcict_atapi_cmd,
1502*68d33a25Syt 		    SATA_ATAPI_MAX_CDB_LEN);
1503*68d33a25Syt 		/*
1504*68d33a25Syt 		 * For ATAPI command, scmd->satacmd_addr_type is 0
1505*68d33a25Syt 		 */
1506*68d33a25Syt 		SET_FIS_CYL_LOW(h2d_register_fisp, scmd->satacmd_lba_mid_lsb);
1507*68d33a25Syt 		SET_FIS_CYL_HI(h2d_register_fisp, scmd->satacmd_lba_high_lsb);
15082fcbc377Syt 	}
15092fcbc377Syt 
15102fcbc377Syt 	/* Set Command Header in Command List */
1511*68d33a25Syt 	cmd_header = &ahci_portp->ahciport_cmd_list[cmd_slot];
15122fcbc377Syt 	BZERO_DESCR_INFO(cmd_header);
15132fcbc377Syt 	BZERO_PRD_BYTE_COUNT(cmd_header);
1514*68d33a25Syt 
1515*68d33a25Syt 	/* Set the number of entries in the PRD table */
15162fcbc377Syt 	SET_PRD_TABLE_LENGTH(cmd_header, ncookies);
1517*68d33a25Syt 
1518*68d33a25Syt 	/* Set the length of the command in the CFIS area */
15192fcbc377Syt 	SET_COMMAND_FIS_LENGTH(cmd_header, AHCI_H2D_REGISTER_FIS_LENGTH);
15202fcbc377Syt 
15212fcbc377Syt 	AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, "command data direction is "
15222fcbc377Syt 	    "sata_data_direction = 0x%x",
1523*68d33a25Syt 	    scmd->satacmd_flags.sata_data_direction);
1524*68d33a25Syt 
1525*68d33a25Syt 	/* Set A bit if it is an ATAPI command */
1526*68d33a25Syt 	if (scmd->satacmd_cmd_reg == SATAC_PACKET)
1527*68d33a25Syt 		SET_ATAPI(cmd_header, AHCI_CMDHEAD_ATAPI);
15282fcbc377Syt 
1529*68d33a25Syt 	/* Set W bit if data is going to the device */
1530*68d33a25Syt 	if (scmd->satacmd_flags.sata_data_direction == SATA_DIR_WRITE)
15312fcbc377Syt 		SET_WRITE(cmd_header, AHCI_CMDHEAD_DATA_WRITE);
15322fcbc377Syt 
1533*68d33a25Syt 	/*
1534*68d33a25Syt 	 * Set the prefetchable bit - this bit is only valid if the PRDTL
1535*68d33a25Syt 	 * field is non-zero or the ATAPI 'A' bit is set in the command
1536*68d33a25Syt 	 * header. This bit cannot be set when using native command
1537*68d33a25Syt 	 * queuing commands or when using FIS-based switching with a Port
1538*68d33a25Syt 	 * Multiplier. At the moment, the driver doesn't support these two
1539*68d33a25Syt 	 * functions, so it's always setting the 'P' bit.
1540*68d33a25Syt 	 */
15412fcbc377Syt 	SET_PREFETCHABLE(cmd_header, AHCI_CMDHEAD_PREFETCHABLE);
15422fcbc377Syt 
15432fcbc377Syt 	/* Now remember the sata packet in ahciport_slot_pkts[]. */
1544*68d33a25Syt 	ahci_portp->ahciport_slot_pkts[cmd_slot] = spkt;
15452fcbc377Syt 
15462fcbc377Syt 	/*
15472fcbc377Syt 	 * We are overloading satapkt_hba_driver_private with
15482fcbc377Syt 	 * watched_cycle count.
15492fcbc377Syt 	 */
15502fcbc377Syt 	spkt->satapkt_hba_driver_private = (void *)(intptr_t)0;
15512fcbc377Syt 
1552*68d33a25Syt #if AHCI_DEBUG
1553*68d33a25Syt 	/* Dump the command header and table */
1554*68d33a25Syt 	AHCIDBG0(AHCIDBG_COMMAND, ahci_ctlp, "\n");
1555*68d33a25Syt 	AHCIDBG3(AHCIDBG_COMMAND, ahci_ctlp,
1556*68d33a25Syt 	    "Command header&table for spkt 0x%p cmd_reg 0x%x port%d",
1557*68d33a25Syt 	    spkt, scmd->satacmd_cmd_reg, port);
1558*68d33a25Syt 	ptr = (uint32_t *)cmd_header;
1559*68d33a25Syt 	AHCIDBG4(AHCIDBG_COMMAND, ahci_ctlp,
1560*68d33a25Syt 	    "  Command Header:%8x %8x %8x %8x",
1561*68d33a25Syt 	    ptr[0], ptr[1], ptr[2], ptr[3]);
1562*68d33a25Syt 
1563*68d33a25Syt 	/* Dump the H2D register FIS */
1564*68d33a25Syt 	ptr = (uint32_t *)h2d_register_fisp;
1565*68d33a25Syt 	AHCIDBG4(AHCIDBG_COMMAND, ahci_ctlp,
1566*68d33a25Syt 	    "  Command FIS:   %8x %8x %8x %8x",
1567*68d33a25Syt 	    ptr[0], ptr[1], ptr[2], ptr[3]);
1568*68d33a25Syt 
1569*68d33a25Syt 	/* Dump the ACMD register FIS */
1570*68d33a25Syt 	ptr2 = (uint8_t *)&(cmd_table->ahcict_atapi_cmd);
1571*68d33a25Syt 	for (i = 0; i < SATA_ATAPI_MAX_CDB_LEN/8; i++)
1572*68d33a25Syt 		if (ahci_debug_flags & AHCIDBG_COMMAND)
1573*68d33a25Syt 			ahci_log(ahci_ctlp, CE_WARN,
1574*68d33a25Syt 			    "  ATAPI command: %2x %2x %2x %2x "
1575*68d33a25Syt 			    "%2x %2x %2x %2x",
1576*68d33a25Syt 			    ptr2[8 * i], ptr2[8 * i + 1],
1577*68d33a25Syt 			    ptr2[8 * i + 2], ptr2[8 * i + 3],
1578*68d33a25Syt 			    ptr2[8 * i + 4], ptr2[8 * i + 5],
1579*68d33a25Syt 			    ptr2[8 * i + 6], ptr2[8 * i + 7]);
1580*68d33a25Syt 
1581*68d33a25Syt 	/* Dump the PRDT */
1582*68d33a25Syt 	for (i = 0; i < ncookies; i++) {
1583*68d33a25Syt 		ptr = (uint32_t *)&(cmd_table->ahcict_prdt[i]);
1584*68d33a25Syt 		AHCIDBG5(AHCIDBG_COMMAND, ahci_ctlp,
1585*68d33a25Syt 		    "  Cookie %d:      %8x %8x %8x %8x",
1586*68d33a25Syt 		    i, ptr[0], ptr[1], ptr[2], ptr[3]);
1587*68d33a25Syt 	}
1588*68d33a25Syt #endif
1589*68d33a25Syt 
1590*68d33a25Syt 	(void) ddi_dma_sync(
1591*68d33a25Syt 	    ahci_portp->ahciport_cmd_tables_dma_handle[cmd_slot],
15922fcbc377Syt 	    0,
15932fcbc377Syt 	    ahci_cmd_table_size,
15942fcbc377Syt 	    DDI_DMA_SYNC_FORDEV);
15952fcbc377Syt 
15962fcbc377Syt 	(void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle,
1597*68d33a25Syt 	    cmd_slot * sizeof (ahci_cmd_header_t),
15982fcbc377Syt 	    sizeof (ahci_cmd_header_t),
15992fcbc377Syt 	    DDI_DMA_SYNC_FORDEV);
16002fcbc377Syt 
16012fcbc377Syt 	/* Indicate to the HBA that a command is active. */
16022fcbc377Syt 	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
16032fcbc377Syt 	    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port),
1604*68d33a25Syt 	    (0x1 << cmd_slot));
16052fcbc377Syt 
16062fcbc377Syt 	AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, "ahci_deliver_satapkt "
16072fcbc377Syt 	    "exit: port %d", port);
16082fcbc377Syt 
1609*68d33a25Syt 	return (cmd_slot);
16102fcbc377Syt }
16112fcbc377Syt 
16122fcbc377Syt /*
16132fcbc377Syt  * Called by the sata framework to abort the previously sent packet(s).
16142fcbc377Syt  *
16152fcbc377Syt  * Reset device to abort commands.
16162fcbc377Syt  */
16172fcbc377Syt static int
16182fcbc377Syt ahci_tran_abort(dev_info_t *dip, sata_pkt_t *spkt, int flag)
16192fcbc377Syt {
16202fcbc377Syt 	ahci_ctl_t *ahci_ctlp;
16212fcbc377Syt 	ahci_port_t *ahci_portp;
16222fcbc377Syt 	uint32_t slot_status;
16232fcbc377Syt 	uint32_t aborted_tags, finished_tags;
16242fcbc377Syt 	uint8_t cport = spkt->satapkt_device.satadev_addr.cport;
16252fcbc377Syt 	uint8_t port;
16262fcbc377Syt 	int tmp_slot;
16272fcbc377Syt 
16282fcbc377Syt 	ahci_ctlp = ddi_get_soft_state(ahci_statep, ddi_get_instance(dip));
16292fcbc377Syt 	port = ahci_ctlp->ahcictl_cport_to_port[cport];
16302fcbc377Syt 
16312fcbc377Syt 	AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp,
1632*68d33a25Syt 	    "ahci_tran_abort enter: port %d", port);
16332fcbc377Syt 
16342fcbc377Syt 	ahci_portp = ahci_ctlp->ahcictl_ports[port];
16352fcbc377Syt 	mutex_enter(&ahci_portp->ahciport_mutex);
16362fcbc377Syt 
16372fcbc377Syt 	/*
1638*68d33a25Syt 	 * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending
1639*68d33a25Syt 	 * commands are being mopped, therefore there is nothing else to do
16402fcbc377Syt 	 */
1641*68d33a25Syt 	if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
16422fcbc377Syt 		AHCIDBG1(AHCIDBG_INFO, ahci_ctlp,
16432fcbc377Syt 		    "ahci_tran_abort: port %d is in "
16442fcbc377Syt 		    "mopping process, so just return directly ", port);
16452fcbc377Syt 		mutex_exit(&ahci_portp->ahciport_mutex);
16462fcbc377Syt 		return (SATA_SUCCESS);
16472fcbc377Syt 	}
16482fcbc377Syt 
16492fcbc377Syt 	if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED |
16502fcbc377Syt 	    ahci_portp->ahciport_port_state & SATA_PSTATE_SHUTDOWN |
16512fcbc377Syt 	    ahci_portp->ahciport_port_state & SATA_PSTATE_PWROFF) {
16522fcbc377Syt 		/*
16532fcbc377Syt 		 * In case the targer driver would send the request before
16542fcbc377Syt 		 * sata framework can have the opportunity to process those
16552fcbc377Syt 		 * event reports.
16562fcbc377Syt 		 */
16572fcbc377Syt 		spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
16582fcbc377Syt 		spkt->satapkt_device.satadev_state =
16592fcbc377Syt 		    ahci_portp->ahciport_port_state;
16602fcbc377Syt 		ahci_update_sata_registers(ahci_ctlp, port,
16612fcbc377Syt 		    &spkt->satapkt_device);
16622fcbc377Syt 		AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp,
16632fcbc377Syt 		    "ahci_tran_abort returning SATA_FAILURE while "
16642fcbc377Syt 		    "port in FAILED/SHUTDOWN/PWROFF state: "
1665*68d33a25Syt 		    "port: %d", port);
16662fcbc377Syt 		mutex_exit(&ahci_portp->ahciport_mutex);
16672fcbc377Syt 		return (SATA_FAILURE);
16682fcbc377Syt 	}
16692fcbc377Syt 
16702fcbc377Syt 	if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
16712fcbc377Syt 		/*
16722fcbc377Syt 		 * ahci_intr_phyrdy_change() may have rendered it to
16732fcbc377Syt 		 * AHCI_PORT_TYPE_NODEV.
16742fcbc377Syt 		 */
16752fcbc377Syt 		spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
16762fcbc377Syt 		spkt->satapkt_device.satadev_type = SATA_DTYPE_NONE;
16772fcbc377Syt 		spkt->satapkt_device.satadev_state =
16782fcbc377Syt 		    ahci_portp->ahciport_port_state;
16792fcbc377Syt 		ahci_update_sata_registers(ahci_ctlp, port,
16802fcbc377Syt 		    &spkt->satapkt_device);
16812fcbc377Syt 		AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp,
16822fcbc377Syt 		    "ahci_tran_abort returning SATA_FAILURE while "
1683*68d33a25Syt 		    "no device attached: port: %d", port);
16842fcbc377Syt 		mutex_exit(&ahci_portp->ahciport_mutex);
16852fcbc377Syt 		return (SATA_FAILURE);
16862fcbc377Syt 	}
16872fcbc377Syt 
16882fcbc377Syt 	if (flag == SATA_ABORT_ALL_PACKETS) {
16892fcbc377Syt 		aborted_tags = ahci_portp->ahciport_pending_tags;
16902fcbc377Syt 		cmn_err(CE_NOTE, "!ahci port %d abort all packets", port);
16912fcbc377Syt 	} else {
16922fcbc377Syt 		aborted_tags = 0xffffffff;
16932fcbc377Syt 		/*
16942fcbc377Syt 		 * Aborting one specific packet, first search our
16952fcbc377Syt 		 * ahciport_slot_pkts[] list for matching spkt.
16962fcbc377Syt 		 */
16972fcbc377Syt 		for (tmp_slot = 0;
16982fcbc377Syt 		    tmp_slot < ahci_ctlp->ahcictl_num_cmd_slots; tmp_slot++) {
16992fcbc377Syt 			if (ahci_portp->ahciport_slot_pkts[tmp_slot] == spkt) {
17002fcbc377Syt 				aborted_tags = (0x1 << tmp_slot);
17012fcbc377Syt 				break;
17022fcbc377Syt 			}
17032fcbc377Syt 		}
17042fcbc377Syt 
17052fcbc377Syt 		if (aborted_tags == 0xffffffff) {
17062fcbc377Syt 			/* request packet is not on the pending list */
17072fcbc377Syt 			AHCIDBG1(AHCIDBG_INFO, ahci_ctlp,
17082fcbc377Syt 			    "Cannot find the aborting pkt 0x%p on the "
17092fcbc377Syt 			    "pending list", (void *)spkt);
17102fcbc377Syt 			ahci_update_sata_registers(ahci_ctlp, port,
17112fcbc377Syt 			    &spkt->satapkt_device);
17122fcbc377Syt 			mutex_exit(&ahci_portp->ahciport_mutex);
17132fcbc377Syt 			return (SATA_FAILURE);
17142fcbc377Syt 		}
17152fcbc377Syt 		cmn_err(CE_NOTE, "!ahci port %d abort satapkt 0x%p",
17162fcbc377Syt 		    port, (void *)spkt);
17172fcbc377Syt 	}
17182fcbc377Syt 
17192fcbc377Syt 	slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
17202fcbc377Syt 	    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
17212fcbc377Syt 
1722*68d33a25Syt 	ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
1723*68d33a25Syt 	ahci_portp->ahciport_mop_in_progress++;
17242fcbc377Syt 
17252fcbc377Syt 	/*
17262fcbc377Syt 	 * To abort the packet(s), first we are trying to clear PxCMD.ST
1727*68d33a25Syt 	 * to stop the port, and if the port can be stopped
17282fcbc377Syt 	 * successfully with PxTFD.STS.BSY and PxTFD.STS.DRQ cleared to '0',
17292fcbc377Syt 	 * then we just send back the aborted packet(s) with ABORTED flag
17302fcbc377Syt 	 * and then restart the port by setting PxCMD.ST and PxCMD.FRE.
17312fcbc377Syt 	 * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then we
17322fcbc377Syt 	 * perform a COMRESET.
17332fcbc377Syt 	 */
17342fcbc377Syt 	(void) ahci_restart_port_wait_till_ready(ahci_ctlp,
1735*68d33a25Syt 	    ahci_portp, port, NULL, NULL);
17362fcbc377Syt 
17372fcbc377Syt 	/*
17382fcbc377Syt 	 * Compute which have finished and which need to be retried.
17392fcbc377Syt 	 *
17402fcbc377Syt 	 * The finished tags are ahciport_pending_tags minus the slot_status.
17412fcbc377Syt 	 * The aborted_tags have to be reduced by finished_tags since we
17422fcbc377Syt 	 * can't possibly abort a tag which had finished already.
17432fcbc377Syt 	 */
17442fcbc377Syt 	finished_tags = ahci_portp->ahciport_pending_tags &
17452fcbc377Syt 	    ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
17462fcbc377Syt 
17472fcbc377Syt 	aborted_tags &= ~finished_tags;
17482fcbc377Syt 
17492fcbc377Syt 	ahci_mop_commands(ahci_ctlp,
17502fcbc377Syt 	    ahci_portp,
17512fcbc377Syt 	    port,
17522fcbc377Syt 	    slot_status,
17532fcbc377Syt 	    0, /* failed tags */
17542fcbc377Syt 	    0, /* timeout tags */
17552fcbc377Syt 	    aborted_tags,
17562fcbc377Syt 	    0); /* reset tags */
17572fcbc377Syt 
17582fcbc377Syt 	ahci_update_sata_registers(ahci_ctlp, port, &spkt->satapkt_device);
17592fcbc377Syt 	mutex_exit(&ahci_portp->ahciport_mutex);
17602fcbc377Syt 
17612fcbc377Syt 	return (SATA_SUCCESS);
17622fcbc377Syt }
17632fcbc377Syt 
17642fcbc377Syt /*
17652fcbc377Syt  * Used to do device reset and reject all the pending packets on a device
17662fcbc377Syt  * during the reset operation.
17672fcbc377Syt  *
17682fcbc377Syt  * WARNING!!! ahciport_mutex should be acquired before the function is called.
17692fcbc377Syt  */
17702fcbc377Syt static int
17712fcbc377Syt ahci_reset_device_reject_pkts(ahci_ctl_t *ahci_ctlp,
17722fcbc377Syt     ahci_port_t *ahci_portp, uint8_t port)
17732fcbc377Syt {
17742fcbc377Syt 	uint32_t slot_status;
17752fcbc377Syt 	uint32_t reset_tags, finished_tags;
17762fcbc377Syt 	sata_device_t sdevice;
17772fcbc377Syt 	int ret;
17782fcbc377Syt 
17792fcbc377Syt 	AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp,
17802fcbc377Syt 	    "ahci_reset_device_reject_pkts on port: %d", port);
17812fcbc377Syt 
17822fcbc377Syt 	/*
1783*68d33a25Syt 	 * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending
1784*68d33a25Syt 	 * commands are being mopped, therefore there is nothing else to do
17852fcbc377Syt 	 */
1786*68d33a25Syt 	if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
17872fcbc377Syt 		AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp,
17882fcbc377Syt 		    "ahci_reset_device_reject_pkts: port %d is in "
17892fcbc377Syt 		    "mopping process, so return directly ", port);
17902fcbc377Syt 		return (SATA_SUCCESS);
17912fcbc377Syt 	}
17922fcbc377Syt 
17932fcbc377Syt 	slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
17942fcbc377Syt 	    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
17952fcbc377Syt 
17962fcbc377Syt 	reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp);
17972fcbc377Syt 
17982fcbc377Syt 	if (ahci_software_reset(ahci_ctlp, ahci_portp, port)
17992fcbc377Syt 	    != AHCI_SUCCESS) {
18002fcbc377Syt 		AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp,
18012fcbc377Syt 		    "Try to do a port reset after software "
18022fcbc377Syt 		    "reset failed", port);
18032fcbc377Syt 		ret = ahci_port_reset(ahci_ctlp, ahci_portp, port);
18042fcbc377Syt 		if (ret != AHCI_SUCCESS) {
18052fcbc377Syt 			AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp,
18062fcbc377Syt 			    "ahci_reset_device_reject_pkts: port %d "
18072fcbc377Syt 			    "failed", port);
18082fcbc377Syt 			return (SATA_FAILURE);
18092fcbc377Syt 		}
18102fcbc377Syt 	}
18112fcbc377Syt 	/* Set the reset in progress flag */
18122fcbc377Syt 	ahci_portp->ahciport_reset_in_progress = 1;
18132fcbc377Syt 
1814*68d33a25Syt 	ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
1815*68d33a25Syt 	ahci_portp->ahciport_mop_in_progress++;
18162fcbc377Syt 
18172fcbc377Syt 	/* Indicate to the framework that a reset has happened */
18182fcbc377Syt 	bzero((void *)&sdevice, sizeof (sata_device_t));
181909121340Syt 	sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];
1820*68d33a25Syt 	sdevice.satadev_addr.pmport = 0;
1821*68d33a25Syt 	sdevice.satadev_addr.qual = SATA_ADDR_DCPORT;
18222fcbc377Syt 
18232fcbc377Syt 	sdevice.satadev_state = SATA_DSTATE_RESET |
18242fcbc377Syt 	    SATA_DSTATE_PWR_ACTIVE;
18252fcbc377Syt 	mutex_exit(&ahci_portp->ahciport_mutex);
18262fcbc377Syt 	sata_hba_event_notify(
18272fcbc377Syt 	    ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
18282fcbc377Syt 	    &sdevice,
18292fcbc377Syt 	    SATA_EVNT_DEVICE_RESET);
18302fcbc377Syt 	mutex_enter(&ahci_portp->ahciport_mutex);
18312fcbc377Syt 
18322fcbc377Syt 	AHCIDBG1(AHCIDBG_EVENT, ahci_ctlp,
18332fcbc377Syt 	    "port %d sending event up: SATA_EVNT_RESET", port);
18342fcbc377Syt 
18352fcbc377Syt 	/* Next try to mop the pending commands */
18362fcbc377Syt 	finished_tags = ahci_portp->ahciport_pending_tags &
18372fcbc377Syt 	    ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
18382fcbc377Syt 
18392fcbc377Syt 	reset_tags &= ~finished_tags;
18402fcbc377Syt 
18412fcbc377Syt 	ahci_mop_commands(ahci_ctlp,
18422fcbc377Syt 	    ahci_portp,
18432fcbc377Syt 	    port,
18442fcbc377Syt 	    slot_status,
18452fcbc377Syt 	    0, /* failed tags */
18462fcbc377Syt 	    0, /* timeout tags */
18472fcbc377Syt 	    0, /* aborted tags */
18482fcbc377Syt 	    reset_tags); /* reset tags */
18492fcbc377Syt 
18502fcbc377Syt 	return (SATA_SUCCESS);
18512fcbc377Syt }
18522fcbc377Syt 
18532fcbc377Syt /*
18542fcbc377Syt  * Used to do port reset and reject all the pending packets on a port during
18552fcbc377Syt  * the reset operation.
18562fcbc377Syt  *
18572fcbc377Syt  * WARNING!!! ahciport_mutex should be acquired before the function is called.
18582fcbc377Syt  */
18592fcbc377Syt static int
18602fcbc377Syt ahci_reset_port_reject_pkts(ahci_ctl_t *ahci_ctlp,
18612fcbc377Syt     ahci_port_t *ahci_portp, uint8_t port)
18622fcbc377Syt {
18632fcbc377Syt 	uint32_t slot_status;
18642fcbc377Syt 	uint32_t reset_tags, finished_tags;
18652fcbc377Syt 
18662fcbc377Syt 	AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp,
18672fcbc377Syt 	    "ahci_reset_port_reject_pkts on port: %d", port);
18682fcbc377Syt 
18692fcbc377Syt 	/*
1870*68d33a25Syt 	 * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending
1871*68d33a25Syt 	 * commands are being mopped, therefore there is nothing else to do
18722fcbc377Syt 	 */
1873*68d33a25Syt 	if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
18742fcbc377Syt 		AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp,
18752fcbc377Syt 		    "ahci_reset_port_reject_pkts: port %d is in "
18762fcbc377Syt 		    "mopping process, so return directly ", port);
18772fcbc377Syt 		return (SATA_SUCCESS);
18782fcbc377Syt 	}
18792fcbc377Syt 
1880*68d33a25Syt 	ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
1881*68d33a25Syt 	ahci_portp->ahciport_mop_in_progress++;
18822fcbc377Syt 
18832fcbc377Syt 	slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
18842fcbc377Syt 	    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
18852fcbc377Syt 
18862fcbc377Syt 	reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp);
18872fcbc377Syt 
18882fcbc377Syt 	if (ahci_restart_port_wait_till_ready(ahci_ctlp,
1889*68d33a25Syt 	    ahci_portp, port, AHCI_PORT_RESET, NULL) != AHCI_SUCCESS)
18902fcbc377Syt 		return (SATA_FAILURE);
18912fcbc377Syt 
18922fcbc377Syt 	finished_tags = ahci_portp->ahciport_pending_tags &
18932fcbc377Syt 	    ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
18942fcbc377Syt 
18952fcbc377Syt 	reset_tags &= ~finished_tags;
18962fcbc377Syt 
18972fcbc377Syt 	ahci_mop_commands(ahci_ctlp,
18982fcbc377Syt 	    ahci_portp,
18992fcbc377Syt 	    port,
19002fcbc377Syt 	    slot_status,
19012fcbc377Syt 	    0, /* failed tags */
19022fcbc377Syt 	    0, /* timeout tags */
19032fcbc377Syt 	    0, /* aborted tags */
19042fcbc377Syt 	    reset_tags); /* reset tags */
19052fcbc377Syt 
19062fcbc377Syt 	return (SATA_SUCCESS);
19072fcbc377Syt }
19082fcbc377Syt 
19092fcbc377Syt /*
19102fcbc377Syt  * Used to do hba reset and reject all the pending packets on all ports
19112fcbc377Syt  * during the reset operation.
19122fcbc377Syt  */
19132fcbc377Syt static int
19142fcbc377Syt ahci_reset_hba_reject_pkts(ahci_ctl_t *ahci_ctlp)
19152fcbc377Syt {
19162fcbc377Syt 	ahci_port_t *ahci_portp;
19172fcbc377Syt 	uint32_t slot_status[AHCI_MAX_PORTS];
19182fcbc377Syt 	uint32_t reset_tags[AHCI_MAX_PORTS];
19192fcbc377Syt 	uint32_t finished_tags[AHCI_MAX_PORTS];
19202fcbc377Syt 	sata_device_t sdevice[AHCI_MAX_PORTS];
19212fcbc377Syt 	uint8_t port;
19222fcbc377Syt 	int ret = SATA_SUCCESS;
19232fcbc377Syt 
19242fcbc377Syt 	AHCIDBG0(AHCIDBG_ENTRY, ahci_ctlp,
19252fcbc377Syt 	    "ahci_reset_hba_reject_pkts enter");
19262fcbc377Syt 
19272fcbc377Syt 	for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
19282fcbc377Syt 		if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
19292fcbc377Syt 			continue;
19302fcbc377Syt 		}
19312fcbc377Syt 
19322fcbc377Syt 		ahci_portp = ahci_ctlp->ahcictl_ports[port];
19332fcbc377Syt 
19342fcbc377Syt 		mutex_enter(&ahci_portp->ahciport_mutex);
19352fcbc377Syt 		slot_status[port] = ddi_get32(
19362fcbc377Syt 		    ahci_ctlp->ahcictl_ahci_acc_handle,
19372fcbc377Syt 		    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
19382fcbc377Syt 		reset_tags[port] = slot_status[port]
19392fcbc377Syt 		    & AHCI_SLOT_MASK(ahci_ctlp);
19402fcbc377Syt 		mutex_exit(&ahci_portp->ahciport_mutex);
19412fcbc377Syt 	}
19422fcbc377Syt 
19432fcbc377Syt 	if (ahci_hba_reset(ahci_ctlp) != AHCI_SUCCESS) {
19442fcbc377Syt 		ret = SATA_FAILURE;
19452fcbc377Syt 		goto out;
19462fcbc377Syt 	}
19472fcbc377Syt 
19482fcbc377Syt 	for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
19492fcbc377Syt 		if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
19502fcbc377Syt 			continue;
19512fcbc377Syt 		}
19522fcbc377Syt 
19532fcbc377Syt 		ahci_portp = ahci_ctlp->ahcictl_ports[port];
19542fcbc377Syt 
19552fcbc377Syt 		mutex_enter(&ahci_portp->ahciport_mutex);
19562fcbc377Syt 		/*
19572fcbc377Syt 		 * To prevent recursive enter to ahci_mop_commands, we need
1958*68d33a25Syt 		 * check AHCI_PORT_FLAG_MOPPING flag.
19592fcbc377Syt 		 */
1960*68d33a25Syt 		if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
19612fcbc377Syt 			AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp,
19622fcbc377Syt 			    "ahci_reset_hba_reject_pkts: port %d is in "
19632fcbc377Syt 			    "mopping process, so return directly ", port);
19642fcbc377Syt 			mutex_exit(&ahci_portp->ahciport_mutex);
19652fcbc377Syt 			continue;
19662fcbc377Syt 		}
19672fcbc377Syt 
1968*68d33a25Syt 		ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
1969*68d33a25Syt 		ahci_portp->ahciport_mop_in_progress++;
19702fcbc377Syt 
19712fcbc377Syt 		/* Indicate to the framework that a reset has happened */
19722fcbc377Syt 		bzero((void *)&sdevice[port], sizeof (sata_device_t));
197309121340Syt 		sdevice[port].satadev_addr.cport =
197409121340Syt 		    ahci_ctlp->ahcictl_port_to_cport[port];
1975*68d33a25Syt 		sdevice[port].satadev_addr.pmport = 0;
1976*68d33a25Syt 		sdevice[port].satadev_addr.qual = SATA_ADDR_DCPORT;
19772fcbc377Syt 
19782fcbc377Syt 		sdevice[port].satadev_state = SATA_DSTATE_RESET |
19792fcbc377Syt 		    SATA_DSTATE_PWR_ACTIVE;
19802fcbc377Syt 		mutex_exit(&ahci_portp->ahciport_mutex);
19812fcbc377Syt 		sata_hba_event_notify(
19822fcbc377Syt 		    ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
19832fcbc377Syt 		    &sdevice[port],
19842fcbc377Syt 		    SATA_EVNT_DEVICE_RESET);
19852fcbc377Syt 		mutex_enter(&ahci_portp->ahciport_mutex);
19862fcbc377Syt 
19872fcbc377Syt 		AHCIDBG1(AHCIDBG_EVENT, ahci_ctlp,
19882fcbc377Syt 		    "port %d sending event up: SATA_EVNT_RESET",
19892fcbc377Syt 		    port);
19902fcbc377Syt 
19912fcbc377Syt 		finished_tags[port]  = ahci_portp->ahciport_pending_tags &
19922fcbc377Syt 		    ~slot_status[port] & AHCI_SLOT_MASK(ahci_ctlp);
19932fcbc377Syt 
19942fcbc377Syt 		reset_tags[port] &= ~finished_tags[port];
19952fcbc377Syt 
19962fcbc377Syt 		ahci_mop_commands(ahci_ctlp,
19972fcbc377Syt 		    ahci_portp,
19982fcbc377Syt 		    port,
19992fcbc377Syt 		    slot_status[port],
20002fcbc377Syt 		    0, /* failed tags */
20012fcbc377Syt 		    0, /* timeout tags */
20022fcbc377Syt 		    0, /* aborted tags */
20032fcbc377Syt 		    reset_tags[port]); /* reset tags */
2004*68d33a25Syt 		mutex_exit(&ahci_portp->ahciport_mutex);
20052fcbc377Syt 	}
20062fcbc377Syt out:
20072fcbc377Syt 	return (ret);
20082fcbc377Syt }
20092fcbc377Syt 
20102fcbc377Syt /*
20112fcbc377Syt  * Called by sata framework to reset a port(s) or device.
20122fcbc377Syt  */
20132fcbc377Syt static int
20142fcbc377Syt ahci_tran_reset_dport(dev_info_t *dip, sata_device_t *sd)
20152fcbc377Syt {
20162fcbc377Syt 	ahci_ctl_t *ahci_ctlp;
20172fcbc377Syt 	ahci_port_t *ahci_portp;
20182fcbc377Syt 	uint8_t cport = sd->satadev_addr.cport;
20192fcbc377Syt 	uint8_t port;
20202fcbc377Syt 	int ret = SATA_SUCCESS;
20212fcbc377Syt 
20222fcbc377Syt 	ahci_ctlp = ddi_get_soft_state(ahci_statep, ddi_get_instance(dip));
20232fcbc377Syt 	port = ahci_ctlp->ahcictl_cport_to_port[cport];
20242fcbc377Syt 
20252fcbc377Syt 	AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp,
2026*68d33a25Syt 	    "ahci_tran_reset_port enter: cport: %d", cport);
20272fcbc377Syt 
20282fcbc377Syt 	switch (sd->satadev_addr.qual) {
20292fcbc377Syt 	case SATA_ADDR_CPORT:
20302fcbc377Syt 		/* Port reset */
20312fcbc377Syt 		ahci_portp = ahci_ctlp->ahcictl_ports[port];
2032*68d33a25Syt 		cmn_err(CE_NOTE, "!ahci_tran_reset_dport: port %d "
2033*68d33a25Syt 		    "reset port", port);
20342fcbc377Syt 
20352fcbc377Syt 		mutex_enter(&ahci_portp->ahciport_mutex);
20362fcbc377Syt 		ret = ahci_reset_port_reject_pkts(ahci_ctlp, ahci_portp, port);
20372fcbc377Syt 		mutex_exit(&ahci_portp->ahciport_mutex);
20382fcbc377Syt 
20392fcbc377Syt 		break;
20402fcbc377Syt 
20412fcbc377Syt 	case SATA_ADDR_DCPORT:
20422fcbc377Syt 		/* Device reset */
20432fcbc377Syt 		ahci_portp = ahci_ctlp->ahcictl_ports[port];
2044*68d33a25Syt 		cmn_err(CE_NOTE, "!ahci_tran_reset_dport: port %d "
2045*68d33a25Syt 		    "reset device", port);
20462fcbc377Syt 
20472fcbc377Syt 		mutex_enter(&ahci_portp->ahciport_mutex);
20482fcbc377Syt 		if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED |
20492fcbc377Syt 		    ahci_portp->ahciport_port_state & SATA_PSTATE_SHUTDOWN |
20502fcbc377Syt 		    ahci_portp->ahciport_port_state & SATA_PSTATE_PWROFF) {
20512fcbc377Syt 			/*
20522fcbc377Syt 			 * In case the targer driver would send the request
20532fcbc377Syt 			 * before sata framework can have the opportunity to
20542fcbc377Syt 			 * process those event reports.
20552fcbc377Syt 			 */
20562fcbc377Syt 			sd->satadev_state = ahci_portp->ahciport_port_state;
20572fcbc377Syt 			ahci_update_sata_registers(ahci_ctlp, port, sd);
20582fcbc377Syt 			AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp,
20592fcbc377Syt 			    "ahci_tran_reset_dport returning SATA_FAILURE "
20602fcbc377Syt 			    "while port in FAILED/SHUTDOWN/PWROFF state: "
2061*68d33a25Syt 			    "port: %d", port);
20622fcbc377Syt 			mutex_exit(&ahci_portp->ahciport_mutex);
20632fcbc377Syt 			ret = SATA_FAILURE;
20642fcbc377Syt 			break;
20652fcbc377Syt 		}
20662fcbc377Syt 
20672fcbc377Syt 		if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
20682fcbc377Syt 			/*
20692fcbc377Syt 			 * ahci_intr_phyrdy_change() may have rendered it to
20702fcbc377Syt 			 * AHCI_PORT_TYPE_NODEV.
20712fcbc377Syt 			 */
20722fcbc377Syt 			sd->satadev_type = SATA_DTYPE_NONE;
20732fcbc377Syt 			sd->satadev_state = ahci_portp->ahciport_port_state;
20742fcbc377Syt 			ahci_update_sata_registers(ahci_ctlp, port, sd);
20752fcbc377Syt 			AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp,
20762fcbc377Syt 			    "ahci_tran_reset_dport returning SATA_FAILURE "
2077*68d33a25Syt 			    "while no device attached: port: %d", port);
20782fcbc377Syt 			mutex_exit(&ahci_portp->ahciport_mutex);
20792fcbc377Syt 			ret = SATA_FAILURE;
20802fcbc377Syt 			break;
20812fcbc377Syt 		}
20822fcbc377Syt 
20832fcbc377Syt 		ret = ahci_reset_device_reject_pkts(ahci_ctlp,
20842fcbc377Syt 		    ahci_portp, port);
20852fcbc377Syt 		mutex_exit(&ahci_portp->ahciport_mutex);
20862fcbc377Syt 		break;
20872fcbc377Syt 
20882fcbc377Syt 	case SATA_ADDR_CNTRL:
20892fcbc377Syt 		/* Reset the whole controller */
2090*68d33a25Syt 		cmn_err(CE_NOTE, "!ahci_tran_reset_dport: port %d "
2091*68d33a25Syt 		    "reset the whole hba", port);
20922fcbc377Syt 		ret = ahci_reset_hba_reject_pkts(ahci_ctlp);
20932fcbc377Syt 		break;
20942fcbc377Syt 
20952fcbc377Syt 	case SATA_ADDR_PMPORT:
20962fcbc377Syt 	case SATA_ADDR_DPMPORT:
20972fcbc377Syt 		AHCIDBG0(AHCIDBG_INFO, ahci_ctlp,
2098*68d33a25Syt 		    "ahci_tran_reset_dport: port multiplier will be "
2099*68d33a25Syt 		    "supported later");
21002fcbc377Syt 		/* FALLSTHROUGH */
21012fcbc377Syt 	default:
21022fcbc377Syt 		ret = SATA_FAILURE;
21032fcbc377Syt 	}
21042fcbc377Syt 
21052fcbc377Syt 	return (ret);
21062fcbc377Syt }
21072fcbc377Syt 
21082fcbc377Syt /*
21092fcbc377Syt  * Called by sata framework to activate a port as part of hotplug.
21102fcbc377Syt  * (cfgadm -c connect satax/y)
21112fcbc377Syt  * Note: Not port-mult aware.
21122fcbc377Syt  */
21132fcbc377Syt static int
21142fcbc377Syt ahci_tran_hotplug_port_activate(dev_info_t *dip, sata_device_t *satadev)
21152fcbc377Syt {
21162fcbc377Syt 	ahci_ctl_t *ahci_ctlp;
21172fcbc377Syt 	ahci_port_t *ahci_portp;
21182fcbc377Syt 	uint8_t	cport = satadev->satadev_addr.cport;
21192fcbc377Syt 	uint8_t port;
21202fcbc377Syt 
21212fcbc377Syt 	ahci_ctlp = ddi_get_soft_state(ahci_statep, ddi_get_instance(dip));
21222fcbc377Syt 	port = ahci_ctlp->ahcictl_cport_to_port[cport];
21232fcbc377Syt 
21242fcbc377Syt 	AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp,
21252fcbc377Syt 	    "ahci_tran_hotplug_port_activate cport %d enter", cport);
21262fcbc377Syt 
21272fcbc377Syt 	ahci_portp = ahci_ctlp->ahcictl_ports[port];
21282fcbc377Syt 
21292fcbc377Syt 	mutex_enter(&ahci_portp->ahciport_mutex);
2130*68d33a25Syt 	cmn_err(CE_NOTE, "!ahci port %d is activated", port);
21312fcbc377Syt 
21322fcbc377Syt 	/*
21332fcbc377Syt 	 * Reset the port so that the PHY communication would be re-established.
2134*68d33a25Syt 	 * But this reset is an internal operation and the sata module doesn't
2135*68d33a25Syt 	 * need to know about it. Moreover, the port with a device attached will
2136*68d33a25Syt 	 * be started too.
21372fcbc377Syt 	 */
21382fcbc377Syt 	(void) ahci_restart_port_wait_till_ready(ahci_ctlp,
2139*68d33a25Syt 	    ahci_portp, port,
2140*68d33a25Syt 	    AHCI_PORT_RESET|AHCI_RESET_NO_EVENTS_UP,
2141*68d33a25Syt 	    NULL);
21422fcbc377Syt 
21432fcbc377Syt 	/*
21442fcbc377Syt 	 * Need to check the link status and device status of the port
21452fcbc377Syt 	 * and consider raising power if the port was in D3 state
21462fcbc377Syt 	 */
2147*68d33a25Syt 	ahci_portp->ahciport_port_state |= SATA_PSTATE_PWRON;
2148*68d33a25Syt 	ahci_portp->ahciport_port_state &= ~SATA_PSTATE_PWROFF;
2149*68d33a25Syt 	ahci_portp->ahciport_port_state &= ~SATA_PSTATE_SHUTDOWN;
2150*68d33a25Syt 
2151*68d33a25Syt 	satadev->satadev_state = ahci_portp->ahciport_port_state;
21522fcbc377Syt 
21532fcbc377Syt 	ahci_update_sata_registers(ahci_ctlp, port, satadev);
21542fcbc377Syt 
21552fcbc377Syt 	mutex_exit(&ahci_portp->ahciport_mutex);
21562fcbc377Syt 	return (SATA_SUCCESS);
21572fcbc377Syt }
21582fcbc377Syt 
21592fcbc377Syt /*
21602fcbc377Syt  * Called by sata framework to deactivate a port as part of hotplug.
21612fcbc377Syt  * (cfgadm -c disconnect satax/y)
21622fcbc377Syt  * Note: Not port-mult aware.
21632fcbc377Syt  */
21642fcbc377Syt static int
21652fcbc377Syt ahci_tran_hotplug_port_deactivate(dev_info_t *dip, sata_device_t *satadev)
21662fcbc377Syt {
21672fcbc377Syt 	ahci_ctl_t *ahci_ctlp;
21682fcbc377Syt 	ahci_port_t *ahci_portp;
21692fcbc377Syt 	uint8_t cport = satadev->satadev_addr.cport;
21702fcbc377Syt 	uint8_t port;
2171*68d33a25Syt 	uint32_t port_scontrol;
21722fcbc377Syt 
21732fcbc377Syt 	ahci_ctlp = ddi_get_soft_state(ahci_statep, ddi_get_instance(dip));
21742fcbc377Syt 	port = ahci_ctlp->ahcictl_cport_to_port[cport];
21752fcbc377Syt 
21762fcbc377Syt 	AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp,
21772fcbc377Syt 	    "ahci_tran_hotplug_port_deactivate cport %d enter", cport);
21782fcbc377Syt 
21792fcbc377Syt 	ahci_portp = ahci_ctlp->ahcictl_ports[port];
21802fcbc377Syt 
21812fcbc377Syt 	mutex_enter(&ahci_portp->ahciport_mutex);
2182*68d33a25Syt 	cmn_err(CE_NOTE, "!ahci port %d is deactivated", port);
21832fcbc377Syt 
2184*68d33a25Syt 	/* Disable the interrupts on the port */
21852fcbc377Syt 	ahci_disable_port_intrs(ahci_ctlp, ahci_portp, port);
21862fcbc377Syt 
2187*68d33a25Syt 	if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
2188*68d33a25Syt 		goto phy_offline;
2189*68d33a25Syt 	}
2190*68d33a25Syt 
21912fcbc377Syt 	/* First to abort all the pending commands */
21922fcbc377Syt 	ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port);
21932fcbc377Syt 
2194*68d33a25Syt 	/* Then stop the port */
2195*68d33a25Syt 	(void) ahci_put_port_into_notrunning_state(ahci_ctlp,
21962fcbc377Syt 	    ahci_portp, port);
21972fcbc377Syt 
2198*68d33a25Syt 	/* Next put the PHY offline */
2199*68d33a25Syt 
2200*68d33a25Syt phy_offline:
2201*68d33a25Syt 	port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2202*68d33a25Syt 	    (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port));
2203*68d33a25Syt 	AHCI_SCONTROL_SET_DET(port_scontrol, AHCI_SCONTROL_DET_PHYOFFLINE);
2204*68d33a25Syt 
22052fcbc377Syt 	/* Update ahciport_port_state */
22062fcbc377Syt 	ahci_portp->ahciport_port_state = SATA_PSTATE_SHUTDOWN;
2207*68d33a25Syt 	satadev->satadev_state = ahci_portp->ahciport_port_state;
22082fcbc377Syt 
22092fcbc377Syt 	ahci_update_sata_registers(ahci_ctlp, port, satadev);
22102fcbc377Syt 
22112fcbc377Syt 	mutex_exit(&ahci_portp->ahciport_mutex);
22122fcbc377Syt 	return (SATA_SUCCESS);
22132fcbc377Syt }
22142fcbc377Syt 
22152fcbc377Syt /*
2216*68d33a25Syt  * To be used to mark all the outstanding pkts with SATA_PKT_ABORTED
22172fcbc377Syt  * when a device is unplugged or a port is deactivated.
22182fcbc377Syt  *
22192fcbc377Syt  * WARNING!!! ahciport_mutex should be acquired before the function is called.
22202fcbc377Syt  */
22212fcbc377Syt static void
22222fcbc377Syt ahci_reject_all_abort_pkts(ahci_ctl_t *ahci_ctlp,
22232fcbc377Syt     ahci_port_t *ahci_portp, uint8_t port)
22242fcbc377Syt {
22252fcbc377Syt 	uint32_t slot_status;
22262fcbc377Syt 	uint32_t abort_tags;
22272fcbc377Syt 
2228*68d33a25Syt 	AHCIDBG1(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp,
22292fcbc377Syt 	    "ahci_reject_all_abort_pkts on port: %d", port);
22302fcbc377Syt 
22312fcbc377Syt 	slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
22322fcbc377Syt 	    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
22332fcbc377Syt 	abort_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp);
22342fcbc377Syt 
2235*68d33a25Syt 	/* No need to do mop when there is no outstanding commands */
2236*68d33a25Syt 	if (slot_status != 0) {
2237*68d33a25Syt 		/*
2238*68d33a25Syt 		 * We need to do the mop even AHCI_PORT_FLAG_MOPPING is
2239*68d33a25Syt 		 * already set because during error recovery process, the
2240*68d33a25Syt 		 * REQUEST SENSE command can be delivered to HBA to get
2241*68d33a25Syt 		 * sense data, so when the device is removed, the command
2242*68d33a25Syt 		 * need to be aborted too. And for this kind of condition,
2243*68d33a25Syt 		 * we can make sure the abort_tags is just the REQUEST
2244*68d33a25Syt 		 * SENSE slot number.
2245*68d33a25Syt 		 */
2246*68d33a25Syt 		ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
2247*68d33a25Syt 		ahci_portp->ahciport_mop_in_progress++;
22482fcbc377Syt 
2249*68d33a25Syt 		ahci_mop_commands(ahci_ctlp,
2250*68d33a25Syt 		    ahci_portp,
2251*68d33a25Syt 		    port,
2252*68d33a25Syt 		    slot_status,
2253*68d33a25Syt 		    0, /* failed tags */
2254*68d33a25Syt 		    0, /* timeout tags */
2255*68d33a25Syt 		    abort_tags, /* aborting tags */
2256*68d33a25Syt 		    0); /* reset tags */
2257*68d33a25Syt 	}
2258*68d33a25Syt }
2259*68d33a25Syt 
2260*68d33a25Syt #if defined(__lock_lint)
2261*68d33a25Syt static int
2262*68d33a25Syt ahci_selftest(dev_info_t *dip, sata_device_t *device)
2263*68d33a25Syt {
22642fcbc377Syt 	return (SATA_SUCCESS);
22652fcbc377Syt }
22662fcbc377Syt #endif
22672fcbc377Syt 
22682fcbc377Syt /*
2269*68d33a25Syt  * Allocate the ports structure, only called by ahci_attach
2270*68d33a25Syt  */
2271*68d33a25Syt static int
2272*68d33a25Syt ahci_alloc_ports_state(ahci_ctl_t *ahci_ctlp)
2273*68d33a25Syt {
2274*68d33a25Syt 	int port, cport = 0;
2275*68d33a25Syt 
2276*68d33a25Syt 	AHCIDBG0(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
2277*68d33a25Syt 	    "ahci_alloc_ports_state enter");
2278*68d33a25Syt 
2279*68d33a25Syt 	mutex_enter(&ahci_ctlp->ahcictl_mutex);
2280*68d33a25Syt 
2281*68d33a25Syt 	/* Allocate structures only for the implemented ports */
2282*68d33a25Syt 	for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
2283*68d33a25Syt 		if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
2284*68d33a25Syt 			AHCIDBG1(AHCIDBG_INIT, ahci_ctlp,
2285*68d33a25Syt 			    "hba port %d not implemented", port);
2286*68d33a25Syt 			continue;
2287*68d33a25Syt 		}
2288*68d33a25Syt 
2289*68d33a25Syt #ifndef __lock_lint
2290*68d33a25Syt 		ahci_ctlp->ahcictl_cport_to_port[cport] = (uint8_t)port;
2291*68d33a25Syt 		ahci_ctlp->ahcictl_port_to_cport[port] =
2292*68d33a25Syt 		    (uint8_t)cport++;
2293*68d33a25Syt #endif /* __lock_lint */
2294*68d33a25Syt 
2295*68d33a25Syt 		if (ahci_alloc_port_state(ahci_ctlp, port) != AHCI_SUCCESS) {
2296*68d33a25Syt 			goto err_out;
2297*68d33a25Syt 		}
2298*68d33a25Syt 	}
2299*68d33a25Syt 
2300*68d33a25Syt 	mutex_exit(&ahci_ctlp->ahcictl_mutex);
2301*68d33a25Syt 	return (AHCI_SUCCESS);
2302*68d33a25Syt 
2303*68d33a25Syt err_out:
2304*68d33a25Syt 	for (port--; port >= 0; port--) {
2305*68d33a25Syt 		if (AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
2306*68d33a25Syt 			ahci_dealloc_port_state(ahci_ctlp, port);
2307*68d33a25Syt 		}
2308*68d33a25Syt 	}
2309*68d33a25Syt 
2310*68d33a25Syt 	mutex_exit(&ahci_ctlp->ahcictl_mutex);
2311*68d33a25Syt 	return (AHCI_FAILURE);
2312*68d33a25Syt }
2313*68d33a25Syt 
2314*68d33a25Syt /*
2315*68d33a25Syt  * Reverse of ahci_alloc_ports_state(), only called by ahci_detach
2316*68d33a25Syt  */
2317*68d33a25Syt static void
2318*68d33a25Syt ahci_dealloc_ports_state(ahci_ctl_t *ahci_ctlp)
2319*68d33a25Syt {
2320*68d33a25Syt 	int port;
2321*68d33a25Syt 
2322*68d33a25Syt 	mutex_enter(&ahci_ctlp->ahcictl_mutex);
2323*68d33a25Syt 	for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
2324*68d33a25Syt 		/* if this port is implemented by the HBA */
2325*68d33a25Syt 		if (AHCI_PORT_IMPLEMENTED(ahci_ctlp, port))
2326*68d33a25Syt 			ahci_dealloc_port_state(ahci_ctlp, port);
2327*68d33a25Syt 	}
2328*68d33a25Syt 	mutex_exit(&ahci_ctlp->ahcictl_mutex);
2329*68d33a25Syt }
2330*68d33a25Syt 
2331*68d33a25Syt /*
2332*68d33a25Syt  * Initialize the controller.
23332fcbc377Syt  *
23342fcbc377Syt  * This routine can be called from three seperate cases: DDI_ATTACH,
23352fcbc377Syt  * PM_LEVEL_D0 and DDI_RESUME. The DDI_ATTACH case is different from
2336*68d33a25Syt  * other two cases; device signature probing are attempted only during
2337*68d33a25Syt  * DDI_ATTACH case.
23382fcbc377Syt  *
23392fcbc377Syt  * WARNING!!! Disable the whole controller's interrupts before calling and
23402fcbc377Syt  * the interrupts will be enabled upon successfully return.
23412fcbc377Syt  */
23422fcbc377Syt static int
23432fcbc377Syt ahci_initialize_controller(ahci_ctl_t *ahci_ctlp)
23442fcbc377Syt {
23452fcbc377Syt 	ahci_port_t *ahci_portp;
23462fcbc377Syt 	uint32_t ghc_control;
2347*68d33a25Syt 	int port;
23482fcbc377Syt 
23492fcbc377Syt 	AHCIDBG0(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
23502fcbc377Syt 	    "ahci_initialize_controller enter");
23512fcbc377Syt 
23522fcbc377Syt 	mutex_enter(&ahci_ctlp->ahcictl_mutex);
23532fcbc377Syt 
23542fcbc377Syt 	/*
23552fcbc377Syt 	 * Indicate that system software is AHCI aware by setting
23562fcbc377Syt 	 * GHC.AE to 1
23572fcbc377Syt 	 */
23582fcbc377Syt 	ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
23592fcbc377Syt 	    (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
23602fcbc377Syt 
23612fcbc377Syt 	ghc_control |= AHCI_HBA_GHC_AE;
23622fcbc377Syt 	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
23632fcbc377Syt 	    (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp),
23642fcbc377Syt 	    ghc_control);
23652fcbc377Syt 
23662fcbc377Syt 	/* Initialize the implemented ports and structures */
23672fcbc377Syt 	for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
23682fcbc377Syt 		if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
23692fcbc377Syt 			continue;
23702fcbc377Syt 		}
23712fcbc377Syt 
23722fcbc377Syt 		ahci_portp = ahci_ctlp->ahcictl_ports[port];
23732fcbc377Syt 		mutex_enter(&ahci_portp->ahciport_mutex);
23742fcbc377Syt 
23752fcbc377Syt 		/*
23762fcbc377Syt 		 * Ensure that the controller is not in the running state
23772fcbc377Syt 		 * by checking every implemented port's PxCMD register
23782fcbc377Syt 		 */
23792fcbc377Syt 		if (ahci_initialize_port(ahci_ctlp, ahci_portp, port)
23802fcbc377Syt 		    != AHCI_SUCCESS) {
23812fcbc377Syt 			AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp,
23822fcbc377Syt 			    "ahci_initialize_controller: failed to "
23832fcbc377Syt 			    "initialize port %d", port);
23842fcbc377Syt 			/*
23852fcbc377Syt 			 * Set the port state to SATA_PSTATE_FAILED if
23862fcbc377Syt 			 * failed to initialize it.
23872fcbc377Syt 			 */
2388*68d33a25Syt 			ahci_portp->ahciport_port_state = SATA_PSTATE_FAILED;
23892fcbc377Syt 		}
23902fcbc377Syt 
23912fcbc377Syt 		mutex_exit(&ahci_portp->ahciport_mutex);
23922fcbc377Syt 	}
23932fcbc377Syt 
23942fcbc377Syt 	/* Enable the whole controller interrupts */
23952fcbc377Syt 	ahci_enable_all_intrs(ahci_ctlp);
23962fcbc377Syt 	mutex_exit(&ahci_ctlp->ahcictl_mutex);
23972fcbc377Syt 
23982fcbc377Syt 	return (AHCI_SUCCESS);
23992fcbc377Syt }
24002fcbc377Syt 
24012fcbc377Syt /*
2402*68d33a25Syt  * Reverse of ahci_initialize_controller()
24032fcbc377Syt  *
24042fcbc377Syt  * WARNING!!! ahcictl_mutex should be acquired before the function is called.
24052fcbc377Syt  */
24062fcbc377Syt static void
2407*68d33a25Syt ahci_uninitialize_controller(ahci_ctl_t *ahci_ctlp)
24082fcbc377Syt {
24092fcbc377Syt 	AHCIDBG0(AHCIDBG_INIT, ahci_ctlp,
2410*68d33a25Syt 	    "ahci_uninitialize_controller enter");
24112fcbc377Syt 
24122fcbc377Syt 	/* disable all the interrupts. */
24132fcbc377Syt 	ahci_disable_all_intrs(ahci_ctlp);
24142fcbc377Syt }
24152fcbc377Syt 
24162fcbc377Syt /*
24172fcbc377Syt  * The routine is to initialize the port. First put the port in NOTRunning
24182fcbc377Syt  * state, then enable port interrupt and clear Serror register. And under
24192fcbc377Syt  * AHCI_ATTACH case, find device signature and then try to start the port.
24202fcbc377Syt  *
24212fcbc377Syt  * WARNING!!! ahcictl_mutex and ahciport_mutex should be acquired before
24222fcbc377Syt  * the function is called.
24232fcbc377Syt  */
24242fcbc377Syt static int
24252fcbc377Syt ahci_initialize_port(ahci_ctl_t *ahci_ctlp,
24262fcbc377Syt     ahci_port_t *ahci_portp, uint8_t port)
24272fcbc377Syt {
24282fcbc377Syt 	uint32_t port_cmd_status;
24292fcbc377Syt 
24302fcbc377Syt 	port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
24312fcbc377Syt 	    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
24322fcbc377Syt 
24332fcbc377Syt 	AHCIDBG2(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
24342fcbc377Syt 	    "ahci_initialize_port: port %d "
24352fcbc377Syt 	    "port_cmd_status = 0x%x", port, port_cmd_status);
24362fcbc377Syt 	/*
24372fcbc377Syt 	 * Check whether the port is in NotRunning state, if not,
24382fcbc377Syt 	 * put the port in NotRunning state
24392fcbc377Syt 	 */
24402fcbc377Syt 	if (!(port_cmd_status &
24412fcbc377Syt 	    (AHCI_CMD_STATUS_ST |
24422fcbc377Syt 	    AHCI_CMD_STATUS_CR |
24432fcbc377Syt 	    AHCI_CMD_STATUS_FRE |
24442fcbc377Syt 	    AHCI_CMD_STATUS_FR))) {
24452fcbc377Syt 
2446*68d33a25Syt 		goto next;
24472fcbc377Syt 	}
24482fcbc377Syt 
24492fcbc377Syt 	if (ahci_restart_port_wait_till_ready(ahci_ctlp, ahci_portp,
2450*68d33a25Syt 	    port, AHCI_RESET_NO_EVENTS_UP|AHCI_PORT_INIT, NULL) != AHCI_SUCCESS)
24512fcbc377Syt 		return (AHCI_FAILURE);
24522fcbc377Syt 
2453*68d33a25Syt next:
24542fcbc377Syt 	AHCIDBG1(AHCIDBG_INIT, ahci_ctlp,
2455*68d33a25Syt 	    "port %d is in NotRunning state now", port);
24562fcbc377Syt 
24572fcbc377Syt 	/*
2458*68d33a25Syt 	 * At the time being, only probe ports/devices and get the types of
2459*68d33a25Syt 	 * attached devices during attach. In fact, the device can be changed
2460*68d33a25Syt 	 * during power state changes, but I would like to postpone this part
2461*68d33a25Syt 	 * when the power management is supported.
24622fcbc377Syt 	 */
24632fcbc377Syt 	if (ahci_ctlp->ahcictl_flags & AHCI_ATTACH) {
2464*68d33a25Syt 		/* Try to get the device signature */
2465*68d33a25Syt 		ahci_find_dev_signature(ahci_ctlp, ahci_portp, port);
24662fcbc377Syt 
2467*68d33a25Syt 		/* Return directly if no device connected */
2468*68d33a25Syt 		if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
2469*68d33a25Syt 			AHCIDBG1(AHCIDBG_INIT, ahci_ctlp,
2470*68d33a25Syt 			    "No device connected to port %d", port);
2471*68d33a25Syt 			goto out;
24722fcbc377Syt 		}
24732fcbc377Syt 
24742fcbc377Syt 		/* Try to start the port */
2475*68d33a25Syt 		if (ahci_start_port(ahci_ctlp, ahci_portp, port)
2476*68d33a25Syt 		    != AHCI_SUCCESS) {
2477*68d33a25Syt 			AHCIDBG1(AHCIDBG_INIT, ahci_ctlp,
2478*68d33a25Syt 			    "failed to start port %d", port);
24792fcbc377Syt 			return (AHCI_FAILURE);
24802fcbc377Syt 		}
24812fcbc377Syt 	}
2482*68d33a25Syt out:
2483*68d33a25Syt 	/* Enable port interrupts */
2484*68d33a25Syt 	ahci_enable_port_intrs(ahci_ctlp, ahci_portp, port);
24852fcbc377Syt 
24862fcbc377Syt 	return (AHCI_SUCCESS);
24872fcbc377Syt }
24882fcbc377Syt 
24892fcbc377Syt /*
24902fcbc377Syt  * AHCI device reset ...; a single device on one of the ports is reset,
24912fcbc377Syt  * but the HBA and physical communication remain intact. This is the
24922fcbc377Syt  * least intrusive.
24932fcbc377Syt  *
24942fcbc377Syt  * When issuing a software reset sequence, there should not be other
24952fcbc377Syt  * commands in the command list, so we will first clear and then re-set
24962fcbc377Syt  * PxCMD.ST to clear PxCI. And before issuing the software reset,
24972fcbc377Syt  * the port must be idle and PxTFD.STS.BSY and PxTFD.STS.DRQ must be
24982fcbc377Syt  * cleared.
24992fcbc377Syt  *
25002fcbc377Syt  * WARNING!!! ahciport_mutex should be acquired and PxCMD.FRE should be
25012fcbc377Syt  * set before the function is called.
25022fcbc377Syt  */
25032fcbc377Syt static int
25042fcbc377Syt ahci_software_reset(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
25052fcbc377Syt     uint8_t port)
25062fcbc377Syt {
25072fcbc377Syt 	ahci_fis_h2d_register_t *h2d_register_fisp;
25082fcbc377Syt 	ahci_cmd_table_t *cmd_table;
25092fcbc377Syt 	ahci_cmd_header_t *cmd_header;
2510*68d33a25Syt 	uint32_t port_cmd_status, port_cmd_issue, port_task_file;
25112fcbc377Syt 	int slot, loop_count;
25122fcbc377Syt 
25132fcbc377Syt 	AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp,
25142fcbc377Syt 	    "Port %d device resetting", port);
25152fcbc377Syt 
25162fcbc377Syt 	/* First to clear PxCMD.ST */
25172fcbc377Syt 	port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
25182fcbc377Syt 	    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
25192fcbc377Syt 
25202fcbc377Syt 	port_cmd_status &= ~AHCI_CMD_STATUS_ST;
25212fcbc377Syt 
25222fcbc377Syt 	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
25232fcbc377Syt 	    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
25242fcbc377Syt 	    port_cmd_status|AHCI_CMD_STATUS_ST);
25252fcbc377Syt 
25262fcbc377Syt 	/* And then to re-set PxCMD.ST */
25272fcbc377Syt 	port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
25282fcbc377Syt 	    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
25292fcbc377Syt 
25302fcbc377Syt 	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
25312fcbc377Syt 	    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
25322fcbc377Syt 	    port_cmd_status|AHCI_CMD_STATUS_ST);
25332fcbc377Syt 
25342fcbc377Syt 	/* Check PxTFD.STS.BSY and PxTFD.STS.DRQ */
25352fcbc377Syt 	port_task_file = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
25362fcbc377Syt 	    (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
25372fcbc377Syt 
25382fcbc377Syt 	if (port_task_file & AHCI_TFD_STS_BSY ||
25392fcbc377Syt 	    port_task_file & AHCI_TFD_STS_DRQ) {
25402fcbc377Syt 		if (!(port_cmd_status & AHCI_CMD_STATUS_CLO)) {
25412fcbc377Syt 			AHCIDBG0(AHCIDBG_ERRS, ahci_ctlp,
25422fcbc377Syt 			    "PxTFD.STS.BSY or PxTFD.STS.DRQ is still set, "
25432fcbc377Syt 			    "but PxCMD.CLO isn't supported, so a port "
25442fcbc377Syt 			    "reset is needed.");
25452fcbc377Syt 			return (AHCI_FAILURE);
25462fcbc377Syt 		}
25472fcbc377Syt 	}
25482fcbc377Syt 
25492fcbc377Syt 	slot = ahci_claim_free_slot(ahci_ctlp, ahci_portp);
25502fcbc377Syt 	if (slot == AHCI_FAILURE) {
25512fcbc377Syt 		AHCIDBG0(AHCIDBG_INFO, ahci_ctlp,
25522fcbc377Syt 		    "ahci_software_reset: no free slot");
25532fcbc377Syt 		return (AHCI_FAILURE);
25542fcbc377Syt 	}
25552fcbc377Syt 
25562fcbc377Syt 	/* Now send the first R2H FIS with SRST set to 1 */
25572fcbc377Syt 	cmd_table = ahci_portp->ahciport_cmd_tables[slot];
25582fcbc377Syt 	bzero((void *)cmd_table, ahci_cmd_table_size);
25592fcbc377Syt 
25602fcbc377Syt 	h2d_register_fisp =
25612fcbc377Syt 	    &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register);
25622fcbc377Syt 
25632fcbc377Syt 	SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE);
25642fcbc377Syt 	SET_FIS_PMP(h2d_register_fisp, AHCI_PORTMULT_CONTROL_PORT);
25652fcbc377Syt 	SET_FIS_DEVCTL(h2d_register_fisp, SATA_DEVCTL_SRST);
25662fcbc377Syt 
25672fcbc377Syt 	/* Set Command Header in Command List */
25682fcbc377Syt 	cmd_header = &ahci_portp->ahciport_cmd_list[slot];
25692fcbc377Syt 	BZERO_DESCR_INFO(cmd_header);
25702fcbc377Syt 	BZERO_PRD_BYTE_COUNT(cmd_header);
25712fcbc377Syt 	SET_COMMAND_FIS_LENGTH(cmd_header, 5);
25722fcbc377Syt 
25732fcbc377Syt 	SET_CLEAR_BUSY_UPON_R_OK(cmd_header, 1);
25742fcbc377Syt 	SET_RESET(cmd_header, 1);
25752fcbc377Syt 	SET_WRITE(cmd_header, 1);
25762fcbc377Syt 
25772fcbc377Syt 	(void) ddi_dma_sync(ahci_portp->ahciport_cmd_tables_dma_handle[slot],
25782fcbc377Syt 	    0,
25792fcbc377Syt 	    ahci_cmd_table_size,
25802fcbc377Syt 	    DDI_DMA_SYNC_FORDEV);
25812fcbc377Syt 
25822fcbc377Syt 	(void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle,
25832fcbc377Syt 	    slot * sizeof (ahci_cmd_header_t),
25842fcbc377Syt 	    sizeof (ahci_cmd_header_t),
25852fcbc377Syt 	    DDI_DMA_SYNC_FORDEV);
25862fcbc377Syt 
25872fcbc377Syt 	/* Indicate to the HBA that a command is active. */
25882fcbc377Syt 	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
25892fcbc377Syt 	    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port),
25902fcbc377Syt 	    (0x1 << slot));
25912fcbc377Syt 
25922fcbc377Syt 	loop_count = 0;
25932fcbc377Syt 
25942fcbc377Syt 	/* Loop till the first command is finished */
25952fcbc377Syt 	do {
25962fcbc377Syt 		port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
25972fcbc377Syt 		    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
25982fcbc377Syt 
25992fcbc377Syt 		if (loop_count++ > AHCI_POLLRATE_PORT_SOFTRESET) {
26002fcbc377Syt 			/* We are effectively timing out after 0.1 sec. */
26012fcbc377Syt 			break;
26022fcbc377Syt 		}
26032fcbc377Syt 		/* Wait for 10 millisec */
26042fcbc377Syt #ifndef __lock_lint
26052fcbc377Syt 		delay(AHCI_10MS_TICKS);
26062fcbc377Syt #endif /* __lock_lint */
26072fcbc377Syt 	} while (port_cmd_issue	& AHCI_SLOT_MASK(ahci_ctlp) & (0x1 << slot));
26082fcbc377Syt 
26092fcbc377Syt 	AHCIDBG3(AHCIDBG_POLL_LOOP, ahci_ctlp,
26102fcbc377Syt 	    "ahci_software_reset: 1st loop count: %d, "
26112fcbc377Syt 	    "port_cmd_issue = 0x%x, slot = 0x%x",
26122fcbc377Syt 	    loop_count, port_cmd_issue, slot);
26132fcbc377Syt 
26142fcbc377Syt 	CLEAR_BIT(ahci_portp->ahciport_pending_tags, slot);
26152fcbc377Syt 	ahci_portp->ahciport_slot_pkts[slot] = NULL;
26162fcbc377Syt 
26172fcbc377Syt 	/* Now send the second R2H FIS with SRST cleard to zero */
26182fcbc377Syt 	cmd_table = ahci_portp->ahciport_cmd_tables[slot];
26192fcbc377Syt 	bzero((void *)cmd_table, ahci_cmd_table_size);
26202fcbc377Syt 
26212fcbc377Syt 	h2d_register_fisp =
26222fcbc377Syt 	    &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register);
26232fcbc377Syt 
26242fcbc377Syt 	SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE);
26252fcbc377Syt 	SET_FIS_PMP(h2d_register_fisp, AHCI_PORTMULT_CONTROL_PORT);
26262fcbc377Syt 
26272fcbc377Syt 	/* Set Command Header in Command List */
26282fcbc377Syt 	cmd_header = &ahci_portp->ahciport_cmd_list[slot];
26292fcbc377Syt 	BZERO_DESCR_INFO(cmd_header);
26302fcbc377Syt 	BZERO_PRD_BYTE_COUNT(cmd_header);
26312fcbc377Syt 	SET_COMMAND_FIS_LENGTH(cmd_header, 5);
26322fcbc377Syt 
26332fcbc377Syt 	SET_WRITE(cmd_header, 1);
26342fcbc377Syt 
26352fcbc377Syt 	(void) ddi_dma_sync(ahci_portp->ahciport_cmd_tables_dma_handle[slot],
26362fcbc377Syt 	    0,
26372fcbc377Syt 	    ahci_cmd_table_size,
26382fcbc377Syt 	    DDI_DMA_SYNC_FORDEV);
26392fcbc377Syt 
26402fcbc377Syt 	(void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle,
26412fcbc377Syt 	    slot * sizeof (ahci_cmd_header_t),
26422fcbc377Syt 	    sizeof (ahci_cmd_header_t),
26432fcbc377Syt 	    DDI_DMA_SYNC_FORDEV);
26442fcbc377Syt 
26452fcbc377Syt 	/* Indicate to the HBA that a command is active. */
26462fcbc377Syt 	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
26472fcbc377Syt 	    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port),
26482fcbc377Syt 	    (0x1 << slot));
26492fcbc377Syt 
26502fcbc377Syt 	loop_count = 0;
26512fcbc377Syt 
26522fcbc377Syt 	/* Loop till the second command is finished */
26532fcbc377Syt 	do {
26542fcbc377Syt 		port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
26552fcbc377Syt 		    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
26562fcbc377Syt 
26572fcbc377Syt 		if (loop_count++ > AHCI_POLLRATE_PORT_SOFTRESET) {
26582fcbc377Syt 			/* We are effectively timing out after 0.1 sec. */
26592fcbc377Syt 			break;
26602fcbc377Syt 		}
26612fcbc377Syt 		/* Wait for 10 millisec */
26622fcbc377Syt #ifndef __lock_lint
26632fcbc377Syt 		delay(AHCI_10MS_TICKS);
26642fcbc377Syt #endif /* __lock_lint */
26652fcbc377Syt 	} while (port_cmd_issue	& AHCI_SLOT_MASK(ahci_ctlp) & (0x1 << slot));
26662fcbc377Syt 
26672fcbc377Syt 	AHCIDBG3(AHCIDBG_POLL_LOOP, ahci_ctlp,
26682fcbc377Syt 	    "ahci_software_reset: 2nd loop count: %d, "
26692fcbc377Syt 	    "port_cmd_issue = 0x%x, slot = 0x%x",
26702fcbc377Syt 	    loop_count, port_cmd_issue, slot);
26712fcbc377Syt 
26722fcbc377Syt 	CLEAR_BIT(ahci_portp->ahciport_pending_tags, slot);
26732fcbc377Syt 	ahci_portp->ahciport_slot_pkts[slot] = NULL;
26742fcbc377Syt 
26752fcbc377Syt 	return (AHCI_SUCCESS);
26762fcbc377Syt }
26772fcbc377Syt 
26782fcbc377Syt /*
26792fcbc377Syt  * AHCI port reset ...; the physical communication between the HBA and device
26802fcbc377Syt  * on a port are disabled. This is more intrusive.
26812fcbc377Syt  *
2682*68d33a25Syt  * When an HBA or port reset occurs, Phy communication is going to
26832fcbc377Syt  * be re-established with the device through a COMRESET followed by the
26842fcbc377Syt  * normal out-of-band communication sequence defined in Serial ATA. AT
26852fcbc377Syt  * the end of reset, the device, if working properly, will send a D2H
26862fcbc377Syt  * Register FIS, which contains the device signature. When the HBA receives
26872fcbc377Syt  * this FIS, it updates PxTFD.STS and PxTFD.ERR register fields, and updates
26882fcbc377Syt  * the PxSIG register with the signature.
26892fcbc377Syt  *
26902fcbc377Syt  * Staggered spin-up is an optional feature in SATA II, and it enables an HBA
26912fcbc377Syt  * to individually spin-up attached devices. Please refer to chapter 10.9 of
2692*68d33a25Syt  * AHCI 1.0 spec.
26932fcbc377Syt  */
26942fcbc377Syt /*
26952fcbc377Syt  * WARNING!!! ahciport_mutex should be acquired, intr should be disabled,
26962fcbc377Syt  * and PxCMD.ST and PxCMD.FRE should be also cleared before the function
26972fcbc377Syt  * is called.
26982fcbc377Syt  */
26992fcbc377Syt static int
27002fcbc377Syt ahci_port_reset(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, uint8_t port)
27012fcbc377Syt {
27022fcbc377Syt 	uint32_t cap_status, port_cmd_status;
27032fcbc377Syt 	uint32_t port_scontrol, port_sstatus;
27042fcbc377Syt 	uint32_t port_signature, port_intr_status, port_task_file;
27052fcbc377Syt 	int loop_count;
2706*68d33a25Syt 	int rval = AHCI_SUCCESS;
27072fcbc377Syt 
27082fcbc377Syt 	AHCIDBG1(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
27092fcbc377Syt 	    "Port %d port resetting...", port);
2710*68d33a25Syt 	ahci_portp->ahciport_port_state = 0;
27112fcbc377Syt 
27122fcbc377Syt 	cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
27132fcbc377Syt 	    (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp));
27142fcbc377Syt 
27152fcbc377Syt 	port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
27162fcbc377Syt 	    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
27172fcbc377Syt 
27182fcbc377Syt 	if (cap_status & AHCI_HBA_CAP_SSS) {
27192fcbc377Syt 		/*
27202fcbc377Syt 		 * HBA support staggered spin-up, if the port has
27212fcbc377Syt 		 * not spin up yet, then force it to do spin-up
27222fcbc377Syt 		 */
27232fcbc377Syt 		if (!(port_cmd_status & AHCI_CMD_STATUS_SUD)) {
27242fcbc377Syt 			if (!(ahci_portp->ahciport_flags
2725*68d33a25Syt 			    & AHCI_PORT_FLAG_SPINUP)) {
27262fcbc377Syt 				AHCIDBG1(AHCIDBG_INIT, ahci_ctlp,
27272fcbc377Syt 				    "Port %d PxCMD.SUD is zero, force "
27282fcbc377Syt 				    "it to do spin-up", port);
27292fcbc377Syt 				ahci_portp->ahciport_flags |=
2730*68d33a25Syt 				    AHCI_PORT_FLAG_SPINUP;
27312fcbc377Syt 			}
27322fcbc377Syt 		}
27332fcbc377Syt 	} else {
27342fcbc377Syt 		/*
27352fcbc377Syt 		 * HBA doesn't support stagger spin-up, force it
27362fcbc377Syt 		 * to do normal COMRESET
27372fcbc377Syt 		 */
27382fcbc377Syt 		ASSERT(port_cmd_status & AHCI_CMD_STATUS_SUD);
27392fcbc377Syt 		if (ahci_portp->ahciport_flags &
2740*68d33a25Syt 		    AHCI_PORT_FLAG_SPINUP) {
27412fcbc377Syt 			AHCIDBG0(AHCIDBG_INIT, ahci_ctlp,
27422fcbc377Syt 			    "HBA does not support staggered spin-up "
27432fcbc377Syt 			    "force it to do normal COMRESET");
27442fcbc377Syt 			ahci_portp->ahciport_flags &=
2745*68d33a25Syt 			    ~AHCI_PORT_FLAG_SPINUP;
27462fcbc377Syt 		}
27472fcbc377Syt 	}
27482fcbc377Syt 
2749*68d33a25Syt 	if (!(ahci_portp->ahciport_flags & AHCI_PORT_FLAG_SPINUP)) {
27502fcbc377Syt 		/* Do normal COMRESET */
27512fcbc377Syt 		AHCIDBG1(AHCIDBG_INFO, ahci_ctlp,
27522fcbc377Syt 		    "ahci_port_reset: do normal COMRESET", port);
27532fcbc377Syt 
27542fcbc377Syt 		ASSERT(port_cmd_status & AHCI_CMD_STATUS_SUD);
27552fcbc377Syt 
27562fcbc377Syt 		port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
27572fcbc377Syt 		    (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port));
27582fcbc377Syt 		AHCI_SCONTROL_SET_DET(port_scontrol,
27592fcbc377Syt 		    AHCI_SCONTROL_DET_COMRESET);
27602fcbc377Syt 
27612fcbc377Syt 		ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
27622fcbc377Syt 		    (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port),
27632fcbc377Syt 		    port_scontrol);
27642fcbc377Syt 
27652fcbc377Syt 		/* Enable PxCMD.FRE to read device */
27662fcbc377Syt 		ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
27672fcbc377Syt 		    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
27682fcbc377Syt 		    port_cmd_status|AHCI_CMD_STATUS_FRE);
27692fcbc377Syt 
2770*68d33a25Syt 		/*
2771*68d33a25Syt 		 * Give time for COMRESET to percolate, according to the AHCI
2772*68d33a25Syt 		 * spec, software shall wait at least 1 millisecond before
2773*68d33a25Syt 		 * clearing PxSCTL.DET
2774*68d33a25Syt 		 */
27752fcbc377Syt #ifndef __lock_lint
27762fcbc377Syt 		delay(AHCI_1MS_TICKS*2);
27772fcbc377Syt #endif /* __lock_lint */
27782fcbc377Syt 
27792fcbc377Syt 		/* Fetch the SCONTROL again and rewrite the DET part with 0 */
27802fcbc377Syt 		port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
27812fcbc377Syt 		    (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port));
27822fcbc377Syt 		AHCI_SCONTROL_SET_DET(port_scontrol,
27832fcbc377Syt 		    AHCI_SCONTROL_DET_NOACTION);
27842fcbc377Syt 		ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
27852fcbc377Syt 		    (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port),
27862fcbc377Syt 		    port_scontrol);
27872fcbc377Syt 	} else {
27882fcbc377Syt 		/* Do staggered spin-up */
27892fcbc377Syt 		port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
27902fcbc377Syt 		    (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port));
27912fcbc377Syt 		AHCI_SCONTROL_SET_DET(port_scontrol,
27922fcbc377Syt 		    AHCI_SCONTROL_DET_NOACTION);
27932fcbc377Syt 
27942fcbc377Syt 		/* PxSCTL.DET must be 0 */
27952fcbc377Syt 		ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
27962fcbc377Syt 		    (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port),
27972fcbc377Syt 		    port_scontrol);
27982fcbc377Syt 
27992fcbc377Syt 		port_cmd_status &= ~AHCI_CMD_STATUS_SUD;
28002fcbc377Syt 		ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
28012fcbc377Syt 		    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
28022fcbc377Syt 		    port_cmd_status);
28032fcbc377Syt 
28042fcbc377Syt 		/* 0 -> 1 edge */
28052fcbc377Syt #ifndef __lock_lint
28062fcbc377Syt 		delay(AHCI_1MS_TICKS*2);
28072fcbc377Syt #endif /* __lock_lint */
28082fcbc377Syt 
28092fcbc377Syt 		/* Set PxCMD.SUD to 1 */
28102fcbc377Syt 		port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
28112fcbc377Syt 		    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
28122fcbc377Syt 		port_cmd_status |= AHCI_CMD_STATUS_SUD;
28132fcbc377Syt 		ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
28142fcbc377Syt 		    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
28152fcbc377Syt 		    port_cmd_status);
28162fcbc377Syt 
28172fcbc377Syt 		/* Enable PxCMD.FRE to read device */
28182fcbc377Syt 		ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
28192fcbc377Syt 		    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
28202fcbc377Syt 		    port_cmd_status|AHCI_CMD_STATUS_FRE);
28212fcbc377Syt 	}
28222fcbc377Syt 
28232fcbc377Syt 	/*
2824*68d33a25Syt 	 * The port enters P:StartComm state, and HBA tells link layer to
2825*68d33a25Syt 	 * start communication, which involves sending COMRESET to device.
2826*68d33a25Syt 	 * And the HBA resets PxTFD.STS to 7Fh.
2827*68d33a25Syt 	 *
2828*68d33a25Syt 	 * When a COMINIT is received from the device, then the port enters
2829*68d33a25Syt 	 * P:ComInit state. And HBA sets PxTFD.STS to FFh or 80h. HBA sets
2830*68d33a25Syt 	 * PxSSTS.DET to 1h to indicate a device is detected but communication
2831*68d33a25Syt 	 * is not yet established. HBA sets PxSERR.DIAG.X to '1' to indicate
2832*68d33a25Syt 	 * a COMINIT has been received.
28332fcbc377Syt 	 */
28342fcbc377Syt 	/*
28352fcbc377Syt 	 * The DET field is valid only if IPM field indicates
28362fcbc377Syt 	 * that the interface is in active state.
28372fcbc377Syt 	 */
28382fcbc377Syt 	loop_count = 0;
28392fcbc377Syt 	do {
28402fcbc377Syt 		port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
28412fcbc377Syt 		    (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port));
28422fcbc377Syt 
28432fcbc377Syt 		if (AHCI_SSTATUS_GET_IPM(port_sstatus) !=
28442fcbc377Syt 		    AHCI_SSTATUS_IPM_INTERFACE_ACTIVE) {
28452fcbc377Syt 			/*
28462fcbc377Syt 			 * If the interface is not active, the DET field
28472fcbc377Syt 			 * is considered not accurate. So we want to
28482fcbc377Syt 			 * continue looping.
28492fcbc377Syt 			 */
28502fcbc377Syt 			AHCI_SSTATUS_SET_DET(port_sstatus,
28512fcbc377Syt 			    AHCI_SSTATUS_DET_NODEV_NOPHY);
28522fcbc377Syt 		}
28532fcbc377Syt 
28542fcbc377Syt 		if (loop_count++ > AHCI_POLLRATE_PORT_SSTATUS) {
28552fcbc377Syt 			/*
28562fcbc377Syt 			 * We are effectively timing out after 0.1 sec.
28572fcbc377Syt 			 */
28582fcbc377Syt 			break;
28592fcbc377Syt 		}
28602fcbc377Syt 
28612fcbc377Syt 		/* Wait for 10 millisec */
28622fcbc377Syt #ifndef __lock_lint
28632fcbc377Syt 		delay(AHCI_10MS_TICKS);
28642fcbc377Syt #endif /* __lock_lint */
28652fcbc377Syt 
28662fcbc377Syt 	} while (AHCI_SSTATUS_GET_DET(port_sstatus) !=
28672fcbc377Syt 	    AHCI_SSTATUS_DET_DEVPRESENT_PHYONLINE);
28682fcbc377Syt 
2869*68d33a25Syt 	AHCIDBG3(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp,
2870*68d33a25Syt 	    "ahci_port_reset: 1st loop count: %d, "
2871*68d33a25Syt 	    "port_sstatus = 0x%x port %d",
2872*68d33a25Syt 	    loop_count, port_sstatus, port);
28732fcbc377Syt 
28742fcbc377Syt 	if ((AHCI_SSTATUS_GET_IPM(port_sstatus) !=
28752fcbc377Syt 	    AHCI_SSTATUS_IPM_INTERFACE_ACTIVE) ||
28762fcbc377Syt 	    (AHCI_SSTATUS_GET_DET(port_sstatus) !=
28772fcbc377Syt 	    AHCI_SSTATUS_DET_DEVPRESENT_PHYONLINE)) {
28782fcbc377Syt 		/*
28792fcbc377Syt 		 * Either the port is not active or there
28802fcbc377Syt 		 * is no device present.
28812fcbc377Syt 		 */
28822fcbc377Syt 		ahci_portp->ahciport_device_type = SATA_DTYPE_NONE;
2883*68d33a25Syt 		goto out;
28842fcbc377Syt 	}
28852fcbc377Syt 
2886*68d33a25Syt 	/* Now we can make sure there is a device connected to the port */
2887*68d33a25Syt 	port_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2888*68d33a25Syt 	    (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port));
28892fcbc377Syt 
2890*68d33a25Syt 	/* a COMINIT signal is supposed to be received */
2891*68d33a25Syt 	if (!(port_intr_status & AHCI_INTR_STATUS_PCS)) {
2892*68d33a25Syt 		cmn_err(CE_WARN, "ahci_port_reset: port %d COMINIT signal "
2893*68d33a25Syt 		    "from the device not received", port);
2894*68d33a25Syt 		ahci_portp->ahciport_port_state |= SATA_PSTATE_FAILED;
2895*68d33a25Syt 		rval = AHCI_FAILURE;
2896*68d33a25Syt 		goto out;
2897*68d33a25Syt 	}
28982fcbc377Syt 
2899*68d33a25Syt 	/* PxTFD.STS.BSY is supposed to be set */
2900*68d33a25Syt 	port_task_file = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2901*68d33a25Syt 	    (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
2902*68d33a25Syt 	if (!(port_task_file & AHCI_TFD_STS_BSY)) {
2903*68d33a25Syt 		cmn_err(CE_WARN, "ahci_port_reset: port %d BSY bit "
2904*68d33a25Syt 		    "is not set after COMINIT signal is received", port);
2905*68d33a25Syt 		ahci_portp->ahciport_port_state |= SATA_PSTATE_FAILED;
2906*68d33a25Syt 		rval = AHCI_FAILURE;
2907*68d33a25Syt 		goto out;
2908*68d33a25Syt 	}
29092fcbc377Syt 
2910*68d33a25Syt 	/*
2911*68d33a25Syt 	 * PxSERR.DIAG.X has to be cleared in order to update PxTFD with
2912*68d33a25Syt 	 * the D2H FIS received by HBA.
2913*68d33a25Syt 	 */
2914*68d33a25Syt 	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
2915*68d33a25Syt 	    (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
2916*68d33a25Syt 	    AHCI_SERROR_DIAG_X);
29172fcbc377Syt 
2918*68d33a25Syt 	/*
2919*68d33a25Syt 	 * Next check whether COMRESET is completed successfully
2920*68d33a25Syt 	 */
2921*68d33a25Syt 	loop_count = 0;
2922*68d33a25Syt 	do {
2923*68d33a25Syt 		port_task_file =
2924*68d33a25Syt 		    ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2925*68d33a25Syt 		    (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
29262fcbc377Syt 
29272fcbc377Syt 		/*
2928*68d33a25Syt 		 * The Error bit '1' means COMRESET is finished successfully
2929*68d33a25Syt 		 * The device hardware has been initialized and the power-up
2930*68d33a25Syt 		 * diagnostics successfully completed.
29312fcbc377Syt 		 */
2932*68d33a25Syt 		if (((port_task_file & AHCI_TFD_ERR_MASK)
2933*68d33a25Syt 		    >> AHCI_TFD_ERR_SHIFT) == 0x1) {
2934*68d33a25Syt 
2935*68d33a25Syt 			port_signature = ddi_get32(
2936*68d33a25Syt 			    ahci_ctlp->ahcictl_ahci_acc_handle,
2937*68d33a25Syt 			    (uint32_t *)AHCI_PORT_PxSIG(ahci_ctlp, port));
2938*68d33a25Syt 			AHCIDBG2(AHCIDBG_INFO, ahci_ctlp,
2939*68d33a25Syt 			    "COMRESET success, D2H register FIS "
2940*68d33a25Syt 			    "post to received FIS structure "
2941*68d33a25Syt 			    "port %d signature = 0x%x",
2942*68d33a25Syt 			    port, port_signature);
2943*68d33a25Syt 			goto out_check;
2944*68d33a25Syt 		}
29452fcbc377Syt 
2946*68d33a25Syt 		if (loop_count++ > AHCI_POLLRATE_PORT_TFD_ERROR) {
2947*68d33a25Syt 			/*
2948*68d33a25Syt 			 * We are effectively timing out after 11 sec.
2949*68d33a25Syt 			 */
2950*68d33a25Syt 			break;
2951*68d33a25Syt 		}
29522fcbc377Syt 
2953*68d33a25Syt 		/* Wait for 10 millisec */
29542fcbc377Syt #ifndef __lock_lint
2955*68d33a25Syt 		delay(AHCI_10MS_TICKS);
29562fcbc377Syt #endif /* __lock_lint */
29572fcbc377Syt 
2958*68d33a25Syt 	} while (((port_task_file & AHCI_TFD_ERR_MASK)
2959*68d33a25Syt 	    >> AHCI_TFD_ERR_SHIFT) != 0x1);
29602fcbc377Syt 
2961*68d33a25Syt 	AHCIDBG3(AHCIDBG_ERRS, ahci_ctlp, "ahci_port_reset: 2nd loop "
2962*68d33a25Syt 	    "count: %d, port_task_file = 0x%x port %d",
2963*68d33a25Syt 	    loop_count, port_task_file, port);
29642fcbc377Syt 
2965*68d33a25Syt 	cmn_err(CE_WARN, "ahci_port_reset: port %d the device hardware "
2966*68d33a25Syt 	    "has been initialized and the power-up diagnostics failed",
2967*68d33a25Syt 	    port);
29682fcbc377Syt 
2969*68d33a25Syt 	ahci_portp->ahciport_port_state |= SATA_PSTATE_FAILED;
2970*68d33a25Syt 	rval = AHCI_FAILURE;
29712fcbc377Syt 
2972*68d33a25Syt out:
2973*68d33a25Syt 	/* Clear port serror register for the port */
2974*68d33a25Syt 	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
2975*68d33a25Syt 	    (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
2976*68d33a25Syt 	    AHCI_SERROR_CLEAR_ALL);
2977*68d33a25Syt 
2978*68d33a25Syt 	return (rval);
2979*68d33a25Syt 
2980*68d33a25Syt out_check:
2981*68d33a25Syt 	/*
2982*68d33a25Syt 	 * Check device status, if keep busy or COMRESET error
2983*68d33a25Syt 	 * do device reset to patch some SATA disks' issue
2984*68d33a25Syt 	 *
2985*68d33a25Syt 	 * For VT8251, sometimes need to do the device reset
2986*68d33a25Syt 	 */
2987*68d33a25Syt 	if ((port_task_file & AHCI_TFD_STS_BSY) ||
2988*68d33a25Syt 	    (port_task_file & AHCI_TFD_STS_DRQ)) {
2989*68d33a25Syt 		AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "port %d keep BSY/DRQ set "
2990*68d33a25Syt 		    "need to do device reset", port);
29912fcbc377Syt 
2992*68d33a25Syt 		(void) ahci_software_reset(ahci_ctlp, ahci_portp, port);
29932fcbc377Syt 
2994*68d33a25Syt 		port_task_file =
2995*68d33a25Syt 		    ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2996*68d33a25Syt 		    (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
2997*68d33a25Syt 
2998*68d33a25Syt 		if (port_task_file & AHCI_TFD_STS_BSY ||
2999*68d33a25Syt 		    port_task_file & AHCI_TFD_STS_DRQ) {
3000*68d33a25Syt 			cmn_err(CE_WARN, "ahci_port_reset: port %d "
3001*68d33a25Syt 			    "BSY/DRQ still set after device reset "
3002*68d33a25Syt 			    "port_task_file = 0x%x",
3003*68d33a25Syt 			    port, port_task_file);
3004*68d33a25Syt 			ahci_portp->ahciport_port_state |= SATA_PSTATE_FAILED;
3005*68d33a25Syt 			rval = AHCI_FAILURE;
30062fcbc377Syt 		}
30072fcbc377Syt 	}
30082fcbc377Syt 
3009*68d33a25Syt 	goto out;
30102fcbc377Syt }
30112fcbc377Syt 
30122fcbc377Syt /*
30132fcbc377Syt  * AHCI HBA reset ...; the entire HBA is reset, and all ports are disabled.
30142fcbc377Syt  * This is the most intrusive.
30152fcbc377Syt  *
3016*68d33a25Syt  * When an HBA reset occurs, Phy communication will be re-established with
3017*68d33a25Syt  * the device through a COMRESET followed by the normal out-of-band
3018*68d33a25Syt  * communication sequence defined in Serial ATA. AT the end of reset, the
3019*68d33a25Syt  * device, if working properly, will send a D2H Register FIS, which contains
3020*68d33a25Syt  * the device signature. When the HBA receives this FIS, it updates PxTFD.STS
3021*68d33a25Syt  * and PxTFD.ERR register fields, and updates the PxSIG register with the
3022*68d33a25Syt  * signature.
30232fcbc377Syt  *
30242fcbc377Syt  * Remember to set GHC.AE to 1 before calling ahci_hba_reset.
30252fcbc377Syt  */
30262fcbc377Syt static int
30272fcbc377Syt ahci_hba_reset(ahci_ctl_t *ahci_ctlp)
30282fcbc377Syt {
30292fcbc377Syt 	ahci_port_t *ahci_portp;
30302fcbc377Syt 	uint32_t ghc_control;
30312fcbc377Syt 	uint8_t port;
30322fcbc377Syt 	int loop_count;
30332fcbc377Syt 	int rval = AHCI_SUCCESS;
30342fcbc377Syt 
30352fcbc377Syt 	AHCIDBG0(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, "HBA resetting");
30362fcbc377Syt 
3037*68d33a25Syt 	mutex_enter(&ahci_ctlp->ahcictl_mutex);
3038*68d33a25Syt 
30392fcbc377Syt 	ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
30402fcbc377Syt 	    (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
30412fcbc377Syt 
30422fcbc377Syt 	/* Setting GHC.HR to 1, remember GHC.AE is already set to 1 before */
30432fcbc377Syt 	ghc_control |= AHCI_HBA_GHC_HR;
30442fcbc377Syt 	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
30452fcbc377Syt 	    (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
30462fcbc377Syt 
30472fcbc377Syt 	/*
30482fcbc377Syt 	 * Wait until HBA Reset complete or timeout
30492fcbc377Syt 	 */
30502fcbc377Syt 	loop_count = 0;
30512fcbc377Syt 	do {
30522fcbc377Syt 		ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
30532fcbc377Syt 		    (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
30542fcbc377Syt 
30552fcbc377Syt 		if (loop_count++ > AHCI_POLLRATE_HBA_RESET) {
30562fcbc377Syt 			AHCIDBG1(AHCIDBG_INIT, ahci_ctlp,
30572fcbc377Syt 			    "ahci hba reset is timing out, "
30582fcbc377Syt 			    "ghc_control = 0x%x", ghc_control);
30592fcbc377Syt 			/* We are effectively timing out after 1 sec. */
30602fcbc377Syt 			break;
30612fcbc377Syt 		}
30622fcbc377Syt 
30632fcbc377Syt 		/* Wait for 10 millisec */
30642fcbc377Syt #ifndef __lock_lint
30652fcbc377Syt 		delay(AHCI_10MS_TICKS);
30662fcbc377Syt #endif /* __lock_lint */
30672fcbc377Syt 
30682fcbc377Syt 	} while (ghc_control & AHCI_HBA_GHC_HR);
30692fcbc377Syt 
30702fcbc377Syt 	AHCIDBG2(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp,
30712fcbc377Syt 	    "ahci_hba_reset: 1st loop count: %d, "
30722fcbc377Syt 	    "ghc_control = 0x%x", loop_count, ghc_control);
30732fcbc377Syt 
30742fcbc377Syt 	if (ghc_control & AHCI_HBA_GHC_HR) {
30752fcbc377Syt 		/* The hba is not reset for some reasons */
30762fcbc377Syt 		AHCIDBG0(AHCIDBG_INIT, ahci_ctlp,
30772fcbc377Syt 		    "hba reset failed: HBA in a hung or locked state");
3078*68d33a25Syt 		mutex_exit(&ahci_ctlp->ahcictl_mutex);
30792fcbc377Syt 		return (AHCI_FAILURE);
30802fcbc377Syt 	}
30812fcbc377Syt 
30822fcbc377Syt 	for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
30832fcbc377Syt 		/* Only check implemented ports */
30842fcbc377Syt 		if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
30852fcbc377Syt 			continue;
30862fcbc377Syt 		}
30872fcbc377Syt 
30882fcbc377Syt 		ahci_portp = ahci_ctlp->ahcictl_ports[port];
30892fcbc377Syt 		mutex_enter(&ahci_portp->ahciport_mutex);
30902fcbc377Syt 
30912fcbc377Syt 		if (ahci_port_reset(ahci_ctlp, ahci_portp, port)
30922fcbc377Syt 		    != AHCI_SUCCESS) {
30932fcbc377Syt 			rval = AHCI_FAILURE;
30942fcbc377Syt 			AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp,
30952fcbc377Syt 			    "ahci_hba_reset: port %d failed", port);
30962fcbc377Syt 		}
30972fcbc377Syt 
30982fcbc377Syt 		mutex_exit(&ahci_portp->ahciport_mutex);
30992fcbc377Syt 	}
31002fcbc377Syt 
31012fcbc377Syt 	/*
31022fcbc377Syt 	 * Indicate that system software is AHCI aware by setting
31032fcbc377Syt 	 * GHC.AE to 1
31042fcbc377Syt 	 */
31052fcbc377Syt 	ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
31062fcbc377Syt 	    (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
31072fcbc377Syt 
31082fcbc377Syt 	ghc_control |= AHCI_HBA_GHC_AE;
31092fcbc377Syt 	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
31102fcbc377Syt 	    (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
31112fcbc377Syt 
3112*68d33a25Syt 	mutex_exit(&ahci_ctlp->ahcictl_mutex);
3113*68d33a25Syt 
31142fcbc377Syt 	return (rval);
31152fcbc377Syt }
31162fcbc377Syt 
31172fcbc377Syt /*
31182fcbc377Syt  * This routine is only called from AHCI_ATTACH or phyrdy change
31192fcbc377Syt  * case. It first calls port reset to initialize port, probe port and probe
3120*68d33a25Syt  * device, then try to read PxSIG register to find the type of device
3121*68d33a25Syt  * attached to the port.
31222fcbc377Syt  *
31232fcbc377Syt  * WARNING!!! ahciport_mutex should be acquired before the function
3124*68d33a25Syt  * is called. And the port interrupt is disabled.
31252fcbc377Syt  */
3126*68d33a25Syt static void
31272fcbc377Syt ahci_find_dev_signature(ahci_ctl_t *ahci_ctlp,
31282fcbc377Syt     ahci_port_t *ahci_portp, uint8_t port)
31292fcbc377Syt {
31302fcbc377Syt 	uint32_t signature;
31312fcbc377Syt 
31322fcbc377Syt 	AHCIDBG1(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
3133*68d33a25Syt 	    "ahci_find_dev_signature enter: port %d", port);
31342fcbc377Syt 
3135*68d33a25Syt 	ahci_portp->ahciport_device_type = SATA_DTYPE_UNKNOWN;
31362fcbc377Syt 
31372fcbc377Syt 	/* Call port reset to check link status and get device signature */
3138*68d33a25Syt 	(void) ahci_port_reset(ahci_ctlp, ahci_portp, port);
3139*68d33a25Syt 
3140*68d33a25Syt 	if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
3141*68d33a25Syt 		AHCIDBG1(AHCIDBG_INFO, ahci_ctlp,
3142*68d33a25Syt 		    "ahci_find_dev_signature: No device is found "
3143*68d33a25Syt 		    "at port %d", port);
3144*68d33a25Syt 		return;
31452fcbc377Syt 	}
31462fcbc377Syt 
3147*68d33a25Syt 	/* Check the port state */
3148*68d33a25Syt 	if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED) {
3149*68d33a25Syt 		AHCIDBG2(AHCIDBG_ERRS, ahci_ctlp,
3150*68d33a25Syt 		    "ahci_find_dev_signature: port %d state 0x%x",
3151*68d33a25Syt 		    port, ahci_portp->ahciport_port_state);
3152*68d33a25Syt 		return;
3153*68d33a25Syt 	}
31542fcbc377Syt 
31552fcbc377Syt 	signature = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
31562fcbc377Syt 	    (uint32_t *)AHCI_PORT_PxSIG(ahci_ctlp, port));
31572fcbc377Syt 
31582fcbc377Syt 	AHCIDBG2(AHCIDBG_INIT|AHCIDBG_INFO, ahci_ctlp,
3159*68d33a25Syt 	    "ahci_find_dev_signature: port %d signature = 0x%x",
3160*68d33a25Syt 	    port, signature);
31612fcbc377Syt 
31622fcbc377Syt 	switch (signature) {
31632fcbc377Syt 
31642fcbc377Syt 	case AHCI_SIGNATURE_DISK:
31652fcbc377Syt 		ahci_portp->ahciport_device_type = SATA_DTYPE_ATADISK;
31662fcbc377Syt 		AHCIDBG1(AHCIDBG_INFO, ahci_ctlp,
31672fcbc377Syt 		    "Disk is found at port: %d", port);
31682fcbc377Syt 		break;
31692fcbc377Syt 
31702fcbc377Syt 	case AHCI_SIGNATURE_ATAPI:
31712fcbc377Syt 		ahci_portp->ahciport_device_type = SATA_DTYPE_ATAPICD;
31722fcbc377Syt 		AHCIDBG1(AHCIDBG_INFO, ahci_ctlp,
31732fcbc377Syt 		    "ATAPI device is found at port: %d", port);
31742fcbc377Syt 		break;
31752fcbc377Syt 
31762fcbc377Syt 	case AHCI_SIGNATURE_PORT_MULTIPLIER:
31772fcbc377Syt 		ahci_portp->ahciport_device_type = SATA_DTYPE_PMULT;
31782fcbc377Syt 		AHCIDBG1(AHCIDBG_INFO, ahci_ctlp,
31792fcbc377Syt 		    "Port Multiplier is found at port: %d", port);
31802fcbc377Syt 		break;
31812fcbc377Syt 
31822fcbc377Syt 	default:
31832fcbc377Syt 		ahci_portp->ahciport_device_type = SATA_DTYPE_UNKNOWN;
31842fcbc377Syt 		AHCIDBG1(AHCIDBG_INFO, ahci_ctlp,
31852fcbc377Syt 		    "Unknown device is found at port: %d", port);
31862fcbc377Syt 	}
31872fcbc377Syt }
31882fcbc377Syt 
31892fcbc377Syt /*
3190*68d33a25Syt  * Start the port - set PxCMD.ST to 1, if PxCMD.FRE is not set
3191*68d33a25Syt  * to 1, then set it firstly.
3192*68d33a25Syt  *
3193*68d33a25Syt  * Each port contains two major DMA engines. One DMA engine walks through
3194*68d33a25Syt  * the command list, and is controlled by PxCMD.ST. The second DMA engine
3195*68d33a25Syt  * copies received FISes into system memory, and is controlled by PxCMD.FRE.
3196*68d33a25Syt  *
3197*68d33a25Syt  * Software shall not set PxCMD.ST to '1' until it verifies that PxCMD.CR
3198*68d33a25Syt  * is '0' and has set PxCMD.FRE is '1'. And software shall not clear
3199*68d33a25Syt  * PxCMD.FRE while PxCMD.ST or PxCMD.CR is set '1'.
3200*68d33a25Syt  *
3201*68d33a25Syt  * Software shall not set PxCMD.ST to '1' unless a functional device is
3202*68d33a25Syt  * present on the port(as determined by PxTFD.STS.BSY = '0',
3203*68d33a25Syt  * PxTFD.STS.DRQ = '0', and PxSSTS.DET = 3h).
32042fcbc377Syt  *
32052fcbc377Syt  * WARNING!!! ahciport_mutex should be acquired before the function
32062fcbc377Syt  * is called.
32072fcbc377Syt  */
32082fcbc377Syt static int
3209*68d33a25Syt ahci_start_port(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, uint8_t port)
32102fcbc377Syt {
3211*68d33a25Syt 	uint32_t port_cmd_status;
32122fcbc377Syt 
3213*68d33a25Syt 	AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp, "ahci_start_port: %d enter", port);
32142fcbc377Syt 
3215*68d33a25Syt 	if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED) {
3216*68d33a25Syt 		AHCIDBG2(AHCIDBG_ERRS, ahci_ctlp, "ahci_start_port failed "
3217*68d33a25Syt 		    "the state for port %d is 0x%x",
3218*68d33a25Syt 		    port, ahci_portp->ahciport_port_state);
32192fcbc377Syt 		return (AHCI_FAILURE);
3220*68d33a25Syt 	}
32212fcbc377Syt 
3222*68d33a25Syt 	if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
3223*68d33a25Syt 		AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "ahci_start_port failed "
3224*68d33a25Syt 		    "no device is attached at port %d", port);
3225*68d33a25Syt 		return (AHCI_FAILURE);
3226*68d33a25Syt 	}
32272fcbc377Syt 
3228*68d33a25Syt 	/* First to set PxCMD.FRE before setting PxCMD.ST. */
3229*68d33a25Syt 	port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3230*68d33a25Syt 	    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
32312fcbc377Syt 
3232*68d33a25Syt 	if (!(port_cmd_status & AHCI_CMD_STATUS_FRE)) {
3233*68d33a25Syt 		port_cmd_status |= AHCI_CMD_STATUS_FRE;
32342fcbc377Syt 		ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
32352fcbc377Syt 		    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
32362fcbc377Syt 		    port_cmd_status);
32372fcbc377Syt 	}
3238*68d33a25Syt 
3239*68d33a25Syt 	port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3240*68d33a25Syt 	    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
3241*68d33a25Syt 
3242*68d33a25Syt 	port_cmd_status |= AHCI_CMD_STATUS_ST;
3243*68d33a25Syt 
3244*68d33a25Syt 	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
3245*68d33a25Syt 	    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
3246*68d33a25Syt 	    port_cmd_status);
3247*68d33a25Syt 
3248*68d33a25Syt 	ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_STARTED;
3249*68d33a25Syt 
3250*68d33a25Syt 	return (AHCI_SUCCESS);
32512fcbc377Syt }
32522fcbc377Syt 
32532fcbc377Syt /*
32542fcbc377Syt  * Allocate the ahci_port_t including Received FIS and Command List.
32552fcbc377Syt  * The argument - port is the physical port number, and not logical
32562fcbc377Syt  * port number seen by the SATA framework.
32572fcbc377Syt  *
32582fcbc377Syt  * WARNING!!! ahcictl_mutex should be acquired before the function
32592fcbc377Syt  * is called.
32602fcbc377Syt  */
32612fcbc377Syt static int
32622fcbc377Syt ahci_alloc_port_state(ahci_ctl_t *ahci_ctlp, uint8_t port)
32632fcbc377Syt {
32642fcbc377Syt 	ahci_port_t *ahci_portp;
32652fcbc377Syt 
32662fcbc377Syt 	ahci_portp =
32672fcbc377Syt 	    (ahci_port_t *)kmem_zalloc(sizeof (ahci_port_t), KM_SLEEP);
32682fcbc377Syt 
32692fcbc377Syt #ifndef __lock_lint
32702fcbc377Syt 	ahci_ctlp->ahcictl_ports[port] = ahci_portp;
32712fcbc377Syt #endif /* __lock_lint */
32722fcbc377Syt 
32732fcbc377Syt 	ahci_portp->ahciport_port_num = port;
32742fcbc377Syt 
3275*68d33a25Syt 	/* Intialize the port condition variable */
3276*68d33a25Syt 	cv_init(&ahci_portp->ahciport_cv, NULL, CV_DRIVER, NULL);
3277*68d33a25Syt 
3278*68d33a25Syt 	/* Initialize the port mutex */
32792fcbc377Syt 	mutex_init(&ahci_portp->ahciport_mutex, NULL, MUTEX_DRIVER,
32802fcbc377Syt 	    (void *)(uintptr_t)ahci_ctlp->ahcictl_intr_pri);
3281*68d33a25Syt 
32822fcbc377Syt 	mutex_enter(&ahci_portp->ahciport_mutex);
32832fcbc377Syt 
32842fcbc377Syt 	/*
32852fcbc377Syt 	 * Allocate memory for received FIS structure and
32862fcbc377Syt 	 * command list for this port
32872fcbc377Syt 	 */
32882fcbc377Syt 	if (ahci_alloc_rcvd_fis(ahci_ctlp, ahci_portp, port) != AHCI_SUCCESS) {
32892fcbc377Syt 		goto err_case1;
32902fcbc377Syt 	}
32912fcbc377Syt 
32922fcbc377Syt 	if (ahci_alloc_cmd_list(ahci_ctlp, ahci_portp, port) != AHCI_SUCCESS) {
32932fcbc377Syt 		goto err_case2;
32942fcbc377Syt 	}
32952fcbc377Syt 
3296*68d33a25Syt 	ahci_portp->ahciport_event_args =
3297*68d33a25Syt 	    kmem_zalloc(sizeof (ahci_event_arg_t), KM_SLEEP);
3298*68d33a25Syt 
3299*68d33a25Syt 	if (ahci_portp->ahciport_event_args == NULL)
3300*68d33a25Syt 		goto err_case3;
3301*68d33a25Syt 
33022fcbc377Syt 	mutex_exit(&ahci_portp->ahciport_mutex);
33032fcbc377Syt 
33042fcbc377Syt 	return (AHCI_SUCCESS);
33052fcbc377Syt 
3306*68d33a25Syt err_case3:
3307*68d33a25Syt 	ahci_dealloc_cmd_list(ahci_ctlp, ahci_portp);
3308*68d33a25Syt 
33092fcbc377Syt err_case2:
33102fcbc377Syt 	ahci_dealloc_rcvd_fis(ahci_ctlp, ahci_portp);
33112fcbc377Syt 
33122fcbc377Syt err_case1:
33132fcbc377Syt 	mutex_exit(&ahci_portp->ahciport_mutex);
33142fcbc377Syt 	mutex_destroy(&ahci_portp->ahciport_mutex);
3315*68d33a25Syt 	cv_destroy(&ahci_portp->ahciport_cv);
33162fcbc377Syt 
33172fcbc377Syt 	kmem_free(ahci_portp, sizeof (ahci_port_t));
33182fcbc377Syt 
33192fcbc377Syt 	return (AHCI_FAILURE);
33202fcbc377Syt }
33212fcbc377Syt 
33222fcbc377Syt /*
33232fcbc377Syt  * Reverse of ahci_dealloc_port_state().
33242fcbc377Syt  *
33252fcbc377Syt  * WARNING!!! ahcictl_mutex should be acquired before the function
33262fcbc377Syt  * is called.
33272fcbc377Syt  */
33282fcbc377Syt static void
33292fcbc377Syt ahci_dealloc_port_state(ahci_ctl_t *ahci_ctlp, uint8_t port)
33302fcbc377Syt {
33312fcbc377Syt 	ahci_port_t *ahci_portp = ahci_ctlp->ahcictl_ports[port];
33322fcbc377Syt 
33332fcbc377Syt 	ASSERT(ahci_portp != NULL);
33342fcbc377Syt 
33352fcbc377Syt 	mutex_enter(&ahci_portp->ahciport_mutex);
3336*68d33a25Syt 	kmem_free(ahci_portp->ahciport_event_args, sizeof (ahci_event_arg_t));
3337*68d33a25Syt 	ahci_portp->ahciport_event_args = NULL;
33382fcbc377Syt 	ahci_dealloc_cmd_list(ahci_ctlp, ahci_portp);
33392fcbc377Syt 	ahci_dealloc_rcvd_fis(ahci_ctlp, ahci_portp);
33402fcbc377Syt 	mutex_exit(&ahci_portp->ahciport_mutex);
33412fcbc377Syt 
33422fcbc377Syt 	mutex_destroy(&ahci_portp->ahciport_mutex);
3343*68d33a25Syt 	cv_destroy(&ahci_portp->ahciport_cv);
33442fcbc377Syt 
33452fcbc377Syt 	kmem_free(ahci_portp, sizeof (ahci_port_t));
33462fcbc377Syt 
33472fcbc377Syt #ifndef __lock_lint
33482fcbc377Syt 	ahci_ctlp->ahcictl_ports[port] = NULL;
33492fcbc377Syt #endif /* __lock_lint */
33502fcbc377Syt }
33512fcbc377Syt 
33522fcbc377Syt /*
33532fcbc377Syt  * Allocates memory for the Received FIS Structure
33542fcbc377Syt  *
33552fcbc377Syt  * WARNING!!! ahciport_mutex should be acquired before the function
33562fcbc377Syt  * is called.
33572fcbc377Syt  */
33582fcbc377Syt static int
33592fcbc377Syt ahci_alloc_rcvd_fis(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
33602fcbc377Syt     uint8_t port)
33612fcbc377Syt {
33622fcbc377Syt 	size_t rcvd_fis_size;
33632fcbc377Syt 	size_t ret_len;
33642fcbc377Syt 	ddi_dma_cookie_t rcvd_fis_dma_cookie;
33652fcbc377Syt 	uint_t cookie_count;
33662fcbc377Syt 	uint32_t cap_status;
33672fcbc377Syt 
33682fcbc377Syt 	rcvd_fis_size = sizeof (ahci_rcvd_fis_t);
33692fcbc377Syt 
33702fcbc377Syt 	/* Check whether the HBA can access 64-bit data structures */
33712fcbc377Syt 	cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
33722fcbc377Syt 	    (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp));
33732fcbc377Syt 
33742fcbc377Syt 	ahci_ctlp->ahcictl_rcvd_fis_dma_attr = rcvd_fis_dma_attr;
33752fcbc377Syt 
33762fcbc377Syt 	/*
33772fcbc377Syt 	 * If 64-bit addressing is not supported,
33782fcbc377Syt 	 * change dma_attr_addr_hi of ahcictl_rcvd_fis_dma_attr
33792fcbc377Syt 	 */
33802fcbc377Syt 	if (!(cap_status & AHCI_HBA_CAP_S64A)) {
33812fcbc377Syt 		AHCIDBG0(AHCIDBG_INIT, ahci_ctlp,
33822fcbc377Syt 		    "hba does not support 64-bit addressing");
33832fcbc377Syt 		ahci_ctlp->ahcictl_rcvd_fis_dma_attr.dma_attr_addr_hi =
33842fcbc377Syt 		    0xffffffffull;
33852fcbc377Syt 	}
33862fcbc377Syt 
33872fcbc377Syt 	/* allocate rcvd FIS dma handle. */
33882fcbc377Syt 	if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip,
33892fcbc377Syt 	    &ahci_ctlp->ahcictl_rcvd_fis_dma_attr,
33902fcbc377Syt 	    DDI_DMA_SLEEP,
33912fcbc377Syt 	    NULL,
33922fcbc377Syt 	    &ahci_portp->ahciport_rcvd_fis_dma_handle) !=
33932fcbc377Syt 	    DDI_SUCCESS) {
33942fcbc377Syt 		AHCIDBG0(AHCIDBG_INIT, ahci_ctlp,
33952fcbc377Syt 		    "rcvd FIS dma handle alloc failed");
33962fcbc377Syt 
33972fcbc377Syt 		return (AHCI_FAILURE);
33982fcbc377Syt 	}
33992fcbc377Syt 
34002fcbc377Syt 	if (ddi_dma_mem_alloc(ahci_portp->ahciport_rcvd_fis_dma_handle,
34012fcbc377Syt 	    rcvd_fis_size,
34022fcbc377Syt 	    &accattr,
34032fcbc377Syt 	    DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
34042fcbc377Syt 	    DDI_DMA_SLEEP,
34052fcbc377Syt 	    NULL,
34062fcbc377Syt 	    (caddr_t *)&ahci_portp->ahciport_rcvd_fis,
34072fcbc377Syt 	    &ret_len,
34082fcbc377Syt 	    &ahci_portp->ahciport_rcvd_fis_acc_handle) != NULL) {
34092fcbc377Syt 
34102fcbc377Syt 		AHCIDBG0(AHCIDBG_INIT, ahci_ctlp,
34112fcbc377Syt 		    "rcvd FIS dma mem alloc fail");
34122fcbc377Syt 		/* error.. free the dma handle. */
34132fcbc377Syt 		ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle);
34142fcbc377Syt 		return (AHCI_FAILURE);
34152fcbc377Syt 	}
34162fcbc377Syt 
34172fcbc377Syt 	if (ddi_dma_addr_bind_handle(ahci_portp->ahciport_rcvd_fis_dma_handle,
34182fcbc377Syt 	    NULL,
34192fcbc377Syt 	    (caddr_t)ahci_portp->ahciport_rcvd_fis,
34202fcbc377Syt 	    rcvd_fis_size,
34212fcbc377Syt 	    DDI_DMA_CONSISTENT,
34222fcbc377Syt 	    DDI_DMA_SLEEP,
34232fcbc377Syt 	    NULL,
34242fcbc377Syt 	    &rcvd_fis_dma_cookie,
34252fcbc377Syt 	    &cookie_count) !=  DDI_DMA_MAPPED) {
34262fcbc377Syt 
34272fcbc377Syt 		AHCIDBG0(AHCIDBG_INIT, ahci_ctlp,
34282fcbc377Syt 		    "rcvd FIS dma handle bind fail");
34292fcbc377Syt 		/*  error.. free the dma handle & free the memory. */
34302fcbc377Syt 		ddi_dma_mem_free(&ahci_portp->ahciport_rcvd_fis_acc_handle);
34312fcbc377Syt 		ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle);
34322fcbc377Syt 		return (AHCI_FAILURE);
34332fcbc377Syt 	}
34342fcbc377Syt 
34352fcbc377Syt 	bzero((void *)ahci_portp->ahciport_rcvd_fis, rcvd_fis_size);
34362fcbc377Syt 
34372fcbc377Syt 	/* Config Port Received FIS Base Address */
34382fcbc377Syt 	ddi_put64(ahci_ctlp->ahcictl_ahci_acc_handle,
34392fcbc377Syt 	    (uint64_t *)AHCI_PORT_PxFB(ahci_ctlp, port),
34402fcbc377Syt 	    rcvd_fis_dma_cookie.dmac_laddress);
34412fcbc377Syt 
34422fcbc377Syt 	AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, "64-bit, dma address: 0x%llx",
34432fcbc377Syt 	    rcvd_fis_dma_cookie.dmac_laddress);
34442fcbc377Syt 	AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, "32-bit, dma address: 0x%x",
34452fcbc377Syt 	    rcvd_fis_dma_cookie.dmac_address);
34462fcbc377Syt 
34472fcbc377Syt 	return (AHCI_SUCCESS);
34482fcbc377Syt }
34492fcbc377Syt 
34502fcbc377Syt /*
34512fcbc377Syt  * Deallocates the Received FIS Structure
34522fcbc377Syt  *
34532fcbc377Syt  * WARNING!!! ahciport_mutex should be acquired before the function
34542fcbc377Syt  * is called.
34552fcbc377Syt  */
34562fcbc377Syt static void
34572fcbc377Syt ahci_dealloc_rcvd_fis(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
34582fcbc377Syt {
34592fcbc377Syt 	AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp,
34602fcbc377Syt 	    "ahci_dealloc_rcvd_fis: port %d enter",
34612fcbc377Syt 	    ahci_portp->ahciport_port_num);
34622fcbc377Syt 
34632fcbc377Syt 	/* Unbind the cmd list dma handle first. */
34642fcbc377Syt 	(void) ddi_dma_unbind_handle(ahci_portp->ahciport_rcvd_fis_dma_handle);
34652fcbc377Syt 
34662fcbc377Syt 	/* Then free the underlying memory. */
34672fcbc377Syt 	ddi_dma_mem_free(&ahci_portp->ahciport_rcvd_fis_acc_handle);
34682fcbc377Syt 
34692fcbc377Syt 	/* Now free the handle itself. */
34702fcbc377Syt 	ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle);
34712fcbc377Syt }
34722fcbc377Syt 
34732fcbc377Syt /*
34742fcbc377Syt  * Allocates memory for the Command List, which contains up to 32 entries.
34752fcbc377Syt  * Each entry contains a command header, which is a 32-byte structure that
34762fcbc377Syt  * includes the pointer to the command table.
34772fcbc377Syt  *
34782fcbc377Syt  * WARNING!!! ahciport_mutex should be acquired before the function
34792fcbc377Syt  * is called.
34802fcbc377Syt  */
34812fcbc377Syt static int
34822fcbc377Syt ahci_alloc_cmd_list(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
34832fcbc377Syt     uint8_t port)
34842fcbc377Syt {
34852fcbc377Syt 	size_t cmd_list_size;
34862fcbc377Syt 	size_t ret_len;
34872fcbc377Syt 	ddi_dma_cookie_t cmd_list_dma_cookie;
34882fcbc377Syt 	uint_t cookie_count;
34892fcbc377Syt 	uint32_t cap_status;
34902fcbc377Syt 
34912fcbc377Syt 	cmd_list_size =
34922fcbc377Syt 	    ahci_ctlp->ahcictl_num_cmd_slots * sizeof (ahci_cmd_header_t);
34932fcbc377Syt 
34942fcbc377Syt 	cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
34952fcbc377Syt 	    (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp));
34962fcbc377Syt 
34972fcbc377Syt 	ahci_ctlp->ahcictl_cmd_list_dma_attr = cmd_list_dma_attr;
34982fcbc377Syt 
34992fcbc377Syt 	/*
35002fcbc377Syt 	 * If 64-bit addressing is not supported,
35012fcbc377Syt 	 * change dma_attr_addr_hi of ahcictl_cmd_list_dma_attr
35022fcbc377Syt 	 */
35032fcbc377Syt 	if (!(cap_status & AHCI_HBA_CAP_S64A)) {
35042fcbc377Syt 		AHCIDBG0(AHCIDBG_INIT, ahci_ctlp,
35052fcbc377Syt 		    "hba does not support 64-bit addressing");
35062fcbc377Syt 		ahci_ctlp->ahcictl_cmd_list_dma_attr.dma_attr_addr_hi =
35072fcbc377Syt 		    0xffffffffull;
35082fcbc377Syt 	}
35092fcbc377Syt 
35102fcbc377Syt 	/* allocate cmd list dma handle. */
35112fcbc377Syt 	if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip,
35122fcbc377Syt 	    &ahci_ctlp->ahcictl_cmd_list_dma_attr,
35132fcbc377Syt 	    DDI_DMA_SLEEP,
35142fcbc377Syt 	    NULL,
35152fcbc377Syt 	    &ahci_portp->ahciport_cmd_list_dma_handle) != DDI_SUCCESS) {
35162fcbc377Syt 
35172fcbc377Syt 		AHCIDBG0(AHCIDBG_INIT, ahci_ctlp,
35182fcbc377Syt 		    "cmd list dma handle alloc failed");
35192fcbc377Syt 		return (AHCI_FAILURE);
35202fcbc377Syt 	}
35212fcbc377Syt 
35222fcbc377Syt 	if (ddi_dma_mem_alloc(ahci_portp->ahciport_cmd_list_dma_handle,
35232fcbc377Syt 	    cmd_list_size,
35242fcbc377Syt 	    &accattr,
35252fcbc377Syt 	    DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
35262fcbc377Syt 	    DDI_DMA_SLEEP,
35272fcbc377Syt 	    NULL,
35282fcbc377Syt 	    (caddr_t *)&ahci_portp->ahciport_cmd_list,
35292fcbc377Syt 	    &ret_len,
35302fcbc377Syt 	    &ahci_portp->ahciport_cmd_list_acc_handle) != NULL) {
35312fcbc377Syt 
35322fcbc377Syt 		AHCIDBG0(AHCIDBG_INIT, ahci_ctlp,
35332fcbc377Syt 		    "cmd list dma mem alloc fail");
35342fcbc377Syt 		/* error.. free the dma handle. */
35352fcbc377Syt 		ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle);
35362fcbc377Syt 		return (AHCI_FAILURE);
35372fcbc377Syt 	}
35382fcbc377Syt 
35392fcbc377Syt 	if (ddi_dma_addr_bind_handle(ahci_portp->ahciport_cmd_list_dma_handle,
35402fcbc377Syt 	    NULL,
35412fcbc377Syt 	    (caddr_t)ahci_portp->ahciport_cmd_list,
35422fcbc377Syt 	    cmd_list_size,
35432fcbc377Syt 	    DDI_DMA_CONSISTENT,
35442fcbc377Syt 	    DDI_DMA_SLEEP,
35452fcbc377Syt 	    NULL,
35462fcbc377Syt 	    &cmd_list_dma_cookie,
35472fcbc377Syt 	    &cookie_count) !=  DDI_DMA_MAPPED) {
35482fcbc377Syt 
35492fcbc377Syt 		AHCIDBG0(AHCIDBG_INIT, ahci_ctlp,
35502fcbc377Syt 		    "cmd list dma handle bind fail");
35512fcbc377Syt 		/*  error.. free the dma handle & free the memory. */
35522fcbc377Syt 		ddi_dma_mem_free(&ahci_portp->ahciport_cmd_list_acc_handle);
35532fcbc377Syt 		ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle);
35542fcbc377Syt 		return (AHCI_FAILURE);
35552fcbc377Syt 	}
35562fcbc377Syt 
35572fcbc377Syt 	bzero((void *)ahci_portp->ahciport_cmd_list, cmd_list_size);
35582fcbc377Syt 
35592fcbc377Syt 	/* Config Port Command List Base Address */
35602fcbc377Syt 	ddi_put64(ahci_ctlp->ahcictl_ahci_acc_handle,
35612fcbc377Syt 	    (uint64_t *)AHCI_PORT_PxCLB(ahci_ctlp, port),
35622fcbc377Syt 	    cmd_list_dma_cookie.dmac_laddress);
35632fcbc377Syt 
35642fcbc377Syt 	AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, "64-bit, dma address: 0x%llx",
35652fcbc377Syt 	    cmd_list_dma_cookie.dmac_laddress);
35662fcbc377Syt 
35672fcbc377Syt 	AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, "32-bit, dma address: 0x%x",
35682fcbc377Syt 	    cmd_list_dma_cookie.dmac_address);
35692fcbc377Syt 
35702fcbc377Syt 	if (ahci_alloc_cmd_tables(ahci_ctlp, ahci_portp) != AHCI_SUCCESS) {
35712fcbc377Syt 		ahci_dealloc_cmd_list(ahci_ctlp, ahci_portp);
35722fcbc377Syt 		return (AHCI_FAILURE);
35732fcbc377Syt 	}
35742fcbc377Syt 
35752fcbc377Syt 	return (AHCI_SUCCESS);
35762fcbc377Syt }
35772fcbc377Syt 
35782fcbc377Syt /*
35792fcbc377Syt  * Deallocates the Command List
35802fcbc377Syt  *
35812fcbc377Syt  * WARNING!!! ahciport_mutex should be acquired before the function
35822fcbc377Syt  * is called.
35832fcbc377Syt  */
35842fcbc377Syt static void
35852fcbc377Syt ahci_dealloc_cmd_list(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
35862fcbc377Syt {
35872fcbc377Syt 	/* First dealloc command table */
35882fcbc377Syt 	ahci_dealloc_cmd_tables(ahci_ctlp, ahci_portp);
35892fcbc377Syt 
35902fcbc377Syt 	/* Unbind the cmd list dma handle first. */
35912fcbc377Syt 	(void) ddi_dma_unbind_handle(ahci_portp->ahciport_cmd_list_dma_handle);
35922fcbc377Syt 
35932fcbc377Syt 	/* Then free the underlying memory. */
35942fcbc377Syt 	ddi_dma_mem_free(&ahci_portp->ahciport_cmd_list_acc_handle);
35952fcbc377Syt 
35962fcbc377Syt 	/* Now free the handle itself. */
35972fcbc377Syt 	ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle);
35982fcbc377Syt }
35992fcbc377Syt 
36002fcbc377Syt /*
36012fcbc377Syt  * Allocates memory for all Command Tables, which contains Command FIS,
36022fcbc377Syt  * ATAPI Command and Physical Region Descriptor Table.
36032fcbc377Syt  *
36042fcbc377Syt  * WARNING!!! ahciport_mutex should be acquired before the function
36052fcbc377Syt  * is called.
36062fcbc377Syt  */
36072fcbc377Syt static int
36082fcbc377Syt ahci_alloc_cmd_tables(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
36092fcbc377Syt {
36102fcbc377Syt 	size_t ret_len;
36112fcbc377Syt 	ddi_dma_cookie_t cmd_table_dma_cookie;
36122fcbc377Syt 	uint_t cookie_count;
36132fcbc377Syt 	uint32_t cap_status;
36142fcbc377Syt 	int slot;
36152fcbc377Syt 
36162fcbc377Syt 	AHCIDBG1(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
36172fcbc377Syt 	    "ahci_alloc_cmd_tables: port %d enter",
36182fcbc377Syt 	    ahci_portp->ahciport_port_num);
36192fcbc377Syt 
36202fcbc377Syt 	/* Check whether the HBA can access 64-bit data structures */
36212fcbc377Syt 	cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
36222fcbc377Syt 	    (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp));
36232fcbc377Syt 
36242fcbc377Syt 	ahci_ctlp->ahcictl_cmd_table_dma_attr = cmd_table_dma_attr;
36252fcbc377Syt 
36262fcbc377Syt 	/*
36272fcbc377Syt 	 * If 64-bit addressing is not supported,
36282fcbc377Syt 	 * change dma_attr_addr_hi of ahcictl_cmd_table_dma_attr
36292fcbc377Syt 	 */
36302fcbc377Syt 	if (!(cap_status & AHCI_HBA_CAP_S64A)) {
36312fcbc377Syt 		AHCIDBG0(AHCIDBG_INIT, ahci_ctlp,
36322fcbc377Syt 		    "hba does not support 64-bit addressing");
36332fcbc377Syt 		ahci_ctlp->ahcictl_cmd_table_dma_attr.dma_attr_addr_hi =
36342fcbc377Syt 		    0xffffffffull;
36352fcbc377Syt 	}
36362fcbc377Syt 
36372fcbc377Syt 	for (slot = 0; slot < ahci_ctlp->ahcictl_num_cmd_slots; slot++) {
36382fcbc377Syt 		/* Allocate cmd table dma handle. */
36392fcbc377Syt 		if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip,
36402fcbc377Syt 		    &ahci_ctlp->ahcictl_cmd_table_dma_attr,
36412fcbc377Syt 		    DDI_DMA_SLEEP,
36422fcbc377Syt 		    NULL,
36432fcbc377Syt 		    &ahci_portp->ahciport_cmd_tables_dma_handle[slot]) !=
36442fcbc377Syt 		    DDI_SUCCESS) {
36452fcbc377Syt 
36462fcbc377Syt 			AHCIDBG0(AHCIDBG_INIT, ahci_ctlp,
36472fcbc377Syt 			    "cmd table dma handle alloc failed");
36482fcbc377Syt 
36492fcbc377Syt 			goto err_out;
36502fcbc377Syt 		}
36512fcbc377Syt 
36522fcbc377Syt 		if (ddi_dma_mem_alloc(
36532fcbc377Syt 		    ahci_portp->ahciport_cmd_tables_dma_handle[slot],
36542fcbc377Syt 		    ahci_cmd_table_size,
36552fcbc377Syt 		    &accattr,
36562fcbc377Syt 		    DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
36572fcbc377Syt 		    DDI_DMA_SLEEP,
36582fcbc377Syt 		    NULL,
36592fcbc377Syt 		    (caddr_t *)&ahci_portp->ahciport_cmd_tables[slot],
36602fcbc377Syt 		    &ret_len,
36612fcbc377Syt 		    &ahci_portp->ahciport_cmd_tables_acc_handle[slot]) !=
36622fcbc377Syt 		    NULL) {
36632fcbc377Syt 
36642fcbc377Syt 			AHCIDBG0(AHCIDBG_INIT, ahci_ctlp,
36652fcbc377Syt 			    "cmd table dma mem alloc fail");
36662fcbc377Syt 
36672fcbc377Syt 			/* error.. free the dma handle. */
36682fcbc377Syt 			ddi_dma_free_handle(
36692fcbc377Syt 			    &ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
36702fcbc377Syt 			goto err_out;
36712fcbc377Syt 		}
36722fcbc377Syt 
36732fcbc377Syt 		if (ddi_dma_addr_bind_handle(
36742fcbc377Syt 		    ahci_portp->ahciport_cmd_tables_dma_handle[slot],
36752fcbc377Syt 		    NULL,
36762fcbc377Syt 		    (caddr_t)ahci_portp->ahciport_cmd_tables[slot],
36772fcbc377Syt 		    ahci_cmd_table_size,
36782fcbc377Syt 		    DDI_DMA_CONSISTENT,
36792fcbc377Syt 		    DDI_DMA_SLEEP,
36802fcbc377Syt 		    NULL,
36812fcbc377Syt 		    &cmd_table_dma_cookie,
36822fcbc377Syt 		    &cookie_count) !=  DDI_DMA_MAPPED) {
36832fcbc377Syt 
36842fcbc377Syt 			AHCIDBG0(AHCIDBG_INIT, ahci_ctlp,
36852fcbc377Syt 			    "cmd table dma handle bind fail");
36862fcbc377Syt 			/*  error.. free the dma handle & free the memory. */
36872fcbc377Syt 			ddi_dma_mem_free(
36882fcbc377Syt 			    &ahci_portp->ahciport_cmd_tables_acc_handle[slot]);
36892fcbc377Syt 			ddi_dma_free_handle(
36902fcbc377Syt 			    &ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
36912fcbc377Syt 			goto err_out;
36922fcbc377Syt 		}
36932fcbc377Syt 
36942fcbc377Syt 		bzero((void *)ahci_portp->ahciport_cmd_tables[slot],
36952fcbc377Syt 		    ahci_cmd_table_size);
36962fcbc377Syt 
36972fcbc377Syt 		/* Config Port Command Table Base Address */
36982fcbc377Syt 		SET_COMMAND_TABLE_BASE_ADDR(
36992fcbc377Syt 		    (&ahci_portp->ahciport_cmd_list[slot]),
37002fcbc377Syt 		    cmd_table_dma_cookie.dmac_laddress & 0xffffffffull);
37012fcbc377Syt 
37022fcbc377Syt #ifndef __lock_lint
37032fcbc377Syt 		SET_COMMAND_TABLE_BASE_ADDR_UPPER(
37042fcbc377Syt 		    (&ahci_portp->ahciport_cmd_list[slot]),
37052fcbc377Syt 		    cmd_table_dma_cookie.dmac_laddress >> 32);
37062fcbc377Syt #endif /* __lock_lint */
37072fcbc377Syt 	}
37082fcbc377Syt 
37092fcbc377Syt 	return (AHCI_SUCCESS);
37102fcbc377Syt err_out:
37112fcbc377Syt 
37122fcbc377Syt 	for (slot--; slot >= 0; slot--) {
37132fcbc377Syt 		/* Unbind the cmd table dma handle first */
37142fcbc377Syt 		(void) ddi_dma_unbind_handle(
37152fcbc377Syt 		    ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
37162fcbc377Syt 
37172fcbc377Syt 		/* Then free the underlying memory */
37182fcbc377Syt 		ddi_dma_mem_free(
37192fcbc377Syt 		    &ahci_portp->ahciport_cmd_tables_acc_handle[slot]);
37202fcbc377Syt 
37212fcbc377Syt 		/* Now free the handle itself */
37222fcbc377Syt 		ddi_dma_free_handle(
37232fcbc377Syt 		    &ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
37242fcbc377Syt 	}
37252fcbc377Syt 
37262fcbc377Syt 	return (AHCI_FAILURE);
37272fcbc377Syt }
37282fcbc377Syt 
37292fcbc377Syt /*
37302fcbc377Syt  * Deallocates memory for all Command Tables.
37312fcbc377Syt  *
37322fcbc377Syt  * WARNING!!! ahciport_mutex should be acquired before the function
37332fcbc377Syt  * is called.
37342fcbc377Syt  */
37352fcbc377Syt static void
37362fcbc377Syt ahci_dealloc_cmd_tables(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
37372fcbc377Syt {
37382fcbc377Syt 	int slot;
37392fcbc377Syt 
37402fcbc377Syt 	AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp,
37412fcbc377Syt 	    "ahci_dealloc_cmd_tables: %d enter",
37422fcbc377Syt 	    ahci_portp->ahciport_port_num);
37432fcbc377Syt 
37442fcbc377Syt 	for (slot = 0; slot < ahci_ctlp->ahcictl_num_cmd_slots; slot++) {
37452fcbc377Syt 		/* Unbind the cmd table dma handle first. */
37462fcbc377Syt 		(void) ddi_dma_unbind_handle(
37472fcbc377Syt 		    ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
37482fcbc377Syt 
37492fcbc377Syt 		/* Then free the underlying memory. */
37502fcbc377Syt 		ddi_dma_mem_free(
37512fcbc377Syt 		    &ahci_portp->ahciport_cmd_tables_acc_handle[slot]);
37522fcbc377Syt 
37532fcbc377Syt 		/* Now free the handle itself. */
37542fcbc377Syt 		ddi_dma_free_handle(
37552fcbc377Syt 		    &ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
37562fcbc377Syt 	}
37572fcbc377Syt }
37582fcbc377Syt 
37592fcbc377Syt /*
37602fcbc377Syt  * WARNING!!! ahciport_mutex should be acquired before the function
37612fcbc377Syt  * is called.
37622fcbc377Syt  */
37632fcbc377Syt static void
37642fcbc377Syt ahci_update_sata_registers(ahci_ctl_t *ahci_ctlp, uint8_t port,
37652fcbc377Syt     sata_device_t *sd)
37662fcbc377Syt {
37672fcbc377Syt 	sd->satadev_scr.sstatus =
37682fcbc377Syt 	    ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
37692fcbc377Syt 	    (uint32_t *)(AHCI_PORT_PxSSTS(ahci_ctlp, port)));
37702fcbc377Syt 	sd->satadev_scr.serror =
37712fcbc377Syt 	    ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
37722fcbc377Syt 	    (uint32_t *)(AHCI_PORT_PxSERR(ahci_ctlp, port)));
37732fcbc377Syt 	sd->satadev_scr.scontrol =
37742fcbc377Syt 	    ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
37752fcbc377Syt 	    (uint32_t *)(AHCI_PORT_PxSCTL(ahci_ctlp, port)));
37762fcbc377Syt 	sd->satadev_scr.sactive =
37772fcbc377Syt 	    ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
37782fcbc377Syt 	    (uint32_t *)(AHCI_PORT_PxSACT(ahci_ctlp, port)));
37792fcbc377Syt }
37802fcbc377Syt 
37812fcbc377Syt /*
3782*68d33a25Syt  * For poll mode, ahci_port_intr will be called to emulate the interrupt
37832fcbc377Syt  *
3784*68d33a25Syt  * retrieve_errinfo_slot_status keeps the slot number of REQUEST SENSE command
3785*68d33a25Syt  * which is sent down to get sense data for a failed PACKET command.
37862fcbc377Syt  */
37872fcbc377Syt static void
3788*68d33a25Syt ahci_port_intr(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
3789*68d33a25Syt     uint8_t port, uint32_t retrieve_errinfo_slot_status)
37902fcbc377Syt {
3791*68d33a25Syt 	uint32_t port_intr_status;
3792*68d33a25Syt 	uint32_t port_intr_enable;
3793*68d33a25Syt 
3794*68d33a25Syt 	AHCIDBG1(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp,
3795*68d33a25Syt 	    "ahci_port_intr enter: port %d", port);
3796*68d33a25Syt 
3797*68d33a25Syt 	mutex_enter(&ahci_portp->ahciport_mutex);
3798*68d33a25Syt 	if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_POLLING) {
3799*68d33a25Syt 		/* For SATA_OPMODE_POLLING commands */
3800*68d33a25Syt 		port_intr_enable =
3801*68d33a25Syt 		    (AHCI_INTR_STATUS_DHRS |
3802*68d33a25Syt 		    AHCI_INTR_STATUS_PSS |
3803*68d33a25Syt 		    AHCI_INTR_STATUS_SDBS |
3804*68d33a25Syt 		    AHCI_INTR_STATUS_UFS |
3805*68d33a25Syt 		    AHCI_INTR_STATUS_PCS |
3806*68d33a25Syt 		    AHCI_INTR_STATUS_PRCS |
3807*68d33a25Syt 		    AHCI_INTR_STATUS_OFS |
3808*68d33a25Syt 		    AHCI_INTR_STATUS_INFS |
3809*68d33a25Syt 		    AHCI_INTR_STATUS_IFS |
3810*68d33a25Syt 		    AHCI_INTR_STATUS_HBDS |
3811*68d33a25Syt 		    AHCI_INTR_STATUS_HBFS |
3812*68d33a25Syt 		    AHCI_INTR_STATUS_TFES);
3813*68d33a25Syt 		mutex_exit(&ahci_portp->ahciport_mutex);
3814*68d33a25Syt 		goto next;
3815*68d33a25Syt 	}
3816*68d33a25Syt 	mutex_exit(&ahci_portp->ahciport_mutex);
3817*68d33a25Syt 
3818*68d33a25Syt 	/*
3819*68d33a25Syt 	 * port_intr_enable indicates that the corresponding interrrupt
3820*68d33a25Syt 	 * reporting is enabled.
3821*68d33a25Syt 	 */
3822*68d33a25Syt 	port_intr_enable = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3823*68d33a25Syt 	    (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port));
3824*68d33a25Syt next:
3825*68d33a25Syt 	/*
3826*68d33a25Syt 	 * port_intr_stats indicates that the corresponding interrupt
3827*68d33a25Syt 	 * condition is active.
3828*68d33a25Syt 	 */
3829*68d33a25Syt 	port_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3830*68d33a25Syt 	    (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port));
3831*68d33a25Syt 
3832*68d33a25Syt 	AHCIDBG4(AHCIDBG_INTR, ahci_ctlp,
3833*68d33a25Syt 	    "ahci_port_intr: port %d, port_intr_status = 0x%x, "
3834*68d33a25Syt 	    "port_intr_enable = 0x%x, retrieve_errinfo_slot_status = 0x%x",
3835*68d33a25Syt 	    port, port_intr_status, port_intr_enable,
3836*68d33a25Syt 	    retrieve_errinfo_slot_status);
3837*68d33a25Syt 
3838*68d33a25Syt 	port_intr_status &= port_intr_enable;
3839*68d33a25Syt 
3840*68d33a25Syt 	/* First clear the port interrupts status */
3841*68d33a25Syt 	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
3842*68d33a25Syt 	    (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port),
3843*68d33a25Syt 	    port_intr_status);
3844*68d33a25Syt 
3845*68d33a25Syt 	/* Check the completed commands */
3846*68d33a25Syt 	if (port_intr_status & (AHCI_INTR_STATUS_DHRS |
3847*68d33a25Syt 	    AHCI_INTR_STATUS_PSS)) {
3848*68d33a25Syt 		(void) ahci_intr_cmd_cmplt(ahci_ctlp,
3849*68d33a25Syt 		    ahci_portp, port, port_intr_status,
3850*68d33a25Syt 		    retrieve_errinfo_slot_status);
3851*68d33a25Syt 	}
3852*68d33a25Syt 
3853*68d33a25Syt 	/* Check the asynchronous notification */
3854*68d33a25Syt 	if (port_intr_status & AHCI_INTR_STATUS_SDBS) {
3855*68d33a25Syt 		(void) ahci_intr_set_device_bits(ahci_ctlp,
3856*68d33a25Syt 		    ahci_portp, port);
3857*68d33a25Syt 	}
3858*68d33a25Syt 
3859*68d33a25Syt 	/* Check the port connect change status interrupt bit */
3860*68d33a25Syt 	if (port_intr_status & AHCI_INTR_STATUS_PCS) {
3861*68d33a25Syt 		(void) ahci_intr_port_connect_change(ahci_ctlp,
3862*68d33a25Syt 		    ahci_portp, port);
3863*68d33a25Syt 	}
3864*68d33a25Syt 
3865*68d33a25Syt 	/* Check the device mechanical presence status interrupt bit */
3866*68d33a25Syt 	if (port_intr_status & AHCI_INTR_STATUS_DMPS) {
3867*68d33a25Syt 		(void) ahci_intr_device_mechanical_presence_status(
3868*68d33a25Syt 		    ahci_ctlp, ahci_portp, port);
3869*68d33a25Syt 	}
3870*68d33a25Syt 
3871*68d33a25Syt 	/* Check the PhyRdy change status interrupt bit */
3872*68d33a25Syt 	if (port_intr_status & AHCI_INTR_STATUS_PRCS) {
3873*68d33a25Syt 		(void) ahci_intr_phyrdy_change(ahci_ctlp, ahci_portp,
3874*68d33a25Syt 		    port);
38752fcbc377Syt 	}
3876*68d33a25Syt 
3877*68d33a25Syt 	/*
3878*68d33a25Syt 	 * Check the non-fatal error interrupt bits, there are three
3879*68d33a25Syt 	 * kinds of non-fatal errors at the time being:
3880*68d33a25Syt 	 *
3881*68d33a25Syt 	 *    PxIS.UFS - Unknown FIS Error
3882*68d33a25Syt 	 *    PxIS.OFS - Overflow Error
3883*68d33a25Syt 	 *    PxIS.INFS - Interface Non-Fatal Error
3884*68d33a25Syt 	 *
3885*68d33a25Syt 	 * For these non-fatal errors, the HBA can continue to operate,
3886*68d33a25Syt 	 * so the driver just log the error messages.
3887*68d33a25Syt 	 */
3888*68d33a25Syt 	if (port_intr_status & (AHCI_INTR_STATUS_UFS |
3889*68d33a25Syt 	    AHCI_INTR_STATUS_OFS |
3890*68d33a25Syt 	    AHCI_INTR_STATUS_INFS)) {
3891*68d33a25Syt 		(void) ahci_intr_non_fatal_error(ahci_ctlp, ahci_portp,
3892*68d33a25Syt 		    port, port_intr_status);
3893*68d33a25Syt 	}
3894*68d33a25Syt 
3895*68d33a25Syt 	/*
3896*68d33a25Syt 	 * Check the fatal error interrupt bits, there are four kinds
3897*68d33a25Syt 	 * of fatal errors for AHCI controllers:
3898*68d33a25Syt 	 *
3899*68d33a25Syt 	 *    PxIS.HBFS - Host Bus Fatal Error
3900*68d33a25Syt 	 *    PxIS.HBDS - Host Bus Data Error
3901*68d33a25Syt 	 *    PxIS.IFS - Interface Fatal Error
3902*68d33a25Syt 	 *    PxIS.TFES - Task File Error
3903*68d33a25Syt 	 *
3904*68d33a25Syt 	 * The fatal error means the HBA can not recover from it by
3905*68d33a25Syt 	 * itself, and it will try to abort the transfer, and the software
3906*68d33a25Syt 	 * must intervene to restart the port.
3907*68d33a25Syt 	 */
3908*68d33a25Syt 	if (port_intr_status & (AHCI_INTR_STATUS_IFS |
3909*68d33a25Syt 	    AHCI_INTR_STATUS_HBDS |
3910*68d33a25Syt 	    AHCI_INTR_STATUS_HBFS |
3911*68d33a25Syt 	    AHCI_INTR_STATUS_TFES))
3912*68d33a25Syt 		(void) ahci_intr_fatal_error(ahci_ctlp, ahci_portp,
3913*68d33a25Syt 		    port, port_intr_status, retrieve_errinfo_slot_status);
3914*68d33a25Syt 
3915*68d33a25Syt 	/* Check the cold port detect interrupt bit */
3916*68d33a25Syt 	if (port_intr_status & AHCI_INTR_STATUS_CPDS) {
3917*68d33a25Syt 		(void) ahci_intr_cold_port_detect(ahci_ctlp, ahci_portp, port);
3918*68d33a25Syt 	}
3919*68d33a25Syt 
3920*68d33a25Syt 	/* Second clear the corresponding bit in IS.IPS */
3921*68d33a25Syt 	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
3922*68d33a25Syt 	    (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp), (0x1 << port));
39232fcbc377Syt }
39242fcbc377Syt 
39252fcbc377Syt /*
39262fcbc377Syt  * Interrupt service handler
39272fcbc377Syt  */
39282fcbc377Syt /* ARGSUSED1 */
39292fcbc377Syt static uint_t
39302fcbc377Syt ahci_intr(caddr_t arg1, caddr_t arg2)
39312fcbc377Syt {
39322fcbc377Syt 	ahci_ctl_t *ahci_ctlp = (ahci_ctl_t *)arg1;
39332fcbc377Syt 	ahci_port_t *ahci_portp;
3934*68d33a25Syt 	int32_t global_intr_status;
39352fcbc377Syt 	uint8_t port;
39362fcbc377Syt 
39372fcbc377Syt 	/*
39382fcbc377Syt 	 * global_intr_status indicates that the corresponding port has
39392fcbc377Syt 	 * an interrupt pending.
39402fcbc377Syt 	 */
39412fcbc377Syt 	global_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
39422fcbc377Syt 	    (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp));
39432fcbc377Syt 
39442fcbc377Syt 	if (!(global_intr_status & ahci_ctlp->ahcictl_ports_implemented)) {
39452fcbc377Syt 		/* The interrupt is not ours */
39462fcbc377Syt 		return (DDI_INTR_UNCLAIMED);
39472fcbc377Syt 	}
39482fcbc377Syt 
39492fcbc377Syt 	/* Loop for all the ports */
39502fcbc377Syt 	for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
39512fcbc377Syt 		if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
39522fcbc377Syt 			continue;
39532fcbc377Syt 		}
39542fcbc377Syt 		if (!((0x1 << port) & global_intr_status)) {
39552fcbc377Syt 			continue;
39562fcbc377Syt 		}
39572fcbc377Syt 
39582fcbc377Syt 		ahci_portp = ahci_ctlp->ahcictl_ports[port];
39592fcbc377Syt 
3960*68d33a25Syt 		/* Call ahci_port_intr */
3961*68d33a25Syt 		ahci_port_intr(ahci_ctlp, ahci_portp, port, 0);
39622fcbc377Syt 	}
39632fcbc377Syt 
39642fcbc377Syt 	return (DDI_INTR_CLAIMED);
39652fcbc377Syt }
39662fcbc377Syt 
39672fcbc377Syt /*
3968*68d33a25Syt  * For non-queued commands, when the corresponding bit in the PxCI register
3969*68d33a25Syt  * is cleared, it means the command is completed successfully. And according
3970*68d33a25Syt  * to the HBA state machine, there are three conditions which possibly will
3971*68d33a25Syt  * try to clear the PxCI register bit.
3972*68d33a25Syt  *	1. Receive one D2H Register FIS which is with 'I' bit set
3973*68d33a25Syt  *	2. Update PIO Setup FIS
3974*68d33a25Syt  *	3. Transmit a command and receive R_OK if CTBA.C is set (software reset)
3975*68d33a25Syt  *
3976*68d33a25Syt  * Process completed commands including non-queued commands when the
3977*68d33a25Syt  * interrupt status bit - AHCI_INTR_STATUS_DHRS or AHCI_INTR_STATUS_PSS
3978*68d33a25Syt  * is set, and queued commands when the interrupt bit -
3979*68d33a25Syt  * AHCI_INTR_STATUS_SDBS is set. Currently, the driver doesn't support
3980*68d33a25Syt  * queued commands yet.
39812fcbc377Syt  *
3982*68d33a25Syt  * AHCI_INTR_STATUS_DHRS means a D2H Register FIS has been received
3983*68d33a25Syt  * with the 'I' bit set. And the following commands will send thus
3984*68d33a25Syt  * FIS with 'I' bit set upon the successful completion:
39852fcbc377Syt  * 	1. Non-data commands
39862fcbc377Syt  * 	2. DMA data-in command
39872fcbc377Syt  * 	3. DMA data-out command
39882fcbc377Syt  * 	4. PIO data-out command
3989*68d33a25Syt  *	5. PACKET non-data commands
3990*68d33a25Syt  *	6. PACKET PIO data-in command
3991*68d33a25Syt  *	7. PACKET PIO data-out command
3992*68d33a25Syt  *	8. PACKET DMA data-in command
3993*68d33a25Syt  *	9. PACKET DMA data-out command
3994*68d33a25Syt  *
3995*68d33a25Syt  * AHCI_INTR_STATUS_PSS means a PIO Setup FIS has been received
3996*68d33a25Syt  * with the 'I' bit set. And the following commands will send this
3997*68d33a25Syt  * FIS upon the successful completion:
3998*68d33a25Syt  * 	1. PIO data-in command
39992fcbc377Syt  */
40002fcbc377Syt static int
4001*68d33a25Syt ahci_intr_cmd_cmplt(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
4002*68d33a25Syt     uint8_t port, uint32_t intr_status, uint32_t retrieve_errinfo_slot_status)
40032fcbc377Syt {
4004*68d33a25Syt 	uint32_t port_cmd_issue = 0;
40052fcbc377Syt 	uint32_t finished_tags;
4006*68d33a25Syt 	int finished_slot;
40072fcbc377Syt 	sata_pkt_t *satapkt;
40082fcbc377Syt 	ahci_fis_d2h_register_t *rcvd_fisp;
40092fcbc377Syt 
4010*68d33a25Syt 	if (intr_status & AHCI_INTR_STATUS_DHRS)
4011*68d33a25Syt 		AHCIDBG1(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp,
4012*68d33a25Syt 		    "ahci_intr_cmd_cmplt enter: port %d "
4013*68d33a25Syt 		    "a d2h register fis is received", port);
40142fcbc377Syt 
4015*68d33a25Syt 	if (intr_status & AHCI_INTR_STATUS_PSS)
4016*68d33a25Syt 		AHCIDBG1(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp,
4017*68d33a25Syt 		    "ahci_intr_cmd_cmplt enter: port %d "
4018*68d33a25Syt 		    "a pio setup fis is received", port);
40192fcbc377Syt 
4020*68d33a25Syt 	mutex_enter(&ahci_portp->ahciport_mutex);
40212fcbc377Syt 	if (!ahci_portp->ahciport_pending_tags) {
40222fcbc377Syt 		/*
40232fcbc377Syt 		 * Spurious interrupt. Nothing to be done.
40242fcbc377Syt 		 */
40252fcbc377Syt 		mutex_exit(&ahci_portp->ahciport_mutex);
40262fcbc377Syt 		return (AHCI_SUCCESS);
40272fcbc377Syt 	}
40282fcbc377Syt 
4029*68d33a25Syt 	port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4030*68d33a25Syt 	    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
40312fcbc377Syt 
40322fcbc377Syt 	/*
4033*68d33a25Syt 	 * When AHCI_PORT_FLAG_RQSENSE is set, the port is doing error recovery,
4034*68d33a25Syt 	 * and REQUEST SENSE for sense data retrieval is sent down to get the
4035*68d33a25Syt 	 * sense data. At this time, the only pending command is REQUESR SENSE,
4036*68d33a25Syt 	 * and retrieve_errinfo_slot_status contains the slot number of it.
40372fcbc377Syt 	 */
4038*68d33a25Syt 	if (retrieve_errinfo_slot_status != 0) {
4039*68d33a25Syt 		ASSERT(ahci_portp->ahciport_flags & AHCI_PORT_FLAG_RQSENSE);
4040*68d33a25Syt 		finished_tags = ahci_portp->ahciport_pending_tags &
4041*68d33a25Syt 		    retrieve_errinfo_slot_status &
4042*68d33a25Syt 		    ~port_cmd_issue &
4043*68d33a25Syt 		    AHCI_SLOT_MASK(ahci_ctlp);
4044*68d33a25Syt 	} else {
4045*68d33a25Syt 		finished_tags = ahci_portp->ahciport_pending_tags &
4046*68d33a25Syt 		    ~port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp);
40472fcbc377Syt 	}
40482fcbc377Syt 
4049*68d33a25Syt 	AHCIDBG3(AHCIDBG_INTR, ahci_ctlp,
4050*68d33a25Syt 	    "ahci_intr_cmd_cmplt: pending_tags = 0x%x, port_cmd_issue = 0x%x "
4051*68d33a25Syt 	    "retrieve_errinfo_slot_status = 0x%x",
4052*68d33a25Syt 	    ahci_portp->ahciport_pending_tags, port_cmd_issue,
4053*68d33a25Syt 	    retrieve_errinfo_slot_status);
40542fcbc377Syt 
40552fcbc377Syt 	AHCIDBG1(AHCIDBG_INTR, ahci_ctlp,
4056*68d33a25Syt 	    "ahci_intr_cmd_cmplt: finished_tags = 0x%x", finished_tags);
40572fcbc377Syt 
4058*68d33a25Syt 	while (finished_tags) {
4059*68d33a25Syt 		finished_slot = ddi_ffs(finished_tags) - 1;
4060*68d33a25Syt 		if (finished_slot == -1) {
4061*68d33a25Syt 			goto out;
4062*68d33a25Syt 		}
40632fcbc377Syt 
4064*68d33a25Syt 		satapkt = ahci_portp->ahciport_slot_pkts[finished_slot];
4065*68d33a25Syt 		ASSERT(satapkt != NULL);
40662fcbc377Syt 
40672fcbc377Syt 		/*
4068*68d33a25Syt 		 * For SATAC_SMART command with SATA_SMART_RETURN_STATUS
4069*68d33a25Syt 		 * feature, sata_special_regs flag will be set, and the
4070*68d33a25Syt 		 * driver should copy the status and the other corresponding
4071*68d33a25Syt 		 * register values in the D2H Register FIS received (It's
4072*68d33a25Syt 		 * working on Non-data protocol) from the device back to
4073*68d33a25Syt 		 * the sata_cmd.
4074*68d33a25Syt 		 *
4075*68d33a25Syt 		 * For every AHCI port, there is only one Received FIS
4076*68d33a25Syt 		 * structure, which contains the FISes received from the
4077*68d33a25Syt 		 * device, So we're trying to copy the content of D2H
4078*68d33a25Syt 		 * Register FIS in the Received FIS structure back to
4079*68d33a25Syt 		 * the sata_cmd.
40802fcbc377Syt 		 */
4081*68d33a25Syt 		if (satapkt->satapkt_cmd.satacmd_flags.sata_special_regs) {
4082*68d33a25Syt 			rcvd_fisp = &(ahci_portp->ahciport_rcvd_fis->
4083*68d33a25Syt 			    ahcirf_d2h_register_fis);
4084*68d33a25Syt 			satapkt->satapkt_cmd.satacmd_status_reg =
4085*68d33a25Syt 			    GET_RFIS_STATUS(rcvd_fisp);
4086*68d33a25Syt 			ahci_copy_out_regs(&satapkt->satapkt_cmd, rcvd_fisp);
4087*68d33a25Syt 		}
40882fcbc377Syt 
4089*68d33a25Syt 		AHCIDBG1(AHCIDBG_INTR, ahci_ctlp,
4090*68d33a25Syt 		    "ahci_intr_cmd_cmplt: sending up pkt 0x%p "
4091*68d33a25Syt 		    "with SATA_PKT_COMPLETED", (void *)satapkt);
40922fcbc377Syt 
4093*68d33a25Syt 		CLEAR_BIT(ahci_portp->ahciport_pending_tags, finished_slot);
4094*68d33a25Syt 		CLEAR_BIT(finished_tags, finished_slot);
4095*68d33a25Syt 		ahci_portp->ahciport_slot_pkts[finished_slot] = NULL;
40962fcbc377Syt 
4097*68d33a25Syt 		SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_COMPLETED);
40982fcbc377Syt 	}
40992fcbc377Syt out:
41002fcbc377Syt 	AHCIDBG1(AHCIDBG_PKTCOMP, ahci_ctlp,
4101*68d33a25Syt 	    "ahci_intr_cmd_cmplt: pending_tags = 0x%x",
41022fcbc377Syt 	    ahci_portp->ahciport_pending_tags);
41032fcbc377Syt 
41042fcbc377Syt 	mutex_exit(&ahci_portp->ahciport_mutex);
41052fcbc377Syt 
41062fcbc377Syt 	return (AHCI_SUCCESS);
41072fcbc377Syt }
41082fcbc377Syt 
41092fcbc377Syt /*
4110*68d33a25Syt  * AHCI_INTR_STATUS_SDBS means a Set Device Bits FIS has been received
4111*68d33a25Syt  * with the 'I' bit set and has been copied into system memory.
41122fcbc377Syt  *
4113*68d33a25Syt  * Asynchronous Notification is a feature in SATA II, which allows an
4114*68d33a25Syt  * ATAPI device to send a signal to the host when media is inserted or
4115*68d33a25Syt  * removed and avoids polling the device for media changes. The signal
4116*68d33a25Syt  * sent to the host is a Set Device Bits FIS with the 'I' and 'N' bits
4117*68d33a25Syt  * set to '1'.
41182fcbc377Syt  */
41192fcbc377Syt static int
4120*68d33a25Syt ahci_intr_set_device_bits(ahci_ctl_t *ahci_ctlp,
41212fcbc377Syt     ahci_port_t *ahci_portp, uint8_t port)
41222fcbc377Syt {
4123*68d33a25Syt 	ahci_fis_set_device_bits_t *rcvd_fisp;
41242fcbc377Syt 
4125*68d33a25Syt 	AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp,
4126*68d33a25Syt 	    "ahci_intr_set_device_bits enter: port %d", port);
41272fcbc377Syt 
4128*68d33a25Syt 	rcvd_fisp = &(ahci_portp->ahciport_rcvd_fis->
4129*68d33a25Syt 	    ahcirf_set_device_bits_fis);
4130*68d33a25Syt 
4131*68d33a25Syt 	if (GET_N_BIT_OF_SET_DEV_BITS(rcvd_fisp)) {
4132*68d33a25Syt 		AHCIDBG1(AHCIDBG_INTR, ahci_ctlp,
4133*68d33a25Syt 		    "ahci_intr_set_device_bits N bit is set for "
4134*68d33a25Syt 		    "port %d", port);
4135*68d33a25Syt 	}
41362fcbc377Syt 
41372fcbc377Syt 	return (AHCI_SUCCESS);
41382fcbc377Syt }
41392fcbc377Syt 
41402fcbc377Syt /*
4141*68d33a25Syt  * 1=Change in Current Connect Status. 0=No change in Current Connect Status.
4142*68d33a25Syt  * This bit reflects the state of PxSERR.DIAG.X. This bit is only cleared
4143*68d33a25Syt  * when PxSERR.DIAG.X is cleared. When PxSERR.DIAG.X is set to one, it
4144*68d33a25Syt  * indicates a COMINIT signal was received.
41452fcbc377Syt  *
4146*68d33a25Syt  * Hot plug insertion is detected by reception of a COMINIT signal from the
4147*68d33a25Syt  * device. On reception of unsolicited COMINIT, the HBA shall generate a
4148*68d33a25Syt  * COMRESET. If the COMINIT is in responce to a COMRESET, then the HBA shall
4149*68d33a25Syt  * begin the normal communication negotiation sequence as outlined in the
4150*68d33a25Syt  * Serial ATA 1.0a specification. When a COMRESET is sent to the device the
4151*68d33a25Syt  * PxSSTS.DET field shall be cleared to 0h. When a COMINIT is received, the
4152*68d33a25Syt  * PxSSTS.DET field shall be set to 1h. When the communication negotiation
4153*68d33a25Syt  * sequence is complete and PhyRdy is true the PxSSTS.DET field	shall be set
4154*68d33a25Syt  * to 3h. Therefore, at the moment the ahci driver is going to check PhyRdy
4155*68d33a25Syt  * to handle hot plug insertion. In this interrupt handler, just do nothing
4156*68d33a25Syt  * but print some log message and clear the bit.
41572fcbc377Syt  */
41582fcbc377Syt static int
4159*68d33a25Syt ahci_intr_port_connect_change(ahci_ctl_t *ahci_ctlp,
41602fcbc377Syt     ahci_port_t *ahci_portp, uint8_t port)
41612fcbc377Syt {
41622fcbc377Syt 	uint32_t port_serror;
41632fcbc377Syt 
41642fcbc377Syt 	mutex_enter(&ahci_portp->ahciport_mutex);
41652fcbc377Syt 
41662fcbc377Syt 	port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
41672fcbc377Syt 	    (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port));
41682fcbc377Syt 
41692fcbc377Syt 	AHCIDBG2(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp,
4170*68d33a25Syt 	    "ahci_intr_port_connect_change: port %d, "
4171*68d33a25Syt 	    "port_serror = 0x%x", port, port_serror);
41722fcbc377Syt 
4173*68d33a25Syt 	/* Clear PxSERR.DIAG.X to clear the interrupt bit */
41742fcbc377Syt 	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
41752fcbc377Syt 	    (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
4176*68d33a25Syt 	    AHCI_SERROR_DIAG_X);
41772fcbc377Syt 
41782fcbc377Syt 	mutex_exit(&ahci_portp->ahciport_mutex);
41792fcbc377Syt 
41802fcbc377Syt 	return (AHCI_SUCCESS);
41812fcbc377Syt }
41822fcbc377Syt 
41832fcbc377Syt /*
41842fcbc377Syt  * Hot Plug Operation for platforms that support Mechanical Presence
41852fcbc377Syt  * Switches.
41862fcbc377Syt  *
41872fcbc377Syt  * When set, it indicates that a mechanical presence switch attached to this
41882fcbc377Syt  * port has been opened or closed, which may lead to a change in the connection
41892fcbc377Syt  * state of the device. This bit is only valid if both CAP.SMPS and PxCMD.MPSP
41902fcbc377Syt  * are set to '1'.
41912fcbc377Syt  *
4192*68d33a25Syt  * At the moment, this interrupt is not needed and disabled and we just log
41932fcbc377Syt  * the debug message.
41942fcbc377Syt  */
41952fcbc377Syt /* ARGSUSED */
41962fcbc377Syt static int
41972fcbc377Syt ahci_intr_device_mechanical_presence_status(ahci_ctl_t *ahci_ctlp,
41982fcbc377Syt     ahci_port_t *ahci_portp, uint8_t port)
41992fcbc377Syt {
42002fcbc377Syt 	uint32_t cap_status, port_cmd_status;
42012fcbc377Syt 
42022fcbc377Syt 	AHCIDBG1(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp,
42032fcbc377Syt 	    "ahci_intr_device_mechanical_presence_status enter, "
42042fcbc377Syt 	    "port %d", port);
42052fcbc377Syt 
42062fcbc377Syt 	cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
42072fcbc377Syt 	    (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp));
42082fcbc377Syt 
42092fcbc377Syt 	mutex_enter(&ahci_portp->ahciport_mutex);
42102fcbc377Syt 	port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
42112fcbc377Syt 	    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
42122fcbc377Syt 
42132fcbc377Syt 	if (!(cap_status & AHCI_HBA_CAP_SMPS) ||
42142fcbc377Syt 	    !(port_cmd_status & AHCI_CMD_STATUS_MPSP)) {
42152fcbc377Syt 		AHCIDBG2(AHCIDBG_INTR, ahci_ctlp,
42162fcbc377Syt 		    "CAP.SMPS or PxCMD.MPSP is not set, so just ignore "
42172fcbc377Syt 		    "the interrupt: cap_status = 0x%x, "
42182fcbc377Syt 		    "port_cmd_status = 0x%x", cap_status, port_cmd_status);
42192fcbc377Syt 		mutex_exit(&ahci_portp->ahciport_mutex);
42202fcbc377Syt 
42212fcbc377Syt 		return (AHCI_SUCCESS);
42222fcbc377Syt 	}
42232fcbc377Syt 
42242fcbc377Syt 	if (port_cmd_status & AHCI_CMD_STATUS_MPSS) {
42252fcbc377Syt 		AHCIDBG2(AHCIDBG_INTR, ahci_ctlp,
42262fcbc377Syt 		    "The mechanical presence switch is open: "
42272fcbc377Syt 		    "port %d, port_cmd_status = 0x%x",
42282fcbc377Syt 		    port, port_cmd_status);
42292fcbc377Syt 	} else {
42302fcbc377Syt 		AHCIDBG2(AHCIDBG_INTR, ahci_ctlp,
42312fcbc377Syt 		    "The mechanical presence switch is close: "
42322fcbc377Syt 		    "port %d, port_cmd_status = 0x%x",
42332fcbc377Syt 		    port, port_cmd_status);
42342fcbc377Syt 	}
42352fcbc377Syt 
42362fcbc377Syt 	mutex_exit(&ahci_portp->ahciport_mutex);
42372fcbc377Syt 
42382fcbc377Syt 	return (AHCI_SUCCESS);
42392fcbc377Syt }
42402fcbc377Syt 
42412fcbc377Syt /*
42422fcbc377Syt  * Native Hot Plug Support.
42432fcbc377Syt  *
42442fcbc377Syt  * When set, it indicates that the internal PHYRDY signal changed state.
42452fcbc377Syt  * This bit reflects the state of PxSERR.DIAG.N.
4246*68d33a25Syt  *
4247*68d33a25Syt  * There are three kinds of conditions to generate this interrupt event:
4248*68d33a25Syt  * 1. a device is inserted
4249*68d33a25Syt  * 2. a device is disconnected
4250*68d33a25Syt  * 3. when the link enters/exits a Partial or Slumber interface power
4251*68d33a25Syt  *    management state
4252*68d33a25Syt  *
4253*68d33a25Syt  * If inteface power management is enabled for a port, the PxSERR.DIAG.N
4254*68d33a25Syt  * bit may be set due to the link entering the Partial or Slumber power
4255*68d33a25Syt  * management state, rather than due to a hot plug insertion or removal
4256*68d33a25Syt  * event. So far, the power management is disabled, so the driver can
4257*68d33a25Syt  * reliably get removal detection notification via the PxSERR.DIAG.N bit.
42582fcbc377Syt  */
42592fcbc377Syt static int
42602fcbc377Syt ahci_intr_phyrdy_change(ahci_ctl_t *ahci_ctlp,
42612fcbc377Syt     ahci_port_t *ahci_portp, uint8_t port)
42622fcbc377Syt {
42632fcbc377Syt 	uint32_t port_sstatus = 0; /* No dev present & PHY not established. */
42642fcbc377Syt 	sata_device_t sdevice;
42652fcbc377Syt 	int dev_exists_now = 0;
42662fcbc377Syt 	int dev_existed_previously = 0;
42672fcbc377Syt 
42682fcbc377Syt 	AHCIDBG1(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp,
42692fcbc377Syt 	    "ahci_intr_phyrdy_change enter, port %d", port);
42702fcbc377Syt 
42712fcbc377Syt 	/* Clear PxSERR.DIAG.N to clear the interrupt bit */
42722fcbc377Syt 	mutex_enter(&ahci_portp->ahciport_mutex);
42732fcbc377Syt 	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
42742fcbc377Syt 	    (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
42752fcbc377Syt 	    AHCI_SERROR_DIAG_N);
42762fcbc377Syt 	mutex_exit(&ahci_portp->ahciport_mutex);
42772fcbc377Syt 
42782fcbc377Syt 	mutex_enter(&ahci_ctlp->ahcictl_mutex);
42792fcbc377Syt 	if ((ahci_ctlp->ahcictl_sata_hba_tran == NULL) ||
42802fcbc377Syt 	    (ahci_portp == NULL)) {
42812fcbc377Syt 		/* The whole controller setup is not yet done. */
42822fcbc377Syt 		mutex_exit(&ahci_ctlp->ahcictl_mutex);
42832fcbc377Syt 		return (AHCI_SUCCESS);
42842fcbc377Syt 	}
42852fcbc377Syt 	mutex_exit(&ahci_ctlp->ahcictl_mutex);
42862fcbc377Syt 
42872fcbc377Syt 	mutex_enter(&ahci_portp->ahciport_mutex);
42882fcbc377Syt 
42892fcbc377Syt 	/* SStatus tells the presence of device. */
42902fcbc377Syt 	port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
42912fcbc377Syt 	    (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port));
42922fcbc377Syt 
4293*68d33a25Syt 	if (AHCI_SSTATUS_GET_DET(port_sstatus) ==
4294*68d33a25Syt 	    AHCI_SSTATUS_DET_DEVPRESENT_PHYONLINE) {
4295*68d33a25Syt 		dev_exists_now = 1;
4296*68d33a25Syt 	}
42972fcbc377Syt 
42982fcbc377Syt 	if (ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) {
42992fcbc377Syt 		dev_existed_previously = 1;
43002fcbc377Syt 	}
43012fcbc377Syt 
43022fcbc377Syt 	bzero((void *)&sdevice, sizeof (sata_device_t));
430309121340Syt 	sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];
43042fcbc377Syt 	sdevice.satadev_addr.qual = SATA_ADDR_CPORT;
43052fcbc377Syt 	sdevice.satadev_addr.pmport = 0;
43062fcbc377Syt 	sdevice.satadev_state = SATA_PSTATE_PWRON;
43072fcbc377Syt 	ahci_portp->ahciport_port_state = SATA_PSTATE_PWRON;
43082fcbc377Syt 
43092fcbc377Syt 	if (dev_exists_now) {
43102fcbc377Syt 		if (dev_existed_previously) {
43112fcbc377Syt 			/* Things are fine now. The loss was temporary. */
4312*68d33a25Syt 			AHCIDBG1(AHCIDBG_EVENT, ahci_ctlp,
4313*68d33a25Syt 			    "ahci_intr_phyrdy_change  port %d "
43142fcbc377Syt 			    "device link lost/established", port);
43152fcbc377Syt 
43162fcbc377Syt 			mutex_exit(&ahci_portp->ahciport_mutex);
43172fcbc377Syt 			sata_hba_event_notify(
43182fcbc377Syt 			    ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
43192fcbc377Syt 			    &sdevice,
43202fcbc377Syt 			    SATA_EVNT_LINK_LOST|SATA_EVNT_LINK_ESTABLISHED);
43212fcbc377Syt 			mutex_enter(&ahci_portp->ahciport_mutex);
43222fcbc377Syt 
43232fcbc377Syt 		} else {
4324*68d33a25Syt 			AHCIDBG1(AHCIDBG_EVENT, ahci_ctlp,
4325*68d33a25Syt 			    "ahci_intr_phyrdy_change: port %d "
43262fcbc377Syt 			    "device link established", port);
43272fcbc377Syt 
4328*68d33a25Syt 			ahci_disable_port_intrs(ahci_ctlp, ahci_portp, port);
4329*68d33a25Syt 
43302fcbc377Syt 			/* A new device has been detected. */
4331*68d33a25Syt 			ahci_find_dev_signature(ahci_ctlp, ahci_portp, port);
43322fcbc377Syt 
4333*68d33a25Syt 			ahci_enable_port_intrs(ahci_ctlp, ahci_portp, port);
4334*68d33a25Syt 
4335*68d33a25Syt 			/* Try to start the port */
4336*68d33a25Syt 			if (ahci_start_port(ahci_ctlp, ahci_portp, port)
4337*68d33a25Syt 			    != AHCI_SUCCESS) {
4338*68d33a25Syt 				sdevice.satadev_state |= SATA_PSTATE_FAILED;
43392fcbc377Syt 				AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp,
4340*68d33a25Syt 				    "ahci_intr_phyrdy_change: port %d failed "
43412fcbc377Syt 				    "at start port", port);
43422fcbc377Syt 			}
43432fcbc377Syt 
43442fcbc377Syt 			mutex_exit(&ahci_portp->ahciport_mutex);
43452fcbc377Syt 			sata_hba_event_notify(
43462fcbc377Syt 			    ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
43472fcbc377Syt 			    &sdevice,
43482fcbc377Syt 			    SATA_EVNT_LINK_ESTABLISHED);
43492fcbc377Syt 			mutex_enter(&ahci_portp->ahciport_mutex);
43502fcbc377Syt 
43512fcbc377Syt 		}
43522fcbc377Syt 	} else { /* No device exists now */
43532fcbc377Syt 
43542fcbc377Syt 		if (dev_existed_previously) {
4355*68d33a25Syt 			AHCIDBG1(AHCIDBG_EVENT, ahci_ctlp,
4356*68d33a25Syt 			    "ahci_intr_phyrdy_change: port %d "
43572fcbc377Syt 			    "device link lost", port);
43582fcbc377Syt 
43592fcbc377Syt 			ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port);
4360*68d33a25Syt 			(void) ahci_put_port_into_notrunning_state(ahci_ctlp,
43612fcbc377Syt 			    ahci_portp, port);
43622fcbc377Syt 
43632fcbc377Syt 			/* An existing device is lost. */
43642fcbc377Syt 			ahci_portp->ahciport_device_type = SATA_DTYPE_NONE;
43652fcbc377Syt 
43662fcbc377Syt 			mutex_exit(&ahci_portp->ahciport_mutex);
43672fcbc377Syt 			sata_hba_event_notify(
43682fcbc377Syt 			    ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
43692fcbc377Syt 			    &sdevice,
43702fcbc377Syt 			    SATA_EVNT_LINK_LOST);
43712fcbc377Syt 			mutex_enter(&ahci_portp->ahciport_mutex);
43722fcbc377Syt 
43732fcbc377Syt 		} else {
43742fcbc377Syt 
43752fcbc377Syt 			/* Spurious interrupt */
43762fcbc377Syt 			AHCIDBG0(AHCIDBG_INTR, ahci_ctlp,
4377*68d33a25Syt 			    "ahci_intr_phyrdy_change: "
43782fcbc377Syt 			    "spurious phy ready interrupt");
43792fcbc377Syt 		}
43802fcbc377Syt 	}
43812fcbc377Syt 
43822fcbc377Syt 	mutex_exit(&ahci_portp->ahciport_mutex);
43832fcbc377Syt 
43842fcbc377Syt 	return (AHCI_SUCCESS);
43852fcbc377Syt }
43862fcbc377Syt 
43872fcbc377Syt /*
4388*68d33a25Syt  * PxIS.UFS - Unknown FIS Error
43892fcbc377Syt  *
4390*68d33a25Syt  * This interrupt event means an unknown FIS was received and has been
4391*68d33a25Syt  * copied into system memory. An unknown FIS is not considered an illegal
4392*68d33a25Syt  * FIS, unless the length received is more than 64 bytes. If an unknown
4393*68d33a25Syt  * FIS arrives with length <= 64 bytes, it is posted and the HBA continues
4394*68d33a25Syt  * normal operation. If the unknown FIS is more than 64 bytes, then it
4395*68d33a25Syt  * won't be posted to memory and PxSERR.ERR.P will be set, which is then
4396*68d33a25Syt  * a fatal error.
43972fcbc377Syt  *
4398*68d33a25Syt  * PxIS.OFS - Overflow Error
43992fcbc377Syt  *
4400*68d33a25Syt  * Command list overflow is defined as software building a command table
4401*68d33a25Syt  * that has fewer total bytes than the transaction given to the device.
4402*68d33a25Syt  * On device writes, the HBA will run out of data, and on reads, there
4403*68d33a25Syt  * will be no room to put the data.
44042fcbc377Syt  *
4405*68d33a25Syt  * For an overflow on data read, either PIO or DMA, the HBA will set
4406*68d33a25Syt  * PxIS.OFS, and the HBA will do a best effort to continue, and it's a
4407*68d33a25Syt  * non-fatal error when the HBA can continues. Sometimes, it will cause
4408*68d33a25Syt  * a fatal error and need the software to do something.
44092fcbc377Syt  *
4410*68d33a25Syt  * For an overflow on data write, setting PxIS.OFS is optional for both
4411*68d33a25Syt  * DMA and PIO, and it's a fatal error, and a COMRESET is required by
4412*68d33a25Syt  * software to clean up from this serious error.
44132fcbc377Syt  *
4414*68d33a25Syt  * PxIS.INFS - Interface Non-Fatal Error
4415*68d33a25Syt  *
4416*68d33a25Syt  * This interrupt event indicates that the HBA encountered an error on
4417*68d33a25Syt  * the Serial ATA interface but was able to continue operation. The kind
4418*68d33a25Syt  * of error usually occurred during a non-Data FIS, and under this condition
4419*68d33a25Syt  * the FIS will be re-transmitted by HBA automatically.
44202fcbc377Syt  *
4421*68d33a25Syt  * When the FMA is implemented, there should be a stat structure to
4422*68d33a25Syt  * record how many every kind of error happens.
44232fcbc377Syt  */
44242fcbc377Syt static int
4425*68d33a25Syt ahci_intr_non_fatal_error(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
4426*68d33a25Syt     uint8_t port, uint32_t intr_status)
44272fcbc377Syt {
44282fcbc377Syt 	uint32_t port_serror;
4429*68d33a25Syt #if AHCI_DEBUG
44302fcbc377Syt 	uint32_t port_cmd_status;
44312fcbc377Syt 	int current_slot;
4432*68d33a25Syt 	sata_pkt_t *satapkt;
4433*68d33a25Syt #endif
44342fcbc377Syt 
44352fcbc377Syt 	mutex_enter(&ahci_portp->ahciport_mutex);
44362fcbc377Syt 
44372fcbc377Syt 	port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
44382fcbc377Syt 	    (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port));
44392fcbc377Syt 
4440*68d33a25Syt 	AHCIDBG2(AHCIDBG_INTR|AHCIDBG_ENTRY|AHCIDBG_ERRS, ahci_ctlp,
4441*68d33a25Syt 	    "ahci_intr_non_fatal_error: port %d, "
44422fcbc377Syt 	    "port_serror = 0x%x", port, port_serror);
44432fcbc377Syt 
4444*68d33a25Syt 	ahci_log_serror_message(ahci_ctlp, port, port_serror);
4445*68d33a25Syt 
4446*68d33a25Syt 	if (intr_status & AHCI_INTR_STATUS_UFS) {
4447*68d33a25Syt 		AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp,
4448*68d33a25Syt 		    "ahci port %d has unknown FIS error", port);
44492fcbc377Syt 
4450*68d33a25Syt 		/* Clear the interrupt bit by clearing PxSERR.DIAG.F */
4451*68d33a25Syt 		ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
4452*68d33a25Syt 		    (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
4453*68d33a25Syt 		    AHCI_SERROR_DIAG_F);
4454*68d33a25Syt 	}
4455*68d33a25Syt 
4456*68d33a25Syt #if AHCI_DEBUG
4457*68d33a25Syt 	if (intr_status & AHCI_INTR_STATUS_OFS) {
4458*68d33a25Syt 		AHCIDBG1(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
4459*68d33a25Syt 		    "ahci port %d has overflow error", port);
4460*68d33a25Syt 	}
4461*68d33a25Syt 
4462*68d33a25Syt 	if (intr_status & AHCI_INTR_STATUS_INFS) {
4463*68d33a25Syt 		AHCIDBG1(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
4464*68d33a25Syt 		    "ahci port %d has interface non fatal error", port);
4465*68d33a25Syt 	}
44662fcbc377Syt 
44672fcbc377Syt 	/*
44682fcbc377Syt 	 * Record the error occurred command's slot.
44692fcbc377Syt 	 */
44702fcbc377Syt 	port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
44712fcbc377Syt 	    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
44722fcbc377Syt 
44732fcbc377Syt 	current_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >>
44742fcbc377Syt 	    AHCI_CMD_STATUS_CCS_SHIFT;
44752fcbc377Syt 
44762fcbc377Syt 	satapkt = ahci_portp->ahciport_slot_pkts[current_slot];
44772fcbc377Syt 	if (satapkt != NULL) {
4478*68d33a25Syt 		AHCIDBG2(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
4479*68d33a25Syt 		    "ahci_intr_non_fatal_error: pending_tags 0x%x "
4480*68d33a25Syt 		    "cmd 0x%x", ahci_portp->ahciport_pending_tags,
44812fcbc377Syt 		    satapkt->satapkt_cmd.satacmd_cmd_reg);
44822fcbc377Syt 
4483*68d33a25Syt 		AHCIDBG2(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
4484*68d33a25Syt 		    "ahci_intr_non_fatal_error: port %d, "
4485*68d33a25Syt 		    "satapkt 0x%p is being processed when error occurs",
44862fcbc377Syt 		    port, (void *)satapkt);
44872fcbc377Syt 	}
44882fcbc377Syt #endif
44892fcbc377Syt 	mutex_exit(&ahci_portp->ahciport_mutex);
44902fcbc377Syt 
44912fcbc377Syt 	return (AHCI_SUCCESS);
44922fcbc377Syt }
44932fcbc377Syt 
44942fcbc377Syt /*
4495*68d33a25Syt  * According to the AHCI spec, the error types include system memory
4496*68d33a25Syt  * errors, interface errors, port multiplier errors, device errors,
4497*68d33a25Syt  * command list overflow, command list underflow, native command
4498*68d33a25Syt  * queuing tag errors and pio data transfer errors.
4499*68d33a25Syt  *
4500*68d33a25Syt  * System memory errors such as target abort, master abort, and parity
4501*68d33a25Syt  * may cause the host to stop, and they are serious errors and needed
4502*68d33a25Syt  * to be recovered with software intervention. When system software
4503*68d33a25Syt  * has given a pointer to the HBA that doesn't exist in physical memory,
4504*68d33a25Syt  * a master/target abort error occurs, and PxIS.HBFS will be set. A
4505*68d33a25Syt  * data error such as CRC or parity occurs, the HBA aborts the transfer
4506*68d33a25Syt  * (if necessary) and PxIS.HBDS will be set.
4507*68d33a25Syt  *
4508*68d33a25Syt  * Interface errors are errors that occur due to electrical issues on
4509*68d33a25Syt  * the interface, or protocol miscommunication between the device and
4510*68d33a25Syt  * HBA, and the respective PxSERR register bit will be set. And PxIS.IFS
4511*68d33a25Syt  * (fatal) or PxIS.INFS (non-fatal) will be set. The conditions that
4512*68d33a25Syt  * causes PxIS.IFS/PxIS.INFS to be set are
4513*68d33a25Syt  * 	1. in PxSERR.ERR, P bit is set to '1'
4514*68d33a25Syt  *	2. in PxSERR.DIAG, C or H bit is set to '1'
4515*68d33a25Syt  *	3. PhyRdy drop unexpectly, N bit is set to '1'
4516*68d33a25Syt  * If the error occurred during a non-data FIS, the FIS must be
4517*68d33a25Syt  * retransmitted, and the error is non-fatal and PxIS.INFS is set. If
4518*68d33a25Syt  * the error occurred during a data FIS, the transfer will stop, so
4519*68d33a25Syt  * the error is fatal and PxIS.IFS is set.
4520*68d33a25Syt  *
4521*68d33a25Syt  * When a FIS arrives that updates the taskfile, the HBA checks to see
4522*68d33a25Syt  * if PxTFD.STS.ERR is set. If yes, PxIS.TFES will be set and the HBA
4523*68d33a25Syt  * stops processing any more commands.
45242fcbc377Syt  *
4525*68d33a25Syt  * Command list overflow is defined as software building a command table
4526*68d33a25Syt  * that has fewer total bytes than the transaction given to the device.
4527*68d33a25Syt  * On device writes, the HBA will run out of data, and on reads, there
4528*68d33a25Syt  * will be no room to put the data. For an overflow on data read, either
4529*68d33a25Syt  * PIO or DMA, the HBA will set PxIS.OFS, and it's a non-fatal error.
4530*68d33a25Syt  * For an overflow on data write, setting PxIS.OFS is optional for both
4531*68d33a25Syt  * DMA and PIO, and a COMRESET is required by software to clean up from
4532*68d33a25Syt  * this serious error.
45332fcbc377Syt  *
4534*68d33a25Syt  * Command list underflow is defined as software building a command
4535*68d33a25Syt  * table that has more total bytes than the transaction given to the
4536*68d33a25Syt  * device. For data writes, both PIO and DMA, the device will detect
4537*68d33a25Syt  * an error and end the transfer. And these errors are most likely going
4538*68d33a25Syt  * to be fatal errors that will cause the port to be restarted. For
4539*68d33a25Syt  * data reads, the HBA updates its PRD byte count, and may be
4540*68d33a25Syt  * able to continue normally, but is not required to. And The HBA is
4541*68d33a25Syt  * not required to detect underflow conditions for native command
4542*68d33a25Syt  * queuing command.
4543*68d33a25Syt  *
4544*68d33a25Syt  * The HBA does not actively check incoming DMA Setup FISes to ensure
4545*68d33a25Syt  * that the PxSACT register bit for that slot is set. Existing error
4546*68d33a25Syt  * mechanisms, such as host bus failure, or bad protocol, are used to
4547*68d33a25Syt  * recover from this case.
4548*68d33a25Syt  *
4549*68d33a25Syt  * In accordance with Serial ATA 1.0a, DATA FISes prior to the final
4550*68d33a25Syt  * DATA FIS must be an integral number of Dwords. If the HBA receives
4551*68d33a25Syt  * a request which is not an integral number of Dwords, the HBA
4552*68d33a25Syt  * set PxSERR.ERR.P to '1', set PxIS.IFS to '1' and stop running until
4553*68d33a25Syt  * software restarts the port. And the HBA ensures that the size
4554*68d33a25Syt  * of the DATA FIS received during a PIO command matches the size in
4555*68d33a25Syt  * the Transfer Cound field of the preceding PIO Setup FIS, if not, the
4556*68d33a25Syt  * HBA sets PxSERR.ERR.P to '1', set PxIS.IFS to '1', and then
4557*68d33a25Syt  * stop running until software restarts the port.
45582fcbc377Syt  */
45592fcbc377Syt /*
4560*68d33a25Syt  * the fatal errors include PxIS.IFS, PxIS.HBDS, PxIS.HBFS and PxIS.TFES.
4561*68d33a25Syt  *
4562*68d33a25Syt  * PxIS.IFS indicates that the hba encountered an error on the serial ata
4563*68d33a25Syt  * interface which caused the transfer to stop.
45642fcbc377Syt  *
4565*68d33a25Syt  * PxIS.HBDS indicates that the hba encountered a data error
4566*68d33a25Syt  * (uncorrectable ecc/parity) when reading from or writing to system memory.
4567*68d33a25Syt  *
4568*68d33a25Syt  * PxIS.HBFS indicates that the hba encountered a host bus error that it
4569*68d33a25Syt  * cannot recover from, such as a bad software pointer.
4570*68d33a25Syt  *
4571*68d33a25Syt  * PxIS.TFES is set whenever the status register is updated by the device
4572*68d33a25Syt  * and the error bit (bit 0) is set.
45732fcbc377Syt  */
45742fcbc377Syt static int
4575*68d33a25Syt ahci_intr_fatal_error(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
4576*68d33a25Syt     uint8_t port, uint32_t intr_status, uint32_t retrieve_errinfo_slot_status)
45772fcbc377Syt {
4578*68d33a25Syt 	uint32_t port_cmd_status;
45792fcbc377Syt 	uint32_t port_serror;
4580*68d33a25Syt 	uint32_t task_file_status;
4581*68d33a25Syt 	int failed_slot;
4582*68d33a25Syt 	sata_pkt_t *spkt;
4583*68d33a25Syt 	uint8_t err_byte;
4584*68d33a25Syt 	ahci_event_arg_t *args;
45852fcbc377Syt 
45862fcbc377Syt 	mutex_enter(&ahci_portp->ahciport_mutex);
45872fcbc377Syt 
4588*68d33a25Syt 	/*
4589*68d33a25Syt 	 * ahci_intr_phyrdy_change() may have rendered it to
4590*68d33a25Syt 	 * SATA_DTYPE_NONE.
4591*68d33a25Syt 	 */
4592*68d33a25Syt 	if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
4593*68d33a25Syt 		AHCIDBG1(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp,
4594*68d33a25Syt 		    "ahci_intr_fatal_error: port %d no device attached, "
4595*68d33a25Syt 		    "and just return without doing anything", port);
4596*68d33a25Syt 		goto out0;
4597*68d33a25Syt 	}
45982fcbc377Syt 
4599*68d33a25Syt 	/*
4600*68d33a25Syt 	 * Read PxCMD.CCS to determine the slot that the HBA
4601*68d33a25Syt 	 * was processing when the error occurred.
4602*68d33a25Syt 	 */
4603*68d33a25Syt 	port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4604*68d33a25Syt 	    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
4605*68d33a25Syt 	failed_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >>
4606*68d33a25Syt 	    AHCI_CMD_STATUS_CCS_SHIFT;
46072fcbc377Syt 
4608*68d33a25Syt 	spkt = ahci_portp->ahciport_slot_pkts[failed_slot];
4609*68d33a25Syt 	AHCIDBG2(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
4610*68d33a25Syt 	    "ahci_intr_fatal_error: spkt 0x%p is being processed when "
4611*68d33a25Syt 	    "fatal error occurred for port %d", spkt, port);
46122fcbc377Syt 
4613*68d33a25Syt 	if (intr_status & AHCI_INTR_STATUS_TFES) {
4614*68d33a25Syt 		task_file_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
4615*68d33a25Syt 		    (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
4616*68d33a25Syt 		AHCIDBG2(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
4617*68d33a25Syt 		    "ahci_intr_fatal_error: port %d "
4618*68d33a25Syt 		    "task_file_status = 0x%x", port, task_file_status);
46192fcbc377Syt 
4620*68d33a25Syt 		err_byte = (task_file_status & AHCI_TFD_ERR_MASK)
4621*68d33a25Syt 		    >> AHCI_TFD_ERR_SHIFT;
46222fcbc377Syt 
4623*68d33a25Syt 		/*
4624*68d33a25Syt 		 * Won't emit the error message if it is an IDENTIFY DEVICE
4625*68d33a25Syt 		 * command sent to an ATAPI device.
4626*68d33a25Syt 		 */
4627*68d33a25Syt 		if ((spkt != NULL) &&
4628*68d33a25Syt 		    (spkt->satapkt_cmd.satacmd_cmd_reg == SATAC_ID_DEVICE) &&
4629*68d33a25Syt 		    (err_byte == SATA_ERROR_ABORT))
4630*68d33a25Syt 			goto out1;
46312fcbc377Syt 
4632*68d33a25Syt 		/*
4633*68d33a25Syt 		 * Won't emit the error message if it is an ATAPI PACKET command
4634*68d33a25Syt 		 */
4635*68d33a25Syt 		if ((spkt != NULL) &&
4636*68d33a25Syt 		    (spkt->satapkt_cmd.satacmd_cmd_reg == SATAC_PACKET))
4637*68d33a25Syt 			goto out1;
4638*68d33a25Syt 	}
46392fcbc377Syt 
4640*68d33a25Syt 	ahci_log_fatal_error_message(ahci_ctlp, port, intr_status);
46412fcbc377Syt 
4642*68d33a25Syt out1:
46432fcbc377Syt 	port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
46442fcbc377Syt 	    (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port));
4645*68d33a25Syt 	ahci_log_serror_message(ahci_ctlp, port, port_serror);
4646*68d33a25Syt 
4647*68d33a25Syt 	/* Prepare the argument for the taskq */
4648*68d33a25Syt 	args = ahci_portp->ahciport_event_args;
4649*68d33a25Syt 	args->ahciea_ctlp = (void *)ahci_ctlp;
4650*68d33a25Syt 	args->ahciea_portp = (void *)ahci_portp;
4651*68d33a25Syt 	args->ahciea_event = intr_status;
4652*68d33a25Syt 	args->ahciea_retrierr_slot = retrieve_errinfo_slot_status;
4653*68d33a25Syt 
4654*68d33a25Syt 	/* Start the taskq to handle error recovery */
4655*68d33a25Syt 	if ((ddi_taskq_dispatch(ahci_ctlp->ahcictl_event_taskq,
4656*68d33a25Syt 	    ahci_events_handler,
4657*68d33a25Syt 	    (void *)args, DDI_NOSLEEP)) != DDI_SUCCESS) {
4658*68d33a25Syt 		cmn_err(CE_WARN, "ahci start taskq for event handler failed");
4659*68d33a25Syt 	}
4660*68d33a25Syt out0:
46612fcbc377Syt 	mutex_exit(&ahci_portp->ahciport_mutex);
46622fcbc377Syt 
46632fcbc377Syt 	return (AHCI_SUCCESS);
46642fcbc377Syt }
46652fcbc377Syt 
46662fcbc377Syt /*
46672fcbc377Syt  * Hot Plug Operation for platforms that support Cold Presence Detect.
46682fcbc377Syt  *
46692fcbc377Syt  * When set, a device status has changed as detected by the cold presence
46702fcbc377Syt  * detect logic. This bit can either be set due to a non-connected port
46712fcbc377Syt  * receiving a device, or a connected port having its device removed.
46722fcbc377Syt  * This bit is only valid if the port supports cold presence detect as
46732fcbc377Syt  * indicated by PxCMD.CPD set to '1'.
46742fcbc377Syt  *
4675*68d33a25Syt  * At the moment, this interrupt is not needed and disabled and we just
4676*68d33a25Syt  * log the debug message.
46772fcbc377Syt  */
46782fcbc377Syt /* ARGSUSED */
46792fcbc377Syt static int
46802fcbc377Syt ahci_intr_cold_port_detect(ahci_ctl_t *ahci_ctlp,
46812fcbc377Syt     ahci_port_t *ahci_portp, uint8_t port)
46822fcbc377Syt {
46832fcbc377Syt 	uint32_t port_cmd_status;
46842fcbc377Syt 	sata_device_t sdevice;
46852fcbc377Syt 
46862fcbc377Syt 	AHCIDBG1(AHCIDBG_INTR, ahci_ctlp,
46872fcbc377Syt 	    "ahci_intr_cold_port_detect enter, port %d", port);
46882fcbc377Syt 
46892fcbc377Syt 	mutex_enter(&ahci_portp->ahciport_mutex);
46902fcbc377Syt 
46912fcbc377Syt 	port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
46922fcbc377Syt 	    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
46932fcbc377Syt 	if (!(port_cmd_status & AHCI_CMD_STATUS_CPD)) {
46942fcbc377Syt 		AHCIDBG1(AHCIDBG_INTR, ahci_ctlp,
46952fcbc377Syt 		    "port %d does not support cold presence detect, so "
46962fcbc377Syt 		    "we just ignore this interrupt", port);
46972fcbc377Syt 		mutex_exit(&ahci_portp->ahciport_mutex);
46982fcbc377Syt 		return (AHCI_SUCCESS);
46992fcbc377Syt 	}
47002fcbc377Syt 
47012fcbc377Syt 	AHCIDBG1(AHCIDBG_INTR, ahci_ctlp,
47022fcbc377Syt 	    "port %d device status has changed", port);
47032fcbc377Syt 
47042fcbc377Syt 	bzero((void *)&sdevice, sizeof (sata_device_t));
470509121340Syt 	sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];
47062fcbc377Syt 	sdevice.satadev_addr.qual = SATA_ADDR_CPORT;
47072fcbc377Syt 	sdevice.satadev_addr.pmport = 0;
47082fcbc377Syt 	sdevice.satadev_state = SATA_PSTATE_PWRON;
47092fcbc377Syt 
47102fcbc377Syt 	if (port_cmd_status & AHCI_CMD_STATUS_CPS) {
47112fcbc377Syt 		AHCIDBG1(AHCIDBG_INTR, ahci_ctlp,
47122fcbc377Syt 		    "port %d: a device is hot plugged", port);
47132fcbc377Syt 		mutex_exit(&ahci_portp->ahciport_mutex);
47142fcbc377Syt 		sata_hba_event_notify(
47152fcbc377Syt 		    ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
47162fcbc377Syt 		    &sdevice,
47172fcbc377Syt 		    SATA_EVNT_DEVICE_ATTACHED);
47182fcbc377Syt 		mutex_enter(&ahci_portp->ahciport_mutex);
47192fcbc377Syt 
47202fcbc377Syt 	} else {
47212fcbc377Syt 		AHCIDBG1(AHCIDBG_INTR, ahci_ctlp,
47222fcbc377Syt 		    "port %d: a device is hot unplugged", port);
47232fcbc377Syt 		mutex_exit(&ahci_portp->ahciport_mutex);
47242fcbc377Syt 		sata_hba_event_notify(
47252fcbc377Syt 		    ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
47262fcbc377Syt 		    &sdevice,
47272fcbc377Syt 		    SATA_EVNT_DEVICE_DETACHED);
47282fcbc377Syt 		mutex_enter(&ahci_portp->ahciport_mutex);
47292fcbc377Syt 	}
47302fcbc377Syt 
47312fcbc377Syt 	mutex_exit(&ahci_portp->ahciport_mutex);
47322fcbc377Syt 
47332fcbc377Syt 	return (AHCI_SUCCESS);
47342fcbc377Syt }
47352fcbc377Syt 
47362fcbc377Syt /*
47372fcbc377Syt  * Enable the interrupts for a particular port.
47382fcbc377Syt  *
47392fcbc377Syt  * WARNING!!! ahciport_mutex should be acquired before the function
47402fcbc377Syt  * is called.
47412fcbc377Syt  */
47422fcbc377Syt /* ARGSUSED */
47432fcbc377Syt static void
47442fcbc377Syt ahci_enable_port_intrs(ahci_ctl_t *ahci_ctlp,
47452fcbc377Syt     ahci_port_t *ahci_portp, uint8_t port)
47462fcbc377Syt {
47472fcbc377Syt 	AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp,
47482fcbc377Syt 	    "ahci_enable_port_intrs enter, port %d", port);
47492fcbc377Syt 
47502fcbc377Syt 	/*
47512fcbc377Syt 	 * Clear port interrupt status before enabling interrupt
47522fcbc377Syt 	 */
47532fcbc377Syt 	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
47542fcbc377Syt 	    (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port),
47552fcbc377Syt 	    AHCI_PORT_INTR_MASK);
47562fcbc377Syt 
47572fcbc377Syt 	/*
47582fcbc377Syt 	 * Clear the pending bit from IS.IPS
47592fcbc377Syt 	 */
47602fcbc377Syt 	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
47612fcbc377Syt 	    (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp), (1 << port));
47622fcbc377Syt 
47632fcbc377Syt 	/*
47642fcbc377Syt 	 * Enable the following interrupts:
47652fcbc377Syt 	 *	Device to Host Register FIS Interrupt (DHRS)
47662fcbc377Syt 	 *	PIO Setup FIS Interrupt (PSS)
47672fcbc377Syt 	 *	Unknown FIS Interrupt (UFS)
47682fcbc377Syt 	 *	Port Connect Change Status (PCS)
47692fcbc377Syt 	 *	PhyRdy Change Status (PRCS)
47702fcbc377Syt 	 *	Overflow Status (OFS)
47712fcbc377Syt 	 *	Interface Non-fatal Error Status (INFS)
47722fcbc377Syt 	 *	Interface Fatal Error Status (IFS)
47732fcbc377Syt 	 *	Host Bus Data Error Status (HBDS)
47742fcbc377Syt 	 *	Host Bus Fatal Error Status (HBFS)
47752fcbc377Syt 	 *	Task File Error Status (TFES)
47762fcbc377Syt 	 */
47772fcbc377Syt 	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
47782fcbc377Syt 	    (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port),
47792fcbc377Syt 	    (AHCI_INTR_STATUS_DHRS |
47802fcbc377Syt 	    AHCI_INTR_STATUS_PSS |
47812fcbc377Syt 	    AHCI_INTR_STATUS_UFS |
4782*68d33a25Syt 	    AHCI_INTR_STATUS_DPS |
47832fcbc377Syt 	    AHCI_INTR_STATUS_PCS |
47842fcbc377Syt 	    AHCI_INTR_STATUS_PRCS |
47852fcbc377Syt 	    AHCI_INTR_STATUS_OFS |
47862fcbc377Syt 	    AHCI_INTR_STATUS_INFS |
47872fcbc377Syt 	    AHCI_INTR_STATUS_IFS |
47882fcbc377Syt 	    AHCI_INTR_STATUS_HBDS |
47892fcbc377Syt 	    AHCI_INTR_STATUS_HBFS |
47902fcbc377Syt 	    AHCI_INTR_STATUS_TFES));
47912fcbc377Syt }
47922fcbc377Syt 
47932fcbc377Syt /*
47942fcbc377Syt  * Enable interrupts for all the ports.
47952fcbc377Syt  *
47962fcbc377Syt  * WARNING!!! ahcictl_mutex should be acquired before the function
47972fcbc377Syt  * is called.
47982fcbc377Syt  */
47992fcbc377Syt static void
48002fcbc377Syt ahci_enable_all_intrs(ahci_ctl_t *ahci_ctlp)
48012fcbc377Syt {
48022fcbc377Syt 	uint32_t ghc_control;
48032fcbc377Syt 
48042fcbc377Syt 	AHCIDBG0(AHCIDBG_ENTRY, ahci_ctlp, "ahci_enable_all_intrs enter");
48052fcbc377Syt 
48062fcbc377Syt 	ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
48072fcbc377Syt 	    (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
48082fcbc377Syt 
48092fcbc377Syt 	ghc_control |= AHCI_HBA_GHC_IE;
48102fcbc377Syt 
48112fcbc377Syt 	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
48122fcbc377Syt 	    (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
48132fcbc377Syt }
48142fcbc377Syt 
48152fcbc377Syt /*
48162fcbc377Syt  * Disable interrupts for a particular port.
48172fcbc377Syt  *
48182fcbc377Syt  * WARNING!!! ahciport_mutex should be acquired before the function
48192fcbc377Syt  * is called.
48202fcbc377Syt  */
48212fcbc377Syt /* ARGSUSED */
48222fcbc377Syt static void
48232fcbc377Syt ahci_disable_port_intrs(ahci_ctl_t *ahci_ctlp,
48242fcbc377Syt     ahci_port_t *ahci_portp, uint8_t port)
48252fcbc377Syt {
48262fcbc377Syt 	AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp,
48272fcbc377Syt 	    "ahci_disable_port_intrs enter, port %d", port);
48282fcbc377Syt 
48292fcbc377Syt 	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
48302fcbc377Syt 	    (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port), 0);
48312fcbc377Syt }
48322fcbc377Syt 
48332fcbc377Syt /*
48342fcbc377Syt  * Disable interrupts for the whole HBA.
48352fcbc377Syt  *
48362fcbc377Syt  * The global bit is cleared, then all interrupt sources from all
48372fcbc377Syt  * ports are disabled.
48382fcbc377Syt  *
48392fcbc377Syt  * WARNING!!! ahcictl_mutex should be acquired before the function
48402fcbc377Syt  * is called.
48412fcbc377Syt  */
48422fcbc377Syt static void
48432fcbc377Syt ahci_disable_all_intrs(ahci_ctl_t *ahci_ctlp)
48442fcbc377Syt {
48452fcbc377Syt 	uint32_t ghc_control;
48462fcbc377Syt 
48472fcbc377Syt 	AHCIDBG0(AHCIDBG_ENTRY, ahci_ctlp, "ahci_disable_all_intrs enter");
48482fcbc377Syt 
48492fcbc377Syt 	ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
48502fcbc377Syt 	    (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
48512fcbc377Syt 
48522fcbc377Syt 	ghc_control &= ~ AHCI_HBA_GHC_IE;
48532fcbc377Syt 
48542fcbc377Syt 	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
48552fcbc377Syt 	    (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
48562fcbc377Syt }
48572fcbc377Syt 
48582fcbc377Syt /*
48592fcbc377Syt  * Handle INTx and legacy interrupts.
48602fcbc377Syt  */
48612fcbc377Syt static int
48622fcbc377Syt ahci_add_legacy_intrs(ahci_ctl_t *ahci_ctlp)
48632fcbc377Syt {
48642fcbc377Syt 	dev_info_t	*dip = ahci_ctlp->ahcictl_dip;
48652fcbc377Syt 	int		actual, count = 0;
48662fcbc377Syt 	int		x, y, rc, inum = 0;
48672fcbc377Syt 
48682fcbc377Syt 	AHCIDBG0(AHCIDBG_ENTRY|AHCIDBG_INIT, ahci_ctlp,
48692fcbc377Syt 	    "ahci_add_legacy_intrs enter");
48702fcbc377Syt 
48712fcbc377Syt 	/* get number of interrupts. */
48722fcbc377Syt 	rc = ddi_intr_get_nintrs(dip, DDI_INTR_TYPE_FIXED, &count);
48732fcbc377Syt 	if ((rc != DDI_SUCCESS) || (count == 0)) {
48742fcbc377Syt 		AHCIDBG2(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
48752fcbc377Syt 		    "ddi_intr_get_nintrs() failed, "
48762fcbc377Syt 		    "rc %d count %d\n", rc, count);
48772fcbc377Syt 		return (DDI_FAILURE);
48782fcbc377Syt 	}
48792fcbc377Syt 
48802fcbc377Syt 	/* Allocate an array of interrupt handles. */
48812fcbc377Syt 	ahci_ctlp->ahcictl_intr_size = count * sizeof (ddi_intr_handle_t);
48822fcbc377Syt 	ahci_ctlp->ahcictl_intr_htable =
48832fcbc377Syt 	    kmem_zalloc(ahci_ctlp->ahcictl_intr_size, KM_SLEEP);
48842fcbc377Syt 
48852fcbc377Syt 	/* call ddi_intr_alloc(). */
48862fcbc377Syt 	rc = ddi_intr_alloc(dip, ahci_ctlp->ahcictl_intr_htable,
48872fcbc377Syt 	    DDI_INTR_TYPE_FIXED,
48882fcbc377Syt 	    inum, count, &actual, DDI_INTR_ALLOC_STRICT);
48892fcbc377Syt 
48902fcbc377Syt 	if ((rc != DDI_SUCCESS) || (actual == 0)) {
48912fcbc377Syt 		AHCIDBG1(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
48922fcbc377Syt 		    "ddi_intr_alloc() failed, rc %d\n", rc);
48932fcbc377Syt 		kmem_free(ahci_ctlp->ahcictl_intr_htable,
48942fcbc377Syt 		    ahci_ctlp->ahcictl_intr_size);
48952fcbc377Syt 		return (DDI_FAILURE);
48962fcbc377Syt 	}
48972fcbc377Syt 
48982fcbc377Syt 	if (actual < count) {
48992fcbc377Syt 		AHCIDBG2(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
49002fcbc377Syt 		    "Requested: %d, Received: %d", count, actual);
49012fcbc377Syt 
49022fcbc377Syt 		for (x = 0; x < actual; x++) {
49032fcbc377Syt 			(void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[x]);
49042fcbc377Syt 		}
49052fcbc377Syt 
49062fcbc377Syt 		kmem_free(ahci_ctlp->ahcictl_intr_htable,
49072fcbc377Syt 		    ahci_ctlp->ahcictl_intr_size);
49082fcbc377Syt 		return (DDI_FAILURE);
49092fcbc377Syt 	}
49102fcbc377Syt 
49112fcbc377Syt 	ahci_ctlp->ahcictl_intr_cnt = actual;
49122fcbc377Syt 
49132fcbc377Syt 	/* Get intr priority. */
49142fcbc377Syt 	if (ddi_intr_get_pri(ahci_ctlp->ahcictl_intr_htable[0],
49152fcbc377Syt 	    &ahci_ctlp->ahcictl_intr_pri) != DDI_SUCCESS) {
49162fcbc377Syt 		AHCIDBG0(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
49172fcbc377Syt 		    "ddi_intr_get_pri() failed");
49182fcbc377Syt 
49192fcbc377Syt 		for (x = 0; x < actual; x++) {
49202fcbc377Syt 			(void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[x]);
49212fcbc377Syt 		}
49222fcbc377Syt 
49232fcbc377Syt 		kmem_free(ahci_ctlp->ahcictl_intr_htable,
49242fcbc377Syt 		    ahci_ctlp->ahcictl_intr_size);
49252fcbc377Syt 		return (DDI_FAILURE);
49262fcbc377Syt 	}
49272fcbc377Syt 
49282fcbc377Syt 	/* Test for high level interrupt. */
49292fcbc377Syt 	if (ahci_ctlp->ahcictl_intr_pri >= ddi_intr_get_hilevel_pri()) {
49302fcbc377Syt 		AHCIDBG0(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
49312fcbc377Syt 		    "ahci_add_legacy_intrs: Hi level intr not supported");
49322fcbc377Syt 
49332fcbc377Syt 		for (x = 0; x < actual; x++) {
49342fcbc377Syt 			(void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[x]);
49352fcbc377Syt 		}
49362fcbc377Syt 
49372fcbc377Syt 		kmem_free(ahci_ctlp->ahcictl_intr_htable,
49382fcbc377Syt 		    sizeof (ddi_intr_handle_t));
49392fcbc377Syt 
49402fcbc377Syt 		return (DDI_FAILURE);
49412fcbc377Syt 	}
49422fcbc377Syt 
49432fcbc377Syt 	/* Call ddi_intr_add_handler(). */
49442fcbc377Syt 	for (x = 0; x < actual; x++) {
49452fcbc377Syt 		if (ddi_intr_add_handler(ahci_ctlp->ahcictl_intr_htable[x],
49462fcbc377Syt 		    ahci_intr, (caddr_t)ahci_ctlp, NULL) != DDI_SUCCESS) {
49472fcbc377Syt 			AHCIDBG0(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
49482fcbc377Syt 			    "ddi_intr_add_handler() failed");
49492fcbc377Syt 
49502fcbc377Syt 			for (y = 0; y < actual; y++) {
49512fcbc377Syt 				(void) ddi_intr_free(
49522fcbc377Syt 				    ahci_ctlp->ahcictl_intr_htable[y]);
49532fcbc377Syt 			}
49542fcbc377Syt 
49552fcbc377Syt 			kmem_free(ahci_ctlp->ahcictl_intr_htable,
49562fcbc377Syt 			    ahci_ctlp->ahcictl_intr_size);
49572fcbc377Syt 			return (DDI_FAILURE);
49582fcbc377Syt 		}
49592fcbc377Syt 	}
49602fcbc377Syt 
49612fcbc377Syt 	/* Call ddi_intr_enable() for legacy interrupts. */
49622fcbc377Syt 	for (x = 0; x < ahci_ctlp->ahcictl_intr_cnt; x++) {
49632fcbc377Syt 		(void) ddi_intr_enable(ahci_ctlp->ahcictl_intr_htable[x]);
49642fcbc377Syt 	}
49652fcbc377Syt 
49662fcbc377Syt 	return (DDI_SUCCESS);
49672fcbc377Syt }
49682fcbc377Syt 
49692fcbc377Syt /*
49702fcbc377Syt  * Handle MSI interrupts.
49712fcbc377Syt  */
49722fcbc377Syt static int
49732fcbc377Syt ahci_add_msi_intrs(ahci_ctl_t *ahci_ctlp)
49742fcbc377Syt {
49752fcbc377Syt 	dev_info_t *dip = ahci_ctlp->ahcictl_dip;
49762fcbc377Syt 	int		count, avail, actual;
49772fcbc377Syt 	int		x, y, rc, inum = 0;
49782fcbc377Syt 
49792fcbc377Syt 	AHCIDBG0(AHCIDBG_ENTRY|AHCIDBG_INIT, ahci_ctlp,
49802fcbc377Syt 	    "ahci_add_msi_intrs enter");
49812fcbc377Syt 
49822fcbc377Syt 	/* get number of interrupts. */
49832fcbc377Syt 	rc = ddi_intr_get_nintrs(dip, DDI_INTR_TYPE_MSI, &count);
49842fcbc377Syt 	if ((rc != DDI_SUCCESS) || (count == 0)) {
49852fcbc377Syt 		AHCIDBG2(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
49862fcbc377Syt 		    "ddi_intr_get_nintrs() failed, "
49872fcbc377Syt 		    "rc %d count %d\n", rc, count);
49882fcbc377Syt 		return (DDI_FAILURE);
49892fcbc377Syt 	}
49902fcbc377Syt 
49912fcbc377Syt 	/* get number of available interrupts. */
49922fcbc377Syt 	rc = ddi_intr_get_navail(dip, DDI_INTR_TYPE_MSI, &avail);
49932fcbc377Syt 	if ((rc != DDI_SUCCESS) || (avail == 0)) {
49942fcbc377Syt 		AHCIDBG2(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
49952fcbc377Syt 		    "ddi_intr_get_navail() failed, "
49962fcbc377Syt 		    "rc %d avail %d\n", rc, avail);
49972fcbc377Syt 		return (DDI_FAILURE);
49982fcbc377Syt 	}
49992fcbc377Syt 
50002fcbc377Syt 	if (avail < count) {
50012fcbc377Syt 		AHCIDBG2(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
50022fcbc377Syt 		    "ddi_intr_get_nvail returned %d, navail() returned %d",
50032fcbc377Syt 		    count, avail);
50042fcbc377Syt 	}
50052fcbc377Syt 
50062fcbc377Syt 	/* Allocate an array of interrupt handles. */
50072fcbc377Syt 	ahci_ctlp->ahcictl_intr_size = count * sizeof (ddi_intr_handle_t);
50082fcbc377Syt 	ahci_ctlp->ahcictl_intr_htable =
50092fcbc377Syt 	    kmem_alloc(ahci_ctlp->ahcictl_intr_size, KM_SLEEP);
50102fcbc377Syt 
50112fcbc377Syt 	/* call ddi_intr_alloc(). */
50122fcbc377Syt 	rc = ddi_intr_alloc(dip, ahci_ctlp->ahcictl_intr_htable,
50132fcbc377Syt 	    DDI_INTR_TYPE_MSI, inum, count, &actual, DDI_INTR_ALLOC_NORMAL);
50142fcbc377Syt 
50152fcbc377Syt 	if ((rc != DDI_SUCCESS) || (actual == 0)) {
50162fcbc377Syt 		AHCIDBG1(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
50172fcbc377Syt 		    "ddi_intr_alloc() failed, rc %d\n", rc);
50182fcbc377Syt 		kmem_free(ahci_ctlp->ahcictl_intr_htable,
50192fcbc377Syt 		    ahci_ctlp->ahcictl_intr_size);
50202fcbc377Syt 		return (DDI_FAILURE);
50212fcbc377Syt 	}
50222fcbc377Syt 
50232fcbc377Syt 	/* use interrupt count returned */
50242fcbc377Syt 	if (actual < count) {
50252fcbc377Syt 		AHCIDBG2(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
50262fcbc377Syt 		    "Requested: %d, Received: %d", count, actual);
50272fcbc377Syt 	}
50282fcbc377Syt 
50292fcbc377Syt 	ahci_ctlp->ahcictl_intr_cnt = actual;
50302fcbc377Syt 
50312fcbc377Syt 	/*
50322fcbc377Syt 	 * Get priority for first msi, assume remaining are all the same.
50332fcbc377Syt 	 */
50342fcbc377Syt 	if (ddi_intr_get_pri(ahci_ctlp->ahcictl_intr_htable[0],
50352fcbc377Syt 	    &ahci_ctlp->ahcictl_intr_pri) != DDI_SUCCESS) {
50362fcbc377Syt 		AHCIDBG0(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
50372fcbc377Syt 		    "ddi_intr_get_pri() failed");
50382fcbc377Syt 
50392fcbc377Syt 		/* Free already allocated intr. */
50402fcbc377Syt 		for (y = 0; y < actual; y++) {
50412fcbc377Syt 			(void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[y]);
50422fcbc377Syt 		}
50432fcbc377Syt 
50442fcbc377Syt 		kmem_free(ahci_ctlp->ahcictl_intr_htable,
50452fcbc377Syt 		    ahci_ctlp->ahcictl_intr_size);
50462fcbc377Syt 		return (DDI_FAILURE);
50472fcbc377Syt 	}
50482fcbc377Syt 
50492fcbc377Syt 	/* Test for high level interrupt. */
50502fcbc377Syt 	if (ahci_ctlp->ahcictl_intr_pri >= ddi_intr_get_hilevel_pri()) {
50512fcbc377Syt 		AHCIDBG0(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
50522fcbc377Syt 		    "ahci_add_msi_intrs: Hi level intr not supported");
50532fcbc377Syt 
50542fcbc377Syt 		/* Free already allocated intr. */
50552fcbc377Syt 		for (y = 0; y < actual; y++) {
50562fcbc377Syt 			(void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[y]);
50572fcbc377Syt 		}
50582fcbc377Syt 
50592fcbc377Syt 		kmem_free(ahci_ctlp->ahcictl_intr_htable,
50602fcbc377Syt 		    sizeof (ddi_intr_handle_t));
50612fcbc377Syt 
50622fcbc377Syt 		return (DDI_FAILURE);
50632fcbc377Syt 	}
50642fcbc377Syt 
50652fcbc377Syt 	/* Call ddi_intr_add_handler(). */
50662fcbc377Syt 	for (x = 0; x < actual; x++) {
50672fcbc377Syt 		if (ddi_intr_add_handler(ahci_ctlp->ahcictl_intr_htable[x],
50682fcbc377Syt 		    ahci_intr, (caddr_t)ahci_ctlp, NULL) != DDI_SUCCESS) {
50692fcbc377Syt 			AHCIDBG0(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
50702fcbc377Syt 			    "ddi_intr_add_handler() failed");
50712fcbc377Syt 
50722fcbc377Syt 			/* Free already allocated intr. */
50732fcbc377Syt 			for (y = 0; y < actual; y++) {
50742fcbc377Syt 				(void) ddi_intr_free(
50752fcbc377Syt 				    ahci_ctlp->ahcictl_intr_htable[y]);
50762fcbc377Syt 			}
50772fcbc377Syt 
50782fcbc377Syt 			kmem_free(ahci_ctlp->ahcictl_intr_htable,
50792fcbc377Syt 			    ahci_ctlp->ahcictl_intr_size);
50802fcbc377Syt 			return (DDI_FAILURE);
50812fcbc377Syt 		}
50822fcbc377Syt 	}
50832fcbc377Syt 
50842fcbc377Syt 
50852fcbc377Syt 	(void) ddi_intr_get_cap(ahci_ctlp->ahcictl_intr_htable[0],
50862fcbc377Syt 	    &ahci_ctlp->ahcictl_intr_cap);
50872fcbc377Syt 
50882fcbc377Syt 	if (ahci_ctlp->ahcictl_intr_cap & DDI_INTR_FLAG_BLOCK) {
50892fcbc377Syt 		/* Call ddi_intr_block_enable() for MSI. */
50902fcbc377Syt 		(void) ddi_intr_block_enable(ahci_ctlp->ahcictl_intr_htable,
50912fcbc377Syt 		    ahci_ctlp->ahcictl_intr_cnt);
50922fcbc377Syt 	} else {
50932fcbc377Syt 		/* Call ddi_intr_enable() for MSI non block enable. */
50942fcbc377Syt 		for (x = 0; x < ahci_ctlp->ahcictl_intr_cnt; x++) {
50952fcbc377Syt 			(void) ddi_intr_enable(
50962fcbc377Syt 			    ahci_ctlp->ahcictl_intr_htable[x]);
50972fcbc377Syt 		}
50982fcbc377Syt 	}
50992fcbc377Syt 
51002fcbc377Syt 	return (DDI_SUCCESS);
51012fcbc377Syt }
51022fcbc377Syt 
51032fcbc377Syt /*
51042fcbc377Syt  * Removes the registered interrupts irrespective of whether they
51052fcbc377Syt  * were legacy or MSI.
51062fcbc377Syt  *
51072fcbc377Syt  * WARNING!!! The controller interrupts must be disabled before calling
51082fcbc377Syt  * this routine.
51092fcbc377Syt  */
51102fcbc377Syt static void
51112fcbc377Syt ahci_rem_intrs(ahci_ctl_t *ahci_ctlp)
51122fcbc377Syt {
51132fcbc377Syt 	int x;
51142fcbc377Syt 
51152fcbc377Syt 	AHCIDBG0(AHCIDBG_ENTRY, ahci_ctlp, "ahci_rem_intrs entered");
51162fcbc377Syt 
51172fcbc377Syt 	/* Disable all interrupts. */
51182fcbc377Syt 	if ((ahci_ctlp->ahcictl_intr_type == DDI_INTR_TYPE_MSI) &&
51192fcbc377Syt 	    (ahci_ctlp->ahcictl_intr_cap & DDI_INTR_FLAG_BLOCK)) {
51202fcbc377Syt 		/* Call ddi_intr_block_disable(). */
51212fcbc377Syt 		(void) ddi_intr_block_disable(ahci_ctlp->ahcictl_intr_htable,
51222fcbc377Syt 		    ahci_ctlp->ahcictl_intr_cnt);
51232fcbc377Syt 	} else {
51242fcbc377Syt 		for (x = 0; x < ahci_ctlp->ahcictl_intr_cnt; x++) {
51252fcbc377Syt 			(void) ddi_intr_disable(
51262fcbc377Syt 			    ahci_ctlp->ahcictl_intr_htable[x]);
51272fcbc377Syt 		}
51282fcbc377Syt 	}
51292fcbc377Syt 
51302fcbc377Syt 	/* Call ddi_intr_remove_handler(). */
51312fcbc377Syt 	for (x = 0; x < ahci_ctlp->ahcictl_intr_cnt; x++) {
51322fcbc377Syt 		(void) ddi_intr_remove_handler(
51332fcbc377Syt 		    ahci_ctlp->ahcictl_intr_htable[x]);
51342fcbc377Syt 		(void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[x]);
51352fcbc377Syt 	}
51362fcbc377Syt 
51372fcbc377Syt 	kmem_free(ahci_ctlp->ahcictl_intr_htable, ahci_ctlp->ahcictl_intr_size);
51382fcbc377Syt }
51392fcbc377Syt 
51402fcbc377Syt /*
5141*68d33a25Syt  * This routine tries to put port into P:NotRunning state by clearing
5142*68d33a25Syt  * PxCMD.ST. HBA will clear PxCI to 0h, PxSACT to 0h, PxCMD.CCS to 0h
5143*68d33a25Syt  * and PxCMD.CR to '0'.
51442fcbc377Syt  *
51452fcbc377Syt  * WARNING!!! ahciport_mutex should be acquired before the function
51462fcbc377Syt  * is called.
51472fcbc377Syt  */
51482fcbc377Syt /* ARGSUSED */
51492fcbc377Syt static int
5150*68d33a25Syt ahci_put_port_into_notrunning_state(ahci_ctl_t *ahci_ctlp,
5151*68d33a25Syt     ahci_port_t *ahci_portp, uint8_t port)
51522fcbc377Syt {
51532fcbc377Syt 	uint32_t port_cmd_status;
51542fcbc377Syt 	int loop_count;
51552fcbc377Syt 
51562fcbc377Syt 	AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp,
5157*68d33a25Syt 	    "ahci_put_port_into_notrunning_state enter: port %d", port);
51582fcbc377Syt 
51592fcbc377Syt 	port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
51602fcbc377Syt 	    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
51612fcbc377Syt 
51622fcbc377Syt 	port_cmd_status &= ~AHCI_CMD_STATUS_ST;
51632fcbc377Syt 	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
51642fcbc377Syt 	    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), port_cmd_status);
51652fcbc377Syt 
51662fcbc377Syt 	/* Wait until PxCMD.CR is cleared */
51672fcbc377Syt 	loop_count = 0;
51682fcbc377Syt 	do {
51692fcbc377Syt 		port_cmd_status =
51702fcbc377Syt 		    ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
51712fcbc377Syt 		    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
51722fcbc377Syt 
51732fcbc377Syt 		if (loop_count++ > AHCI_POLLRATE_PORT_IDLE) {
51742fcbc377Syt 			AHCIDBG2(AHCIDBG_INIT, ahci_ctlp,
51752fcbc377Syt 			    "clearing port %d CMD.CR timeout, "
51762fcbc377Syt 			    "port_cmd_status = 0x%x", port,
51772fcbc377Syt 			    port_cmd_status);
51782fcbc377Syt 			/*
51792fcbc377Syt 			 * We are effectively timing out after 0.5 sec.
51802fcbc377Syt 			 * This value is specified in AHCI spec.
51812fcbc377Syt 			 */
51822fcbc377Syt 			break;
51832fcbc377Syt 		}
51842fcbc377Syt 
51852fcbc377Syt 	/* Wait for 10 millisec */
51862fcbc377Syt #ifndef __lock_lint
51872fcbc377Syt 		delay(AHCI_10MS_TICKS);
51882fcbc377Syt #endif /* __lock_lint */
51892fcbc377Syt 
51902fcbc377Syt 	} while (port_cmd_status & AHCI_CMD_STATUS_CR);
51912fcbc377Syt 
5192*68d33a25Syt 	ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_STARTED;
51932fcbc377Syt 
5194*68d33a25Syt 	if (port_cmd_status & AHCI_CMD_STATUS_CR) {
51952fcbc377Syt 		AHCIDBG2(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp,
5196*68d33a25Syt 		    "ahci_put_port_into_notrunning_state: failed to clear "
5197*68d33a25Syt 		    "PxCMD.CR to '0' after loop count: %d, and "
5198*68d33a25Syt 		    "port_cmd_status = 0x%x", loop_count, port_cmd_status);
51992fcbc377Syt 		return (AHCI_FAILURE);
52002fcbc377Syt 	} else {
5201*68d33a25Syt 		AHCIDBG2(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp,
5202*68d33a25Syt 		    "ahci_put_port_into_notrunning_state: succeeded to clear "
5203*68d33a25Syt 		    "PxCMD.CR to '0' after loop count: %d, and "
5204*68d33a25Syt 		    "port_cmd_status = 0x%x", loop_count, port_cmd_status);
52052fcbc377Syt 		return (AHCI_SUCCESS);
52062fcbc377Syt 	}
52072fcbc377Syt }
52082fcbc377Syt 
52092fcbc377Syt /*
5210*68d33a25Syt  * First clear PxCMD.ST, and then check PxTFD. If both PxTFD.STS.BSY
5211*68d33a25Syt  * and PxTFD.STS.DRQ cleared to '0', it means the device is in a
5212*68d33a25Syt  * stable state, then set PxCMD.ST to '1' to start the port directly.
5213*68d33a25Syt  * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then issue a
5214*68d33a25Syt  * COMRESET to the device to put it in an idle state.
5215*68d33a25Syt  *
5216*68d33a25Syt  * The fifth argument returns whether the port reset is involved during
5217*68d33a25Syt  * the process.
5218*68d33a25Syt  *
5219*68d33a25Syt  * The routine will be called under six scenarios:
52202fcbc377Syt  *	1. Initialize the port
52212fcbc377Syt  *	2. To abort the packet(s)
52222fcbc377Syt  *	3. To reset the port
5223*68d33a25Syt  *	4. To activate the port
5224*68d33a25Syt  *	5. Fatal error recovery
5225*68d33a25Syt  *	6. To abort the timeout packet(s)
52262fcbc377Syt  *
52272fcbc377Syt  * WARNING!!! ahciport_mutex should be acquired before the function
5228*68d33a25Syt  * is called. And ahciport_mutex will be released before the reset
5229*68d33a25Syt  * event is reported to sata module by calling sata_hba_event_notify,
5230*68d33a25Syt  * and then be acquired again later.
52312fcbc377Syt  */
52322fcbc377Syt static int
52332fcbc377Syt ahci_restart_port_wait_till_ready(ahci_ctl_t *ahci_ctlp,
5234*68d33a25Syt     ahci_port_t *ahci_portp, uint8_t port, int flag, int *reset_flag)
52352fcbc377Syt {
52362fcbc377Syt 	uint32_t task_file_status;
52372fcbc377Syt 	sata_device_t sdevice;
52382fcbc377Syt 	int rval;
52392fcbc377Syt 
52402fcbc377Syt 	AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp,
52412fcbc377Syt 	    "ahci_restart_port_wait_till_ready: port %d enter", port);
52422fcbc377Syt 
5243*68d33a25Syt 	/* First disable the interrupt */
52442fcbc377Syt 	ahci_disable_port_intrs(ahci_ctlp, ahci_portp, port);
52452fcbc377Syt 
5246*68d33a25Syt 	/* Then clear PxCMD.ST */
5247*68d33a25Syt 	rval = ahci_put_port_into_notrunning_state(ahci_ctlp, ahci_portp,
5248*68d33a25Syt 	    port);
5249*68d33a25Syt 	if (rval != AHCI_SUCCESS)
5250*68d33a25Syt 		/*
5251*68d33a25Syt 		 * If PxCMD.CR does not clear within a reasonable time, it
5252*68d33a25Syt 		 * may assume the interface is in a hung condition and may
5253*68d33a25Syt 		 * continue with issuing the port reset.
5254*68d33a25Syt 		 */
5255*68d33a25Syt 		goto reset;
52562fcbc377Syt 
5257*68d33a25Syt 	/* Then clear PxSERR */
5258*68d33a25Syt 	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
5259*68d33a25Syt 	    (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
5260*68d33a25Syt 	    AHCI_SERROR_CLEAR_ALL);
52612fcbc377Syt 
5262*68d33a25Syt 	/* The get PxTFD */
5263*68d33a25Syt 	task_file_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5264*68d33a25Syt 	    (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
52652fcbc377Syt 
5266*68d33a25Syt 	/*
5267*68d33a25Syt 	 * Check whether the device is in a stable status, if yes,
5268*68d33a25Syt 	 * then start the port directly. However for ahci_tran_dport_reset,
5269*68d33a25Syt 	 * we may have to perform a port reset.
5270*68d33a25Syt 	 */
5271*68d33a25Syt 	if (!(task_file_status & (AHCI_TFD_STS_BSY | AHCI_TFD_STS_DRQ)) &&
5272*68d33a25Syt 	    !(flag & AHCI_PORT_RESET))
52732fcbc377Syt 		goto out;
52742fcbc377Syt 
5275*68d33a25Syt reset:
52762fcbc377Syt 	/* Set the reset in progress flag */
52772fcbc377Syt 	if (!(flag & AHCI_RESET_NO_EVENTS_UP)) {
52782fcbc377Syt 		ahci_portp->ahciport_reset_in_progress = 1;
52792fcbc377Syt 	}
52802fcbc377Syt 
5281*68d33a25Syt 	/*
5282*68d33a25Syt 	 * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then issue
5283*68d33a25Syt 	 * a COMRESET to the device
5284*68d33a25Syt 	 */
52852fcbc377Syt 	rval = ahci_port_reset(ahci_ctlp, ahci_portp, port);
5286*68d33a25Syt 	if (rval != AHCI_SUCCESS)
52872fcbc377Syt 		AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp,
52882fcbc377Syt 		    "ahci_restart_port_wait_till_ready: port %d failed",
52892fcbc377Syt 		    port);
5290*68d33a25Syt 
5291*68d33a25Syt 	if (reset_flag != NULL)
5292*68d33a25Syt 		*reset_flag = 1;
52932fcbc377Syt 
52942fcbc377Syt 	/* Indicate to the framework that a reset has happened. */
52952fcbc377Syt 	if (!(flag & AHCI_RESET_NO_EVENTS_UP)) {
52962fcbc377Syt 
52972fcbc377Syt 		bzero((void *)&sdevice, sizeof (sata_device_t));
529809121340Syt 		sdevice.satadev_addr.cport =
529909121340Syt 		    ahci_ctlp->ahcictl_port_to_cport[port];
5300*68d33a25Syt 		sdevice.satadev_addr.pmport = 0;
5301*68d33a25Syt 		sdevice.satadev_addr.qual = SATA_ADDR_DCPORT;
53022fcbc377Syt 
53032fcbc377Syt 		sdevice.satadev_state = SATA_DSTATE_RESET |
53042fcbc377Syt 		    SATA_DSTATE_PWR_ACTIVE;
53052fcbc377Syt 		if (ahci_ctlp->ahcictl_sata_hba_tran) {
53062fcbc377Syt 			mutex_exit(&ahci_portp->ahciport_mutex);
53072fcbc377Syt 			sata_hba_event_notify(
53082fcbc377Syt 			    ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
53092fcbc377Syt 			    &sdevice,
53102fcbc377Syt 			    SATA_EVNT_DEVICE_RESET);
53112fcbc377Syt 			mutex_enter(&ahci_portp->ahciport_mutex);
53122fcbc377Syt 		}
53132fcbc377Syt 
5314*68d33a25Syt 		AHCIDBG1(AHCIDBG_EVENT, ahci_ctlp,
5315*68d33a25Syt 		    "port %d sending event up: SATA_EVNT_RESET", port);
53162fcbc377Syt 	}
53172fcbc377Syt out:
5318*68d33a25Syt 	/* Start the port */
53192fcbc377Syt 	if (!(flag & AHCI_PORT_INIT)) {
5320*68d33a25Syt 		(void) ahci_start_port(ahci_ctlp, ahci_portp, port);
53212fcbc377Syt 		ahci_enable_port_intrs(ahci_ctlp, ahci_portp, port);
53222fcbc377Syt 	}
53232fcbc377Syt 
53242fcbc377Syt 	return (rval);
53252fcbc377Syt }
53262fcbc377Syt 
53272fcbc377Syt /*
5328*68d33a25Syt  * This routine may be called under four scenarios:
5329*68d33a25Syt  *	a) do the recovery from fatal error
53302fcbc377Syt  *	b) or we need to timeout some commands
53312fcbc377Syt  *	c) or we need to abort some commands
53322fcbc377Syt  *	d) or we need reset device/port/controller
53332fcbc377Syt  *
53342fcbc377Syt  * In all these scenarios, we need to send any pending unfinished
53352fcbc377Syt  * commands up to sata framework.
53362fcbc377Syt  *
5337*68d33a25Syt  * WARNING!!! ahciport_mutex should be acquired before the function is called.
53382fcbc377Syt  */
53392fcbc377Syt static void
53402fcbc377Syt ahci_mop_commands(ahci_ctl_t *ahci_ctlp,
53412fcbc377Syt     ahci_port_t *ahci_portp,
53422fcbc377Syt     uint8_t port,
53432fcbc377Syt     uint32_t slot_status,
53442fcbc377Syt     uint32_t failed_tags,
53452fcbc377Syt     uint32_t timeout_tags,
53462fcbc377Syt     uint32_t aborted_tags,
53472fcbc377Syt     uint32_t reset_tags)
53482fcbc377Syt {
53492fcbc377Syt 	uint32_t finished_tags, unfinished_tags;
53502fcbc377Syt 	int tmp_slot;
53512fcbc377Syt 	sata_pkt_t *satapkt;
53522fcbc377Syt 
53532fcbc377Syt 	AHCIDBG2(AHCIDBG_ERRS|AHCIDBG_ENTRY, ahci_ctlp,
53542fcbc377Syt 	    "ahci_mop_commands entered: port: %d slot_status: 0x%x",
53552fcbc377Syt 	    port, slot_status);
53562fcbc377Syt 
53572fcbc377Syt 	AHCIDBG4(AHCIDBG_ERRS|AHCIDBG_ENTRY, ahci_ctlp,
53582fcbc377Syt 	    "ahci_mop_commands: failed_tags: 0x%x, "
53592fcbc377Syt 	    "timeout_tags: 0x%x aborted_tags: 0x%x, "
53602fcbc377Syt 	    "reset_tags: 0x%x", failed_tags,
53612fcbc377Syt 	    timeout_tags, aborted_tags, reset_tags);
53622fcbc377Syt 
53632fcbc377Syt 	finished_tags = ahci_portp->ahciport_pending_tags &
53642fcbc377Syt 	    ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
53652fcbc377Syt 
53662fcbc377Syt 	unfinished_tags = slot_status &
53672fcbc377Syt 	    AHCI_SLOT_MASK(ahci_ctlp) &
53682fcbc377Syt 	    ~failed_tags &
53692fcbc377Syt 	    ~aborted_tags &
53702fcbc377Syt 	    ~reset_tags &
53712fcbc377Syt 	    ~timeout_tags;
53722fcbc377Syt 
53732fcbc377Syt 	/*
5374*68d33a25Syt 	 * When AHCI_PORT_FLAG_RQSENSE is set, it means REQUEST SENSE
5375*68d33a25Syt 	 * command doesn't complete successfully due to one of the
5376*68d33a25Syt 	 * following three conditions:
5377*68d33a25Syt 	 *
5378*68d33a25Syt 	 *	1. Fatal error - failed_tags includes its slot
5379*68d33a25Syt 	 *	2. Timed out - timeout_tags includes its slot
5380*68d33a25Syt 	 *	3. Aborted when hot unplug - aborted_tags includes its slot
53812fcbc377Syt 	 */
5382*68d33a25Syt 	if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_RQSENSE) {
5383*68d33a25Syt 		AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands "
5384*68d33a25Syt 		    "is called for port %d while REQUEST SENSE for error "
5385*68d33a25Syt 		    "retrieval is being executed", port);
5386*68d33a25Syt 		ASSERT(ahci_portp->ahciport_mop_in_progress > 1);
5387*68d33a25Syt 		finished_tags = 0;
5388*68d33a25Syt 		unfinished_tags = 0;
53892fcbc377Syt 	}
53902fcbc377Syt 
5391*68d33a25Syt 	/* Send up finished packets with SATA_PKT_COMPLETED */
53922fcbc377Syt 	while (finished_tags) {
53932fcbc377Syt 		tmp_slot = ddi_ffs(finished_tags) - 1;
53942fcbc377Syt 		if (tmp_slot == -1) {
53952fcbc377Syt 			break;
53962fcbc377Syt 		}
53972fcbc377Syt 
53982fcbc377Syt 		satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
53992fcbc377Syt 		ASSERT(satapkt != NULL);
54002fcbc377Syt 
54012fcbc377Syt 		AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, "ahci_mop_commands: "
54022fcbc377Syt 		    "sending up pkt 0x%p with SATA_PKT_COMPLETED",
54032fcbc377Syt 		    (void *)satapkt);
54042fcbc377Syt 
5405*68d33a25Syt 		/*
5406*68d33a25Syt 		 * Cannot fetch the return register content since the port
5407*68d33a25Syt 		 * was restarted, so the corresponding tag will be set to
5408*68d33a25Syt 		 * aborted tags.
5409*68d33a25Syt 		 */
5410*68d33a25Syt 		if (satapkt->satapkt_cmd.satacmd_flags.sata_special_regs) {
5411*68d33a25Syt 			CLEAR_BIT(finished_tags, tmp_slot);
5412*68d33a25Syt 			aborted_tags |= tmp_slot;
5413*68d33a25Syt 			continue;
5414*68d33a25Syt 		}
5415*68d33a25Syt 
54162fcbc377Syt 		CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
54172fcbc377Syt 		CLEAR_BIT(finished_tags, tmp_slot);
54182fcbc377Syt 		ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
54192fcbc377Syt 
54202fcbc377Syt 		SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_COMPLETED);
54212fcbc377Syt 	}
54222fcbc377Syt 
5423*68d33a25Syt 	/* Send up failed packets with SATA_PKT_DEV_ERROR. */
54242fcbc377Syt 	while (failed_tags) {
54252fcbc377Syt 		tmp_slot = ddi_ffs(failed_tags) - 1;
54262fcbc377Syt 		if (tmp_slot == -1) {
54272fcbc377Syt 			break;
54282fcbc377Syt 		}
54292fcbc377Syt 
54302fcbc377Syt 		satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
54312fcbc377Syt 		ASSERT(satapkt != NULL);
54322fcbc377Syt 
54332fcbc377Syt 		AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
54342fcbc377Syt 		    "sending up pkt 0x%p with SATA_PKT_DEV_ERROR",
54352fcbc377Syt 		    (void *)satapkt);
54362fcbc377Syt 
54372fcbc377Syt 		CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
54382fcbc377Syt 		CLEAR_BIT(failed_tags, tmp_slot);
54392fcbc377Syt 		ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
54402fcbc377Syt 
54412fcbc377Syt 		SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_DEV_ERROR);
54422fcbc377Syt 	}
54432fcbc377Syt 
5444*68d33a25Syt 	/* Send up timeout packets with SATA_PKT_TIMEOUT. */
54452fcbc377Syt 	while (timeout_tags) {
54462fcbc377Syt 		tmp_slot = ddi_ffs(timeout_tags) - 1;
54472fcbc377Syt 		if (tmp_slot == -1) {
54482fcbc377Syt 			break;
54492fcbc377Syt 		}
54502fcbc377Syt 
54512fcbc377Syt 		satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
54522fcbc377Syt 		ASSERT(satapkt != NULL);
54532fcbc377Syt 
54542fcbc377Syt 		AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
54552fcbc377Syt 		    "sending up pkt 0x%p with SATA_PKT_TIMEOUT",
54562fcbc377Syt 		    (void *)satapkt);
54572fcbc377Syt 
54582fcbc377Syt 		CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
54592fcbc377Syt 		CLEAR_BIT(timeout_tags, tmp_slot);
54602fcbc377Syt 		ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
54612fcbc377Syt 
54622fcbc377Syt 		SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_TIMEOUT);
54632fcbc377Syt 	}
54642fcbc377Syt 
5465*68d33a25Syt 	/* Send up aborted packets with SATA_PKT_ABORTED */
54662fcbc377Syt 	while (aborted_tags) {
54672fcbc377Syt 		tmp_slot = ddi_ffs(aborted_tags) - 1;
54682fcbc377Syt 		if (tmp_slot == -1) {
54692fcbc377Syt 			break;
54702fcbc377Syt 		}
54712fcbc377Syt 
54722fcbc377Syt 		satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
54732fcbc377Syt 		ASSERT(satapkt != NULL);
54742fcbc377Syt 
54752fcbc377Syt 		AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
54762fcbc377Syt 		    "sending up pkt 0x%p with SATA_PKT_ABORTED",
54772fcbc377Syt 		    (void *)satapkt);
54782fcbc377Syt 
54792fcbc377Syt 		CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
54802fcbc377Syt 		CLEAR_BIT(aborted_tags, tmp_slot);
54812fcbc377Syt 		ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
54822fcbc377Syt 
54832fcbc377Syt 		SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_ABORTED);
54842fcbc377Syt 	}
54852fcbc377Syt 
5486*68d33a25Syt 	/* Send up reset packets with SATA_PKT_RESET. */
54872fcbc377Syt 	while (reset_tags) {
54882fcbc377Syt 		tmp_slot = ddi_ffs(reset_tags) - 1;
54892fcbc377Syt 		if (tmp_slot == -1) {
54902fcbc377Syt 			break;
54912fcbc377Syt 		}
54922fcbc377Syt 
54932fcbc377Syt 		satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
54942fcbc377Syt 		ASSERT(satapkt != NULL);
54952fcbc377Syt 
54962fcbc377Syt 		AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
54972fcbc377Syt 		    "sending up pkt 0x%p with SATA_PKT_RESET",
54982fcbc377Syt 		    (void *)satapkt);
54992fcbc377Syt 
55002fcbc377Syt 		CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
55012fcbc377Syt 		CLEAR_BIT(reset_tags, tmp_slot);
55022fcbc377Syt 		ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
55032fcbc377Syt 
55042fcbc377Syt 		SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_RESET);
55052fcbc377Syt 	}
55062fcbc377Syt 
5507*68d33a25Syt 	/* Send up unfinished packets with SATA_PKT_RESET */
55082fcbc377Syt 	while (unfinished_tags) {
55092fcbc377Syt 		tmp_slot = ddi_ffs(unfinished_tags) - 1;
55102fcbc377Syt 		if (tmp_slot == -1) {
55112fcbc377Syt 			break;
55122fcbc377Syt 		}
55132fcbc377Syt 
55142fcbc377Syt 		satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
55152fcbc377Syt 		ASSERT(satapkt != NULL);
55162fcbc377Syt 
55172fcbc377Syt 		AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
5518*68d33a25Syt 		    "sending up pkt 0x%p with SATA_PKT_RESET",
55192fcbc377Syt 		    (void *)satapkt);
55202fcbc377Syt 
55212fcbc377Syt 		CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
55222fcbc377Syt 		CLEAR_BIT(unfinished_tags, tmp_slot);
55232fcbc377Syt 		ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;
55242fcbc377Syt 
5525*68d33a25Syt 		SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_RESET);
55262fcbc377Syt 	}
55272fcbc377Syt 
5528*68d33a25Syt 	ahci_portp->ahciport_mop_in_progress--;
5529*68d33a25Syt 	ASSERT(ahci_portp->ahciport_mop_in_progress >= 0);
55302fcbc377Syt 
5531*68d33a25Syt 	if (ahci_portp->ahciport_mop_in_progress == 0)
5532*68d33a25Syt 		ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_MOPPING;
5533*68d33a25Syt }
55342fcbc377Syt 
5535*68d33a25Syt /*
5536*68d33a25Syt  * This routine is going to first request a REQUEST SENSE sata pkt from sata
5537*68d33a25Syt  * module, and then deliver it to the HBA to get the sense data and copy
5538*68d33a25Syt  * the sense data back to the orignal failed sata pkt, and free the REQUEST
5539*68d33a25Syt  * SENSE sata pkt later.
5540*68d33a25Syt  */
5541*68d33a25Syt static void
5542*68d33a25Syt ahci_get_rqsense_data(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
5543*68d33a25Syt     uint8_t port, sata_pkt_t *spkt)
5544*68d33a25Syt {
5545*68d33a25Syt 	sata_device_t	sdevice;
5546*68d33a25Syt 	sata_pkt_t	*rs_spkt;
5547*68d33a25Syt 	sata_cmd_t	*sata_cmd;
5548*68d33a25Syt 	ddi_dma_handle_t buf_dma_handle;
5549*68d33a25Syt 	int		loop_count;
5550*68d33a25Syt 	int		rval;
5551*68d33a25Syt #if AHCI_DEBUG
5552*68d33a25Syt 	struct scsi_extended_sense *rqsense;
5553*68d33a25Syt #endif
5554*68d33a25Syt 
5555*68d33a25Syt 	AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp,
5556*68d33a25Syt 	    "ahci_get_rqsense_data enter: port %d", port);
5557*68d33a25Syt 
5558*68d33a25Syt 	/* Prepare the sdevice data */
5559*68d33a25Syt 	bzero((void *)&sdevice, sizeof (sata_device_t));
5560*68d33a25Syt 	sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];
5561*68d33a25Syt 
5562*68d33a25Syt 	sdevice.satadev_addr.qual = SATA_ADDR_DCPORT;
5563*68d33a25Syt 	sdevice.satadev_addr.pmport = 0;
5564*68d33a25Syt 
5565*68d33a25Syt 	sata_cmd = &spkt->satapkt_cmd;
5566*68d33a25Syt 
5567*68d33a25Syt 	/*
5568*68d33a25Syt 	 * Call the sata hba interface to get a rs spkt
5569*68d33a25Syt 	 */
5570*68d33a25Syt 	loop_count = 0;
5571*68d33a25Syt loop:
5572*68d33a25Syt 	rs_spkt = sata_get_error_retrieval_pkt(ahci_ctlp->ahcictl_dip,
5573*68d33a25Syt 	    &sdevice, SATA_ERR_RETR_PKT_TYPE_ATAPI);
5574*68d33a25Syt 	if (rs_spkt == NULL) {
5575*68d33a25Syt 		if (loop_count++ < AHCI_POLLRATE_GET_SPKT) {
5576*68d33a25Syt 			/* Sleep for a while */
5577*68d33a25Syt 			delay(AHCI_10MS_TICKS);
5578*68d33a25Syt 			goto loop;
5579*68d33a25Syt 
5580*68d33a25Syt 		}
5581*68d33a25Syt 		/* Timed out after 1s */
5582*68d33a25Syt 		AHCIDBG1(AHCIDBG_INFO, ahci_ctlp,
5583*68d33a25Syt 		    "failed to get rs spkt for port %d", port);
5584*68d33a25Syt 		return;
5585*68d33a25Syt 	}
5586*68d33a25Syt 
5587*68d33a25Syt 	ASSERT(rs_spkt->satapkt_op_mode & SATA_OPMODE_SYNCH);
5588*68d33a25Syt 
5589*68d33a25Syt 	/*
5590*68d33a25Syt 	 * This flag is used to handle the specific error recovery when the
5591*68d33a25Syt 	 * REQUEST SENSE command gets a faiure (failure error or time-out).
5592*68d33a25Syt 	 */
5593*68d33a25Syt 	ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_RQSENSE;
5594*68d33a25Syt 
5595*68d33a25Syt 	/* The driver would like to do it in POLL mode */
5596*68d33a25Syt 	rs_spkt->satapkt_op_mode |= SATA_OPMODE_POLLING;
5597*68d33a25Syt 
5598*68d33a25Syt 	/*
5599*68d33a25Syt 	 * This start is not supposed to fail because after port is restarted,
5600*68d33a25Syt 	 * the command slot is empty
5601*68d33a25Syt 	 */
5602*68d33a25Syt 	rval = ahci_do_sync_start(ahci_ctlp, ahci_portp, port, rs_spkt);
5603*68d33a25Syt 	ASSERT(rval != AHCI_FAILURE);
5604*68d33a25Syt 
5605*68d33a25Syt 	/* Remove the flag after REQUEST SENSE command is completed */
5606*68d33a25Syt 	ahci_portp->ahciport_flags &= ~ AHCI_PORT_FLAG_RQSENSE;
5607*68d33a25Syt 
5608*68d33a25Syt 	if (rs_spkt->satapkt_reason == SATA_PKT_COMPLETED) {
5609*68d33a25Syt 		/* Update the request sense data */
5610*68d33a25Syt 		buf_dma_handle = *(ddi_dma_handle_t *)
5611*68d33a25Syt 		    (rs_spkt->satapkt_cmd.satacmd_err_ret_buf_handle);
5612*68d33a25Syt 		rval = ddi_dma_sync(buf_dma_handle, 0, 0,
5613*68d33a25Syt 		    DDI_DMA_SYNC_FORKERNEL);
5614*68d33a25Syt 		if (rval == DDI_SUCCESS) {
5615*68d33a25Syt 			/* Copy the request sense data */
5616*68d33a25Syt 			bcopy(rs_spkt->
5617*68d33a25Syt 			    satapkt_cmd.satacmd_bp->b_un.b_addr,
5618*68d33a25Syt 			    &sata_cmd->satacmd_rqsense,
5619*68d33a25Syt 			    SATA_ATAPI_MIN_RQSENSE_LEN);
5620*68d33a25Syt #if AHCI_DEBUG
5621*68d33a25Syt 			rqsense = (struct scsi_extended_sense *)
5622*68d33a25Syt 			    sata_cmd->satacmd_rqsense;
5623*68d33a25Syt 
5624*68d33a25Syt 			/* Dump the sense data */
5625*68d33a25Syt 			AHCIDBG0(AHCIDBG_SENSEDATA, ahci_ctlp, "\n");
5626*68d33a25Syt 			AHCIDBG2(AHCIDBG_SENSEDATA, ahci_ctlp,
5627*68d33a25Syt 			    "Sense data for satapkt %p ATAPI cmd 0x%x",
5628*68d33a25Syt 			    spkt, sata_cmd->satacmd_acdb[0]);
5629*68d33a25Syt 			AHCIDBG5(AHCIDBG_SENSEDATA, ahci_ctlp,
5630*68d33a25Syt 			    "  es_code 0x%x es_class 0x%x "
5631*68d33a25Syt 			    "es_key 0x%x es_add_code 0x%x "
5632*68d33a25Syt 			    "es_qual_code 0x%x",
5633*68d33a25Syt 			    rqsense->es_code, rqsense->es_class,
5634*68d33a25Syt 			    rqsense->es_key, rqsense->es_add_code,
5635*68d33a25Syt 			    rqsense->es_qual_code);
5636*68d33a25Syt #endif
5637*68d33a25Syt 		}
5638*68d33a25Syt 	}
5639*68d33a25Syt 
5640*68d33a25Syt 	sata_free_error_retrieval_pkt(rs_spkt);
56412fcbc377Syt }
56422fcbc377Syt 
56432fcbc377Syt /*
56442fcbc377Syt  * Fatal errors will cause the HBA to enter the ERR: Fatal state. To recover,
5645*68d33a25Syt  * the port must be restarted. When the HBA detects thus error, it may try
5646*68d33a25Syt  * to abort a transfer. And if the transfer was aborted, the device is
5647*68d33a25Syt  * expected to send a D2H Register FIS with PxTFD.STS.ERR set to '1' and both
5648*68d33a25Syt  * PxTFD.STS.BSY and PxTFD.STS.DRQ cleared to '0'. Then system software knows
5649*68d33a25Syt  * that the device is in a stable status and transfers may be restarted without
5650*68d33a25Syt  * issuing a COMRESET to the device. If PxTFD.STS.BSY or PxTFD.STS.DRQ is set,
5651*68d33a25Syt  * then the software will send the COMRESET to do the port reset.
5652*68d33a25Syt  *
5653*68d33a25Syt  * Software should perform the appropriate error recovery actions based on
5654*68d33a25Syt  * whether non-queued commands were being issued or natived command queuing
5655*68d33a25Syt  * commands were being issued.
5656*68d33a25Syt  *
5657*68d33a25Syt  * And software will complete the command that had the error with error mark
5658*68d33a25Syt  * to higher level software.
56592fcbc377Syt  *
56602fcbc377Syt  * Fatal errors include the following:
56612fcbc377Syt  * 	PxIS.IFS - Interface Fatal Error Status
56622fcbc377Syt  * 	PxIS.HBDS - Host Bus Data Error Status
56632fcbc377Syt  * 	PxIS.HBFS - Host Bus Fatal Error Status
56642fcbc377Syt  *	PxIS.TFES - Task File Error Status
56652fcbc377Syt  *
56662fcbc377Syt  * WARNING!!! ahciport_mutex should be acquired before the function is called.
56672fcbc377Syt  */
5668*68d33a25Syt static void
5669*68d33a25Syt ahci_fatal_error_recovery_handler(ahci_ctl_t *ahci_ctlp,
5670*68d33a25Syt     ahci_port_t *ahci_portp, uint8_t port, uint32_t intr_status,
5671*68d33a25Syt     uint32_t retrieve_errinfo_slot_status)
56722fcbc377Syt {
5673*68d33a25Syt 	uint32_t	port_cmd_status, port_cmd_issue;
5674*68d33a25Syt 	uint32_t	failed_tags = 0;
5675*68d33a25Syt 	int		failed_slot;
5676*68d33a25Syt 	int		reset_flag = 0;
5677*68d33a25Syt 	ahci_fis_d2h_register_t	*ahci_rcvd_fisp;
5678*68d33a25Syt 	sata_cmd_t	*sata_cmd;
5679*68d33a25Syt 	sata_pkt_t	*spkt;
56802fcbc377Syt 
56812fcbc377Syt 	AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp,
5682*68d33a25Syt 	    "ahci_fatal_error_recovery_handler enter: port %d", port);
56832fcbc377Syt 
5684*68d33a25Syt 	/* Read PxCI to see which commands are still outstanding */
5685*68d33a25Syt 	port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5686*68d33a25Syt 	    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
56872fcbc377Syt 
5688*68d33a25Syt 	/*
5689*68d33a25Syt 	 * Read PxCMD.CCS to determine the slot that the HBA
5690*68d33a25Syt 	 * was processing when the error occurred.
5691*68d33a25Syt 	 */
56922fcbc377Syt 	port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
56932fcbc377Syt 	    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5694*68d33a25Syt 	failed_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >>
56952fcbc377Syt 	    AHCI_CMD_STATUS_CCS_SHIFT;
56962fcbc377Syt 
5697*68d33a25Syt 	spkt = ahci_portp->ahciport_slot_pkts[failed_slot];
56982fcbc377Syt 	if (spkt == NULL) {
5699*68d33a25Syt 		/* This may happen when interface errors occur */
5700*68d33a25Syt 		goto next;
57012fcbc377Syt 	}
5702*68d33a25Syt 	sata_cmd = &spkt->satapkt_cmd;
57032fcbc377Syt 
57042fcbc377Syt 	/*
5705*68d33a25Syt 	 * We need to set retrieve_errinfo_slot_status to failed_slot
5706*68d33a25Syt 	 * in case it's an interface error before REQUEST SENSE slot
5707*68d33a25Syt 	 * is executed, that is spkt is NULL.
57082fcbc377Syt 	 */
5709*68d33a25Syt 	if (retrieve_errinfo_slot_status != 0) {
5710*68d33a25Syt 		failed_slot = retrieve_errinfo_slot_status;
57112fcbc377Syt 	}
57122fcbc377Syt 
5713*68d33a25Syt 	/* The failed command must be one of the outstanding commands */
5714*68d33a25Syt 	failed_tags = 0x1 << failed_slot;
5715*68d33a25Syt 	ASSERT(failed_tags & port_cmd_issue);
57162fcbc377Syt 
5717*68d33a25Syt 	/* Update the sata registers, especially PxSERR register */
5718*68d33a25Syt 	ahci_update_sata_registers(ahci_ctlp, port,
5719*68d33a25Syt 	    &spkt->satapkt_device);
57202fcbc377Syt 
5721*68d33a25Syt 	/* Fill out the status and error registers for PxIS.TFES */
5722*68d33a25Syt 	if (intr_status & AHCI_INTR_STATUS_TFES) {
5723*68d33a25Syt 		ahci_rcvd_fisp = &(ahci_portp->ahciport_rcvd_fis->
5724*68d33a25Syt 		    ahcirf_d2h_register_fis);
57252fcbc377Syt 
5726*68d33a25Syt 		/* Copy the error context back to the sata_cmd */
5727*68d33a25Syt 		ahci_copy_err_cnxt(sata_cmd, ahci_rcvd_fisp);
5728*68d33a25Syt 	}
57292fcbc377Syt 
5730*68d33a25Syt next:
57312fcbc377Syt 
5732*68d33a25Syt #if AHCI_DEBUG
5733*68d33a25Syt 	if (failed_tags == 0) {
5734*68d33a25Syt 		AHCIDBG2(AHCIDBG_ERRS, ahci_ctlp,
5735*68d33a25Syt 		    "ahci_fatal_error_recovery_handler: port %d interface "
5736*68d33a25Syt 		    "error occurred while no command was processing "
5737*68d33a25Syt 		    "port_cmd_issue = 0x%x", port, port_cmd_issue);
5738*68d33a25Syt 	} else {
5739*68d33a25Syt 		AHCIDBG3(AHCIDBG_ERRS, ahci_ctlp,
5740*68d33a25Syt 		    "ahci_fatal_error_recovery_handler: port %d "
5741*68d33a25Syt 		    "failed_tags = 0x%x, port_cmd_issue = 0x%x",
5742*68d33a25Syt 		    port, failed_tags, port_cmd_issue);
5743*68d33a25Syt 	}
57442fcbc377Syt 
5745*68d33a25Syt 	/*
5746*68d33a25Syt 	 * retrieve_errinfo_slot_status contains the slot number of REQUEST
5747*68d33a25Syt 	 * SENSE command. If it is not 0, it means a fatal error happens after
5748*68d33a25Syt 	 * REQUEST SENSE command is delivered to the HBA during the error
5749*68d33a25Syt 	 * recovery process. At this time, AHCI_PORT_FLAG_RQSENSE is supposed
5750*68d33a25Syt 	 * to be set, and the only outstanding command is REQUEST SENSE.
5751*68d33a25Syt 	 */
5752*68d33a25Syt 	if (retrieve_errinfo_slot_status != 0) {
57532fcbc377Syt 		AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp,
5754*68d33a25Syt 		    "ahci_fatal_error_recovery_handler: port %d REQUEST SENSE "
5755*68d33a25Syt 		    "command for error retrieval failed", port);
5756*68d33a25Syt 		ASSERT(ahci_portp->ahciport_flags & AHCI_PORT_FLAG_RQSENSE);
5757*68d33a25Syt 		ASSERT(retrieve_errinfo_slot_status == port_cmd_issue);
5758*68d33a25Syt 		ASSERT(spkt && spkt->satapkt_cmd.satacmd_acdb[0] ==
5759*68d33a25Syt 		    SCMD_REQUEST_SENSE);
57602fcbc377Syt 	}
5761*68d33a25Syt #endif
57622fcbc377Syt 
5763*68d33a25Syt 	ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
5764*68d33a25Syt 	ahci_portp->ahciport_mop_in_progress++;
57652fcbc377Syt 
5766*68d33a25Syt 	(void) ahci_restart_port_wait_till_ready(ahci_ctlp, ahci_portp,
5767*68d33a25Syt 	    port, NULL, &reset_flag);
57682fcbc377Syt 
5769*68d33a25Syt 	/*
5770*68d33a25Syt 	 * Won't retrieve error information:
5771*68d33a25Syt 	 * 1. Port reset was involved to recover
5772*68d33a25Syt 	 * 2. No spkt is being executed
5773*68d33a25Syt 	 * 3. IDENTIFY DEVICE command sent to ATAPI device
5774*68d33a25Syt 	 * 4. REQUEST SENSE command during error recovery
5775*68d33a25Syt 	 */
5776*68d33a25Syt 	if (reset_flag || spkt == NULL ||
5777*68d33a25Syt 	    spkt->satapkt_cmd.satacmd_cmd_reg == SATAC_ID_DEVICE ||
5778*68d33a25Syt 	    retrieve_errinfo_slot_status != 0)
5779*68d33a25Syt 		goto out;
57802fcbc377Syt 
5781*68d33a25Syt 	/*
5782*68d33a25Syt 	 * Deliver REQUEST SENSE for ATAPI command to gather information about
5783*68d33a25Syt 	 * the error when a COMRESET has not been performed as part of the
5784*68d33a25Syt 	 * error recovery.
5785*68d33a25Syt 	 */
5786*68d33a25Syt 	if (ahci_portp->ahciport_device_type == SATA_DTYPE_ATAPICD)
5787*68d33a25Syt 		ahci_get_rqsense_data(ahci_ctlp, ahci_portp, port, spkt);
5788*68d33a25Syt 
5789*68d33a25Syt out:
57902fcbc377Syt 	ahci_mop_commands(ahci_ctlp,
57912fcbc377Syt 	    ahci_portp,
57922fcbc377Syt 	    port,
5793*68d33a25Syt 	    port_cmd_issue,
57942fcbc377Syt 	    failed_tags, /* failed tags */
57952fcbc377Syt 	    0, /* timeout tags */
57962fcbc377Syt 	    0, /* aborted tags */
57972fcbc377Syt 	    0); /* reset tags */
5798*68d33a25Syt }
5799*68d33a25Syt 
5800*68d33a25Syt /*
5801*68d33a25Syt  * Handle events - fatal error recovery
5802*68d33a25Syt  */
5803*68d33a25Syt static void
5804*68d33a25Syt ahci_events_handler(void *args)
5805*68d33a25Syt {
5806*68d33a25Syt 	ahci_event_arg_t *ahci_event_arg;
5807*68d33a25Syt 	ahci_ctl_t *ahci_ctlp;
5808*68d33a25Syt 	ahci_port_t *ahci_portp;
5809*68d33a25Syt 	uint32_t event;
5810*68d33a25Syt 	uint32_t retrieve_errinfo_slot_status;
5811*68d33a25Syt 	uint8_t port;
5812*68d33a25Syt 
5813*68d33a25Syt 	ahci_event_arg = (ahci_event_arg_t *)args;
5814*68d33a25Syt 	ahci_ctlp = ahci_event_arg->ahciea_ctlp;
5815*68d33a25Syt 	ahci_portp = ahci_event_arg->ahciea_portp;
5816*68d33a25Syt 	event = ahci_event_arg->ahciea_event;
5817*68d33a25Syt 	retrieve_errinfo_slot_status = ahci_event_arg->ahciea_retrierr_slot;
5818*68d33a25Syt 	port = ahci_portp->ahciport_port_num;
5819*68d33a25Syt 
5820*68d33a25Syt 	AHCIDBG3(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp,
5821*68d33a25Syt 	    "ahci_events_handler enter: "
5822*68d33a25Syt 	    "port %d intr_status = 0x%x retrieve_errinfo_slot_status = 0x%x",
5823*68d33a25Syt 	    port, event, retrieve_errinfo_slot_status);
5824*68d33a25Syt 
58252fcbc377Syt 	mutex_enter(&ahci_portp->ahciport_mutex);
58262fcbc377Syt 
5827*68d33a25Syt 	/*
5828*68d33a25Syt 	 * ahci_intr_phyrdy_change() may have rendered it to
5829*68d33a25Syt 	 * SATA_DTYPE_NONE.
5830*68d33a25Syt 	 */
5831*68d33a25Syt 	if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
5832*68d33a25Syt 		AHCIDBG1(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp,
5833*68d33a25Syt 		    "ahci_events_handler: port %d no device attached, "
5834*68d33a25Syt 		    "and just return without doing anything", port);
5835*68d33a25Syt 		goto out;
5836*68d33a25Syt 	}
5837*68d33a25Syt 
5838*68d33a25Syt 	if (event & (AHCI_INTR_STATUS_IFS |
5839*68d33a25Syt 	    AHCI_INTR_STATUS_HBDS |
5840*68d33a25Syt 	    AHCI_INTR_STATUS_HBFS |
5841*68d33a25Syt 	    AHCI_INTR_STATUS_TFES))
5842*68d33a25Syt 		ahci_fatal_error_recovery_handler(ahci_ctlp, ahci_portp,
5843*68d33a25Syt 		    port, event, retrieve_errinfo_slot_status);
5844*68d33a25Syt 
5845*68d33a25Syt out:
5846*68d33a25Syt 	mutex_exit(&ahci_portp->ahciport_mutex);
58472fcbc377Syt }
58482fcbc377Syt 
58492fcbc377Syt /*
5850*68d33a25Syt  * ahci_watchdog_handler() and ahci_do_sync_start will call us if they
5851*68d33a25Syt  * detect there are some commands which are timed out.
58522fcbc377Syt  */
58532fcbc377Syt static void
58542fcbc377Syt ahci_timeout_pkts(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
5855*68d33a25Syt     uint8_t port, uint32_t tmp_timeout_tags,
5856*68d33a25Syt     uint32_t retrieve_errinfo_slot_status)
58572fcbc377Syt {
5858*68d33a25Syt 	uint32_t port_cmd_issue;
58592fcbc377Syt 	uint32_t finished_tags, timeout_tags;
5860*68d33a25Syt #ifndef __lock_lint
5861*68d33a25Syt 	_NOTE(ARGUNUSED(retrieve_errinfo_slot_status))
5862*68d33a25Syt #endif
58632fcbc377Syt 
5864*68d33a25Syt 	AHCIDBG1(AHCIDBG_TIMEOUT|AHCIDBG_ENTRY, ahci_ctlp,
5865*68d33a25Syt 	    "ahci_timeout_pkts enter: port %d", port);
58662fcbc377Syt 
58672fcbc377Syt 	mutex_enter(&ahci_portp->ahciport_mutex);
58682fcbc377Syt 
5869*68d33a25Syt 	/* Read PxCI to see which commands are still outstanding */
5870*68d33a25Syt 	port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5871*68d33a25Syt 	    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
5872*68d33a25Syt 	ASSERT(port_cmd_issue != 0);
5873*68d33a25Syt 	ASSERT(tmp_timeout_tags & port_cmd_issue);
5874*68d33a25Syt 
5875*68d33a25Syt #if AHCI_DEBUG
58762fcbc377Syt 	/*
5877*68d33a25Syt 	 * Retrieve_errinfo_slot_status contains the slot number for
5878*68d33a25Syt 	 * REQUEST SENSE, and if it is not 0, it means REQUEST SENSE
5879*68d33a25Syt 	 * gets time out. At this time, AHCI_PORT_FLAG_RQSENSE is
5880*68d33a25Syt 	 * supposed to be set, and REQUEST SENSE is supposed to be
5881*68d33a25Syt 	 * the only outstanding command. And we can make sure the timed
5882*68d33a25Syt 	 * out command must be REQUEST SENSE when AHCI_PORT_FLAG_RQSENSE
5883*68d33a25Syt 	 * is set.
58842fcbc377Syt 	 */
5885*68d33a25Syt 	if (retrieve_errinfo_slot_status != 0) {
5886*68d33a25Syt 		AHCIDBG2(AHCIDBG_ERRS|AHCIDBG_TIMEOUT, ahci_ctlp,
5887*68d33a25Syt 		    "ahci_timeout_pkts called while REQUEST SENSE "
5888*68d33a25Syt 		    "command for errror recovery timed out "
5889*68d33a25Syt 		    "timeout_tags = 0x%x port_cmd_issue = 0x%x",
5890*68d33a25Syt 		    tmp_timeout_tags, port_cmd_issue);
5891*68d33a25Syt 		ASSERT(ahci_portp->ahciport_flags & AHCI_PORT_FLAG_RQSENSE);
5892*68d33a25Syt 		ASSERT(port_cmd_issue == retrieve_errinfo_slot_status);
58932fcbc377Syt 	}
5894*68d33a25Syt #endif
58952fcbc377Syt 
5896*68d33a25Syt 	ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
5897*68d33a25Syt 	ahci_portp->ahciport_mop_in_progress++;
58982fcbc377Syt 
58992fcbc377Syt 	(void) ahci_restart_port_wait_till_ready(ahci_ctlp, ahci_portp,
5900*68d33a25Syt 	    port, NULL, NULL);
5901*68d33a25Syt 
59022fcbc377Syt 	/*
59032fcbc377Syt 	 * Re-identify timeout tags because some previously checked commands
59042fcbc377Syt 	 * could already complete.
59052fcbc377Syt 	 */
59062fcbc377Syt 	finished_tags = ahci_portp->ahciport_pending_tags &
5907*68d33a25Syt 	    ~port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp);
59082fcbc377Syt 	timeout_tags = tmp_timeout_tags & ~finished_tags;
59092fcbc377Syt 
5910*68d33a25Syt 	AHCIDBG5(AHCIDBG_TIMEOUT, ahci_ctlp,
59112fcbc377Syt 	    "ahci_timeout_pkts: port %d, finished_tags = 0x%x, "
5912*68d33a25Syt 	    "timeout_tags = 0x%x port_cmd_issue = 0x%x pending_tags = 0x%x ",
5913*68d33a25Syt 	    port, finished_tags, timeout_tags,
5914*68d33a25Syt 	    port_cmd_issue, ahci_portp->ahciport_pending_tags);
59152fcbc377Syt 
59162fcbc377Syt 	ahci_mop_commands(ahci_ctlp,
59172fcbc377Syt 	    ahci_portp,
59182fcbc377Syt 	    port,
5919*68d33a25Syt 	    port_cmd_issue,
59202fcbc377Syt 	    0, /* failed tags */
59212fcbc377Syt 	    timeout_tags, /* timeout tags */
59222fcbc377Syt 	    0, /* aborted tags */
59232fcbc377Syt 	    0); /* reset tags */
5924*68d33a25Syt 
5925*68d33a25Syt 	mutex_exit(&ahci_portp->ahciport_mutex);
59262fcbc377Syt }
59272fcbc377Syt 
59282fcbc377Syt /*
59292fcbc377Syt  * Watchdog handler kicks in every 5 seconds to timeout any commands pending
59302fcbc377Syt  * for long time.
59312fcbc377Syt  */
59322fcbc377Syt static void
59332fcbc377Syt ahci_watchdog_handler(ahci_ctl_t *ahci_ctlp)
59342fcbc377Syt {
59352fcbc377Syt 	ahci_port_t *ahci_portp;
5936*68d33a25Syt 	sata_pkt_t *spkt;
59372fcbc377Syt 	uint32_t pending_tags = 0;
59382fcbc377Syt 	uint32_t timeout_tags = 0;
5939*68d33a25Syt 	uint32_t port_cmd_status;
59402fcbc377Syt 	uint8_t port;
59412fcbc377Syt 	int tmp_slot;
5942*68d33a25Syt 	int current_slot;
59432fcbc377Syt 	/* max number of cycles this packet should survive */
59442fcbc377Syt 	int max_life_cycles;
59452fcbc377Syt 
59462fcbc377Syt 	/* how many cycles this packet survived so far */
59472fcbc377Syt 	int watched_cycles;
59482fcbc377Syt 
59492fcbc377Syt 	mutex_enter(&ahci_ctlp->ahcictl_mutex);
59502fcbc377Syt 
59512fcbc377Syt 	AHCIDBG0(AHCIDBG_TIMEOUT|AHCIDBG_ENTRY, ahci_ctlp,
59522fcbc377Syt 	    "ahci_watchdog_handler entered");
59532fcbc377Syt 
59542fcbc377Syt 	for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
59552fcbc377Syt 		if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
59562fcbc377Syt 			continue;
59572fcbc377Syt 		}
59582fcbc377Syt 
59592fcbc377Syt 		ahci_portp = ahci_ctlp->ahcictl_ports[port];
59602fcbc377Syt 
59612fcbc377Syt 		mutex_enter(&ahci_portp->ahciport_mutex);
59622fcbc377Syt 		if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
59632fcbc377Syt 			mutex_exit(&ahci_portp->ahciport_mutex);
59642fcbc377Syt 			continue;
59652fcbc377Syt 		}
59662fcbc377Syt 
5967*68d33a25Syt 		/* Skip the check for those ports in error recovery */
5968*68d33a25Syt 		if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
5969*68d33a25Syt 			mutex_exit(&ahci_portp->ahciport_mutex);
5970*68d33a25Syt 			continue;
5971*68d33a25Syt 		}
5972*68d33a25Syt 
5973*68d33a25Syt 		port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
5974*68d33a25Syt 		    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
5975*68d33a25Syt 		current_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >>
5976*68d33a25Syt 		    AHCI_CMD_STATUS_CCS_SHIFT;
5977*68d33a25Syt 
59782fcbc377Syt 		pending_tags = ahci_portp->ahciport_pending_tags;
59792fcbc377Syt 		timeout_tags = 0;
59802fcbc377Syt 		while (pending_tags) {
59812fcbc377Syt 			tmp_slot = ddi_ffs(pending_tags) - 1;
59822fcbc377Syt 			if (tmp_slot == -1) {
59832fcbc377Syt 				break;
59842fcbc377Syt 			}
59852fcbc377Syt 
5986*68d33a25Syt 			spkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
5987*68d33a25Syt 			if ((spkt != NULL) && spkt->satapkt_time &&
5988*68d33a25Syt 			    !(spkt->satapkt_op_mode & SATA_OPMODE_POLLING)) {
59892fcbc377Syt 				/*
59902fcbc377Syt 				 * We are overloading satapkt_hba_driver_private
59912fcbc377Syt 				 * with watched_cycle count.
59922fcbc377Syt 				 *
59932fcbc377Syt 				 * If a packet has survived for more than it's
59942fcbc377Syt 				 * max life cycles, it is a candidate for time
59952fcbc377Syt 				 * out.
59962fcbc377Syt 				 */
59972fcbc377Syt 				watched_cycles = (int)(intptr_t)
5998*68d33a25Syt 				    spkt->satapkt_hba_driver_private;
59992fcbc377Syt 				watched_cycles++;
6000*68d33a25Syt 				max_life_cycles = (spkt->satapkt_time +
60012fcbc377Syt 				    ahci_watchdog_timeout - 1) /
60022fcbc377Syt 				    ahci_watchdog_timeout;
6003*68d33a25Syt 
6004*68d33a25Syt 				spkt->satapkt_hba_driver_private =
6005*68d33a25Syt 				    (void *)(intptr_t)watched_cycles;
6006*68d33a25Syt 
6007*68d33a25Syt 				if (watched_cycles <= max_life_cycles)
6008*68d33a25Syt 					goto next;
6009*68d33a25Syt 
6010*68d33a25Syt 				AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp,
6011*68d33a25Syt 				    "the current slot is %d", current_slot);
6012*68d33a25Syt 				/*
6013*68d33a25Syt 				 * We need to check whether the HBA has
6014*68d33a25Syt 				 * begun to execute the command, if not,
6015*68d33a25Syt 				 * then re-set the timer of the command.
6016*68d33a25Syt 				 */
6017*68d33a25Syt 				if (tmp_slot != current_slot) {
6018*68d33a25Syt 					spkt->satapkt_hba_driver_private =
6019*68d33a25Syt 					    (void *)(intptr_t)0;
6020*68d33a25Syt 				} else {
60212fcbc377Syt 					timeout_tags |= (0x1 << tmp_slot);
60222fcbc377Syt 					cmn_err(CE_NOTE, "!ahci watchdog: "
60232fcbc377Syt 					    "port %d satapkt 0x%p timed out\n",
6024*68d33a25Syt 					    port, (void *)spkt);
60252fcbc377Syt 				}
60262fcbc377Syt 			}
6027*68d33a25Syt next:
60282fcbc377Syt 			CLEAR_BIT(pending_tags, tmp_slot);
60292fcbc377Syt 		}
60302fcbc377Syt 
60312fcbc377Syt 		if (timeout_tags) {
60322fcbc377Syt 			mutex_exit(&ahci_portp->ahciport_mutex);
60332fcbc377Syt 			mutex_exit(&ahci_ctlp->ahcictl_mutex);
60342fcbc377Syt 			ahci_timeout_pkts(ahci_ctlp, ahci_portp,
6035*68d33a25Syt 			    port, timeout_tags, 0);
60362fcbc377Syt 			mutex_enter(&ahci_ctlp->ahcictl_mutex);
60372fcbc377Syt 			mutex_enter(&ahci_portp->ahciport_mutex);
60382fcbc377Syt 		}
60392fcbc377Syt 
60402fcbc377Syt 		mutex_exit(&ahci_portp->ahciport_mutex);
60412fcbc377Syt 	}
60422fcbc377Syt 
60432fcbc377Syt 	/* Re-install the watchdog timeout handler */
60442fcbc377Syt 	if (ahci_ctlp->ahcictl_timeout_id != 0) {
60452fcbc377Syt 		ahci_ctlp->ahcictl_timeout_id =
60462fcbc377Syt 		    timeout((void (*)(void *))ahci_watchdog_handler,
60472fcbc377Syt 		    (caddr_t)ahci_ctlp, ahci_watchdog_tick);
60482fcbc377Syt 	}
60492fcbc377Syt 
60502fcbc377Syt 	mutex_exit(&ahci_ctlp->ahcictl_mutex);
60512fcbc377Syt }
60522fcbc377Syt 
60532fcbc377Syt /*
6054*68d33a25Syt  * Fill the error context into sata_cmd for non-queued command error.
6055*68d33a25Syt  */
6056*68d33a25Syt static void
6057*68d33a25Syt ahci_copy_err_cnxt(sata_cmd_t *scmd, ahci_fis_d2h_register_t *rfisp)
6058*68d33a25Syt {
6059*68d33a25Syt 	scmd->satacmd_status_reg = GET_RFIS_STATUS(rfisp);
6060*68d33a25Syt 	scmd->satacmd_error_reg = GET_RFIS_ERROR(rfisp);
6061*68d33a25Syt 	scmd->satacmd_sec_count_lsb = GET_RFIS_SECTOR_COUNT(rfisp);
6062*68d33a25Syt 	scmd->satacmd_lba_low_lsb = GET_RFIS_CYL_LOW(rfisp);
6063*68d33a25Syt 	scmd->satacmd_lba_mid_lsb = GET_RFIS_CYL_MID(rfisp);
6064*68d33a25Syt 	scmd->satacmd_lba_high_lsb = GET_RFIS_CYL_HI(rfisp);
6065*68d33a25Syt 	scmd->satacmd_device_reg = GET_RFIS_DEV_HEAD(rfisp);
6066*68d33a25Syt 
6067*68d33a25Syt 	if (scmd->satacmd_addr_type == ATA_ADDR_LBA48) {
6068*68d33a25Syt 		scmd->satacmd_sec_count_msb = GET_RFIS_SECTOR_COUNT_EXP(rfisp);
6069*68d33a25Syt 		scmd->satacmd_lba_low_msb = GET_RFIS_CYL_LOW_EXP(rfisp);
6070*68d33a25Syt 		scmd->satacmd_lba_mid_msb = GET_RFIS_CYL_MID_EXP(rfisp);
6071*68d33a25Syt 		scmd->satacmd_lba_high_msb = GET_RFIS_CYL_HI_EXP(rfisp);
6072*68d33a25Syt 	}
6073*68d33a25Syt }
6074*68d33a25Syt 
6075*68d33a25Syt /*
6076*68d33a25Syt  * Put the respective register value to sata_cmd_t for satacmd_flags.
60772fcbc377Syt  */
60782fcbc377Syt static void
60792fcbc377Syt ahci_copy_out_regs(sata_cmd_t *scmd, ahci_fis_d2h_register_t *rfisp)
60802fcbc377Syt {
60812fcbc377Syt 	if (scmd->satacmd_flags.sata_copy_out_sec_count_msb)
60822fcbc377Syt 		scmd->satacmd_sec_count_msb = GET_RFIS_SECTOR_COUNT_EXP(rfisp);
60832fcbc377Syt 	if (scmd->satacmd_flags.sata_copy_out_lba_low_msb)
60842fcbc377Syt 		scmd->satacmd_lba_low_msb = GET_RFIS_CYL_LOW_EXP(rfisp);
60852fcbc377Syt 	if (scmd->satacmd_flags.sata_copy_out_lba_mid_msb)
60862fcbc377Syt 		scmd->satacmd_lba_mid_msb = GET_RFIS_CYL_MID_EXP(rfisp);
60872fcbc377Syt 	if (scmd->satacmd_flags.sata_copy_out_lba_high_msb)
60882fcbc377Syt 		scmd->satacmd_lba_high_msb = GET_RFIS_CYL_HI_EXP(rfisp);
60892fcbc377Syt 	if (scmd->satacmd_flags.sata_copy_out_sec_count_lsb)
60902fcbc377Syt 		scmd->satacmd_sec_count_lsb = GET_RFIS_SECTOR_COUNT(rfisp);
60912fcbc377Syt 	if (scmd->satacmd_flags.sata_copy_out_lba_low_lsb)
60922fcbc377Syt 		scmd->satacmd_lba_low_lsb = GET_RFIS_CYL_LOW(rfisp);
60932fcbc377Syt 	if (scmd->satacmd_flags.sata_copy_out_lba_mid_lsb)
60942fcbc377Syt 		scmd->satacmd_lba_mid_lsb = GET_RFIS_CYL_MID(rfisp);
60952fcbc377Syt 	if (scmd->satacmd_flags.sata_copy_out_lba_high_lsb)
60962fcbc377Syt 		scmd->satacmd_lba_high_lsb = GET_RFIS_CYL_HI(rfisp);
60972fcbc377Syt 	if (scmd->satacmd_flags.sata_copy_out_device_reg)
60982fcbc377Syt 		scmd->satacmd_device_reg = GET_RFIS_DEV_HEAD(rfisp);
60992fcbc377Syt 	if (scmd->satacmd_flags.sata_copy_out_error_reg)
61002fcbc377Syt 		scmd->satacmd_error_reg = GET_RFIS_ERROR(rfisp);
61012fcbc377Syt }
61022fcbc377Syt 
6103*68d33a25Syt static void
6104*68d33a25Syt ahci_log_fatal_error_message(ahci_ctl_t *ahci_ctlp, uint8_t port,
6105*68d33a25Syt     uint32_t intr_status)
6106*68d33a25Syt {
6107*68d33a25Syt #ifndef __lock_lint
6108*68d33a25Syt 	_NOTE(ARGUNUSED(ahci_ctlp))
6109*68d33a25Syt #endif
6110*68d33a25Syt 
6111*68d33a25Syt 	if (intr_status & AHCI_INTR_STATUS_IFS)
6112*68d33a25Syt 		cmn_err(CE_NOTE, "!ahci port %d has interface fatal "
6113*68d33a25Syt 		    "error", port);
6114*68d33a25Syt 
6115*68d33a25Syt 	if (intr_status & AHCI_INTR_STATUS_HBDS)
6116*68d33a25Syt 		cmn_err(CE_NOTE, "!ahci port %d has bus data error", port);
6117*68d33a25Syt 
6118*68d33a25Syt 	if (intr_status & AHCI_INTR_STATUS_HBFS)
6119*68d33a25Syt 		cmn_err(CE_NOTE, "!ahci port %d has bus fatal error", port);
6120*68d33a25Syt 
6121*68d33a25Syt 	if (intr_status & AHCI_INTR_STATUS_TFES)
6122*68d33a25Syt 		cmn_err(CE_NOTE, "!ahci port %d has task file error", port);
6123*68d33a25Syt 
6124*68d33a25Syt 	cmn_err(CE_NOTE, "!ahci port %d is trying to do error "
6125*68d33a25Syt 	    "recovery", port);
6126*68d33a25Syt }
6127*68d33a25Syt 
61282fcbc377Syt /*
61292fcbc377Syt  * Dump the serror message to the log.
61302fcbc377Syt  */
61312fcbc377Syt static void
6132*68d33a25Syt ahci_log_serror_message(ahci_ctl_t *ahci_ctlp, uint8_t port,
61332fcbc377Syt     uint32_t port_serror)
61342fcbc377Syt {
6135*68d33a25Syt #ifndef __lock_lint
6136*68d33a25Syt 	_NOTE(ARGUNUSED(ahci_ctlp))
6137*68d33a25Syt #endif
61382fcbc377Syt 	char *err_str;
61392fcbc377Syt 
61402fcbc377Syt 	if (port_serror & AHCI_SERROR_ERR_I) {
61412fcbc377Syt 		err_str = "Recovered Data Integrity Error (I)";
61422fcbc377Syt 		AHCIDBG2(AHCIDBG_ERRS, ahci_ctlp,
61432fcbc377Syt 		    "command error: port: %d, error: %s",
61442fcbc377Syt 		    port, err_str);
61452fcbc377Syt 	}
61462fcbc377Syt 
61472fcbc377Syt 	if (port_serror & AHCI_SERROR_ERR_M) {
61482fcbc377Syt 		err_str = "Recovered Communication Error (M)";
61492fcbc377Syt 		AHCIDBG2(AHCIDBG_ERRS, ahci_ctlp,
61502fcbc377Syt 		    "command error: port: %d, error: %s",
61512fcbc377Syt 		    port, err_str);
61522fcbc377Syt 	}
61532fcbc377Syt 
61542fcbc377Syt 	if (port_serror & AHCI_SERROR_ERR_T) {
61552fcbc377Syt 		err_str = "Transient Data Integrity Error (T)";
61562fcbc377Syt 		AHCIDBG2(AHCIDBG_ERRS, ahci_ctlp,
61572fcbc377Syt 		    "command error: port: %d, error: %s",
61582fcbc377Syt 		    port, err_str);
61592fcbc377Syt 	}
61602fcbc377Syt 
61612fcbc377Syt 	if (port_serror & AHCI_SERROR_ERR_C) {
61622fcbc377Syt 		err_str =
61632fcbc377Syt 		    "Persistent Communication or Data Integrity Error (C)";
61642fcbc377Syt 		AHCIDBG2(AHCIDBG_ERRS, ahci_ctlp,
61652fcbc377Syt 		    "command error: port: %d, error: %s",
61662fcbc377Syt 		    port, err_str);
61672fcbc377Syt 	}
61682fcbc377Syt 
61692fcbc377Syt 	if (port_serror & AHCI_SERROR_ERR_P) {
61702fcbc377Syt 		err_str = "Protocol Error (P)";
61712fcbc377Syt 		AHCIDBG2(AHCIDBG_ERRS, ahci_ctlp,
61722fcbc377Syt 		    "command error: port: %d, error: %s",
61732fcbc377Syt 		    port, err_str);
61742fcbc377Syt 	}
61752fcbc377Syt 
61762fcbc377Syt 	if (port_serror & AHCI_SERROR_ERR_E) {
61772fcbc377Syt 		err_str = "Internal Error (E)";
61782fcbc377Syt 		AHCIDBG2(AHCIDBG_ERRS, ahci_ctlp,
61792fcbc377Syt 		    "command error: port: %d, error: %s",
61802fcbc377Syt 		    port, err_str);
61812fcbc377Syt 	}
61822fcbc377Syt 
61832fcbc377Syt 	if (port_serror & AHCI_SERROR_DIAG_N) {
61842fcbc377Syt 		err_str = "PhyRdy Change (N)";
61852fcbc377Syt 		AHCIDBG2(AHCIDBG_ERRS, ahci_ctlp,
61862fcbc377Syt 		    "command error: port: %d, error: %s",
61872fcbc377Syt 		    port, err_str);
61882fcbc377Syt 	}
61892fcbc377Syt 
61902fcbc377Syt 	if (port_serror & AHCI_SERROR_DIAG_I) {
61912fcbc377Syt 		err_str = "Phy Internal Error (I)";
61922fcbc377Syt 		AHCIDBG2(AHCIDBG_ERRS, ahci_ctlp,
61932fcbc377Syt 		    "command error: port: %d, error: %s",
61942fcbc377Syt 		    port, err_str);
61952fcbc377Syt 	}
61962fcbc377Syt 
61972fcbc377Syt 	if (port_serror & AHCI_SERROR_DIAG_W) {
61982fcbc377Syt 		err_str = "Comm Wake (W)";
61992fcbc377Syt 		AHCIDBG2(AHCIDBG_ERRS, ahci_ctlp,
62002fcbc377Syt 		    "command error: port: %d, error: %s",
62012fcbc377Syt 		    port, err_str);
62022fcbc377Syt 	}
62032fcbc377Syt 
62042fcbc377Syt 	if (port_serror & AHCI_SERROR_DIAG_B) {
62052fcbc377Syt 		err_str = "10B to 8B Decode Error (B)";
62062fcbc377Syt 		AHCIDBG2(AHCIDBG_ERRS, ahci_ctlp,
62072fcbc377Syt 		    "command error: port: %d, error: %s",
62082fcbc377Syt 		    port, err_str);
62092fcbc377Syt 	}
62102fcbc377Syt 
62112fcbc377Syt 	if (port_serror & AHCI_SERROR_DIAG_D) {
62122fcbc377Syt 		err_str = "Disparity Error (D)";
62132fcbc377Syt 		AHCIDBG2(AHCIDBG_ERRS, ahci_ctlp,
62142fcbc377Syt 		    "command error: port: %d, error: %s",
62152fcbc377Syt 		    port, err_str);
62162fcbc377Syt 	}
62172fcbc377Syt 
62182fcbc377Syt 	if (port_serror & AHCI_SERROR_DIAG_C) {
62192fcbc377Syt 		err_str = "CRC Error (C)";
62202fcbc377Syt 		AHCIDBG2(AHCIDBG_ERRS, ahci_ctlp,
62212fcbc377Syt 		    "command error: port: %d, error: %s",
62222fcbc377Syt 		    port, err_str);
62232fcbc377Syt 	}
62242fcbc377Syt 
62252fcbc377Syt 	if (port_serror & AHCI_SERROR_DIAG_H) {
62262fcbc377Syt 		err_str = "Handshake Error (H)";
62272fcbc377Syt 		AHCIDBG2(AHCIDBG_ERRS, ahci_ctlp,
62282fcbc377Syt 		    "command error: port: %d, error: %s",
62292fcbc377Syt 		    port, err_str);
62302fcbc377Syt 	}
62312fcbc377Syt 
62322fcbc377Syt 	if (port_serror & AHCI_SERROR_DIAG_S) {
62332fcbc377Syt 		err_str = "Link Sequence Error (S)";
62342fcbc377Syt 		AHCIDBG2(AHCIDBG_ERRS, ahci_ctlp,
62352fcbc377Syt 		    "command error: port: %d, error: %s",
62362fcbc377Syt 		    port, err_str);
62372fcbc377Syt 	}
62382fcbc377Syt 
62392fcbc377Syt 	if (port_serror & AHCI_SERROR_DIAG_T) {
62402fcbc377Syt 		err_str = "Transport state transition error (T)";
62412fcbc377Syt 		AHCIDBG2(AHCIDBG_ERRS, ahci_ctlp,
62422fcbc377Syt 		    "command error: port: %d, error: %s",
62432fcbc377Syt 		    port, err_str);
62442fcbc377Syt 	}
62452fcbc377Syt 
62462fcbc377Syt 	if (port_serror & AHCI_SERROR_DIAG_F) {
62472fcbc377Syt 		err_str = "Unknown FIS Type (F)";
62482fcbc377Syt 		AHCIDBG2(AHCIDBG_ERRS, ahci_ctlp,
62492fcbc377Syt 		    "command error: port: %d, error: %s",
62502fcbc377Syt 		    port, err_str);
62512fcbc377Syt 	}
62522fcbc377Syt 
62532fcbc377Syt 	if (port_serror & AHCI_SERROR_DIAG_X) {
62542fcbc377Syt 		err_str = "Exchanged (X)";
62552fcbc377Syt 		AHCIDBG2(AHCIDBG_ERRS, ahci_ctlp,
62562fcbc377Syt 		    "command error: port: %d, error: %s",
62572fcbc377Syt 		    port, err_str);
62582fcbc377Syt 	}
62592fcbc377Syt }
62602fcbc377Syt 
62612fcbc377Syt /*
62622fcbc377Syt  * This routine is to calculate the total number of ports implemented
62632fcbc377Syt  * by the HBA.
62642fcbc377Syt  */
62652fcbc377Syt static int
62662fcbc377Syt ahci_get_num_implemented_ports(uint32_t ports_implemented)
62672fcbc377Syt {
62682fcbc377Syt 	uint8_t i;
62692fcbc377Syt 	int num = 0;
62702fcbc377Syt 
62712fcbc377Syt 	for (i = 0; i < AHCI_MAX_PORTS; i++) {
62722fcbc377Syt 		if (((uint32_t)0x1 << i) & ports_implemented)
62732fcbc377Syt 			num++;
62742fcbc377Syt 	}
62752fcbc377Syt 
62762fcbc377Syt 	return (num);
62772fcbc377Syt }
62782fcbc377Syt 
62792fcbc377Syt static void
62802fcbc377Syt ahci_log(ahci_ctl_t *ahci_ctlp, uint_t level, char *fmt, ...)
62812fcbc377Syt {
62822fcbc377Syt 	va_list ap;
62832fcbc377Syt 
62842fcbc377Syt 	mutex_enter(&ahci_log_mutex);
62852fcbc377Syt 
62862fcbc377Syt 	va_start(ap, fmt);
62872fcbc377Syt 	if (ahci_ctlp) {
62882fcbc377Syt 		(void) sprintf(ahci_log_buf, "%s-[%d]:",
62892fcbc377Syt 		    ddi_get_name(ahci_ctlp->ahcictl_dip),
62902fcbc377Syt 		    ddi_get_instance(ahci_ctlp->ahcictl_dip));
62912fcbc377Syt 	} else {
62922fcbc377Syt 		(void) sprintf(ahci_log_buf, "ahci:");
62932fcbc377Syt 	}
62942fcbc377Syt 
62952fcbc377Syt 	(void) vsprintf(ahci_log_buf, fmt, ap);
62962fcbc377Syt 	va_end(ap);
62972fcbc377Syt 
62982fcbc377Syt 	cmn_err(level, "%s", ahci_log_buf);
62992fcbc377Syt 
63002fcbc377Syt 	mutex_exit(&ahci_log_mutex);
63012fcbc377Syt }
6302