xref: /illumos-gate/usr/src/uts/common/io/scsi/adapters/smartpqi/smartpqi_hba.c (revision 8226594f)

/*
 * This file and its contents are supplied under the terms of the
 * Common Development and Distribution License ("CDDL"), version 1.0.
 * You may only use this file in accordance with the terms of version
 * 1.0 of the CDDL.
 *
 * A full copy of the text of the CDDL should have accompanied this
 * source.  A copy of the CDDL is also available via the Internet at
 * http://www.illumos.org/license/CDDL.
 */

/*
 * Copyright 2023 Tintri by DDN, Inc. All rights reserved.
 * Copyright 2021 RackTop Systems, Inc.
 */

/*
 * This file contains all routines necessary to interface with SCSA trans.
 */
#include <smartpqi.h>

/*
 * []------------------------------------------------------------------[]
 * | Forward declarations for SCSA trans routines.			|
 * []------------------------------------------------------------------[]
 */
static int pqi_scsi_tgt_init(dev_info_t *hba_dip, dev_info_t *tgt_dip,
    scsi_hba_tran_t *hba_tran, struct scsi_device *sd);
static void pqi_scsi_tgt_free(dev_info_t *hba_dip, dev_info_t *tgt_dip,
    scsi_hba_tran_t *hba_tran, struct scsi_device *sd);
static int pqi_start(struct scsi_address *ap, struct scsi_pkt *pkt);
static int pqi_scsi_reset(struct scsi_address *ap, int level);
static int pqi_scsi_abort(struct scsi_address *ap, struct scsi_pkt *pkt);
static int pqi_scsi_getcap(struct scsi_address *ap, char *cap, int tgtonly);
static int pqi_scsi_setcap(struct scsi_address *ap, char *cap, int value,
    int tgtonly);
static struct scsi_pkt *pqi_init_pkt(struct scsi_address *ap,
    struct scsi_pkt *pkt, struct buf *bp, int cmdlen, int statuslen, int tgtlen,
    int flags,  int (*callback)(), caddr_t arg);
static void pqi_destroy_pkt(struct scsi_address *ap, struct scsi_pkt *pkt);
static void pqi_dmafree(struct scsi_address *ap, struct scsi_pkt *pkt);
static void pqi_sync_pkt(struct scsi_address *ap, struct scsi_pkt *pkt);
static int pqi_reset_notify(struct scsi_address *ap, int flag,
    void (*callback)(caddr_t), caddr_t arg);
static int pqi_quiesce(dev_info_t *dip);
static int pqi_unquiesce(dev_info_t *dip);
static int pqi_bus_config(dev_info_t *pdip, uint_t flag,
    ddi_bus_config_op_t op, void *arg, dev_info_t **childp);

/* ---- Support method declaration ---- */
static int config_one(dev_info_t *pdip, pqi_state_t *s,  pqi_device_t *,
    dev_info_t **childp);
static void abort_all(struct scsi_address *ap, pqi_state_t *s);
static int cmd_ext_alloc(pqi_cmd_t *cmd, int kf);
static void cmd_ext_free(pqi_cmd_t *cmd);
static boolean_t is_physical_dev(pqi_device_t *d);
static void cmd_timeout_scan(void *);

boolean_t
smartpqi_register_hba(pqi_state_t *s)
{
	scsi_hba_tran_t		*tran;
	int			flags;
	char			iport_str[16];
	int			instance = ddi_get_instance(s->s_dip);

	tran = scsi_hba_tran_alloc(s->s_dip, SCSI_HBA_CANSLEEP);
	if (tran == NULL)
		return (B_FALSE);
	s->s_tran = tran;

	tran->tran_hba_private		= s;
	tran->tran_tgt_private		= NULL;

	tran->tran_tgt_init		= pqi_scsi_tgt_init;
	tran->tran_tgt_free		= pqi_scsi_tgt_free;
	tran->tran_tgt_probe		= scsi_hba_probe;

	tran->tran_start		= pqi_start;
	tran->tran_reset		= pqi_scsi_reset;
	tran->tran_abort		= pqi_scsi_abort;
	tran->tran_getcap		= pqi_scsi_getcap;
	tran->tran_setcap		= pqi_scsi_setcap;
	tran->tran_bus_config		= pqi_bus_config;

	tran->tran_init_pkt		= pqi_init_pkt;
	tran->tran_destroy_pkt		= pqi_destroy_pkt;
	tran->tran_dmafree		= pqi_dmafree;
	tran->tran_sync_pkt		= pqi_sync_pkt;

	tran->tran_reset_notify		= pqi_reset_notify;
	tran->tran_quiesce		= pqi_quiesce;
	tran->tran_unquiesce		= pqi_unquiesce;
	tran->tran_bus_reset		= NULL;

	tran->tran_add_eventcall	= NULL;
	tran->tran_get_eventcookie	= NULL;
	tran->tran_post_event		= NULL;
	tran->tran_remove_eventcall	= NULL;
	tran->tran_bus_config		= pqi_bus_config;
	tran->tran_interconnect_type	= INTERCONNECT_SAS;

	/*
	 * scsi_vhci needs to have "initiator-port" set, but doesn't
	 * seem to care what it's set to. iSCSI uses the InitiatorName
	 * whereas mpt_sas uses the WWN port id, but this HBA doesn't
	 * have such a value. So, for now the instance number will be used.
	 */
	(void) snprintf(iport_str, sizeof (iport_str), "0x%x", instance);
	if (ddi_prop_update_string(DDI_DEV_T_NONE, s->s_dip,
	    SCSI_ADDR_PROP_INITIATOR_PORT, iport_str) != DDI_PROP_SUCCESS) {
		cmn_err(CE_WARN, "%s: Failed to create prop (%s) on %d\n",
		    __func__, SCSI_ADDR_PROP_INITIATOR_PORT, instance);
	}

	flags = SCSI_HBA_ADDR_COMPLEX | SCSI_HBA_TRAN_SCB;
	if (scsi_hba_attach_setup(s->s_dip, &s->s_msg_dma_attr, tran,
	    flags) != DDI_SUCCESS) {
		dev_err(s->s_dip, CE_NOTE, "scsi_hba_attach_setup failed");
		scsi_hba_tran_free(s->s_tran);
		s->s_tran = NULL;
		return (B_FALSE);
	}

	if (!s->s_disable_mpxio) {
		if (mdi_phci_register(MDI_HCI_CLASS_SCSI, s->s_dip, 0) !=
		    MDI_SUCCESS) {
			cmn_err(CE_WARN, "%s: Failed to register with mpxio",
			    __func__);
			s->s_disable_mpxio = B_TRUE;
		}
	}

	s->s_cmd_timeout = timeout(cmd_timeout_scan, s,
	    CMD_TIMEOUT_SCAN_SECS * drv_usectohz(MICROSEC));

	return (B_TRUE);
}

void
smartpqi_unregister_hba(pqi_state_t *s)
{
	if (!s->s_disable_mpxio)
		(void) mdi_phci_unregister(s->s_dip, 0);

	if (s->s_cmd_timeout != NULL) {
		(void) untimeout(s->s_cmd_timeout);
		s->s_cmd_timeout = NULL;
	}

	if (s->s_tran == NULL)
		return;
	scsi_hba_tran_free(s->s_tran);
	s->s_tran = NULL;
}

static int
pqi_scsi_tgt_init(dev_info_t *hba_dip __unused, dev_info_t *tgt_dip,
    scsi_hba_tran_t *hba_tran, struct scsi_device *sd)
{
	pqi_device_t	*d;
	pqi_state_t	*s	= hba_tran->tran_hba_private;
	mdi_pathinfo_t	*pip;
	int		type;
	char		*ua;

	if ((ua = scsi_device_unit_address(sd)) == NULL) {
		return (DDI_FAILURE);
	}

	if ((d = pqi_find_target_ua(s, ua)) == NULL) {
		return (DDI_FAILURE);
	}

	scsi_device_hba_private_set(sd, d);

	type = mdi_get_component_type(tgt_dip);
	if (type == MDI_COMPONENT_CLIENT) {
		char	wwid_str[64];

		if ((pip = (mdi_pathinfo_t *)sd->sd_private) == NULL)
			return (DDI_NOT_WELL_FORMED);

		(void) snprintf(wwid_str, sizeof (wwid_str), "%" PRIx64,
		    d->pd_wwid);
		(void) mdi_prop_update_string(pip, SCSI_ADDR_PROP_TARGET_PORT,
		    wwid_str);
	}

	return (DDI_SUCCESS);
}

static void
pqi_scsi_tgt_free(dev_info_t *hba_dip __unused, dev_info_t *tgt_dip __unused,
    scsi_hba_tran_t *hba_tran __unused, struct scsi_device *sd __unused)
{
}

/*
 * Notes:
 *      - transport the command to the addressed SCSI target/lun device
 *      - normal operation is to schedule the command to be transported,
 *        and return TRAN_ACCEPT if this is successful.
 *      - if NO_INTR, tran_start must poll device for command completion
 */
static int
pqi_start(struct scsi_address *ap, struct scsi_pkt *pkt)
{
	boolean_t	poll	= ((pkt->pkt_flags & FLAG_NOINTR) != 0);
	int		rc;
	pqi_cmd_t	*cmd	= PKT2CMD(pkt);
	pqi_state_t	*s	= ap->a_hba_tran->tran_hba_private;

	ASSERT3P(cmd->pc_pkt, ==, pkt);
	ASSERT3P(cmd->pc_softc, ==, s);

	if (pqi_is_offline(s) || !cmd->pc_device->pd_online)
		return (TRAN_FATAL_ERROR);

	/*
	 * Reinitialize some fields because the packet may have been
	 * resubmitted.
	 */
	pkt->pkt_reason = CMD_CMPLT;
	pkt->pkt_state = 0;
	pkt->pkt_statistics = 0;

	/* ---- Zero status byte ---- */
	*(pkt->pkt_scbp) = 0;

	if ((cmd->pc_flags & PQI_FLAG_DMA_VALID) != 0) {
		ASSERT(cmd->pc_dma_count);
		pkt->pkt_resid = cmd->pc_dma_count;

		/* ---- Sync consistent packets first (only write data) ---- */
		if (((cmd->pc_flags & PQI_FLAG_IO_IOPB) != 0) ||
		    ((cmd->pc_flags & PQI_FLAG_IO_READ) == 0)) {
			(void) ddi_dma_sync(cmd->pc_dmahdl, 0, 0,
			    DDI_DMA_SYNC_FORDEV);
		}
	}

	mutex_enter(&s->s_mutex);
	if (HBA_IS_QUIESCED(s) && !poll) {
		mutex_exit(&s->s_mutex);
		return (TRAN_BUSY);
	}
	mutex_exit(&s->s_mutex);

	rc = pqi_transport_command(s, cmd);

	if (poll) {
		boolean_t	qnotify;

		if (rc == TRAN_ACCEPT) {
			uint32_t	old_state;
			int		timeo;

			timeo = pkt->pkt_time ? pkt->pkt_time :
			    SCSI_POLL_TIMEOUT;
			timeo *= MILLISEC / 2;
			old_state = pqi_disable_intr(s);
			do {
				drv_usecwait(MILLISEC / 2);
				pqi_process_io_intr(s, &s->s_queue_groups[0]);
				if (--timeo == 0) {
					pkt->pkt_state |= STAT_TIMEOUT;
					pkt->pkt_reason = CMD_TIMEOUT;
					break;
				}
			} while (pkt->pkt_state == 0);
			pqi_enable_intr(s, old_state);
		}

		scsi_hba_pkt_comp(pkt);

		mutex_enter(&s->s_mutex);
		qnotify = HBA_QUIESCED_PENDING(s);
		mutex_exit(&s->s_mutex);

		if (qnotify)
			pqi_quiesced_notify(s);
	}

	return (rc);
}

static int
pqi_scsi_reset(struct scsi_address *ap, int level)
{
	pqi_device_t	*d;
	pqi_state_t	*s;
	int		rval = FALSE;

	s = ap->a_hba_tran->tran_hba_private;
	switch (level) {
	case RESET_TARGET:
	case RESET_LUN:
		if ((d = scsi_device_hba_private_get(ap->a.a_sd)) == NULL)
			break;

		pqi_lun_reset(s, d);
		rval = TRUE;
		break;

	case RESET_BUS:
	case RESET_ALL:
		mutex_enter(&s->s_mutex);
		for (d = list_head(&s->s_devnodes); d != NULL;
		    d = list_next(&s->s_devnodes, d)) {
			pqi_lun_reset(s, d);
		}
		mutex_exit(&s->s_mutex);
		rval = TRUE;
		break;
	}
	return (rval);
}

/*
 * abort handling:
 *
 * Notes:
 *      - if pkt is not NULL, abort just that command
 *      - if pkt is NULL, abort all outstanding commands for target
 */
static int
pqi_scsi_abort(struct scsi_address *ap, struct scsi_pkt *pkt)
{
	boolean_t	qnotify	= B_FALSE;
	pqi_state_t	*s	= ADDR2PQI(ap);

	if (pkt != NULL) {
		/* ---- Abort single command ---- */
		pqi_cmd_t	*cmd = PKT2CMD(pkt);

		mutex_enter(&cmd->pc_device->pd_mutex);
		(void) pqi_fail_cmd(cmd, CMD_ABORTED, STAT_ABORTED);
		mutex_exit(&cmd->pc_device->pd_mutex);
	} else {
		abort_all(ap, s);
	}
	qnotify = HBA_QUIESCED_PENDING(s);

	if (qnotify)
		pqi_quiesced_notify(s);
	return (1);
}

/*
 * capability handling:
 * (*tran_getcap).  Get the capability named, and return its value.
 */
static int
pqi_scsi_getcap(struct scsi_address *ap, char *cap, int tgtonly __unused)
{
	pqi_state_t *s = ap->a_hba_tran->tran_hba_private;

	if (cap == NULL)
		return (-1);
	switch (scsi_hba_lookup_capstr(cap)) {
	case SCSI_CAP_LUN_RESET:
		return ((s->s_flags & PQI_HBA_LUN_RESET_CAP) != 0);
	case SCSI_CAP_ARQ:
		return ((s->s_flags & PQI_HBA_AUTO_REQUEST_SENSE) != 0);
	case SCSI_CAP_UNTAGGED_QING:
		return (1);
	default:
		return (-1);
	}
}

/*
 * (*tran_setcap).  Set the capability named to the value given.
 */
static int
pqi_scsi_setcap(struct scsi_address *ap, char *cap, int value,
    int tgtonly __unused)
{
	pqi_state_t	*s	= ADDR2PQI(ap);
	int		rval	= FALSE;

	if (cap == NULL)
		return (-1);

	switch (scsi_hba_lookup_capstr(cap)) {
	case SCSI_CAP_ARQ:
		if (value)
			s->s_flags |= PQI_HBA_AUTO_REQUEST_SENSE;
		else
			s->s_flags &= ~PQI_HBA_AUTO_REQUEST_SENSE;
		rval = 1;
		break;

	case SCSI_CAP_LUN_RESET:
		if (value)
			s->s_flags |= PQI_HBA_LUN_RESET_CAP;
		else
			s->s_flags &= ~PQI_HBA_LUN_RESET_CAP;
		break;

	default:
		break;
	}

	return (rval);
}

int
pqi_cache_constructor(void *buf, void *un, int flags)
{
	pqi_cmd_t		*c	= (pqi_cmd_t *)buf;
	pqi_state_t		*s	= un;
	int			(*callback)(caddr_t);

	bzero(c, sizeof (*c));
	c->pc_softc = s;
	callback = (flags == KM_SLEEP) ? DDI_DMA_SLEEP : DDI_DMA_DONTWAIT;

	/* ---- Allocate a DMA handle for data transfers ---- */
	if (ddi_dma_alloc_handle(s->s_dip, &s->s_msg_dma_attr, callback,
	    NULL, &c->pc_dmahdl) != DDI_SUCCESS) {
		dev_err(s->s_dip, CE_WARN, "Failed to alloc dma handle");
		return (-1);
	}

	return (0);
}

void
pqi_cache_destructor(void *buf, void *un __unused)
{
	pqi_cmd_t	*cmd = buf;
	if (cmd->pc_dmahdl != NULL) {
		(void) ddi_dma_unbind_handle(cmd->pc_dmahdl);
		ddi_dma_free_handle(&cmd->pc_dmahdl);
		cmd->pc_dmahdl = NULL;
	}
}

/*
 * tran_init_pkt(9E) - allocate scsi_pkt(9S) for command
 *
 * One of three possibilities:
 *      - allocate scsi_pkt
 *      - allocate scsi_pkt and DMA resources
 *      - allocate DMA resources to an already-allocated pkt
 */
static struct scsi_pkt *
pqi_init_pkt(struct scsi_address *ap, struct scsi_pkt *pkt,
    struct buf *bp, int cmdlen, int statuslen, int tgtlen, int flags,
    int (*callback)(), caddr_t arg)
{
	pqi_cmd_t	*cmd;
	pqi_state_t	*s;
	int		kf = (callback == SLEEP_FUNC) ? KM_SLEEP : KM_NOSLEEP;
	boolean_t	is_new = B_FALSE;
	int		rc;
	int		i;
	pqi_device_t	*devp;

	s = ap->a_hba_tran->tran_hba_private;

	if (pkt == NULL) {
		ddi_dma_handle_t	saved_dmahdl;
		pqi_cmd_action_t	saved_action;

		if ((devp = scsi_device_hba_private_get(ap->a.a_sd)) == NULL)
			return (NULL);
		if ((cmd = kmem_cache_alloc(s->s_cmd_cache, kf)) == NULL)
			return (NULL);

		is_new = B_TRUE;
		saved_dmahdl = cmd->pc_dmahdl;
		saved_action = cmd->pc_last_action;

		(void) memset(cmd, 0, sizeof (*cmd));
		mutex_init(&cmd->pc_mutex, NULL, MUTEX_DRIVER, NULL);

		cmd->pc_dmahdl = saved_dmahdl;
		cmd->pc_last_action = saved_action;

		cmd->pc_device = devp;
		cmd->pc_pkt = &cmd->pc_cached_pkt;
		cmd->pc_softc = s;
		cmd->pc_tgtlen = tgtlen;
		cmd->pc_statuslen = statuslen;
		cmd->pc_cmdlen = cmdlen;
		cmd->pc_dma_count = 0;

		pkt = cmd->pc_pkt;
		pkt->pkt_ha_private = cmd;
		pkt->pkt_address = *ap;
		pkt->pkt_scbp = (uint8_t *)&cmd->pc_cmd_scb;
		pkt->pkt_cdbp = cmd->pc_cdb;
		pkt->pkt_private = (opaque_t)cmd->pc_tgt_priv;
		if (pkt->pkt_time == 0)
			pkt->pkt_time = SCSI_POLL_TIMEOUT;

		if (cmdlen > sizeof (cmd->pc_cdb) ||
		    statuslen > sizeof (cmd->pc_cmd_scb) ||
		    tgtlen > sizeof (cmd->pc_tgt_priv)) {
			if (cmd_ext_alloc(cmd, kf) != DDI_SUCCESS) {
				dev_err(s->s_dip, CE_WARN,
				    "extent allocation failed");
				goto out;
			}
		}
	} else {
		cmd = PKT2CMD(pkt);
		cmd->pc_flags &= PQI_FLAGS_PERSISTENT;
	}

	/* ---- Handle partial DMA transfer ---- */
	if (cmd->pc_nwin > 0) {
		if (++cmd->pc_winidx >= cmd->pc_nwin)
			return (NULL);
		if (ddi_dma_getwin(cmd->pc_dmahdl, cmd->pc_winidx,
		    &cmd->pc_dma_offset, &cmd->pc_dma_len, &cmd->pc_dmac,
		    &cmd->pc_dmaccount) == DDI_FAILURE)
			return (NULL);
		goto handle_dma_cookies;
	}

	/* ---- Setup data buffer ---- */
	if (bp != NULL && bp->b_bcount > 0 &&
	    (cmd->pc_flags & PQI_FLAG_DMA_VALID) == 0) {
		int	dma_flags;

		ASSERT(cmd->pc_dmahdl != NULL);

		if ((bp->b_flags & B_READ) != 0) {
			cmd->pc_flags |= PQI_FLAG_IO_READ;
			dma_flags = DDI_DMA_READ;
		} else {
			cmd->pc_flags &= ~PQI_FLAG_IO_READ;
			dma_flags = DDI_DMA_WRITE;
		}
		if ((flags & PKT_CONSISTENT) != 0) {
			cmd->pc_flags |= PQI_FLAG_IO_IOPB;
			dma_flags |= DDI_DMA_CONSISTENT;
		}
		if ((flags & PKT_DMA_PARTIAL) != 0) {
			dma_flags |= DDI_DMA_PARTIAL;
		}
		rc = ddi_dma_buf_bind_handle(cmd->pc_dmahdl, bp,
		    dma_flags, callback, arg, &cmd->pc_dmac,
		    &cmd->pc_dmaccount);

		if (rc == DDI_DMA_PARTIAL_MAP) {
			(void) ddi_dma_numwin(cmd->pc_dmahdl, &cmd->pc_nwin);
			cmd->pc_winidx = 0;
			(void) ddi_dma_getwin(cmd->pc_dmahdl, cmd->pc_winidx,
			    &cmd->pc_dma_offset, &cmd->pc_dma_len,
			    &cmd->pc_dmac, &cmd->pc_dmaccount);
		} else if (rc != 0 && rc != DDI_DMA_MAPPED) {
			switch (rc) {
			case DDI_DMA_NORESOURCES:
				bioerror(bp, 0);
				break;
			case DDI_DMA_BADATTR:
			case DDI_DMA_NOMAPPING:
				bioerror(bp, EFAULT);
				break;
			case DDI_DMA_TOOBIG:
			default:
				bioerror(bp, EINVAL);
				break;
			}
			goto out;
		}

handle_dma_cookies:
		ASSERT(cmd->pc_dmaccount > 0);
		if (cmd->pc_dmaccount >
		    (sizeof (cmd->pc_cached_cookies) /
		    sizeof (ddi_dma_cookie_t))) {
			dev_err(s->s_dip, CE_WARN,
			    "invalid cookie count: %d", cmd->pc_dmaccount);
			goto out;
		}
		if (cmd->pc_dmaccount >
		    (s->s_sg_chain_buf_length / sizeof (pqi_sg_entry_t))) {
			dev_err(s->s_dip, CE_WARN,
			    "Cookie(0x%x) verses SG(0x%" PRIx64 ") mismatch",
			    cmd->pc_dmaccount,
			    s->s_sg_chain_buf_length / sizeof (pqi_sg_entry_t));
			goto out;
		}

		cmd->pc_flags |= PQI_FLAG_DMA_VALID;
		cmd->pc_dma_count = cmd->pc_dmac.dmac_size;
		cmd->pc_cached_cookies[0] = cmd->pc_dmac;

		for (i = 1; i < cmd->pc_dmaccount; i++) {
			ddi_dma_nextcookie(cmd->pc_dmahdl, &cmd->pc_dmac);
			cmd->pc_cached_cookies[i] = cmd->pc_dmac;
			cmd->pc_dma_count += cmd->pc_dmac.dmac_size;
		}

		pkt->pkt_resid = bp->b_bcount - cmd->pc_dma_count;
	}

	return (pkt);

out:
	if (is_new == B_TRUE)
		pqi_destroy_pkt(ap, pkt);
	return (NULL);
}

/*
 * tran_destroy_pkt(9E) - scsi_pkt(9s) deallocation
 *
 * Notes:
 *      - also frees DMA resources if allocated
 *      - implicit DMA synchonization
 */
static void
pqi_destroy_pkt(struct scsi_address *ap, struct scsi_pkt *pkt)
{
	pqi_cmd_t	*c = PKT2CMD(pkt);
	pqi_state_t	*s = ADDR2PQI(ap);

	if ((c->pc_flags & PQI_FLAG_DMA_VALID) != 0) {
		c->pc_flags &= ~PQI_FLAG_DMA_VALID;
		(void) ddi_dma_unbind_handle(c->pc_dmahdl);
	}
	cmd_ext_free(c);

	kmem_cache_free(s->s_cmd_cache, c);
}

/*
 * tran_dmafree(9E) - deallocate DMA resources allocated for command
 */
static void
pqi_dmafree(struct scsi_address *ap __unused, struct scsi_pkt *pkt)
{
	pqi_cmd_t	*cmd = PKT2CMD(pkt);

	if (cmd->pc_flags & PQI_FLAG_DMA_VALID) {
		cmd->pc_flags &= ~PQI_FLAG_DMA_VALID;
		(void) ddi_dma_unbind_handle(cmd->pc_dmahdl);
	}
}

/*
 * tran_sync_pkt(9E) - explicit DMA synchronization
 */
static void
pqi_sync_pkt(struct scsi_address *ap __unused, struct scsi_pkt *pkt)
{
	pqi_cmd_t	*cmd = PKT2CMD(pkt);

	if (cmd->pc_dmahdl != NULL) {
		(void) ddi_dma_sync(cmd->pc_dmahdl, 0, 0,
		    (cmd->pc_flags & PQI_FLAG_IO_READ) ? DDI_DMA_SYNC_FORCPU :
		    DDI_DMA_SYNC_FORDEV);
	}
}

static int
pqi_reset_notify(struct scsi_address *ap, int flag,
    void (*callback)(caddr_t), caddr_t arg)
{
	pqi_state_t	*s = ADDR2PQI(ap);

	return (scsi_hba_reset_notify_setup(ap, flag, callback, arg,
	    &s->s_mutex, &s->s_reset_notify_listf));
}

/*
 * Device / Hotplug control
 */
static int
pqi_quiesce(dev_info_t *dip)
{
	pqi_state_t	*s;
	scsi_hba_tran_t	*tran;

	if ((tran = ddi_get_driver_private(dip)) == NULL ||
	    (s = TRAN2PQI(tran)) == NULL)
		return (-1);

	mutex_enter(&s->s_mutex);
	if (!HBA_IS_QUIESCED(s))
		s->s_flags |= PQI_HBA_QUIESCED;

	if (s->s_cmd_queue_len != 0) {
		/* ---- Outstanding commands present, wait ---- */
		s->s_flags |= PQI_HBA_QUIESCED_PENDING;
		cv_wait(&s->s_quiescedvar, &s->s_mutex);
		ASSERT0(s->s_cmd_queue_len);
	}
	mutex_exit(&s->s_mutex);

	return (0);
}

static int
pqi_unquiesce(dev_info_t *dip)
{
	pqi_state_t	*s;
	scsi_hba_tran_t	*tran;

	if ((tran = ddi_get_driver_private(dip)) == NULL ||
	    (s = TRAN2PQI(tran)) == NULL)
		return (-1);

	mutex_enter(&s->s_mutex);
	if (!HBA_IS_QUIESCED(s)) {
		mutex_exit(&s->s_mutex);
		return (0);
	}
	ASSERT0(s->s_cmd_queue_len);
	s->s_flags &= ~PQI_HBA_QUIESCED;
	mutex_exit(&s->s_mutex);

	return (0);
}

static int
pqi_bus_config(dev_info_t *pdip, uint_t flag,
    ddi_bus_config_op_t op, void *arg, dev_info_t **childp)
{
	scsi_hba_tran_t	*tran;
	pqi_state_t	*s;
	int		ret	= NDI_FAILURE;
	pqi_device_t	*d;
	char		*ua;

	tran = ddi_get_driver_private(pdip);
	s = tran->tran_hba_private;
	if (pqi_is_offline(s))
		return (NDI_FAILURE);

	ndi_devi_enter(scsi_vhci_dip);
	ndi_devi_enter(pdip);
	switch (op) {
	case BUS_CONFIG_ONE:
		if ((ua = strrchr((char *)arg, '@')) != NULL) {
			ua++;
			d = pqi_find_target_ua(s, ua);
			if (d != NULL)
				ret = config_one(pdip, s, d, childp);
		} else {
			dev_err(s->s_dip, CE_WARN, "Couldn't decode %s",
			    (char *)arg);
		}
		flag |= NDI_MDI_FALLBACK;
		break;

	case BUS_CONFIG_DRIVER:
	case BUS_CONFIG_ALL:
		ret = pqi_config_all(pdip, s);
		break;
	default:
		ret = NDI_FAILURE;
	}
	if (ret == NDI_SUCCESS)
		ret = ndi_busop_bus_config(pdip, flag, op, arg, childp, 0);
	ndi_devi_exit(pdip);
	ndi_devi_exit(scsi_vhci_dip);

	return (ret);
}

pqi_device_t *
pqi_find_target_ua(pqi_state_t *s, char *ua)
{
	pqi_device_t *d;

	mutex_enter(&s->s_mutex);
	for (d = list_head(&s->s_devnodes); d != NULL;
	    d = list_next(&s->s_devnodes, d)) {
		if (d->pd_online && strcmp(ua, d->pd_unit_address) == 0)
			break;
	}
	mutex_exit(&s->s_mutex);
	return (d);
}

int
pqi_config_all(dev_info_t *pdip, pqi_state_t *s)
{
	pqi_device_t *d;

	/*
	 * Make sure we bring the available devices into play first. These
	 * might be brand new devices just hotplugged into the system or
	 * they could be devices previously offlined because either they
	 * were pulled from an enclosure or a cable to the enclosure was
	 * pulled.
	 */
	/* ---- XXX Grab s_mutex ---- */
	for (d = list_head(&s->s_devnodes); d != NULL;
	    d = list_next(&s->s_devnodes, d)) {
		if (d->pd_online)
			(void) config_one(pdip, s, d, NULL);
	}

	/*
	 * Now deal with devices that we had previously known about, but are
	 * no longer available.
	 */
	for (d = list_head(&s->s_devnodes); d != NULL;
	    d = list_next(&s->s_devnodes, d)) {
		if (!d->pd_online)
			(void) config_one(pdip, s, d, NULL);
	}

	return (NDI_SUCCESS);
}

void
pqi_quiesced_notify(pqi_state_t *s)
{
	mutex_enter(&s->s_mutex);
	if (s->s_cmd_queue_len == 0 &&
	    (s->s_flags & PQI_HBA_QUIESCED_PENDING) != 0) {
		s->s_flags &= ~PQI_HBA_QUIESCED_PENDING;
		cv_broadcast(&s->s_quiescedvar);
	}
	mutex_exit(&s->s_mutex);
}

/*
 * []------------------------------------------------------------------[]
 * | Support routines used only by the trans_xxx routines		|
 * []------------------------------------------------------------------[]
 */
#ifdef DEBUG
int	pqi_force_timeout;
#endif	/* DEBUG */

static void
cmd_timeout_drive(pqi_device_t *d)
{
	uint32_t	timed_out_cnt = 0;
	pqi_cmd_t	*c, *next_c;
	hrtime_t	now = gethrtime();

	mutex_enter(&d->pd_mutex);

rescan:
	c = list_head(&d->pd_cmd_list);
	while (c != NULL) {
		next_c = list_next(&d->pd_cmd_list, c);
#ifdef DEBUG
		if (c->pc_expiration < now || pqi_force_timeout != 0) {
			pqi_force_timeout = 0;
#else
		if (c->pc_expiration < now) {
#endif	/* DEBUG */
			struct scsi_pkt	*pkt = CMD2PKT(c);

			if (pkt != NULL) {
				pkt->pkt_reason = CMD_TIMEOUT;
				pkt->pkt_statistics = STAT_TIMEOUT;
			}
			ASSERT(c->pc_io_rqst != NULL);
			/*
			 * If the i/o has not been serviced yet,
			 * mark the i/o as timed out and clear it out
			 */
			if (pqi_timeout_io(c->pc_io_rqst)) {
				(void) pqi_cmd_action_nolock(c,
				    PQI_CMD_TIMEOUT);
				timed_out_cnt++;
				/*
				 * We dropped pd_mutex so the cmd
				 * list could have changed, restart the
				 * scan of the cmds.  This will terminate
				 * since timed out cmds are removed from
				 * the list.
				 */
				goto rescan;
			}
		}
		c = next_c;
	}

	d->pd_timedout += timed_out_cnt;
	mutex_exit(&d->pd_mutex);
}

static void
cmd_timeout_scan(void *v)
{
	pqi_state_t		*s = v;
	pqi_device_t		*d;

	mutex_enter(&s->s_mutex);

	for (d = list_head(&s->s_devnodes); d != NULL;
	    d = list_next(&s->s_devnodes, d)) {
		cmd_timeout_drive(d);
	}
	cmd_timeout_drive(&s->s_special_device);

	mutex_exit(&s->s_mutex);
	s->s_cmd_timeout = timeout(cmd_timeout_scan, s,
	    CMD_TIMEOUT_SCAN_SECS * drv_usectohz(MICROSEC));
}

static void
abort_all(struct scsi_address *ap, pqi_state_t *s __unused)
{
	pqi_device_t	*devp;

	if ((devp = scsi_device_hba_private_get(ap->a.a_sd)) == NULL)
		return;

	pqi_fail_drive_cmds(devp, CMD_ABORTED);
}

static boolean_t
create_phys_lun(pqi_state_t *s, pqi_device_t *d,
    struct scsi_inquiry *inq, dev_info_t **childp)
{
	char		**compatible	= NULL;
	char		*nodename	= NULL;
	int		ncompatible	= 0;
	dev_info_t	*dip;

	/* ---- At this point we have a new device not in our list ---- */
	scsi_hba_nodename_compatible_get(inq, NULL,
	    inq->inq_dtype, NULL, &nodename, &compatible, &ncompatible);
	if (nodename == NULL)
		return (B_FALSE);

	if (ndi_devi_alloc(s->s_dip, nodename, DEVI_SID_NODEID, &dip) !=
	    NDI_SUCCESS) {
		dev_err(s->s_dip, CE_WARN, "failed to alloc device instance");
		goto free_nodename;
	}

	d->pd_dip = dip;
	d->pd_pip = NULL;

	if (ndi_prop_update_int64(DDI_DEV_T_NONE, dip, LUN64_PROP,
	    d->pd_lun) != DDI_PROP_SUCCESS) {
		goto free_devi;
	}

	if (ndi_prop_update_string_array(DDI_DEV_T_NONE, dip, COMPAT_PROP,
	    compatible, ncompatible) != DDI_PROP_SUCCESS) {
		goto free_devi;
	}

	if (d->pd_wwid != 0) {
		char		wwn_str[20];
		(void) snprintf(wwn_str, 20, "w%016" PRIx64, d->pd_wwid);
		if (ndi_prop_update_string(DDI_DEV_T_NONE, dip,
		    SCSI_ADDR_PROP_TARGET_PORT, wwn_str) != DDI_PROP_SUCCESS) {
			goto free_devi;
		}
	} else {
		if (ndi_prop_update_int(DDI_DEV_T_NONE, dip, TARGET_PROP,
		    d->pd_target) != DDI_PROP_SUCCESS) {
			goto free_devi;
		}
	}

	if (d->pd_guid != NULL) {
		if (ddi_prop_update_string(DDI_DEV_T_NONE, dip, NDI_GUID,
		    d->pd_guid) != DDI_PROP_SUCCESS) {
			goto free_devi;
		}
	}

	if (ndi_prop_update_int(DDI_DEV_T_NONE, dip, "pm-capable", 1) !=
	    DDI_PROP_SUCCESS) {
		goto free_devi;
	}

	if (ndi_devi_online(dip, NDI_ONLINE_ATTACH) != NDI_SUCCESS)
		goto free_devi;

	if (childp != NULL)
		*childp = dip;

	scsi_hba_nodename_compatible_free(nodename, compatible);

	return (B_TRUE);

free_devi:
	ndi_prop_remove_all(dip);
	(void) ndi_devi_free(dip);
	d->pd_dip = NULL;
free_nodename:
	scsi_hba_nodename_compatible_free(nodename, compatible);
	return (B_FALSE);
}

static boolean_t
create_virt_lun(pqi_state_t *s, pqi_device_t *d, struct scsi_inquiry *inq,
    dev_info_t **childp)
{
	char		*nodename;
	char		**compatible;
	int		ncompatible;
	int		rval;
	mdi_pathinfo_t	*pip		= NULL;
	char		*guid_ptr;
	char		wwid_str[17];
	dev_info_t	*lun_dip;
	char		*old_guid;

	if (d->pd_pip_offlined != NULL) {
		lun_dip = mdi_pi_get_client(d->pd_pip_offlined);
		ASSERT(lun_dip != NULL);

		if (ddi_prop_lookup_string(DDI_DEV_T_ANY, lun_dip,
		    (DDI_PROP_DONTPASS | DDI_PROP_NOTPROM),
		    MDI_CLIENT_GUID_PROP, &old_guid) == DDI_SUCCESS) {
			if (strncmp(d->pd_guid, old_guid,
			    strlen(d->pd_guid)) == 0) {
				/* ---- Same path came back online ---- */
				(void) ddi_prop_free(old_guid);
				if (mdi_pi_online(d->pd_pip_offlined, 0) ==
				    DDI_SUCCESS) {
					d->pd_pip = d->pd_pip_offlined;
					d->pd_pip_offlined = NULL;
					return (B_TRUE);
				} else {
					return (B_FALSE);
				}
			} else {
				/* ---- Different device in slot ---- */
				(void) ddi_prop_free(old_guid);
				if (mdi_pi_offline(d->pd_pip_offlined, 0) !=
				    DDI_SUCCESS) {
					return (B_FALSE);
				}
				if (mdi_pi_free(d->pd_pip_offlined, 0) !=
				    MDI_SUCCESS) {
					return (B_FALSE);
				}
				d->pd_pip_offlined = NULL;
			}
		} else {
			dev_err(s->s_dip, CE_WARN, "Can't get client-guid "
			    "property for lun %lx", d->pd_wwid);
			return (B_FALSE);
		}
	}

	scsi_hba_nodename_compatible_get(inq, NULL, inq->inq_dtype, NULL,
	    &nodename, &compatible, &ncompatible);
	if (nodename == NULL)
		return (B_FALSE);

	if (d->pd_guid != NULL) {
		guid_ptr = d->pd_guid;
	} else {
		(void) snprintf(wwid_str, sizeof (wwid_str), "%" PRIx64,
		    d->pd_wwid);
		guid_ptr = wwid_str;
	}
	rval = mdi_pi_alloc_compatible(s->s_dip, nodename, guid_ptr,
	    d->pd_unit_address, compatible, ncompatible, 0, &pip);
	if (rval == MDI_SUCCESS) {
		mdi_pi_set_phci_private(pip, (caddr_t)d);

		if (mdi_prop_update_string(pip, MDI_GUID, guid_ptr) !=
		    DDI_SUCCESS) {
			dev_err(s->s_dip, CE_WARN,
			    "unable to create property (MDI_GUID) for %s",
			    guid_ptr);
			goto cleanup;
		}

		/*
		 * For MPxIO, we actually don't really need to care
		 * about the LUN or target property, because nothing
		 * really uses them.
		 */
		if (mdi_prop_update_int64(pip, LUN64_PROP, d->pd_lun) !=
		    DDI_SUCCESS) {
			dev_err(s->s_dip, CE_WARN,
			    "unable to create property (%s) for %s",
			    LUN64_PROP, guid_ptr);
			goto cleanup;
		}

		if (mdi_prop_update_string_array(pip, COMPAT_PROP,
		    compatible, ncompatible) != DDI_SUCCESS) {
			dev_err(s->s_dip, CE_WARN,
			    "unable to create property (%s) for %s",
			    COMPAT_PROP, guid_ptr);
			goto cleanup;
		}

		if (mdi_pi_online(pip, 0) == MDI_NOT_SUPPORTED)
			goto cleanup;

		d->pd_dip = NULL;
		d->pd_pip = pip;
	}

	scsi_hba_nodename_compatible_free(nodename, compatible);
	if (childp != NULL)
		*childp = mdi_pi_get_client(pip);
	return (B_TRUE);
cleanup:
	scsi_hba_nodename_compatible_free(nodename, compatible);
	d->pd_pip = NULL;
	d->pd_dip = NULL;
	(void) mdi_prop_remove(pip, NULL);
	(void) mdi_pi_free(pip, 0);
	return (B_FALSE);
}

static int
config_one(dev_info_t *pdip, pqi_state_t *s, pqi_device_t *d,
    dev_info_t **childp)
{
	struct scsi_inquiry	inq;
	boolean_t		rval = B_FALSE;

	/* ---- Inquiry target ---- */
	if (!d->pd_online ||
	    pqi_scsi_inquiry(s, d, 0, &inq, sizeof (inq)) == B_FALSE) {
		pqi_fail_drive_cmds(d, CMD_DEV_GONE);

		if (d->pd_dip != NULL) {
			(void) ndi_devi_offline(d->pd_dip,
			    NDI_DEVFS_CLEAN | NDI_DEVI_REMOVE);
			d->pd_dip = NULL;
		} else if (d->pd_pip != NULL) {
			(void) mdi_pi_offline(d->pd_pip, 0);
			d->pd_pip_offlined = d->pd_pip;
			d->pd_pip = NULL;
		}
		return (NDI_FAILURE);
	} else if (d->pd_dip != NULL) {
		if (childp != NULL)
			*childp = d->pd_dip;
		return (NDI_SUCCESS);
	} else if (d->pd_pip != NULL) {
		if (childp != NULL)
			*childp = mdi_pi_get_client(d->pd_pip);
		return (NDI_SUCCESS);
	}

	d->pd_parent = pdip;
	if ((!s->s_disable_mpxio) && is_physical_dev(d))
		rval = create_virt_lun(s, d, &inq, childp);

	if (rval == B_FALSE)
		rval = create_phys_lun(s, d, &inq, childp);

	return ((rval == B_TRUE) ? NDI_SUCCESS : NDI_FAILURE);
}

static void
cmd_ext_free(pqi_cmd_t *cmd)
{
	struct scsi_pkt *pkt = CMD2PKT(cmd);

	if ((cmd->pc_flags & PQI_FLAG_CDB_EXT) != 0) {
		kmem_free(pkt->pkt_cdbp, cmd->pc_cmdlen);
		cmd->pc_flags &= ~PQI_FLAG_CDB_EXT;
	}
	if ((cmd->pc_flags & PQI_FLAG_SCB_EXT) != 0) {
		kmem_free(pkt->pkt_scbp, cmd->pc_statuslen);
		cmd->pc_flags &= ~PQI_FLAG_SCB_EXT;
	}
	if ((cmd->pc_flags & PQI_FLAG_PRIV_EXT) != 0) {
		kmem_free(pkt->pkt_private, cmd->pc_tgtlen);
		cmd->pc_flags &= ~PQI_FLAG_PRIV_EXT;
	}
}

static int
cmd_ext_alloc(pqi_cmd_t *cmd, int kf)
{
	struct scsi_pkt		*pkt = CMD2PKT(cmd);
	void			*buf;

	if (cmd->pc_cmdlen > sizeof (cmd->pc_cdb)) {
		if ((buf = kmem_zalloc(cmd->pc_cmdlen, kf)) == NULL)
			return (DDI_FAILURE);
		pkt->pkt_cdbp = buf;
		cmd->pc_flags |= PQI_FLAG_CDB_EXT;
	}

	if (cmd->pc_statuslen > sizeof (cmd->pc_cmd_scb)) {
		if ((buf = kmem_zalloc(cmd->pc_statuslen, kf)) == NULL)
			goto out;
		pkt->pkt_scbp = buf;
		cmd->pc_flags |= PQI_FLAG_SCB_EXT;
	}

	if (cmd->pc_tgtlen > sizeof (cmd->pc_tgt_priv)) {
		if ((buf = kmem_zalloc(cmd->pc_tgtlen, kf)) == NULL)
			goto out;
		pkt->pkt_private = buf;
		cmd->pc_flags |= PQI_FLAG_PRIV_EXT;
	}

	return (DDI_SUCCESS);

out:
	cmd_ext_free(cmd);

	return (DDI_FAILURE);
}

static boolean_t
is_physical_dev(pqi_device_t *d)
{
	return (d->pd_phys_dev ? B_TRUE : B_FALSE);
}