xref: /illumos-gate/usr/src/uts/common/io/comstar/lu/stmf_sbd/ats_copy_mgr.c (revision 61dfa509)

/*
 * This file and its contents are supplied under the terms of the
 * Common Development and Distribution License ("CDDL"), version 1.0.
 * You may only use this file in accordance with the terms of version
 * 1.0 of the CDDL.
 *
 * A full copy of the text of the CDDL should have accompanied this
 * source.  A copy of the CDDL is also available via the Internet at
 * http://www.illumos.org/license/CDDL.
 */

/*
 * Copyright 2019 Nexenta Systems, Inc.  All rights reserved.
 */

#include <sys/conf.h>
#include <sys/file.h>
#include <sys/ddi.h>
#include <sys/sunddi.h>
#include <sys/modctl.h>
#include <sys/scsi/scsi.h>
#include <sys/scsi/impl/scsi_reset_notify.h>
#include <sys/scsi/generic/mode.h>
#include <sys/disp.h>
#include <sys/byteorder.h>
#include <sys/atomic.h>
#include <sys/sdt.h>
#include <sys/dkio.h>

#include <sys/dmu.h>
#include <sys/txg.h>
#include <sys/refcount.h>
#include <sys/zvol.h>

#include <sys/stmf.h>
#include <sys/lpif.h>
#include <sys/portif.h>
#include <sys/stmf_ioctl.h>
#include <sys/stmf_sbd_ioctl.h>

#include "stmf_sbd.h"
#include "sbd_impl.h"

/* ATS tuning parameters */
#define	OVERLAP_OFF 0
#define	OVERLAP_LOW 1
#define	OVERLAP_MEDIUM 2
#define	OVERLAP_HIGH 3
uint8_t ats_overlap_check = OVERLAP_LOW; /* check for rw overlap with ATS */

uint8_t HardwareAcceleratedLocking = 1; /* 0 for disabled */
uint8_t HardwareAcceleratedMove = 1;
uint64_t sbd_list_length = 0;

#define	SBD_ATS_MAX_NBLKS	32
/* ATS routines. */
uint8_t
sbd_ats_max_nblks(void)
{
	if (HardwareAcceleratedLocking == 0)
		return (0);
	return (SBD_ATS_MAX_NBLKS);
}

#define	is_overlapping(start1, len1, start2, len2) \
	((start2) > (start1) ? ((start2) - (start1)) < (len1) : \
	((start1) - (start2)) < (len2))

/*ARGSUSED*/
static sbd_status_t
sbd_ats_do_handling_before_io(scsi_task_t *task, struct sbd_lu *sl,
    uint64_t lba, uint64_t count, uint32_t flags)
{
	sbd_status_t ret = SBD_SUCCESS;
	ats_state_t *ats_state, *ats_state_ret;
	sbd_cmd_t *scmd = (sbd_cmd_t *)task->task_lu_private;
	uint8_t cdb0 = task->task_cdb[0];

	if (scmd == NULL)
		return (SBD_SUCCESS);

	if (HardwareAcceleratedLocking == 0)
		return (SBD_SUCCESS);
	/*
	 * if ATS overlap checking is disabled just return.  The check
	 * is not done in the function to remove items from the list which
	 * allows this value to be changed at runtime.  If it is turned on
	 * at runtime the remove will just start taking items off the list.
	 * If it is turned off at runtime the list is still cleaned up.
	 */
	if (ats_overlap_check == OVERLAP_OFF)
		return (SBD_SUCCESS);

	/* overlap checking for compare and write only */
	if (ats_overlap_check == OVERLAP_LOW) {
		if (cdb0 != SCMD_COMPARE_AND_WRITE)
			return (SBD_SUCCESS);
	}

	/* overlap checking for compare and write and write only */
	if (ats_overlap_check == OVERLAP_MEDIUM) {
		if ((cdb0 != SCMD_COMPARE_AND_WRITE) && (cdb0 != SCMD_WRITE))
			return (SBD_SUCCESS);
	}

	mutex_enter(&sl->sl_lock);
	/*
	 * if the list is empty then just add the element to the list and
	 * return success. There is no overlap.  This is done for every
	 * read, write or compare and write.
	 */
	if (list_is_empty(&sl->sl_ats_io_list)) {
		goto done;
	}

	/*
	 * There are inflight operations.  As a result the list must be scanned
	 * and if there are any overlaps then SBD_BUSY should be returned.
	 *
	 * Duplicate reads and writes are allowed and kept on the list
	 * since there is no reason that overlapping IO operations should
	 * be delayed.
	 *
	 * A command that conflicts with a running compare and write will
	 * be rescheduled and rerun.  This is handled by stmf_task_poll_lu.
	 * There is a possibility that a command can be starved and still
	 * return busy, which is valid in the SCSI protocol.
	 */

	for (ats_state = list_head(&sl->sl_ats_io_list); ats_state != NULL;
	    ats_state = list_next(&sl->sl_ats_io_list, ats_state)) {

		if (is_overlapping(ats_state->as_cur_ats_lba,
		    ats_state->as_cur_ats_len, lba, count) == 0)
			continue;

		/* if the task is already listed just return */
		if (task == ats_state->as_cur_ats_task) {
			cmn_err(CE_WARN, "sbd_ats_handling_before_io: "
			    "task %p already on list", (void *) task);
			ret = SBD_SUCCESS;
			goto exit;
		}
		/*
		 * the current command is a compare and write, if there is any
		 * overlap return error
		 */

		if ((cdb0 == SCMD_COMPARE_AND_WRITE) ||
		    (ats_state->as_cmd == SCMD_COMPARE_AND_WRITE)) {
			ret = SBD_BUSY;
			goto exit;
		}
	}
done:
	ats_state_ret =
	    (ats_state_t *)kmem_zalloc(sizeof (ats_state_t), KM_SLEEP);
	ats_state_ret->as_cur_ats_lba = lba;
	ats_state_ret->as_cur_ats_len = count;
	ats_state_ret->as_cmd = cdb0;
	ats_state_ret->as_cur_ats_task = task;
	if (list_is_empty(&sl->sl_ats_io_list)) {
		list_insert_head(&sl->sl_ats_io_list, ats_state_ret);
	} else {
		list_insert_tail(&sl->sl_ats_io_list, ats_state_ret);
	}
	scmd->flags |= SBD_SCSI_CMD_ATS_RELATED;
	scmd->ats_state = ats_state;
	sbd_list_length++;
	mutex_exit(&sl->sl_lock);
	return (SBD_SUCCESS);

exit:
	mutex_exit(&sl->sl_lock);
	return (ret);
}

sbd_status_t
sbd_ats_handling_before_io(scsi_task_t *task, struct sbd_lu *sl,
    uint64_t lba, uint64_t count)
{
	return (sbd_ats_do_handling_before_io(task, sl, lba, count, 0));
}

void
sbd_ats_remove_by_task(scsi_task_t *task)
{
	ats_state_t *ats_state;
	sbd_lu_t *sl = (sbd_lu_t *)task->task_lu->lu_provider_private;
	sbd_cmd_t *scmd = task->task_lu_private;

	if (scmd == NULL)
		return;
	/*
	 * Scan the list and take the task off of the list. It is possible
	 * that the call is made in a situation where the task is not
	 * listed.  That is a valid but unlikely case. If it happens
	 * just fall through and return.  The list removal is done by
	 * task not LBA range and a task cannot be active for more than
	 * one command so there is never an issue about removing the
	 * wrong element.
	 */
	mutex_enter(&sl->sl_lock);
	if (list_is_empty(&sl->sl_ats_io_list)) {
		mutex_exit(&sl->sl_lock);
		return;
	}

	for (ats_state = list_head(&sl->sl_ats_io_list); ats_state != NULL;
	    ats_state = list_next(&sl->sl_ats_io_list, ats_state)) {

		if (ats_state->as_cur_ats_task == task) {
			list_remove(&sl->sl_ats_io_list, ats_state);
			kmem_free(ats_state, sizeof (ats_state_t));
			scmd->flags &= ~SBD_SCSI_CMD_ATS_RELATED;
			scmd->ats_state = NULL;
			sbd_list_length--;
			break;
		}
	}
	mutex_exit(&sl->sl_lock);
}

static sbd_status_t
sbd_compare_and_write(struct scsi_task *task, sbd_cmd_t *scmd,
    uint32_t *ret_off)
{
	sbd_lu_t *sl = (sbd_lu_t *)task->task_lu->lu_provider_private;
	uint8_t *buf;
	sbd_status_t ret;
	uint64_t addr;
	uint32_t len, i;

	addr = READ_SCSI64(&task->task_cdb[2], uint64_t);
	len = (uint32_t)task->task_cdb[13];

	addr <<= sl->sl_data_blocksize_shift;
	len <<= sl->sl_data_blocksize_shift;
	buf = kmem_alloc(len, KM_SLEEP);
	ret = sbd_data_read(sl, task, addr, (uint64_t)len, buf);
	if (ret != SBD_SUCCESS) {
		goto compare_and_write_done;
	}
	/*
	 * Can't use bcmp here. We need mismatch offset.
	 */
	for (i = 0; i < len; i++) {
		if (buf[i] != scmd->trans_data[i])
			break;
	}
	if (i != len) {
		*ret_off = i;
		ret = SBD_COMPARE_FAILED;
		goto compare_and_write_done;
	}

	ret = sbd_data_write(sl, task, addr, (uint64_t)len,
	    scmd->trans_data + len);

compare_and_write_done:
	kmem_free(buf, len);
	return (ret);
}

static void
sbd_send_miscompare_status(struct scsi_task *task, uint32_t miscompare_off)
{
	uint8_t sd[18];

	task->task_scsi_status = STATUS_CHECK;
	bzero(sd, 18);
	sd[0] = 0xF0;
	sd[2] = 0xe;
	SCSI_WRITE32(&sd[3], miscompare_off);
	sd[7] = 10;
	sd[12] = 0x1D;
	task->task_sense_data = sd;
	task->task_sense_length = 18;
	(void) stmf_send_scsi_status(task, STMF_IOF_LU_DONE);
}

static void
sbd_ats_release_resources(struct scsi_task *task)
{
	sbd_cmd_t *scmd = (sbd_cmd_t *)task->task_lu_private;

	/*
	 * a few basic check here to be sure that there are not multiple
	 * calls going on.  If scmd is null just return.  This is very
	 * unlikely, but could happed if the task is freed by an abort.
	 * If nbufs is invalid warn but ignore the error.  Last if the
	 * trans_data is either null or the lenght is zero just blow
	 * off the operation and leak the memory buffer.
	 */
	if (scmd == NULL)
		return;

	if (scmd->nbufs == 0xFF)
		cmn_err(CE_WARN, "%s invalid buffer count %x", __func__,
		    scmd->nbufs);

	if ((scmd->trans_data != NULL) && (scmd->trans_data_len != 0))
		kmem_free(scmd->trans_data, scmd->trans_data_len);

	scmd->trans_data = NULL; /* force panic later if re-entered */
	scmd->trans_data_len = 0;
	scmd->flags &= ~SBD_SCSI_CMD_TRANS_DATA;
}

void
sbd_handle_ats_xfer_completion(struct scsi_task *task, sbd_cmd_t *scmd,
    struct stmf_data_buf *dbuf, uint8_t dbuf_reusable)
{
	uint64_t laddr;
	uint32_t buflen, iolen, miscompare_off;
	int ndx;
	sbd_status_t ret;

	if (ATOMIC8_GET(scmd->nbufs) > 0) {
		atomic_dec_8(&scmd->nbufs);
	}

	if (dbuf->db_xfer_status != STMF_SUCCESS) {
		sbd_ats_remove_by_task(task);
		sbd_ats_release_resources(task);
		stmf_abort(STMF_QUEUE_TASK_ABORT, task,
		    dbuf->db_xfer_status, NULL);
		return;
	}

	if (scmd->flags & SBD_SCSI_CMD_XFER_FAIL) {
		goto ATS_XFER_DONE;
	}

	/* if state is confused drop the command */
	if ((scmd->trans_data == NULL) ||
	    ((scmd->flags & SBD_SCSI_CMD_TRANS_DATA) == 0) ||
	    ((scmd->flags & SBD_SCSI_CMD_ACTIVE) == 0)) {
		scmd->flags &= ~SBD_SCSI_CMD_ACTIVE;
		return;
	}

	if (ATOMIC32_GET(scmd->len) != 0) {
		/*
		 * Initiate the next port xfer to occur in parallel
		 * with writing this buf.  A side effect of sbd_do_ats_xfer is
		 * it may set scmd_len to 0.  This means all the data
		 * transfers have been started, not that they are done.
		 */
		sbd_do_ats_xfer(task, scmd, NULL, 0);
	}

	/*
	 * move the most recent data transfer to the temporary buffer
	 * used for the compare and write function.
	 */
	laddr = dbuf->db_relative_offset;
	for (buflen = 0, ndx = 0; (buflen < dbuf->db_data_size) &&
	    (ndx < dbuf->db_sglist_length); ndx++) {
		iolen = min(dbuf->db_data_size - buflen,
		    dbuf->db_sglist[ndx].seg_length);
		if (iolen == 0)
			break;
		bcopy(dbuf->db_sglist[ndx].seg_addr, &scmd->trans_data[laddr],
		    iolen);
		buflen += iolen;
		laddr += (uint64_t)iolen;
	}
	task->task_nbytes_transferred += buflen;

ATS_XFER_DONE:
	if (ATOMIC32_GET(scmd->len) == 0 ||
	    scmd->flags & SBD_SCSI_CMD_XFER_FAIL) {
		stmf_free_dbuf(task, dbuf);
		/*
		 * if this is not the last buffer to be transfered then exit
		 * and wait for the next buffer.  Once nbufs is 0 then all the
		 * data has arrived and the compare can be done.
		 */
		if (ATOMIC8_GET(scmd->nbufs) > 0)
			return;
		scmd->flags &= ~SBD_SCSI_CMD_ACTIVE;
		if (scmd->flags & SBD_SCSI_CMD_XFER_FAIL) {
			sbd_ats_remove_by_task(task);
			stmf_scsilib_send_status(task, STATUS_CHECK,
			    STMF_SAA_WRITE_ERROR);
		} else {
			ret = sbd_compare_and_write(task, scmd,
			    &miscompare_off);
			sbd_ats_remove_by_task(task);
			sbd_ats_release_resources(task);
			if (ret != SBD_SUCCESS) {
				if (ret != SBD_COMPARE_FAILED) {
					stmf_scsilib_send_status(task,
					    STATUS_CHECK, STMF_SAA_WRITE_ERROR);
				} else {
					sbd_send_miscompare_status(task,
					    miscompare_off);
				}
			} else {
				stmf_scsilib_send_status(task, STATUS_GOOD, 0);
			}
		}
		return;
	}
	sbd_do_ats_xfer(task, scmd, dbuf, dbuf_reusable);
}

void
sbd_do_ats_xfer(struct scsi_task *task, sbd_cmd_t *scmd,
    struct stmf_data_buf *dbuf, uint8_t dbuf_reusable)
{
	uint32_t len;

	if (ATOMIC32_GET(scmd->len) == 0) {
		if (dbuf != NULL) {
			stmf_free_dbuf(task, dbuf);
		}
		return;
	}

	if ((dbuf != NULL) &&
	    ((dbuf->db_flags & DB_DONT_REUSE) || (dbuf_reusable == 0))) {
		/* free current dbuf and allocate a new one */
		stmf_free_dbuf(task, dbuf);
		dbuf = NULL;
	}
	if (dbuf == NULL) {
		uint32_t maxsize, minsize, old_minsize;

		maxsize = (ATOMIC32_GET(scmd->len) > (128*1024)) ? 128*1024 :
		    ATOMIC32_GET(scmd->len);
		minsize = maxsize >> 2;
		do {
			old_minsize = minsize;
			dbuf = stmf_alloc_dbuf(task, maxsize, &minsize, 0);
		} while ((dbuf == NULL) && (old_minsize > minsize) &&
		    (minsize >= 512));
		if (dbuf == NULL) {
			if (ATOMIC8_GET(scmd->nbufs) == 0) {
				sbd_ats_remove_by_task(task);
				sbd_ats_release_resources(task);
				stmf_abort(STMF_QUEUE_TASK_ABORT, task,
				    STMF_ALLOC_FAILURE, NULL);
			}
			return;
		}
	}

	len = ATOMIC32_GET(scmd->len) > dbuf->db_buf_size ? dbuf->db_buf_size :
	    ATOMIC32_GET(scmd->len);

	dbuf->db_relative_offset = scmd->current_ro;
	dbuf->db_data_size = len;
	dbuf->db_flags = DB_DIRECTION_FROM_RPORT;
	(void) stmf_xfer_data(task, dbuf, 0);
	/*
	 * scmd->nbufs is the outstanding transfers
	 * scmd->len is the number of bytes that are remaing for requests
	 */
	atomic_inc_8(&scmd->nbufs);
	atomic_add_32(&scmd->len, -len);
	scmd->current_ro += len;
}

void
sbd_handle_ats(scsi_task_t *task, struct stmf_data_buf *initial_dbuf)
{
	sbd_lu_t *sl = (sbd_lu_t *)task->task_lu->lu_provider_private;
	uint64_t addr, len;
	sbd_cmd_t *scmd;
	stmf_data_buf_t *dbuf;
	uint8_t do_immediate_data = 0;
	/* int ret; */

	if (HardwareAcceleratedLocking == 0) {
		stmf_scsilib_send_status(task, STATUS_CHECK,
		    STMF_SAA_INVALID_OPCODE);
		return;
	}

	task->task_cmd_xfer_length = 0;
	if (task->task_additional_flags &
	    TASK_AF_NO_EXPECTED_XFER_LENGTH) {
		task->task_expected_xfer_length = 0;
	}
	if (sl->sl_flags & SL_WRITE_PROTECTED) {
		stmf_scsilib_send_status(task, STATUS_CHECK,
		    STMF_SAA_WRITE_PROTECTED);
		return;
	}
	addr = READ_SCSI64(&task->task_cdb[2], uint64_t);
	len = (uint64_t)task->task_cdb[13];

	if ((task->task_cdb[1]) || (len > SBD_ATS_MAX_NBLKS)) {
		stmf_scsilib_send_status(task, STATUS_CHECK,
		    STMF_SAA_INVALID_FIELD_IN_CDB);
		return;
	}
	if (len == 0) {
		stmf_scsilib_send_status(task, STATUS_GOOD, 0);
		return;
	}

	/*
	 * This can be called again. It will return the same handle again.
	 */
	if (sbd_ats_handling_before_io(task, sl, addr, len) != SBD_SUCCESS) {
		if (stmf_task_poll_lu(task, 10) != STMF_SUCCESS) {
			stmf_scsilib_send_status(task, STATUS_BUSY, 0);
		}
		return;
	}

	addr <<= sl->sl_data_blocksize_shift;
	len <<= sl->sl_data_blocksize_shift;

	task->task_cmd_xfer_length = len << 1;	/* actual amt of data is 2x */
	if (task->task_additional_flags &
	    TASK_AF_NO_EXPECTED_XFER_LENGTH) {
		task->task_expected_xfer_length = task->task_cmd_xfer_length;
	}
	if ((addr + len) > sl->sl_lu_size) {
		sbd_ats_remove_by_task(task);
		stmf_scsilib_send_status(task, STATUS_CHECK,
		    STMF_SAA_LBA_OUT_OF_RANGE);
		return;
	}

	len <<= 1;

	if (len != task->task_expected_xfer_length) {
		sbd_ats_remove_by_task(task);
		stmf_scsilib_send_status(task, STATUS_CHECK,
		    STMF_SAA_INVALID_FIELD_IN_CDB);
		return;
	}

	if ((initial_dbuf != NULL) && (task->task_flags & TF_INITIAL_BURST)) {
		if (initial_dbuf->db_data_size > len) {
			if (initial_dbuf->db_data_size >
			    task->task_expected_xfer_length) {
				/* protocol error */
				sbd_ats_remove_by_task(task);
				stmf_abort(STMF_QUEUE_TASK_ABORT, task,
				    STMF_INVALID_ARG, NULL);
				return;
			}
			ASSERT(len <= 0xFFFFFFFFull);
			initial_dbuf->db_data_size = (uint32_t)len;
		}
		do_immediate_data = 1;
	}
	dbuf = initial_dbuf;

	if (task->task_lu_private) {
		scmd = (sbd_cmd_t *)task->task_lu_private;
	} else {
		scmd = (sbd_cmd_t *)kmem_alloc(sizeof (sbd_cmd_t), KM_SLEEP);
		task->task_lu_private = scmd;
	}

	/* We dont set the ATS_RELATED flag here */
	scmd->flags = SBD_SCSI_CMD_ACTIVE | SBD_SCSI_CMD_TRANS_DATA;
	scmd->cmd_type = SBD_CMD_SCSI_WRITE;
	scmd->nbufs = 0;
	ASSERT(len <= 0xFFFFFFFFull);
	scmd->len = (uint32_t)len;
	scmd->trans_data_len = (uint32_t)len;
	scmd->trans_data = kmem_alloc((size_t)len, KM_SLEEP);
	scmd->current_ro = 0;

	if (do_immediate_data) {
		/*
		 * Account for data passed in this write command
		 */
		(void) stmf_xfer_data(task, dbuf, STMF_IOF_STATS_ONLY);
		atomic_add_32(&scmd->len, -dbuf->db_data_size);
		scmd->current_ro += dbuf->db_data_size;
		dbuf->db_xfer_status = STMF_SUCCESS;
		sbd_handle_ats_xfer_completion(task, scmd, dbuf, 0);
	} else {
		sbd_do_ats_xfer(task, scmd, dbuf, 0);
	}
}

/*
 * SCSI Copy Manager
 *
 * SCSI copy manager is the state machine which implements
 * SCSI extended copy functionality (SPC). There is one
 * cpmgr instance per extended copy command.
 *
 * Exported block-copy functions:
 *   cpmgr_create()  - Creates the state machine.
 *   cpmgr_destroy() - Cleans up a completed cpmgr.
 *   cpmgr_run()     - Performs time bound copy.
 *   cpmgr_abort()   - Aborts a cpmgr(if not already completed).
 *   cpmgr_done()    - Tests if the copy is done.
 */

static void cpmgr_completion_cleanup(cpmgr_t *cm);
int sbd_check_reservation_conflict(sbd_lu_t *sl, scsi_task_t *task);

static uint8_t sbd_recv_copy_results_op_params[] = {
    0, 0, 0, 42, 1, 0, 0, 0,
    0, 2, 0, 1, 0, 0, 0xFF, 0xFF, 0, 0, 0, 0,
    0, 0, 0, 0, 0, 0, 0, 0,
    0, 0, 0, 0, 0, 0,
    0xFF, 0xFF, 0, 9, 0, 0, 0, 0, 0,
    2, 2, 0xE4
};

cpmgr_handle_t
cpmgr_create(scsi_task_t *task, uint8_t *params)
{
	cpmgr_t *cm = NULL;
	uint8_t *p;
	uint32_t plist_len;
	uint32_t dbl;
	int i;
	uint16_t tdlen;
	uint16_t n;

	cm = kmem_zalloc(sizeof (*cm), KM_NOSLEEP);
	if (cm == NULL)
		return (CPMGR_INVALID_HANDLE);

	cm->cm_task = task;
	p = task->task_cdb;
	plist_len = READ_SCSI32(&p[10], uint32_t);

	/*
	 * In case of error. Otherwise we will change this to CM_COPYING.
	 */
	cm->cm_state = CM_COMPLETE;

	if (plist_len == 0) {
		cm->cm_status = 0;
		goto cpmgr_create_done;
	}

	if (plist_len < CPMGR_PARAM_HDR_LEN) {
		cm->cm_status = CPMGR_PARAM_LIST_LEN_ERROR;
		goto cpmgr_create_done;
	} else if ((params[0] != 0) || ((params[1] & 0x18) != 0x18)) {
		/*
		 * Current implementation does not allow the use
		 * of list ID field.
		 */
		cm->cm_status = CPMGR_INVALID_FIELD_IN_PARAM_LIST;
		goto cpmgr_create_done;
	}
	/* No inline data either */
	if (*((uint32_t *)(&params[12])) != 0) {
		cm->cm_status = CPMGR_INVALID_FIELD_IN_PARAM_LIST;
		goto cpmgr_create_done;
	}

	tdlen = READ_SCSI16(&params[2], uint16_t);
	if ((tdlen == 0) || (tdlen % CPMGR_TARGET_DESCRIPTOR_SIZE) ||
	    (plist_len < (CPMGR_PARAM_HDR_LEN + tdlen))) {
		cm->cm_status = CPMGR_PARAM_LIST_LEN_ERROR;
		goto cpmgr_create_done;
	}
	cm->cm_td_count = tdlen / CPMGR_TARGET_DESCRIPTOR_SIZE;
	if (cm->cm_td_count > CPMGR_MAX_TARGET_DESCRIPTORS) {
		cm->cm_status = CPMGR_TOO_MANY_TARGET_DESCRIPTORS;
		goto cpmgr_create_done;
	}
	if (plist_len != (CPMGR_PARAM_HDR_LEN + tdlen +
	    CPMGR_B2B_SEGMENT_DESCRIPTOR_SIZE)) {
		cm->cm_status = CPMGR_PARAM_LIST_LEN_ERROR;
		goto cpmgr_create_done;
	}
	for (i = 0; i < cm->cm_td_count; i++) {
		p = params + CPMGR_PARAM_HDR_LEN;
		p += i * CPMGR_TARGET_DESCRIPTOR_SIZE;
		if ((p[0] != CPMGR_IDENT_TARGET_DESCRIPTOR) ||
		    ((p[5] & 0x30) != 0) || (p[7] != 16)) {
			cm->cm_status = CPMGR_UNSUPPORTED_TARGET_DESCRIPTOR;
			goto cpmgr_create_done;
		}
		/*
		 * stmf should be able to find this LU and lock it. Also
		 * make sure that is indeed a sbd lu.
		 */
		if (((cm->cm_tds[i].td_lu =
		    stmf_check_and_hold_lu(task, &p[8])) == NULL) ||
		    (!sbd_is_valid_lu(cm->cm_tds[i].td_lu))) {
			cm->cm_status = CPMGR_COPY_TARGET_NOT_REACHABLE;
			goto cpmgr_create_done;
		}
		dbl = p[29];
		dbl <<= 8;
		dbl |= p[30];
		dbl <<= 8;
		dbl |= p[31];
		cm->cm_tds[i].td_disk_block_len = dbl;
		cm->cm_tds[i].td_lbasize_shift =
		    sbd_get_lbasize_shift(cm->cm_tds[i].td_lu);
	}
	/* p now points to segment descriptor */
	p += CPMGR_TARGET_DESCRIPTOR_SIZE;

	if (p[0] != CPMGR_B2B_SEGMENT_DESCRIPTOR) {
		cm->cm_status = CPMGR_UNSUPPORTED_SEGMENT_DESCRIPTOR;
		goto cpmgr_create_done;
	}
	n = READ_SCSI16(&p[2], uint16_t);
	if (n != (CPMGR_B2B_SEGMENT_DESCRIPTOR_SIZE - 4)) {
		cm->cm_status = CPMGR_INVALID_FIELD_IN_PARAM_LIST;
		goto cpmgr_create_done;
	}

	n = READ_SCSI16(&p[4], uint16_t);
	if (n >= cm->cm_td_count) {
		cm->cm_status = CPMGR_INVALID_FIELD_IN_PARAM_LIST;
		goto cpmgr_create_done;
	}
	cm->cm_src_td_ndx = n;

	n = READ_SCSI16(&p[6], uint16_t);
	if (n >= cm->cm_td_count) {
		cm->cm_status = CPMGR_INVALID_FIELD_IN_PARAM_LIST;
		goto cpmgr_create_done;
	}
	cm->cm_dst_td_ndx = n;

	cm->cm_copy_size = READ_SCSI16(&p[10], uint64_t);
	cm->cm_copy_size *= (uint64_t)(cm->cm_tds[(p[1] & 2) ?
	    cm->cm_dst_td_ndx : cm->cm_src_td_ndx].td_disk_block_len);
	cm->cm_src_offset = (READ_SCSI64(&p[12], uint64_t)) <<
	    cm->cm_tds[cm->cm_src_td_ndx].td_lbasize_shift;
	cm->cm_dst_offset = (READ_SCSI64(&p[20], uint64_t)) <<
	    cm->cm_tds[cm->cm_dst_td_ndx].td_lbasize_shift;

	/* Allocate the xfer buffer. */
	cm->cm_xfer_buf = kmem_alloc(CPMGR_XFER_BUF_SIZE, KM_NOSLEEP);
	if (cm->cm_xfer_buf == NULL) {
		cm->cm_status = CPMGR_INSUFFICIENT_RESOURCES;
		goto cpmgr_create_done;
	}

	/*
	 * No need to check block limits. cpmgr_run() will
	 * take care of that.
	 */

	/* All checks passed */
	cm->cm_state = CM_COPYING;

cpmgr_create_done:
	if (cm->cm_state == CM_COMPLETE) {
		cpmgr_completion_cleanup(cm);
	}
	return (cm);
}

void
cpmgr_destroy(cpmgr_handle_t h)
{
	cpmgr_t *cm = (cpmgr_t *)h;

	ASSERT(cm->cm_state == CM_COMPLETE);
	kmem_free(cm, sizeof (*cm));
}

static void
cpmgr_completion_cleanup(cpmgr_t *cm)
{
	int i;

	for (i = 0; i < cm->cm_td_count; i++) {
		if (cm->cm_tds[i].td_lu) {
			stmf_release_lu(cm->cm_tds[i].td_lu);
			cm->cm_tds[i].td_lu = NULL;
		}
	}
	if (cm->cm_xfer_buf) {
		kmem_free(cm->cm_xfer_buf, CPMGR_XFER_BUF_SIZE);
		cm->cm_xfer_buf = NULL;
	}
}

void
cpmgr_run(cpmgr_t *cm, clock_t preemption_point)
{
	stmf_lu_t *lu;
	sbd_lu_t *src_slu, *dst_slu;
	uint64_t xfer_size, start, end;
	sbd_status_t ret;

	/*
	 * XXX: Handle reservations and read-only LU here.
	 */
	ASSERT(cm->cm_state == CM_COPYING);
	lu = cm->cm_tds[cm->cm_src_td_ndx].td_lu;
	src_slu = (sbd_lu_t *)lu->lu_provider_private;
	if (sbd_check_reservation_conflict(src_slu, cm->cm_task)) {
		cpmgr_abort(cm, CPMGR_RESERVATION_CONFLICT);
		return;
	}

	lu = cm->cm_tds[cm->cm_dst_td_ndx].td_lu;
	dst_slu = (sbd_lu_t *)lu->lu_provider_private;
	if (sbd_check_reservation_conflict(dst_slu, cm->cm_task)) {
		cpmgr_abort(cm, CPMGR_RESERVATION_CONFLICT);
		return;
	}
	if (dst_slu->sl_flags & SL_WRITE_PROTECTED) {
		cpmgr_abort(cm, STMF_SAA_WRITE_PROTECTED);
		return;
	}

	while (cm->cm_size_done < cm->cm_copy_size) {
		xfer_size = ((cm->cm_copy_size - cm->cm_size_done) >
		    CPMGR_XFER_BUF_SIZE) ? CPMGR_XFER_BUF_SIZE :
		    (cm->cm_copy_size - cm->cm_size_done);
		start = cm->cm_src_offset + cm->cm_size_done;
		ret = sbd_data_read(src_slu, cm->cm_task, start, xfer_size,
		    cm->cm_xfer_buf);
		if (ret != SBD_SUCCESS) {
			if (ret == SBD_IO_PAST_EOF) {
				cpmgr_abort(cm, CPMGR_LBA_OUT_OF_RANGE);
			} else {
				cpmgr_abort(cm,
				    CPMGR_THIRD_PARTY_DEVICE_FAILURE);
			}
			break;
		}
		end = cm->cm_dst_offset + cm->cm_size_done;
		ret = sbd_data_write(dst_slu, cm->cm_task, end, xfer_size,
		    cm->cm_xfer_buf);
		if (ret != SBD_SUCCESS) {
			if (ret == SBD_IO_PAST_EOF) {
				cpmgr_abort(cm, CPMGR_LBA_OUT_OF_RANGE);
			} else {
				cpmgr_abort(cm,
				    CPMGR_THIRD_PARTY_DEVICE_FAILURE);
			}
			break;
		}
		cm->cm_size_done += xfer_size;
		if (ddi_get_lbolt() >= preemption_point)
			break;
	}
	if (cm->cm_size_done == cm->cm_copy_size) {
		cm->cm_state = CM_COMPLETE;
		cm->cm_status = 0;
		cpmgr_completion_cleanup(cm);
	}
}

void
cpmgr_abort(cpmgr_t *cm, uint32_t s)
{
	if (cm->cm_state == CM_COPYING) {
		cm->cm_state = CM_COMPLETE;
		cm->cm_status = s;
		cpmgr_completion_cleanup(cm);
	}
}

void
sbd_handle_xcopy(scsi_task_t *task, stmf_data_buf_t *dbuf)
{
	uint32_t cmd_xfer_len;

	if (HardwareAcceleratedMove == 0) {
		stmf_scsilib_send_status(task, STATUS_CHECK,
		    STMF_SAA_INVALID_OPCODE);
		return;
	}

	cmd_xfer_len = READ_SCSI32(&task->task_cdb[10], uint32_t);

	if (cmd_xfer_len == 0) {
		task->task_cmd_xfer_length = 0;
		if (task->task_additional_flags &
		    TASK_AF_NO_EXPECTED_XFER_LENGTH) {
			task->task_expected_xfer_length = 0;
		}
		stmf_scsilib_send_status(task, STATUS_GOOD, 0);
		return;
	}

	sbd_handle_short_write_transfers(task, dbuf, cmd_xfer_len);
}

void
sbd_handle_xcopy_xfer(scsi_task_t *task, uint8_t *buf)
{
	cpmgr_handle_t h;
	uint32_t s;
	clock_t tic, end;

	/*
	 * No need to pass buf size. Its taken from cdb.
	 */
	h = cpmgr_create(task, buf);
	if (h == CPMGR_INVALID_HANDLE) {
		stmf_scsilib_send_status(task, STATUS_CHECK,
		    CPMGR_INSUFFICIENT_RESOURCES);
		return;
	}
	tic = drv_usectohz(1000000);
	end = ddi_get_lbolt() + (CPMGR_DEFAULT_TIMEOUT * tic);
	while (!cpmgr_done(h)) {
		if (stmf_is_task_being_aborted(task) || (ddi_get_lbolt() > end))
			cpmgr_abort(h, CPMGR_THIRD_PARTY_DEVICE_FAILURE);
		else
			cpmgr_run(h, ddi_get_lbolt() + tic);
	}
	s = cpmgr_status(h);
	if (s) {
		if (s == CPMGR_RESERVATION_CONFLICT) {
			stmf_scsilib_send_status(task,
			    STATUS_RESERVATION_CONFLICT, 0);
		} else {
			stmf_scsilib_send_status(task, STATUS_CHECK, s);
		}
	} else {
		stmf_scsilib_send_status(task, STATUS_GOOD, 0);
	}
	cpmgr_destroy(h);
}

void
sbd_handle_recv_copy_results(struct scsi_task *task,
    struct stmf_data_buf *initial_dbuf)
{
	uint32_t cdb_len;

	cdb_len = READ_SCSI32(&task->task_cdb[10], uint32_t);
	if ((task->task_cdb[1] & 0x1F) != 3) {
		stmf_scsilib_send_status(task, STATUS_CHECK,
		    STMF_SAA_INVALID_FIELD_IN_CDB);
		return;
	}
	sbd_handle_short_read_transfers(task, initial_dbuf,
	    sbd_recv_copy_results_op_params, cdb_len,
	    sizeof (sbd_recv_copy_results_op_params));
}