xref: /illumos-gate/usr/src/uts/common/io/usb/hcd/ehci/ehci_isoch.c (revision d3d50737)

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */


/*
 * EHCI Host Controller Driver (EHCI)
 *
 * The EHCI driver is a software driver which interfaces to the Universal
 * Serial Bus layer (USBA) and the Host Controller (HC). The interface to
 * the Host Controller is defined by the EHCI Host Controller Interface.
 *
 * This module contains the EHCI driver isochronous code, which handles all
 * Checking of status of USB transfers, error recovery and callbacks.
 */
#include <sys/usb/hcd/ehci/ehcid.h>
#include <sys/usb/hcd/ehci/ehci_xfer.h>
#include <sys/usb/hcd/ehci/ehci_util.h>
#include <sys/usb/hcd/ehci/ehci_isoch.h>
#include <sys/usb/hcd/ehci/ehci_isoch_util.h>
#include <sys/strsun.h>

/*
 * Isochronous initialization functions
 */
int ehci_isoc_init(
	ehci_state_t		*ehcip);
void ehci_isoc_cleanup(
	ehci_state_t		*ehcip);
void ehci_isoc_pipe_cleanup(
	ehci_state_t		*ehcip,
	usba_pipe_handle_data_t *ph);
static void ehci_wait_for_isoc_completion(
	ehci_state_t		*ehcip,
	ehci_pipe_private_t	*pp);

/*
 * Isochronous request functions
 */
ehci_isoc_xwrapper_t *ehci_allocate_isoc_resources(
	ehci_state_t		*ehcip,
	usba_pipe_handle_data_t *ph,
	usb_isoc_req_t		*isoc_reqp,
	usb_flags_t		usb_flags);
int ehci_insert_isoc_req(
	ehci_state_t		*ehcip,
	ehci_pipe_private_t	*pp,
	ehci_isoc_xwrapper_t	*itw,
	usb_flags_t		usb_flags);
static int ehci_insert_itd_req(
	ehci_state_t		*ehcip,
	ehci_pipe_private_t	*pp,
	ehci_isoc_xwrapper_t	*itw,
	usb_flags_t		usb_flags);
static int ehci_insert_sitd_req(
	ehci_state_t		*ehcip,
	ehci_pipe_private_t	*pp,
	ehci_isoc_xwrapper_t	*itw,
	usb_flags_t		usb_flags);
static void ehci_remove_isoc_itds(
	ehci_state_t		*ehcip,
	ehci_pipe_private_t	*pp);
static void ehci_mark_reclaim_isoc(
	ehci_state_t		*ehcip,
	ehci_pipe_private_t	*pp);
static void ehci_reclaim_isoc(
	ehci_state_t		*ehcip,
	ehci_isoc_xwrapper_t	*itw,
	ehci_itd_t		*itd,
	ehci_pipe_private_t	*pp);
int	ehci_start_isoc_polling(
	ehci_state_t		*ehcip,
	usba_pipe_handle_data_t	*ph,
	usb_flags_t		flags);

/*
 * Isochronronous handling functions.
 */
void ehci_traverse_active_isoc_list(
	ehci_state_t		*ehcip);
static void ehci_handle_isoc(
	ehci_state_t		*ehcip,
	ehci_isoc_xwrapper_t	*itw,
	ehci_itd_t		*itd);
static void ehci_handle_itd(
	ehci_state_t		*ehcip,
	ehci_pipe_private_t	*pp,
	ehci_isoc_xwrapper_t	*itw,
	ehci_itd_t		*itd,
	void			*tw_handle_callback_value);
static void ehci_sendup_itd_message(
	ehci_state_t		*ehcip,
	ehci_pipe_private_t	*pp,
	ehci_isoc_xwrapper_t	*itw,
	ehci_itd_t		*td,
	usb_cr_t		error);
void ehci_hcdi_isoc_callback(
	usba_pipe_handle_data_t	*ph,
	ehci_isoc_xwrapper_t	*itw,
	usb_cr_t		completion_reason);


/*
 * Isochronous initialization functions
 */
/*
 * Initialize all the needed resources needed by isochronous pipes.
 */
int
ehci_isoc_init(
	ehci_state_t		*ehcip)
{
	return (ehci_allocate_isoc_pools(ehcip));
}


/*
 * Cleanup isochronous resources.
 */
void
ehci_isoc_cleanup(
	ehci_state_t		*ehcip)
{
	ehci_isoc_xwrapper_t	*itw;
	ehci_pipe_private_t	*pp;
	ehci_itd_t		*itd;
	int			i, ctrl, rval;

	/* Free all the buffers */
	if (ehcip->ehci_itd_pool_addr && ehcip->ehci_itd_pool_mem_handle) {
		for (i = 0; i < ehci_get_itd_pool_size(); i ++) {
			itd = &ehcip->ehci_itd_pool_addr[i];
			ctrl = Get_ITD(ehcip->
			    ehci_itd_pool_addr[i].itd_state);

			if ((ctrl != EHCI_ITD_FREE) &&
			    (ctrl != EHCI_ITD_DUMMY) &&
			    (itd->itd_trans_wrapper)) {

				mutex_enter(&ehcip->ehci_int_mutex);

				itw = (ehci_isoc_xwrapper_t *)
				    EHCI_LOOKUP_ID((uint32_t)
				    Get_ITD(itd->itd_trans_wrapper));

				/* Obtain the pipe private structure */
				pp = itw->itw_pipe_private;

				ehci_deallocate_itd(ehcip, itw, itd);
				ehci_deallocate_itw(ehcip, pp, itw);

				mutex_exit(&ehcip->ehci_int_mutex);
			}
		}

		/*
		 * If EHCI_ITD_POOL_BOUND flag is set, then unbind
		 * the handle for ITD pools.
		 */
		if ((ehcip->ehci_dma_addr_bind_flag &
		    EHCI_ITD_POOL_BOUND) == EHCI_ITD_POOL_BOUND) {

			rval = ddi_dma_unbind_handle(
			    ehcip->ehci_itd_pool_dma_handle);

			ASSERT(rval == DDI_SUCCESS);
		}
		ddi_dma_mem_free(&ehcip->ehci_itd_pool_mem_handle);
	}

	/* Free the ITD pool */
	if (ehcip->ehci_itd_pool_dma_handle) {
		ddi_dma_free_handle(&ehcip->ehci_itd_pool_dma_handle);
	}
}


/*
 * ehci_isoc_pipe_cleanup
 *
 * Cleanup ehci isoc pipes.
 */
void ehci_isoc_pipe_cleanup(
	ehci_state_t		*ehcip,
	usba_pipe_handle_data_t *ph) {
	ehci_pipe_private_t	*pp = (ehci_pipe_private_t *)ph->p_hcd_private;
	uint_t			pipe_state = pp->pp_state;
	usb_cr_t		completion_reason;

	USB_DPRINTF_L4(PRINT_MASK_LISTS, ehcip->ehci_log_hdl,
	    "ehci_isoc_pipe_cleanup: ph = 0x%p", (void *)ph);

	ASSERT(mutex_owned(&ehcip->ehci_int_mutex));

	/* Stop all further processing */
	ehci_mark_reclaim_isoc(ehcip, pp);

	/*
	 * Wait for processing all completed transfers
	 * and send result upstream/
	 */
	ehci_wait_for_isoc_completion(ehcip, pp);

	/* Go ahead and remove all remaining itds if there are any */
	ehci_remove_isoc_itds(ehcip, pp);

	switch (pipe_state) {
	case EHCI_PIPE_STATE_CLOSE:
		completion_reason = USB_CR_PIPE_CLOSING;
		break;
	case EHCI_PIPE_STATE_RESET:
	case EHCI_PIPE_STATE_STOP_POLLING:
		/* Set completion reason */
		completion_reason = (pipe_state ==
		    EHCI_PIPE_STATE_RESET) ?
		    USB_CR_PIPE_RESET: USB_CR_STOPPED_POLLING;

		/* Set pipe state to idle */
		pp->pp_state = EHCI_PIPE_STATE_IDLE;

		break;
	}

	/*
	 * Do the callback for the original client
	 * periodic IN request.
	 */
	if ((ph->p_ep.bEndpointAddress & USB_EP_DIR_MASK) ==
	    USB_EP_DIR_IN) {

		ehci_do_client_periodic_in_req_callback(
		    ehcip, pp, completion_reason);
	}
}


/*
 * ehci_wait_for_transfers_completion:
 *
 * Wait for processing all completed transfers and to send results
 * to upstream.
 */
static void
ehci_wait_for_isoc_completion(
	ehci_state_t		*ehcip,
	ehci_pipe_private_t	*pp)
{
	ASSERT(mutex_owned(&ehcip->ehci_int_mutex));

	if (pp->pp_itw_head == NULL) {

		return;
	}

	(void) cv_reltimedwait(&pp->pp_xfer_cmpl_cv, &ehcip->ehci_int_mutex,
	    drv_usectohz(EHCI_XFER_CMPL_TIMEWAIT * 1000000), TR_CLOCK_TICK);

	if (pp->pp_itw_head) {
		USB_DPRINTF_L2(PRINT_MASK_LISTS, ehcip->ehci_log_hdl,
		    "ehci_wait_for_isoc_completion: "
		    "No transfers completion confirmation received");
	}
}


/*
 *  Isochronous request functions
 */
/*
 * ehci_allocate_isoc_resources:
 *
 * Calculates the number of tds necessary for a isoch transfer, and
 * allocates all the necessary resources.
 *
 * Returns NULL if there is insufficient resources otherwise ITW.
 */
ehci_isoc_xwrapper_t *
ehci_allocate_isoc_resources(
	ehci_state_t		*ehcip,
	usba_pipe_handle_data_t *ph,
	usb_isoc_req_t		*isoc_reqp,
	usb_flags_t		usb_flags)
{
	ehci_pipe_private_t	*pp = (ehci_pipe_private_t *)ph->p_hcd_private;
	int			pipe_dir, i;
	uint_t			max_ep_pkt_size, max_isoc_xfer_size;
	usb_isoc_pkt_descr_t	*isoc_pkt_descr;
	size_t			isoc_pkt_count, isoc_pkts_length;
	size_t			itw_xfer_size = 0;
	ehci_isoc_xwrapper_t	*itw;

	USB_DPRINTF_L4(PRINT_MASK_LISTS, ehcip->ehci_log_hdl,
	    "ehci_allocate_isoc_resources: flags = 0x%x", usb_flags);

	ASSERT(mutex_owned(&ehcip->ehci_int_mutex));

	/*
	 * Check whether pipe is in halted state.
	 */
	if (pp->pp_state == EHCI_PIPE_STATE_ERROR) {
		USB_DPRINTF_L2(PRINT_MASK_LISTS, ehcip->ehci_log_hdl,
		    "ehci_allocate_isoc_resources:"
		    "Pipe is in error state, need pipe reset to continue");

		return (NULL);
	}

	/* Calculate the maximum isochronous transfer size we allow */
	max_ep_pkt_size = (ph->p_ep.wMaxPacketSize &
	    EHCI_ITD_CTRL_MAX_PACKET_MASK) *
	    CalculateITDMultiField(ph->p_ep.wMaxPacketSize);

	max_isoc_xfer_size = EHCI_MAX_ISOC_PKTS_PER_XFER * max_ep_pkt_size;

	/* Get the packet descriptor and number of packets to send */
	if (isoc_reqp) {
		isoc_pkt_descr = isoc_reqp->isoc_pkt_descr;
		isoc_pkt_count = isoc_reqp->isoc_pkts_count;
		isoc_pkts_length = isoc_reqp->isoc_pkts_length;
	} else {
		isoc_pkt_descr = ((usb_isoc_req_t *)
		    pp->pp_client_periodic_in_reqp)->isoc_pkt_descr;

		isoc_pkt_count = ((usb_isoc_req_t *)
		    pp->pp_client_periodic_in_reqp)->isoc_pkts_count;

		isoc_pkts_length = ((usb_isoc_req_t *)
		    pp->pp_client_periodic_in_reqp)->isoc_pkts_length;
	}

	/* Calculate the size of the transfer. */
	pipe_dir = ph->p_ep.bEndpointAddress & USB_EP_DIR_MASK;
	if (pipe_dir == USB_EP_DIR_IN) {
		for (i = 0; i < isoc_pkt_count; i++) {
			/*
			 * isoc_pkt_length is used as Transaction Length and
			 * according to EHCI spec Table 3-3, the maximum value
			 * allowed is 3072
			 */
			if (isoc_pkt_descr->isoc_pkt_length > 3072) {

				return (NULL);
			}

			itw_xfer_size += isoc_pkt_descr->isoc_pkt_length;

			isoc_pkt_descr++;
		}

		if ((isoc_pkts_length) &&
		    (isoc_pkts_length != itw_xfer_size)) {

			USB_DPRINTF_L2(PRINT_MASK_LISTS, ehcip->ehci_log_hdl,
			    "ehci_allocate_isoc_resources: "
			    "isoc_pkts_length 0x%lx is not equal to the sum of "
			    "all pkt lengths 0x%lx in an isoc request",
			    isoc_pkts_length, itw_xfer_size);

			return (NULL);
		}

	} else {
		ASSERT(isoc_reqp != NULL);
		itw_xfer_size = MBLKL(isoc_reqp->isoc_data);
	}

	/* Check the size of isochronous request */
	if (itw_xfer_size > max_isoc_xfer_size) {

		USB_DPRINTF_L2(PRINT_MASK_LISTS, ehcip->ehci_log_hdl,
		    "ehci_allocate_isoc_resources: Maximum isoc request "
		    "size 0x%x Given isoc request size 0x%lx",
		    max_isoc_xfer_size, itw_xfer_size);

		return (NULL);
	}

	USB_DPRINTF_L4(PRINT_MASK_LISTS, ehcip->ehci_log_hdl,
	    "ehci_allocate_isoc_resources: length = 0x%lx", itw_xfer_size);

	/* Allocate the itw for this request */
	if ((itw = ehci_allocate_itw_resources(ehcip, pp, itw_xfer_size,
	    usb_flags, isoc_pkt_count)) == NULL) {

		return (NULL);
	}

	itw->itw_handle_callback_value = NULL;

	if (pipe_dir == USB_EP_DIR_IN) {
		if (ehci_allocate_isoc_in_resource(ehcip, pp, itw, usb_flags) !=
		    USB_SUCCESS) {

			ehci_deallocate_itw(ehcip, pp, itw);

			return (NULL);
		}
	} else {
		if (itw->itw_length) {
			ASSERT(isoc_reqp->isoc_data != NULL);

			/* Copy the data into the buffer */
			bcopy(isoc_reqp->isoc_data->b_rptr,
			    itw->itw_buf, itw->itw_length);

			Sync_IO_Buffer_for_device(itw->itw_dmahandle,
			    itw->itw_length);
		}
		itw->itw_curr_xfer_reqp = isoc_reqp;
	}

	return (itw);
}


/*
 * ehci_insert_isoc_req:
 *
 * Insert an isochronous request into the Host Controller's
 * isochronous list.
 */
int
ehci_insert_isoc_req(
	ehci_state_t			*ehcip,
	ehci_pipe_private_t		*pp,
	ehci_isoc_xwrapper_t		*itw,
	usb_flags_t			usb_flags)
{
	int			error;
	ehci_itd_t		*new_itd, *temp_itd;

	USB_DPRINTF_L4(PRINT_MASK_LISTS, ehcip->ehci_log_hdl,
	    "ehci_insert_isoc_req: flags = 0x%x port status = 0x%x",
	    usb_flags, itw->itw_port_status);

	ASSERT(mutex_owned(&ehcip->ehci_int_mutex));

	ASSERT(itw->itw_curr_xfer_reqp != NULL);
	ASSERT(itw->itw_curr_xfer_reqp->isoc_pkt_descr != NULL);

	/*
	 * Save address of first usb isochronous packet descriptor.
	 */
	itw->itw_curr_isoc_pktp = itw->itw_curr_xfer_reqp->isoc_pkt_descr;

	if (itw->itw_port_status == USBA_HIGH_SPEED_DEV) {
		error = ehci_insert_itd_req(ehcip, pp, itw, usb_flags);
	} else {
		error = ehci_insert_sitd_req(ehcip, pp, itw, usb_flags);
	}

	/* Either all the isocs will be added or none of them will */
	error = ehci_insert_isoc_to_pfl(ehcip, pp, itw);

	if (error != USB_SUCCESS) {
		/*
		 * Deallocate all the ITDs, otherwise they will be
		 * lost forever.
		 */
		new_itd = itw->itw_itd_head;
		while (new_itd) {
			temp_itd = ehci_itd_iommu_to_cpu(ehcip,
			    Get_ITD(new_itd->itd_itw_next_itd));
			ehci_deallocate_itd(ehcip, itw, new_itd);
			new_itd = temp_itd;
		}
		if ((itw->itw_direction == USB_EP_DIR_IN)) {
			ehci_deallocate_isoc_in_resource(ehcip, pp, itw);

			if (pp->pp_cur_periodic_req_cnt) {
				/*
				 * Set pipe state to stop polling and
				 * error to no resource. Don't insert
				 * any more isoch polling requests.
				 */
				pp->pp_state =
				    EHCI_PIPE_STATE_STOP_POLLING;
				pp->pp_error = error;
			} else {
				/* Set periodic in pipe state to idle */
				pp->pp_state = EHCI_PIPE_STATE_IDLE;
			}

			return (error);
		}

		/* Save how many packets and data actually went */
		itw->itw_num_itds = 0;
		itw->itw_length  = 0;
	}

	/*
	 * Reset back to the address of first usb isochronous
	 * packet descriptor.
	 */
	itw->itw_curr_isoc_pktp = itw->itw_curr_xfer_reqp->isoc_pkt_descr;

	/* Reset the CONTINUE flag */
	pp->pp_flag &= ~EHCI_ISOC_XFER_CONTINUE;

	return (error);
}


/*
 * ehci_insert_itd_req:
 *
 * Insert an ITD request into the Host Controller's isochronous list.
 */
/* ARGSUSED */
static int
ehci_insert_itd_req(
	ehci_state_t		*ehcip,
	ehci_pipe_private_t	*pp,
	ehci_isoc_xwrapper_t	*itw,
	usb_flags_t		usb_flags)
{
	usba_pipe_handle_data_t	*ph = pp->pp_pipe_handle;
	usb_isoc_req_t		*curr_isoc_reqp;
	usb_isoc_pkt_descr_t	*curr_isoc_pkt_descr;
	size_t			curr_isoc_xfer_offset;
	size_t			isoc_pkt_length;
	uint_t			count, xactcount;
	uint32_t		xact_status;
	uint32_t		page, pageselected;
	uint32_t		buf[EHCI_ITD_BUFFER_LIST_SIZE];
	uint16_t		index = 0;
	uint16_t		multi = 0;
	ehci_itd_t		*new_itd;

	/*
	 * Get the current isochronous request and packet
	 * descriptor pointers.
	 */
	curr_isoc_reqp = (usb_isoc_req_t *)itw->itw_curr_xfer_reqp;

	page = itw->itw_cookie.dmac_address;
	ASSERT((page % EHCI_4K_ALIGN) == 0);

	USB_DPRINTF_L3(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
	    "ehci_insert_itd_req: itw_curr_xfer_reqp = 0x%p page = 0x%x,"
	    " pagesize = 0x%lx", (void *)itw->itw_curr_xfer_reqp, page,
	    itw->itw_cookie.dmac_size);

	/* Insert all the isochronous TDs */
	count = 0;
	curr_isoc_xfer_offset = 0;

	while (count < curr_isoc_reqp->isoc_pkts_count) {

		/* Grab a new itd */
		new_itd = itw->itw_itd_free_list;

		ASSERT(new_itd != NULL);

		itw->itw_itd_free_list = ehci_itd_iommu_to_cpu(ehcip,
		    Get_ITD(new_itd->itd_link_ptr));
		Set_ITD(new_itd->itd_link_ptr, NULL);

		bzero(buf, EHCI_ITD_BUFFER_LIST_SIZE * sizeof (uint32_t));

		multi = CalculateITDMultiField(ph->p_ep.wMaxPacketSize);

		if (multi > EHCI_ITD_CTRL_MULTI_MASK) {
			USB_DPRINTF_L2(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
			    "ehci_insert_itd_req: Wrong multi value.");

			return (USB_FAILURE);
		}

		/* Fill 8 transaction for every iTD */
		for (xactcount = 0, pageselected = 0;
		    xactcount < EHCI_ITD_CTRL_LIST_SIZE; xactcount++) {

			curr_isoc_pkt_descr = itw->itw_curr_isoc_pktp;

			isoc_pkt_length =
			    curr_isoc_pkt_descr->isoc_pkt_length;

			curr_isoc_pkt_descr->isoc_pkt_actual_length
			    = (ushort_t)isoc_pkt_length;

			xact_status = 0;

			if (pageselected < EHCI_ITD_BUFFER_LIST_SIZE) {

				buf[pageselected] |= page;
			} else {
				USB_DPRINTF_L2(PRINT_MASK_INTR,
				    ehcip->ehci_log_hdl,
				    "ehci_insert_itd_req: "
				    "Error in buffer pointer.");

				return (USB_FAILURE);
			}

			xact_status = (uint32_t)curr_isoc_xfer_offset;
			xact_status |= (pageselected << 12);
			xact_status |= isoc_pkt_length << 16;
			xact_status |= EHCI_ITD_XFER_ACTIVE;

			/* Set IOC on the last TD. */
			if (count == (curr_isoc_reqp->isoc_pkts_count - 1)) {
				xact_status |= EHCI_ITD_XFER_IOC_ON;
			}

			USB_DPRINTF_L3(PRINT_MASK_INTR,
			    ehcip->ehci_log_hdl,
			    "ehci_insert_itd_req: count = 0x%x multi = %d"
			    "status = 0x%x page = 0x%x index = %d "
			    "pageselected = %d isoc_pkt_length = 0x%lx",
			    xactcount, multi, xact_status, page,
			    index, pageselected, isoc_pkt_length);

			/* Fill in the new itd */
			Set_ITD_BODY(new_itd, xactcount, xact_status);

			itw->itw_curr_isoc_pktp++;
			Set_ITD_INDEX(new_itd, xactcount, index++);

			curr_isoc_xfer_offset += isoc_pkt_length;

			if (curr_isoc_xfer_offset >= EHCI_4K_ALIGN) {
				pageselected ++;
				page += EHCI_4K_ALIGN;
				curr_isoc_xfer_offset -= EHCI_4K_ALIGN;
			}

			count ++;
			if (count >= curr_isoc_reqp->isoc_pkts_count) {

				break;
			}
		}

		buf[0] |= (itw->itw_endpoint_num << 8);
		buf[0] |= itw->itw_device_addr;
		buf[1] |= ph->p_ep.wMaxPacketSize &
		    EHCI_ITD_CTRL_MAX_PACKET_MASK;

		if (itw->itw_direction == USB_EP_DIR_IN) {
			buf[1] |= EHCI_ITD_CTRL_DIR_IN;
		}
		buf[2] |= multi;

		Set_ITD_BODY(new_itd, EHCI_ITD_BUFFER0, buf[0]);
		Set_ITD_BODY(new_itd, EHCI_ITD_BUFFER1, buf[1]);
		Set_ITD_BODY(new_itd, EHCI_ITD_BUFFER2, buf[2]);
		Set_ITD_BODY(new_itd, EHCI_ITD_BUFFER3, buf[3]);
		Set_ITD_BODY(new_itd, EHCI_ITD_BUFFER4, buf[4]);
		Set_ITD_BODY(new_itd, EHCI_ITD_BUFFER5, buf[5]);
		Set_ITD_BODY(new_itd, EHCI_ITD_BUFFER6, buf[6]);

		Set_ITD(new_itd->itd_state, EHCI_ITD_ACTIVE);
		ehci_print_itd(ehcip, new_itd);

		/*
		 * Add this itd to the itw before we add it in the PFL
		 * If adding it to the PFL fails, we will have to cleanup.
		 */
		ehci_insert_itd_on_itw(ehcip, itw, new_itd);

	}

	return (USB_SUCCESS);
}


/*
 * ehci_insert_sitd_req:
 *
 * Insert an SITD request into the Host Controller's isochronous list.
 */
/* ARGSUSED */
static int
ehci_insert_sitd_req(
	ehci_state_t		*ehcip,
	ehci_pipe_private_t	*pp,
	ehci_isoc_xwrapper_t	*itw,
	usb_flags_t		usb_flags)
{
	usba_pipe_handle_data_t	*ph = pp->pp_pipe_handle;
	usb_isoc_req_t		*curr_isoc_reqp;
	usb_isoc_pkt_descr_t	*curr_isoc_pkt_descr;
	size_t			curr_isoc_xfer_offset;
	size_t			isoc_pkt_length;
	uint_t			count;
	uint32_t		ctrl, uframe_sched, xfer_state;
	uint32_t		page0, page1, prev_sitd;
	uint32_t		ssplit_count;
	ehci_itd_t		*new_sitd;

	/*
	 * Get the current isochronous request and packet
	 * descriptor pointers.
	 */
	curr_isoc_reqp = (usb_isoc_req_t *)itw->itw_curr_xfer_reqp;

	/* Set the ctrl field */
	ctrl = 0;
	if (itw->itw_direction == USB_EP_DIR_IN) {
		ctrl |= EHCI_SITD_CTRL_DIR_IN;
	} else {
		ctrl |= EHCI_SITD_CTRL_DIR_OUT;
	}

	ctrl |= (itw->itw_hub_port << EHCI_SITD_CTRL_PORT_SHIFT) &
	    EHCI_SITD_CTRL_PORT_MASK;
	ctrl |= (itw->itw_hub_addr << EHCI_SITD_CTRL_HUB_SHIFT) &
	    EHCI_SITD_CTRL_HUB_MASK;
	ctrl |= (itw->itw_endpoint_num << EHCI_SITD_CTRL_END_PT_SHIFT) &
	    EHCI_SITD_CTRL_END_PT_MASK;
	ctrl |= (itw->itw_device_addr << EHCI_SITD_CTRL_DEVICE_SHIFT) &
	    EHCI_SITD_CTRL_DEVICE_MASK;

	/* Set the micro frame schedule */
	uframe_sched = 0;
	uframe_sched |= (pp->pp_smask << EHCI_SITD_UFRAME_SMASK_SHIFT) &
	    EHCI_SITD_UFRAME_SMASK_MASK;
	uframe_sched |= (pp->pp_cmask << EHCI_SITD_UFRAME_CMASK_SHIFT) &
	    EHCI_SITD_UFRAME_CMASK_MASK;

	/* Set the default page information */
	page0 = itw->itw_cookie.dmac_address;
	page1 = 0;

	prev_sitd = EHCI_ITD_LINK_PTR_INVALID;

	/*
	 * Save the number of isochronous TDs needs
	 * to be insert to complete current isochronous request.
	 */
	itw->itw_num_itds = curr_isoc_reqp->isoc_pkts_count;

	/* Insert all the isochronous TDs */
	for (count = 0, curr_isoc_xfer_offset = 0;
	    count < itw->itw_num_itds; count++) {

		curr_isoc_pkt_descr = itw->itw_curr_isoc_pktp;

		isoc_pkt_length = curr_isoc_pkt_descr->isoc_pkt_length;
		curr_isoc_pkt_descr->isoc_pkt_actual_length =
		    (ushort_t)isoc_pkt_length;

		/* Set the transfer state information */
		xfer_state = 0;

		if (itw->itw_direction == USB_EP_DIR_IN) {
			/* Set the size to the max packet size */
			xfer_state |= (ph->p_ep.wMaxPacketSize <<
			    EHCI_SITD_XFER_TOTAL_SHIFT) &
			    EHCI_SITD_XFER_TOTAL_MASK;
		} else {
			/* Set the size to the packet length */
			xfer_state |= (isoc_pkt_length <<
			    EHCI_SITD_XFER_TOTAL_SHIFT) &
			    EHCI_SITD_XFER_TOTAL_MASK;
		}
		xfer_state |=  EHCI_SITD_XFER_ACTIVE;

		/* Set IOC on the last TD. */
		if (count == (itw->itw_num_itds - 1)) {
			xfer_state |= EHCI_SITD_XFER_IOC_ON;
		}

		ssplit_count = isoc_pkt_length / MAX_UFRAME_SITD_XFER;
		if (isoc_pkt_length % MAX_UFRAME_SITD_XFER) {
			ssplit_count++;
		}

		page1 = (ssplit_count & EHCI_SITD_XFER_TCOUNT_MASK) <<
		    EHCI_SITD_XFER_TCOUNT_SHIFT;
		if (ssplit_count > 1) {
			page1 |= EHCI_SITD_XFER_TP_BEGIN;
		} else {
			page1 |= EHCI_SITD_XFER_TP_ALL;
		}

		/* Grab a new sitd */
		new_sitd = itw->itw_itd_free_list;

		ASSERT(new_sitd != NULL);

		itw->itw_itd_free_list = ehci_itd_iommu_to_cpu(ehcip,
		    Get_ITD(new_sitd->itd_link_ptr));
		Set_ITD(new_sitd->itd_link_ptr, NULL);

		/* Fill in the new sitd */
		Set_ITD_BODY(new_sitd, EHCI_SITD_CTRL, ctrl);
		Set_ITD_BODY(new_sitd, EHCI_SITD_UFRAME_SCHED, uframe_sched);
		Set_ITD_BODY(new_sitd, EHCI_SITD_XFER_STATE, xfer_state);
		Set_ITD_BODY(new_sitd, EHCI_SITD_BUFFER0,
		    page0 + curr_isoc_xfer_offset);
		Set_ITD_BODY(new_sitd, EHCI_SITD_BUFFER1, page1);
		Set_ITD_BODY(new_sitd, EHCI_SITD_PREV_SITD, prev_sitd);

		Set_ITD(new_sitd->itd_state, EHCI_ITD_ACTIVE);

		/*
		 * Add this itd to the itw before we add it in the PFL
		 * If adding it to the PFL fails, we will have to cleanup.
		 */
		ehci_insert_itd_on_itw(ehcip, itw, new_sitd);

		itw->itw_curr_isoc_pktp++;
		curr_isoc_xfer_offset += isoc_pkt_length;
	}

	return (USB_SUCCESS);
}


/*
 * ehci_remove_isoc_itds:
 *
 * Remove all itds from the PFL.
 */
static void
ehci_remove_isoc_itds(
	ehci_state_t		*ehcip,
	ehci_pipe_private_t	*pp)
{
	ehci_isoc_xwrapper_t	*curr_itw, *next_itw;
	ehci_itd_t		*curr_itd, *next_itd;

	USB_DPRINTF_L4(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
	    "ehci_remove_isoc_itds: pp = 0x%p", (void *)pp);

	ASSERT(mutex_owned(&ehcip->ehci_int_mutex));

	curr_itw = pp->pp_itw_head;
	while (curr_itw) {
		USB_DPRINTF_L4(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
		    "ehci_remove_isoc_itds: itw = 0x%p num itds = %d",
		    (void *)curr_itw, curr_itw->itw_num_itds);

		next_itw = curr_itw->itw_next;

		curr_itd = curr_itw->itw_itd_head;
		while (curr_itd) {
			next_itd = ehci_itd_iommu_to_cpu(ehcip,
			    Get_ITD(curr_itd->itd_itw_next_itd));

			ehci_reclaim_isoc(ehcip, curr_itw, curr_itd, pp);

			curr_itd = next_itd;
		}

		ehci_deallocate_itw(ehcip, pp, curr_itw);

		curr_itw = next_itw;
	}
}


/*
 * ehci_mark_reclaim_isoc:
 *
 * Set active ITDs to RECLAIM.
 * Return number of ITD that need to be processed.
 */
static void
ehci_mark_reclaim_isoc(
	ehci_state_t		*ehcip,
	ehci_pipe_private_t	*pp)
{
	usb_frame_number_t	current_frame_number;
	ehci_isoc_xwrapper_t	*curr_itw, *next_itw;
	ehci_itd_t		*curr_itd, *next_itd;
	uint_t			ctrl;
	uint_t			isActive;
	int			i;

	USB_DPRINTF_L4(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
	    "ehci_mark_reclaim_isoc: pp = 0x%p", (void *)pp);

	if (pp->pp_itw_head == NULL) {

		return;
	}

	/* Get the current frame number. */
	current_frame_number = ehci_get_current_frame_number(ehcip);

	/* Traverse the list of transfer descriptors */
	curr_itw = pp->pp_itw_head;
	while (curr_itw) {
		next_itw = curr_itw->itw_next;

		USB_DPRINTF_L4(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
		    "ehci_mark_reclaim_isoc: itw = 0x%p num itds = %d",
		    (void *)curr_itw, curr_itw->itw_num_itds);

		curr_itd = curr_itw->itw_itd_head;
		while (curr_itd) {
			next_itd = ehci_itd_iommu_to_cpu(ehcip,
			    Get_ITD(curr_itd->itd_itw_next_itd));

			if (curr_itw->itw_port_status == USBA_HIGH_SPEED_DEV) {

				for (i = 0; i < EHCI_ITD_CTRL_LIST_SIZE; i++) {
					ctrl = Get_ITD_BODY(curr_itd,
					    EHCI_ITD_CTRL0 + i);
					isActive = ctrl & EHCI_ITD_XFER_ACTIVE;
					/* If still active, deactivate it */
					if (isActive) {
						ctrl &= ~EHCI_ITD_XFER_ACTIVE;
						Set_ITD_BODY(curr_itd,
						    EHCI_ITD_CTRL0 + i,
						    ctrl);
						break;
					}
				}
			} else {
				ctrl = Get_ITD_BODY(curr_itd,
				    EHCI_SITD_XFER_STATE);
				isActive = ctrl & EHCI_SITD_XFER_ACTIVE;
				/* If it is still active deactivate it */
				if (isActive) {
					ctrl &= ~EHCI_SITD_XFER_ACTIVE;
					Set_ITD_BODY(curr_itd,
					    EHCI_SITD_XFER_STATE,
					    ctrl);
				}
			}

			/*
			 * If the itd was active put it on the reclaim status,
			 * so the interrupt handler will know not to process it.
			 * Otherwise leave it alone and let the interrupt
			 * handler process it normally.
			 */
			if (isActive) {
				Set_ITD(curr_itd->itd_state, EHCI_ITD_RECLAIM);
				Set_ITD_FRAME(curr_itd->itd_reclaim_number,
				    current_frame_number);
				ehci_remove_isoc_from_pfl(ehcip, curr_itd);
			}
			curr_itd = next_itd;
		}
		curr_itw = next_itw;
	}
}


/*
 * ehci_reclaim_isoc:
 *
 * "Reclaim" itds that were marked as RECLAIM.
 */
static void
ehci_reclaim_isoc(
	ehci_state_t		*ehcip,
	ehci_isoc_xwrapper_t	*itw,
	ehci_itd_t		*itd,
	ehci_pipe_private_t	*pp)
{
	USB_DPRINTF_L4(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
	    "ehci_reclaim_isoc: itd = 0x%p", (void *)itd);

	/*
	 * These are itds that were marked "RECLAIM"
	 * by the pipe cleanup.
	 *
	 * Decrement the num_itds and the periodic in
	 * request count if necessary.
	 */
	if ((--itw->itw_num_itds == 0) && (itw->itw_curr_xfer_reqp)) {
		if (itw->itw_direction == USB_EP_DIR_IN) {

			pp->pp_cur_periodic_req_cnt--;

			ehci_deallocate_isoc_in_resource(ehcip, pp, itw);
		} else {
			ehci_hcdi_isoc_callback(pp->pp_pipe_handle, itw,
			    USB_CR_FLUSHED);
		}
	}

	/* Deallocate this transfer descriptor */
	ehci_deallocate_itd(ehcip, itw, itd);
}


/*
 * ehci_start_isoc_polling:
 *
 * Insert the number of periodic requests corresponding to polling
 * interval as calculated during pipe open.
 */
int
ehci_start_isoc_polling(
	ehci_state_t		*ehcip,
	usba_pipe_handle_data_t	*ph,
	usb_flags_t		flags)
{
	ehci_pipe_private_t	*pp = (ehci_pipe_private_t *)ph->p_hcd_private;
	ehci_isoc_xwrapper_t	*itw_list, *itw;
	int			i, total_itws;
	int			error = USB_SUCCESS;

	USB_DPRINTF_L4(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
	    "ehci_start_isoc_polling:");

	/* Allocate all the necessary resources for the IN transfer */
	itw_list = NULL;
	total_itws = pp->pp_max_periodic_req_cnt - pp->pp_cur_periodic_req_cnt;
	for (i = 0; i < total_itws; i += 1) {
		itw = ehci_allocate_isoc_resources(ehcip, ph, NULL, flags);
		if (itw == NULL) {
			error = USB_NO_RESOURCES;
			/* There are not enough resources deallocate the ITWs */
			itw = itw_list;
			while (itw != NULL) {
				itw_list = itw->itw_next;
				ehci_deallocate_isoc_in_resource(
				    ehcip, pp, itw);
				ehci_deallocate_itw(ehcip, pp, itw);
				itw = itw_list;
			}

			return (error);
		} else {
			if (itw_list == NULL) {
				itw_list = itw;
			}
		}
	}

	i = 0;
	while (pp->pp_cur_periodic_req_cnt < pp->pp_max_periodic_req_cnt) {

		USB_DPRINTF_L3(PRINT_MASK_LISTS, ehcip->ehci_log_hdl,
		    "ehci_start_isoc_polling: max = %d curr = %d itw = %p:",
		    pp->pp_max_periodic_req_cnt, pp->pp_cur_periodic_req_cnt,
		    (void *)itw_list);

		itw = itw_list;
		itw_list = itw->itw_next;

		error = ehci_insert_isoc_req(ehcip, pp, itw, flags);

		if (error == USB_SUCCESS) {
			pp->pp_cur_periodic_req_cnt++;
		} else {
			/*
			 * Deallocate the remaining tw
			 * The current tw should have already been deallocated
			 */
			itw = itw_list;
			while (itw != NULL) {
				itw_list = itw->itw_next;
				ehci_deallocate_isoc_in_resource(
				    ehcip, pp, itw);
				ehci_deallocate_itw(ehcip, pp, itw);
				itw = itw_list;
			}
			/*
			 * If this is the first req return an error.
			 * Otherwise return success.
			 */
			if (i != 0) {
				error = USB_SUCCESS;
			}

			break;
		}
		i++;
	}

	return (error);
}


/*
 * Isochronronous handling functions.
 */
/*
 * ehci_traverse_active_isoc_list:
 */
void
ehci_traverse_active_isoc_list(
	ehci_state_t		*ehcip)
{
	ehci_isoc_xwrapper_t	*curr_itw;
	ehci_itd_t		*curr_itd, *next_itd;
	uint_t			state;
	ehci_pipe_private_t	*pp;

	USB_DPRINTF_L4(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
	    "ehci_traverse_active_isoc_list:");

	ASSERT(mutex_owned(&ehcip->ehci_int_mutex));

	/* Sync ITD pool */
	Sync_ITD_Pool(ehcip);

	/* Traverse the list of done itds */
	curr_itd = ehci_create_done_itd_list(ehcip);
	USB_DPRINTF_L3(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
	    "ehci_traverse_active_isoc_list: current itd = 0x%p",
	    (void *)curr_itd);

	while (curr_itd) {
		/* Save the next_itd */
		next_itd = ehci_itd_iommu_to_cpu(ehcip,
		    Get_ITD(curr_itd->itd_next_active_itd));

		/* Get the transfer wrapper and the pp */
		curr_itw = (ehci_isoc_xwrapper_t *)EHCI_LOOKUP_ID(
		    (uint32_t)Get_ITD(curr_itd->itd_trans_wrapper));
		pp = curr_itw->itw_pipe_private;

		if (curr_itw->itw_port_status == USBA_HIGH_SPEED_DEV) {
			ehci_print_itd(ehcip, curr_itd);
		} else {
			ehci_print_sitd(ehcip, curr_itd);
		}

		/* Get the ITD state */
		state = Get_ITD(curr_itd->itd_state);

		/* Only process the ITDs marked as active. */
		if (state == EHCI_ITD_ACTIVE) {
			ehci_parse_isoc_error(ehcip, curr_itw, curr_itd);
			ehci_handle_isoc(ehcip, curr_itw, curr_itd);
		} else {
			ASSERT(state == EHCI_ITD_RECLAIM);
			ehci_reclaim_isoc(ehcip, curr_itw, curr_itd, pp);
		}

		/*
		 * Deallocate the transfer wrapper if there are no more
		 * ITD's for the transfer wrapper.  ehci_deallocate_itw()
		 * will  not deallocate the tw for a periodic in endpoint
		 * since it will always have a ITD attached to it.
		 */
		ehci_deallocate_itw(ehcip, pp, curr_itw);

		/* Check any ISOC is waiting for transfers completion event */
		if (pp->pp_itw_head == NULL) {
			USB_DPRINTF_L3(PRINT_MASK_LISTS, ehcip->ehci_log_hdl,
			    "ehci_traverse_active_isoc_list: "
			    "Sent transfers completion event pp = 0x%p",
			    (void *)pp);
			cv_signal(&pp->pp_xfer_cmpl_cv);
		}

		curr_itd = next_itd;

		USB_DPRINTF_L3(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
		    "ehci_traverse_active_isoc_list: state = 0x%x "
		    "pp = 0x%p itw = 0x%p itd = 0x%p next_itd = 0x%p",
		    state, (void *)pp, (void *)curr_itw, (void *)curr_itd,
		    (void *)next_itd);
	}
}


static void
ehci_handle_isoc(
	ehci_state_t		*ehcip,
	ehci_isoc_xwrapper_t	*itw,
	ehci_itd_t		*itd)
{
	ehci_pipe_private_t	*pp;	/* Pipe private field */

	ASSERT(mutex_owned(&ehcip->ehci_int_mutex));

	USB_DPRINTF_L4(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
	    "ehci_handle_isoc:");

	/* Obtain the pipe private structure */
	pp = itw->itw_pipe_private;

	ehci_handle_itd(ehcip, pp, itw, itd, itw->itw_handle_callback_value);
}


/*
 * ehci_handle_itd:
 *
 * Handle an (split) isochronous transfer descriptor.
 * This function will deallocate the itd from the list as well.
 */
/* ARGSUSED */
static void
ehci_handle_itd(
	ehci_state_t		*ehcip,
	ehci_pipe_private_t	*pp,
	ehci_isoc_xwrapper_t	*itw,
	ehci_itd_t		*itd,
	void			*tw_handle_callback_value)
{
	usba_pipe_handle_data_t	*ph = pp->pp_pipe_handle;
	usb_isoc_req_t		*curr_isoc_reqp =
	    (usb_isoc_req_t *)itw->itw_curr_xfer_reqp;
	int			error = USB_SUCCESS;
	int			i, index;

	USB_DPRINTF_L4(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
	    "ehci_handle_itd: pp=0x%p itw=0x%p itd=0x%p "
	    "isoc_reqp=0%p data=0x%p", (void *)pp, (void *)itw, (void *)itd,
	    (void *)curr_isoc_reqp, (void *)curr_isoc_reqp->isoc_data);

	if (itw->itw_port_status == USBA_HIGH_SPEED_DEV &&
	    curr_isoc_reqp != NULL) {

		for (i = 0; i < EHCI_ITD_CTRL_LIST_SIZE; i++) {

			index = Get_ITD_INDEX(itd, i);
			if (index == EHCI_ITD_UNUSED_INDEX) {

				continue;
			}
			curr_isoc_reqp->
			    isoc_pkt_descr[index].isoc_pkt_actual_length =
			    (Get_ITD_BODY(itd, i) & EHCI_ITD_XFER_LENGTH) >> 16;
		}
	}

	/*
	 * Decrement the ITDs counter and check whether all the isoc
	 * data has been send or received. If ITDs counter reaches
	 * zero then inform client driver about completion current
	 * isoc request. Otherwise wait for completion of other isoc
	 * ITDs or transactions on this pipe.
	 */
	if (--itw->itw_num_itds != 0) {
		/* Deallocate this transfer descriptor */
		ehci_deallocate_itd(ehcip, itw, itd);

		return;
	}

	/*
	 * If this is a isoc in pipe, return the data to the client.
	 * For a isoc out pipe, there is no need to do anything.
	 */
	if (itw->itw_direction == USB_EP_DIR_OUT) {
		USB_DPRINTF_L3(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
		    "ehci_handle_itd: Isoc out pipe, isoc_reqp=0x%p, data=0x%p",
		    (void *)curr_isoc_reqp, (void *)curr_isoc_reqp->isoc_data);

		/* Do the callback */
		ehci_hcdi_isoc_callback(ph, itw, USB_CR_OK);

		/* Deallocate this transfer descriptor */
		ehci_deallocate_itd(ehcip, itw, itd);

		return;
	}

	/* Decrement number of IN isochronous request count */
	pp->pp_cur_periodic_req_cnt--;

	USB_DPRINTF_L3(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
	    "ehci_handle_itd: pp_cur_periodic_req_cnt = 0x%x ",
	    pp->pp_cur_periodic_req_cnt);

	/* Call ehci_sendup_itd_message to send message to upstream */
	ehci_sendup_itd_message(ehcip, pp, itw, itd, USB_CR_OK);

	/* Deallocate this transfer descriptor */
	ehci_deallocate_itd(ehcip, itw, itd);

	/*
	 * If isochronous pipe state is still active, insert next isochronous
	 * request into the Host Controller's isochronous list.
	 */
	if (pp->pp_state != EHCI_PIPE_STATE_ACTIVE) {

		return;
	}

	if ((error = ehci_allocate_isoc_in_resource(ehcip, pp, itw, 0)) ==
	    USB_SUCCESS) {
		curr_isoc_reqp = (usb_isoc_req_t *)itw->itw_curr_xfer_reqp;

		ASSERT(curr_isoc_reqp != NULL);

		itw->itw_num_itds = ehci_calc_num_itds(itw,
		    curr_isoc_reqp->isoc_pkts_count);

		if (ehci_allocate_itds_for_itw(ehcip, itw, itw->itw_num_itds) !=
		    USB_SUCCESS) {
			ehci_deallocate_isoc_in_resource(ehcip, pp, itw);
			itw->itw_num_itds = 0;
			error = USB_FAILURE;
		}
	}

	if ((error != USB_SUCCESS) ||
	    (ehci_insert_isoc_req(ehcip, pp, itw, 0) != USB_SUCCESS)) {
		/*
		 * Set pipe state to stop polling and error to no
		 * resource. Don't insert any more isoch polling
		 * requests.
		 */
		pp->pp_state = EHCI_PIPE_STATE_STOP_POLLING;
		pp->pp_error = USB_CR_NO_RESOURCES;

	} else {
		/* Increment number of IN isochronous request count */
		pp->pp_cur_periodic_req_cnt++;

		ASSERT(pp->pp_cur_periodic_req_cnt ==
		    pp->pp_max_periodic_req_cnt);
	}
}


/*
 * ehci_sendup_qtd_message:
 *	copy data, if necessary and do callback
 */
/* ARGSUSED */
static void
ehci_sendup_itd_message(
	ehci_state_t		*ehcip,
	ehci_pipe_private_t	*pp,
	ehci_isoc_xwrapper_t	*itw,
	ehci_itd_t		*td,
	usb_cr_t		error)
{
	usb_isoc_req_t		*isoc_reqp = itw->itw_curr_xfer_reqp;
	usba_pipe_handle_data_t	*ph = pp->pp_pipe_handle;
	size_t			length;
	uchar_t			*buf;
	mblk_t			*mp;

	ASSERT(mutex_owned(&ehcip->ehci_int_mutex));

	USB_DPRINTF_L4(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
	    "ehci_sendup_itd_message:");

	ASSERT(itw != NULL);

	length = itw->itw_length;

	/* Copy the data into the mblk_t */
	buf = (uchar_t *)itw->itw_buf;

	USB_DPRINTF_L3(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
	    "ehci_sendup_itd_message: length %ld error %d", length, error);

	/* Get the message block */
	mp = isoc_reqp->isoc_data;

	ASSERT(mp != NULL);

	if (length) {
		/* Sync IO buffer */
		Sync_IO_Buffer(itw->itw_dmahandle, length);

		/* Copy the data into the message */
		ddi_rep_get8(itw->itw_accesshandle,
		    mp->b_rptr, buf, length, DDI_DEV_AUTOINCR);

		/* Increment the write pointer */
		mp->b_wptr = mp->b_wptr + length;
	} else {
		USB_DPRINTF_L3(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
		    "ehci_sendup_itd_message: Zero length packet");
	}

	ehci_hcdi_isoc_callback(ph, itw, error);
}


/*
 * ehci_hcdi_isoc_callback:
 *
 * Convenience wrapper around usba_hcdi_cb() other than root hub.
 */
void
ehci_hcdi_isoc_callback(
	usba_pipe_handle_data_t	*ph,
	ehci_isoc_xwrapper_t	*itw,
	usb_cr_t		completion_reason)
{
	ehci_state_t		*ehcip = ehci_obtain_state(
	    ph->p_usba_device->usb_root_hub_dip);
	ehci_pipe_private_t	*pp = (ehci_pipe_private_t *)ph->p_hcd_private;
	usb_opaque_t		curr_xfer_reqp;
	uint_t			pipe_state = 0;

	USB_DPRINTF_L4(PRINT_MASK_HCDI, ehcip->ehci_log_hdl,
	    "ehci_hcdi_isoc_callback: ph = 0x%p, itw = 0x%p, cr = 0x%x",
	    (void *)ph, (void *)itw, completion_reason);

	ASSERT(mutex_owned(&ehcip->ehci_int_mutex));

	/* Set the pipe state as per completion reason */
	switch (completion_reason) {
	case USB_CR_OK:
		pipe_state = pp->pp_state;
		break;
	case USB_CR_NO_RESOURCES:
	case USB_CR_NOT_SUPPORTED:
	case USB_CR_PIPE_RESET:
	case USB_CR_STOPPED_POLLING:
		pipe_state = EHCI_PIPE_STATE_IDLE;
		break;
	case USB_CR_PIPE_CLOSING:
		break;
	}

	pp->pp_state = pipe_state;

	if (itw && itw->itw_curr_xfer_reqp) {
		curr_xfer_reqp = (usb_opaque_t)itw->itw_curr_xfer_reqp;
		itw->itw_curr_xfer_reqp = NULL;
	} else {
		ASSERT(pp->pp_client_periodic_in_reqp != NULL);

		curr_xfer_reqp = pp->pp_client_periodic_in_reqp;
		pp->pp_client_periodic_in_reqp = NULL;
	}

	ASSERT(curr_xfer_reqp != NULL);

	mutex_exit(&ehcip->ehci_int_mutex);

	usba_hcdi_cb(ph, curr_xfer_reqp, completion_reason);

	mutex_enter(&ehcip->ehci_int_mutex);
}