xref: /illumos-gate/usr/src/uts/common/io/usb/hcd/ehci/ehci_polled.c (revision 0a1044f1)

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * EHCI Host Controller Driver (EHCI)
 *
 * The EHCI driver is a software driver which interfaces to the Universal
 * Serial Bus layer (USBA) and the Host Controller (HC). The interface to
 * the Host Controller is defined by the EHCI Host Controller Interface.
 *
 * This module contains the specific EHCI code used in POLLED mode. This
 * code is in a separate file since it will never become part of the EHCI
 * driver.
 */

#include <sys/usb/usba/usbai_version.h>
#include <sys/usb/hcd/ehci/ehcid.h>
#include <sys/usb/hcd/ehci/ehci_xfer.h>
#include <sys/usb/hcd/ehci/ehci_intr.h>
#include <sys/usb/hcd/ehci/ehci_util.h>
#include <sys/usb/hcd/ehci/ehci_polled.h>

#ifndef __sparc
extern void invalidate_cache();
#endif

/*
 * Internal Function Prototypes
 */

/* Polled initialization routines */
static int	ehci_polled_init(
				usba_pipe_handle_data_t	*ph,
				ehci_state_t		*ehcip,
				usb_console_info_impl_t	*console_input_info);

/* Polled deinitialization routines */
static int	ehci_polled_fini(ehci_polled_t		*ehci_polledp);

/* Polled save state routines */
static void	ehci_polled_save_state(ehci_polled_t	*ehci_polledp);

/* Polled restore state routines */
static void	ehci_polled_restore_state(ehci_polled_t	*ehci_polledp);
static void	ehci_polled_stop_processing(
				ehci_polled_t		*ehci_polledp);
static void	ehci_polled_start_processing(
				ehci_polled_t		*ehci_polledp);

/* Polled read routines */
static int	ehci_polled_process_active_intr_qtd_list(
				ehci_polled_t		*ehci_polledp);
static int	ehci_polled_handle_normal_qtd(
				ehci_polled_t		*ehci_polledp,
				ehci_qtd_t		*qtd);
static void	ehci_polled_insert_intr_qtd(
				ehci_polled_t		*ehci_polledp,
				ehci_qtd_t		*qtd);
static void	ehci_polled_insert_bulk_qtd(
				ehci_polled_t		*ehci_polledp);
static void	ehci_polled_fill_in_qtd(
				ehci_state_t		*ehcip,
				ehci_qtd_t		*qtd,
				uint_t			qtd_ctrl,
				size_t			qtd_dma_offs,
				size_t			qtd_length,
				ehci_trans_wrapper_t	*tw);
static void	ehci_polled_insert_qtd_on_tw(
				ehci_state_t		*ehcip,
				ehci_trans_wrapper_t	*tw,
				ehci_qtd_t		*qtd);
static ehci_qtd_t *ehci_polled_create_done_qtd_list(
				ehci_polled_t		*ehci_polledp);
static void	ehci_polled_insert_qtd_into_active_intr_qtd_list(
				ehci_polled_t		*ehci_polledp,
				ehci_qtd_t		*curr_qtd);
static void	ehci_polled_remove_qtd_from_active_intr_qtd_list(
				ehci_polled_t		*ehci_polledp,
				ehci_qtd_t		*curr_qtd);
static void	ehci_polled_traverse_qtds(
				ehci_polled_t		*ehci_polledp,
				usba_pipe_handle_data_t	*ph);
static void	ehci_polled_finish_interrupt(
				ehci_state_t		*ehcip,
				uint_t			intr);
static int	ehci_polled_create_tw(
				ehci_polled_t		*ehci_polledp,
				usba_pipe_handle_data_t	*ph,
				usb_flags_t		usb_flags);
static void	ehci_polled_insert_async_qh(
				ehci_state_t		*ehcip,
				ehci_pipe_private_t	*pp);
static void	ehci_polled_remove_async_qh(
				ehci_state_t		*ehcip,
				ehci_pipe_private_t	*pp);

/*
 * POLLED entry points
 *
 * These functions are entry points into the POLLED code.
 */

/*
 * ehci_hcdi_polled_input_init:
 *
 * This is the initialization routine for handling the USB input device
 * in POLLED mode.  This routine is not called from POLLED mode, so
 * it is OK to acquire mutexes.
 */
int
ehci_hcdi_polled_input_init(
	usba_pipe_handle_data_t	*ph,
	uchar_t			**polled_buf,
	usb_console_info_impl_t	*console_input_info)
{
	ehci_polled_t		*ehci_polledp;
	ehci_state_t		*ehcip;
	int			ret;

	ehcip = ehci_obtain_state(ph->p_usba_device->usb_root_hub_dip);

	/*
	 * Grab the ehci_int_mutex so that things don't change on us
	 * if an interrupt comes in.
	 */
	mutex_enter(&ehcip->ehci_int_mutex);

	ret = ehci_polled_init(ph, ehcip, console_input_info);

	if (ret != USB_SUCCESS) {

		/* Allow interrupts to continue */
		mutex_exit(&ehcip->ehci_int_mutex);
		return (ret);
	}

	ehci_polledp = (ehci_polled_t *)console_input_info->uci_private;

	/*
	 * Mark the structure so that if we are using it, we don't free
	 * the structures if one of them is unplugged.
	 */
	ehci_polledp->ehci_polled_flags |= POLLED_INPUT_MODE;

	/* increase the counter for keyboard connected */
	ehcip->ehci_polled_kbd_count ++;

	/*
	 * This is the buffer we will copy characters into. It will be
	 * copied into at this layer, so we need to keep track of it.
	 */
	ehci_polledp->ehci_polled_buf =
	    (uchar_t *)kmem_zalloc(POLLED_RAW_BUF_SIZE, KM_SLEEP);

	*polled_buf = ehci_polledp->ehci_polled_buf;

	/*
	 * This is a software workaround to fix schizo hardware bug.
	 * Existence of "no-prom-cdma-sync"  property means consistent
	 * dma sync should not be done while in prom or polled mode.
	 */
	if (ddi_prop_exists(DDI_DEV_T_ANY, ehcip->ehci_dip,
	    DDI_PROP_NOTPROM, "no-prom-cdma-sync")) {
		ehci_polledp->ehci_polled_no_sync_flag = B_TRUE;
	}

	/* Allow interrupts to continue */
	mutex_exit(&ehcip->ehci_int_mutex);

	return (USB_SUCCESS);
}


/*
 * ehci_hcdi_polled_input_fini:
 */
int
ehci_hcdi_polled_input_fini(usb_console_info_impl_t *info)
{
	ehci_polled_t		*ehci_polledp;
	ehci_state_t		*ehcip;
	int			ret;

	ehci_polledp = (ehci_polled_t *)info->uci_private;

	ehcip = ehci_polledp->ehci_polled_ehcip;

	mutex_enter(&ehcip->ehci_int_mutex);

	/*
	 * Reset the POLLED_INPUT_MODE flag so that we can tell if
	 * this structure is in use in the ehci_polled_fini routine.
	 */
	ehci_polledp->ehci_polled_flags &= ~POLLED_INPUT_MODE;

	/* decrease the counter for keyboard disconnected */
	ehcip->ehci_polled_kbd_count --;

	/* Free the buffer that we copied data into */
	kmem_free(ehci_polledp->ehci_polled_buf, POLLED_RAW_BUF_SIZE);

	ret = ehci_polled_fini(ehci_polledp);

	mutex_exit(&ehcip->ehci_int_mutex);

	return (ret);
}


/*
 * ehci_hcdi_polled_input_enter:
 *
 * This is where we enter into POLLED mode.  This routine sets up
 * everything so that calls to	ehci_hcdi_polled_read will return
 * characters.
 */
int
ehci_hcdi_polled_input_enter(usb_console_info_impl_t *info)
{
	ehci_polled_t		*ehci_polledp;
	ehci_state_t		*ehcip;
	usba_pipe_handle_data_t	*ph;
	ehci_pipe_private_t	*pp;
	int			pipe_attr;

#ifndef lint
	_NOTE(NO_COMPETING_THREADS_NOW);
#endif
	ehci_polledp = (ehci_polled_t *)info->uci_private;
	ehcip = ehci_polledp->ehci_polled_ehcip;
	ph = ehci_polledp->ehci_polled_input_pipe_handle;
	pp = (ehci_pipe_private_t *)ph->p_hcd_private;

	pipe_attr = ph->p_ep.bmAttributes & USB_EP_ATTR_MASK;
#ifndef lint
	_NOTE(COMPETING_THREADS_NOW);
#endif

	ehci_polledp->ehci_polled_entry++;
	/*
	 * If the controller is already switched over, just return
	 */
	if (ehci_polledp->ehci_polled_entry > 1) {

		return (USB_SUCCESS);
	}

	switch (pipe_attr) {
	case USB_EP_ATTR_INTR:
		ehci_polled_save_state(ehci_polledp);
		break;
	case USB_EP_ATTR_BULK:
#ifndef lint
	_NOTE(NO_COMPETING_THREADS_NOW);
#endif
		Set_OpReg(ehci_command, (Get_OpReg(ehci_command) &
		    ~(EHCI_CMD_PERIODIC_SCHED_ENABLE |
		    EHCI_CMD_ASYNC_SCHED_ENABLE)));
		/* Wait for few milliseconds */
		drv_usecwait(EHCI_POLLED_TIMEWAIT);

		ehci_polled_insert_async_qh(ehcip, pp);

		Set_OpReg(ehci_command,
		    (Get_OpReg(ehci_command) | EHCI_CMD_ASYNC_SCHED_ENABLE));
#ifndef lint
	_NOTE(COMPETING_THREADS_NOW);
#endif
		/* Wait for few milliseconds */
		drv_usecwait(EHCI_POLLED_TIMEWAIT);
		break;
	default:
		return (USB_FAILURE);
	}

	ehci_polledp->ehci_polled_flags |= POLLED_INPUT_MODE_INUSE;

	return (USB_SUCCESS);
}


/*
 * ehci_hcdi_polled_input_exit:
 *
 * This is where we exit POLLED mode. This routine restores
 * everything that is needed to continue operation.
 */
int
ehci_hcdi_polled_input_exit(usb_console_info_impl_t *info)
{
	ehci_polled_t		*ehci_polledp;
	ehci_state_t		*ehcip;
	ehci_pipe_private_t	*pp;
	int			pipe_attr;

#ifndef lint
	_NOTE(NO_COMPETING_THREADS_NOW);
#endif
	ehci_polledp = (ehci_polled_t *)info->uci_private;
	ehcip = ehci_polledp->ehci_polled_ehcip;
	pp = (ehci_pipe_private_t *)ehci_polledp->
	    ehci_polled_input_pipe_handle->p_hcd_private;

	pipe_attr = ehci_polledp->ehci_polled_input_pipe_handle->
	    p_ep.bmAttributes & USB_EP_ATTR_MASK;
#ifndef lint
	_NOTE(COMPETING_THREADS_NOW);
#endif

	ehci_polledp->ehci_polled_entry--;

	/*
	 * If there are still outstanding "enters", just return
	 */
	if (ehci_polledp->ehci_polled_entry > 0) {

		return (USB_SUCCESS);
	}

	ehci_polledp->ehci_polled_flags &= ~POLLED_INPUT_MODE_INUSE;

	switch (pipe_attr & USB_EP_ATTR_MASK) {
	case USB_EP_ATTR_INTR:
		ehci_polled_restore_state(ehci_polledp);
		break;
	case USB_EP_ATTR_BULK:
#ifndef lint
	_NOTE(NO_COMPETING_THREADS_NOW);
#endif
		Set_OpReg(ehci_command, (Get_OpReg(ehci_command) &
		    ~(EHCI_CMD_PERIODIC_SCHED_ENABLE |
		    EHCI_CMD_ASYNC_SCHED_ENABLE)));
		/* Wait for few milliseconds */
		drv_usecwait(EHCI_POLLED_TIMEWAIT);

		ehci_polled_remove_async_qh(ehcip, pp);

		Set_OpReg(ehci_command,
		    (Get_OpReg(ehci_command) | EHCI_CMD_ASYNC_SCHED_ENABLE |
		    EHCI_CMD_ASYNC_SCHED_ENABLE));
#ifndef lint
	_NOTE(COMPETING_THREADS_NOW);
#endif
		/* Wait for few milliseconds */
		drv_usecwait(EHCI_POLLED_TIMEWAIT);
		break;
	default:
		return (USB_FAILURE);

	}

	return (USB_SUCCESS);
}


/*
 * ehci_hcdi_polled_read:
 *
 * Get a key character
 */
int
ehci_hcdi_polled_read(
	usb_console_info_impl_t	*info,
	uint_t			*num_characters)
{
	ehci_state_t		*ehcip;
	ehci_polled_t		*ehci_polledp;
	uint_t			intr;
	int			pipe_attr;

	ehci_polledp = (ehci_polled_t *)info->uci_private;

	ehcip = ehci_polledp->ehci_polled_ehcip;

#ifndef lint
	_NOTE(NO_COMPETING_THREADS_NOW);
#endif

	*num_characters = 0;

	pipe_attr = ehci_polledp->ehci_polled_input_pipe_handle->
	    p_ep.bmAttributes & USB_EP_ATTR_MASK;

	if (pipe_attr == USB_EP_ATTR_BULK) {
		ehci_polled_insert_bulk_qtd(ehci_polledp);
	}

	intr = ((Get_OpReg(ehci_status) & Get_OpReg(ehci_interrupt)) &
	    (EHCI_INTR_FRAME_LIST_ROLLOVER |
	    EHCI_INTR_USB | EHCI_INTR_USB_ERROR));

	/*
	 * Check whether any frame list rollover interrupt is pending
	 * and if it is pending, process this interrupt.
	 */
	if (intr & EHCI_INTR_FRAME_LIST_ROLLOVER) {
		/* Check any frame list rollover interrupt is pending */
		ehci_handle_frame_list_rollover(ehcip);
		ehci_polled_finish_interrupt(ehcip,
		    EHCI_INTR_FRAME_LIST_ROLLOVER);
	}

	/* Process any QTD's on the active interrupt qtd list */
	*num_characters =
	    ehci_polled_process_active_intr_qtd_list(ehci_polledp);

#ifndef lint
	_NOTE(COMPETING_THREADS_NOW);
#endif

	return (USB_SUCCESS);
}


/*
 * ehci_hcdi_polled_output_init:
 *
 * This is the initialization routine for handling the USB serial output
 * in POLLED mode.  This routine is not called from POLLED mode, so
 * it is OK to acquire mutexes.
 */
int
ehci_hcdi_polled_output_init(
	usba_pipe_handle_data_t	*ph,
	usb_console_info_impl_t	*console_output_info)
{
	ehci_polled_t		*ehci_polledp;
	ehci_state_t		*ehcip;
	ehci_pipe_private_t	*pp;
	int			ret;

	ehcip = ehci_obtain_state(ph->p_usba_device->usb_root_hub_dip);

	/*
	 * Grab the ehci_int_mutex so that things don't change on us
	 * if an interrupt comes in.
	 */
	mutex_enter(&ehcip->ehci_int_mutex);

	ret = ehci_polled_init(ph, ehcip, console_output_info);

	if (ret != USB_SUCCESS) {

		/* Allow interrupts to continue */
		mutex_exit(&ehcip->ehci_int_mutex);

		return (ret);
	}

	ehci_polledp = (ehci_polled_t *)console_output_info->uci_private;
	/*
	 * Mark the structure so that if we are using it, we don't free
	 * the structures if one of them is unplugged.
	 */
	ehci_polledp->ehci_polled_flags |= POLLED_OUTPUT_MODE;

	/*
	 * Insert the Endpoint Descriptor to appropriate endpoint list.
	 */
	pp = (ehci_pipe_private_t *)ehci_polledp->
	    ehci_polled_input_pipe_handle->p_hcd_private;
	ehci_polled_insert_async_qh(ehcip, pp);

	/*
	 * This is a software workaround to fix schizo hardware bug.
	 * Existence of "no-prom-cdma-sync"  property means consistent
	 * dma sync should not be done while in prom or polled mode.
	 */
	if (ddi_prop_exists(DDI_DEV_T_ANY, ehcip->ehci_dip,
	    DDI_PROP_NOTPROM, "no-prom-cdma-sync")) {
		ehci_polledp->ehci_polled_no_sync_flag = B_TRUE;
	}

	/* Allow interrupts to continue */
	mutex_exit(&ehcip->ehci_int_mutex);

	return (USB_SUCCESS);
}


/*
 * ehci_hcdi_polled_output_fini:
 */
int
ehci_hcdi_polled_output_fini(usb_console_info_impl_t *info)
{
	ehci_polled_t		*ehci_polledp;
	ehci_state_t		*ehcip;
	ehci_pipe_private_t	*pp;
	int			ret;

	ehci_polledp = (ehci_polled_t *)info->uci_private;

	ehcip = ehci_polledp->ehci_polled_ehcip;

	mutex_enter(&ehcip->ehci_int_mutex);

	/* Remove the Endpoint Descriptor. */
	pp = (ehci_pipe_private_t *)ehci_polledp->
	    ehci_polled_input_pipe_handle->p_hcd_private;
	ehci_polled_remove_async_qh(ehcip, pp);

	/*
	 * Reset the POLLED_INPUT_MODE flag so that we can tell if
	 * this structure is in use in the ehci_polled_fini routine.
	 */
	ehci_polledp->ehci_polled_flags &= ~POLLED_OUTPUT_MODE;

	ret = ehci_polled_fini(ehci_polledp);

	info->uci_private = NULL;

	mutex_exit(&ehcip->ehci_int_mutex);

	return (ret);
}


/*
 * ehci_hcdi_polled_output_enter:
 *
 * everything is done in input enter
 */
/*ARGSUSED*/
int
ehci_hcdi_polled_output_enter(usb_console_info_impl_t *info)
{
	return (USB_SUCCESS);
}


/*
 * ehci_hcdi_polled_output_exit:
 *
 * everything is done in input exit
 */
/*ARGSUSED*/
int
ehci_hcdi_polled_output_exit(usb_console_info_impl_t *info)
{
	return (USB_SUCCESS);
}


/*
 * ehci_hcdi_polled_write:
 *	Put a key character.
 */
int
ehci_hcdi_polled_write(usb_console_info_impl_t *info, uchar_t *buf,
    uint_t num_characters, uint_t *num_characters_written)
{
	ehci_state_t		*ehcip;
	ehci_polled_t		*ehci_polledp;
	ehci_trans_wrapper_t	*tw;
	ehci_pipe_private_t	*pp;
	usba_pipe_handle_data_t	*ph;
	int			intr;

#ifndef lint
	_NOTE(NO_COMPETING_THREADS_NOW);
#endif
	ehci_polledp = (ehci_polled_t *)info->uci_private;
	ehcip = ehci_polledp->ehci_polled_ehcip;
	ph = ehci_polledp->ehci_polled_input_pipe_handle;
	pp = (ehci_pipe_private_t *)ph->p_hcd_private;

	/* Disable all list processing */
	Set_OpReg(ehci_command, Get_OpReg(ehci_command) &
	    ~(EHCI_CMD_ASYNC_SCHED_ENABLE |
	    EHCI_CMD_PERIODIC_SCHED_ENABLE));

	/* Wait for few milliseconds */
	drv_usecwait(EHCI_POLLED_TIMEWAIT);

	tw = pp->pp_tw_head;
	ASSERT(tw != NULL);

	/* copy transmit buffer */
	if (num_characters > POLLED_RAW_BUF_SIZE) {
		cmn_err(CE_NOTE, "polled write size %d bigger than %d",
		    num_characters, POLLED_RAW_BUF_SIZE);
		num_characters = POLLED_RAW_BUF_SIZE;
	}
	tw->tw_length = num_characters;
	ddi_rep_put8(tw->tw_accesshandle,
	    buf, (uint8_t *)tw->tw_buf,
	    tw->tw_length, DDI_DEV_AUTOINCR);
	Sync_IO_Buffer_for_device(tw->tw_dmahandle, tw->tw_length);

	ehci_polled_insert_bulk_qtd(ehci_polledp);

	/* Enable async list processing */
	Set_OpReg(ehci_command, (Get_OpReg(ehci_command) |
	    EHCI_CMD_ASYNC_SCHED_ENABLE));

	/* Wait for few milliseconds */
	drv_usecwait(EHCI_POLLED_TIMEWAIT);

	while (!((Get_OpReg(ehci_status)) & (EHCI_INTR_USB
	    |EHCI_INTR_FRAME_LIST_ROLLOVER | EHCI_INTR_USB_ERROR))) {
#ifndef __sparc
		invalidate_cache();
#else
		;
#endif
	}

	intr = (Get_OpReg(ehci_status)) &
	    (EHCI_INTR_FRAME_LIST_ROLLOVER |
	    EHCI_INTR_USB | EHCI_INTR_USB_ERROR);

	/*
	 * Check whether any frame list rollover interrupt is pending
	 * and if it is pending, process this interrupt.
	 */
	if (intr & EHCI_INTR_FRAME_LIST_ROLLOVER) {

		ehci_handle_frame_list_rollover(ehcip);
		ehci_polled_finish_interrupt(ehcip,
		    EHCI_INTR_FRAME_LIST_ROLLOVER);
	}

	/* Check for any USB transaction completion notification */
	if (intr & (EHCI_INTR_USB | EHCI_INTR_USB_ERROR)) {

		(void) ehci_polled_process_active_intr_qtd_list(ehci_polledp);

		/* Acknowledge the USB and USB error interrupt */
		ehci_polled_finish_interrupt(ehcip,
		    intr & (EHCI_INTR_USB | EHCI_INTR_USB_ERROR));

	}

	*num_characters_written = num_characters;

#ifndef lint
	_NOTE(COMPETING_THREADS_NOW);
#endif

	return (USB_SUCCESS);
}


/*
 * Internal Functions
 */

/*
 * Polled initialization routines
 */


/*
 * ehci_polled_init:
 *
 * Initialize generic information Uthat is needed to provide USB/POLLED
 * support.
 */
static int
ehci_polled_init(
	usba_pipe_handle_data_t	*ph,
	ehci_state_t		*ehcip,
	usb_console_info_impl_t	*console_info)
{
	ehci_polled_t		*ehci_polledp;
	ehci_pipe_private_t	*pp;
	ehci_qtd_t		*qtd;
	int			pipe_attr;

	ASSERT(mutex_owned(&ehcip->ehci_int_mutex));

	/*
	 * We have already initialized this structure. If the structure
	 * has already been initialized, then we don't need to redo it.
	 */
	if (console_info->uci_private) {

		return (USB_SUCCESS);
	}

	/* Allocate and intitialize a state structure */
	ehci_polledp = (ehci_polled_t *)
	    kmem_zalloc(sizeof (ehci_polled_t), KM_SLEEP);

	console_info->uci_private = (usb_console_info_private_t)ehci_polledp;

	/*
	 * Store away the ehcip so that we can get to it when we are in
	 * POLLED mode. We don't want to have to call ehci_obtain_state
	 * every time we want to access this structure.
	 */
	ehci_polledp->ehci_polled_ehcip = ehcip;
	/*
	 * Save usb device and endpoint number information from the usb
	 * pipe handle.
	 */
	mutex_enter(&ph->p_mutex);
	ehci_polledp->ehci_polled_usb_dev = ph->p_usba_device;
	ehci_polledp->ehci_polled_ep_addr = ph->p_ep.bEndpointAddress;
	mutex_exit(&ph->p_mutex);

	/*
	 * Allocate memory to make duplicate of original usb pipe handle.
	 */
	ehci_polledp->ehci_polled_input_pipe_handle =
	    kmem_zalloc(sizeof (usba_pipe_handle_data_t), KM_SLEEP);

	/*
	 * Copy the USB handle into the new pipe handle. Also
	 * create new lock for the new pipe handle.
	 */
	bcopy((void *)ph,
	    (void *)ehci_polledp->ehci_polled_input_pipe_handle,
	    sizeof (usba_pipe_handle_data_t));

	/*
	 * uint64_t typecast to make sure amd64 can compile
	 */
	mutex_init(&ehci_polledp->ehci_polled_input_pipe_handle->p_mutex,
	    NULL, MUTEX_DRIVER, DDI_INTR_PRI(ehcip->ehci_intr_pri));

	/*
	 * Create a new ehci pipe private structure
	 */
	pp = (ehci_pipe_private_t *)
	    kmem_zalloc(sizeof (ehci_pipe_private_t), KM_SLEEP);

	/*
	 * Store the pointer in the pipe handle. This structure was also
	 * just allocated.
	 */
	mutex_enter(&ehci_polledp->ehci_polled_input_pipe_handle->p_mutex);

	ehci_polledp->ehci_polled_input_pipe_handle->
	    p_hcd_private = (usb_opaque_t)pp;

	mutex_exit(&ehci_polledp->ehci_polled_input_pipe_handle->p_mutex);

	/*
	 * Store a pointer to the pipe handle. This structure was  just
	 * allocated and it is not in use yet.	The locking is there to
	 * satisfy warlock.
	 */
	mutex_enter(&ph->p_mutex);

	bcopy(&ph->p_policy, &pp->pp_policy, sizeof (usb_pipe_policy_t));

	mutex_exit(&ph->p_mutex);

	pp->pp_pipe_handle = ehci_polledp->ehci_polled_input_pipe_handle;

	/*
	 * Allocate a dummy for the interrupt table. This dummy will be
	 * put into the action when we	switch interrupt  tables during
	 * ehci_hcdi_polled_enter. Dummy is placed on the unused lattice
	 * entries. When the QH is allocated we will replace dummy QH by
	 * valid interrupt QH in one or more locations in the interrupt
	 * lattice depending on the requested polling interval. Also we
	 * will hang a dummy QTD to the QH & dummy QTD is used to indicate
	 * the end of the QTD chain.
	 */
	ehci_polledp->ehci_polled_dummy_qh =
	    ehci_alloc_qh(ehcip, NULL, EHCI_POLLED_MODE_FLAG);

	if (ehci_polledp->ehci_polled_dummy_qh == NULL) {

		return (USB_NO_RESOURCES);
	}

	/*
	 * Allocate the endpoint. This QH will be inserted in
	 * to the lattice chain for the device. This endpoint
	 * will have the QTDs hanging off of it for the processing.
	 */
	ehci_polledp->ehci_polled_qh = ehci_alloc_qh(
	    ehcip, ph, EHCI_POLLED_MODE_FLAG);

	if (ehci_polledp->ehci_polled_qh == NULL) {

		return (USB_NO_RESOURCES);
	}

	/* Set the state of pipe as idle */
	pp->pp_state = EHCI_PIPE_STATE_IDLE;

	/* Set polled mode flag */
	pp->pp_flag = EHCI_POLLED_MODE_FLAG;

	/* Insert the endpoint onto the pipe handle */
	pp->pp_qh = ehci_polledp->ehci_polled_qh;

	pipe_attr = ph->p_ep.bmAttributes & USB_EP_ATTR_MASK;

	switch (pipe_attr) {
	case USB_EP_ATTR_INTR:
		/*
		 * Set soft interrupt handler flag in the normal mode usb
		 * pipe handle.
		 */
		mutex_enter(&ph->p_mutex);
		ph->p_spec_flag |= USBA_PH_FLAG_USE_SOFT_INTR;
		mutex_exit(&ph->p_mutex);

		/*
		 * Insert a Interrupt polling request onto the endpoint.
		 *
		 * There will now be two QTDs on the QH, one is the dummy QTD
		 * that was allocated above in the  ehci_alloc_qh and this
		 * new one.
		 */
		if ((ehci_start_periodic_pipe_polling(ehcip,
		    ehci_polledp->ehci_polled_input_pipe_handle,
		    NULL, USB_FLAGS_SLEEP)) != USB_SUCCESS) {

			return (USB_NO_RESOURCES);
		}
		/* Get the given new interrupt qtd */
		qtd = (ehci_qtd_t *)(ehci_qtd_iommu_to_cpu(ehcip,
		    (Get_QH(pp->pp_qh->qh_next_qtd) & EHCI_QH_NEXT_QTD_PTR)));

		/* Insert this qtd into active interrupt QTD list */
		ehci_polled_insert_qtd_into_active_intr_qtd_list(ehci_polledp,
		    qtd);
		break;
	case USB_EP_ATTR_BULK:
		if ((ehci_polled_create_tw(ehci_polledp,
		    ehci_polledp->ehci_polled_input_pipe_handle,
		    USB_FLAGS_SLEEP)) != USB_SUCCESS) {

			return (USB_NO_RESOURCES);
		}
		break;
	default:
		return (USB_FAILURE);
	}

	return (USB_SUCCESS);
}


/*
 * Polled deinitialization routines
 */


/*
 * ehci_polled_fini:
 */
static int
ehci_polled_fini(ehci_polled_t	*ehci_polledp)
{
	ehci_state_t		*ehcip = ehci_polledp->ehci_polled_ehcip;
	ehci_pipe_private_t	*pp;

	ASSERT(mutex_owned(&ehcip->ehci_int_mutex));

	/* If the structure is already in use, then don't free it */
	if (ehci_polledp->ehci_polled_flags & POLLED_INPUT_MODE) {

		return (USB_SUCCESS);
	}

	pp = (ehci_pipe_private_t *)
	    ehci_polledp->ehci_polled_input_pipe_handle->p_hcd_private;

	/* Deallocate all the pre-allocated interrupt requests */
	ehci_handle_outstanding_requests(ehcip, pp);

	/*
	 * Traverse the list of QTD's on this endpoint and these QTD's
	 * have outstanding transfer requests. Since list processing
	 * is stopped, these QTDs can be deallocated.
	 */
	ehci_polled_traverse_qtds(ehci_polledp, pp->pp_pipe_handle);

	/* Free DMA resources */
	ehci_free_dma_resources(ehcip, pp->pp_pipe_handle);

	/*
	 * Deallocate the endpoint descriptors that we allocated
	 * with ehci_alloc_qh.
	 */
	if (ehci_polledp->ehci_polled_dummy_qh) {
		ehci_deallocate_qh(ehcip, ehci_polledp->ehci_polled_dummy_qh);
	}

	if (ehci_polledp->ehci_polled_qh) {
		ehci_deallocate_qh(ehcip, ehci_polledp->ehci_polled_qh);
	}

	mutex_destroy(&ehci_polledp->ehci_polled_input_pipe_handle->p_mutex);

	/*
	 * Destroy everything about the pipe that we allocated in
	 * ehci_polled_duplicate_pipe_handle
	 */
	kmem_free(pp, sizeof (ehci_pipe_private_t));

	kmem_free(ehci_polledp->ehci_polled_input_pipe_handle,
	    sizeof (usba_pipe_handle_data_t));

	/*
	 * We use this field to determine if a QTD is for input or not,
	 * so NULL the pointer so we don't check deallocated data.
	 */
	ehci_polledp->ehci_polled_input_pipe_handle = NULL;

	/*
	 * Finally, free off the structure that we use to keep track
	 * of all this.
	 */
	kmem_free(ehci_polledp, sizeof (ehci_polled_t));

	return (USB_SUCCESS);
}


/*
 * Polled save state routines
 */


/*
 * ehci_polled_save_state:
 */
static void
ehci_polled_save_state(ehci_polled_t	*ehci_polledp)
{
	int				i;
	ehci_state_t			*ehcip;
	uint_t				polled_toggle;
	uint_t				real_toggle;
	ehci_pipe_private_t		*pp = NULL; /* Normal mode Pipe */
	ehci_pipe_private_t		*polled_pp; /* Polled mode Pipe */
	usba_pipe_handle_data_t		*ph;
	uint8_t				ep_addr;
	ehci_regs_t			*ehci_polled_regsp;
	ehci_qh_t			*qh;

#ifndef lint
	_NOTE(NO_COMPETING_THREADS_NOW);
#endif

	/*
	 * If either of these two flags are set, then we have already
	 * saved off the state information and setup the controller.
	 */
	if (ehci_polledp->ehci_polled_flags & POLLED_INPUT_MODE_INUSE) {
#ifndef lint
		_NOTE(COMPETING_THREADS_NOW);
#endif
		return;
	}

	ehcip = ehci_polledp->ehci_polled_ehcip;

	/*
	 * Check if the number of keyboard reach the max number we can
	 * support in polled mode
	 */
	if (++ ehcip->ehci_polled_enter_count > MAX_NUM_FOR_KEYBOARD) {
#ifndef lint
		_NOTE(COMPETING_THREADS_NOW);
#endif
		return;
	}
	ehci_polled_regsp = &ehcip->ehci_polled_save_regs;

	/* Get the endpoint addr. */
	ep_addr = ehci_polledp->ehci_polled_ep_addr;

	/* Get the normal mode usb pipe handle */
	ph = usba_hcdi_get_ph_data(ehci_polledp->ehci_polled_usb_dev, ep_addr);

	/*
	 * The first enter keyboard entry should save info of the normal mode,
	 * disable all list processing and interrupt, initialize the
	 * frame list table with dummy QHs.
	 */
	if (ehcip->ehci_polled_enter_count == 1) {
		/*
		 * Save the current normal mode ehci registers	and later this
		 * saved register copy is used to replace some of required ehci
		 * registers before switching from polled mode to normal mode.
		 */

		bzero((void *)ehci_polled_regsp, sizeof (ehci_regs_t));

		/* Save current ehci registers */
		ehci_polled_regsp->ehci_command = Get_OpReg(ehci_command);
		ehci_polled_regsp->ehci_interrupt = Get_OpReg(ehci_interrupt);
		ehci_polled_regsp->ehci_ctrl_segment =
		    Get_OpReg(ehci_ctrl_segment);
		ehci_polled_regsp->
		    ehci_async_list_addr = Get_OpReg(ehci_async_list_addr);
		ehci_polled_regsp->ehci_config_flag =
		    Get_OpReg(ehci_config_flag);
		ehci_polled_regsp->ehci_periodic_list_base =
		    Get_OpReg(ehci_periodic_list_base);

		/* Disable all list processing and interrupts */
		Set_OpReg(ehci_command, Get_OpReg(ehci_command) &
		    ~(EHCI_CMD_ASYNC_SCHED_ENABLE |
		    EHCI_CMD_PERIODIC_SCHED_ENABLE));

		/* Wait for few milliseconds */
		drv_usecwait(EHCI_POLLED_TIMEWAIT);

		/* Save any unprocessed normal mode ehci interrupts */
		ehcip->ehci_missed_intr_sts = EHCI_INTR_USB;

		/*
		 * Save the current interrupt lattice and  replace this lattice
		 * with an lattice used in POLLED mode. We will restore lattice
		 * back when we exit from the POLLED mode.
		 */
		for (i = 0; i < EHCI_NUM_PERIODIC_FRAME_LISTS; i++) {
			ehcip->ehci_polled_frame_list_table[i] =
			    (ehci_qh_t *)(uintptr_t)Get_PFLT(ehcip->
			    ehci_periodic_frame_list_tablep->
			    ehci_periodic_frame_list_table[i]);
		}

		/*
		 * Fill in the lattice with dummy QHs. These QHs are used so the
		 * controller can tell that it is at the end of the QH list.
		 */
		for (i = 0; i < EHCI_NUM_PERIODIC_FRAME_LISTS; i++) {
			Set_PFLT(ehcip->ehci_periodic_frame_list_tablep->
			    ehci_periodic_frame_list_table[i],
			    ehci_qh_cpu_to_iommu(ehcip,
			    ehci_polledp->ehci_polled_dummy_qh) |
			    (EHCI_QH_LINK_REF_QH | EHCI_QH_LINK_PTR_VALID));
		}

	}

	/* Get the polled mode ehci pipe private structure */
	polled_pp = (ehci_pipe_private_t *)
	    ehci_polledp->ehci_polled_input_pipe_handle->p_hcd_private;

	/*
	 * Before replacing the lattice, adjust the data togggle on the
	 * on the ehci's interrupt ed
	 */
	polled_toggle = (Get_QH(polled_pp->pp_qh->qh_status) &
	    EHCI_QH_STS_DATA_TOGGLE) ? DATA1:DATA0;

	/*
	 * If normal mode interrupt pipe endpoint is active, get the data
	 * toggle from the this interrupt endpoint through the corresponding
	 * interrupt pipe handle. Else get the data toggle information from
	 * the usb device structure and this information is saved during the
	 * normal mode interrupt pipe close. Use this data toggle information
	 * to fix the data toggle of polled mode interrupt endpoint.
	 */
	if (ph) {
		/* Get the normal mode ehci pipe private structure */
		pp = (ehci_pipe_private_t *)ph->p_hcd_private;

		real_toggle = (Get_QH(pp->pp_qh->qh_status) &
		    EHCI_QH_STS_DATA_TOGGLE) ? DATA1:DATA0;
	} else {
		real_toggle = usba_hcdi_get_data_toggle(
		    ehci_polledp->ehci_polled_usb_dev, ep_addr);
	}

	if (polled_toggle != real_toggle) {
		if (real_toggle == DATA0) {
			Set_QH(polled_pp->pp_qh->qh_status,
			    Get_QH(polled_pp->pp_qh->qh_status) &
			    ~EHCI_QH_STS_DATA_TOGGLE);
		} else {
			Set_QH(polled_pp->pp_qh->qh_status,
			    Get_QH(polled_pp->pp_qh->qh_status) |
			    EHCI_QH_STS_DATA_TOGGLE);
		}
	}

	/*
	 * Check whether Halt bit is set in the QH and if so  clear the
	 * halt bit.
	 */
	if (polled_pp->pp_qh->qh_status & EHCI_QH_STS_HALTED) {

		/* Clear the halt bit */
		Set_QH(polled_pp->pp_qh->qh_status,
		    (Get_QH(polled_pp->pp_qh->qh_status) &
		    ~EHCI_QH_STS_HALTED));
	}

	/*
	 * Initialize the qh overlay area
	 */
	qh = ehci_polledp->ehci_polled_qh;
	for (i = 0; i < 5; i++) {
		Set_QH(qh->qh_buf[i], 0);
		Set_QH(qh->qh_buf_high[i], 0);
	}
	Set_QH(qh->qh_next_qtd, ehci_qtd_cpu_to_iommu(ehcip,
	    ehci_polledp->ehci_polled_active_intr_qtd_list));

	/*
	 * Now, add the endpoint to the lattice that we will  hang  our
	 * QTD's off of.  We need to poll this device at  every 8 ms and
	 * hence add this QH needs 4 entries in interrupt lattice.
	 */
	for (i = ehcip->ehci_polled_enter_count - 1;
	    i < EHCI_NUM_PERIODIC_FRAME_LISTS;
	    i = i + LS_MIN_POLL_INTERVAL) {
		Set_PFLT(ehcip->ehci_periodic_frame_list_tablep->
		    ehci_periodic_frame_list_table[i],
		    ehci_qh_cpu_to_iommu(ehcip,
		    ehci_polledp->ehci_polled_qh) | EHCI_QH_LINK_REF_QH);
	}
	/* The first enter keyboard entry enable interrupts and periodic list */
	if (ehcip->ehci_polled_enter_count == 1) {
		/* Enable USB and Frame list rollover interrupts */
		Set_OpReg(ehci_interrupt, (EHCI_INTR_USB |
		    EHCI_INTR_USB_ERROR | EHCI_INTR_FRAME_LIST_ROLLOVER));