1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #pragma ident	"%Z%%M%	%I%	%E% SMI"
27 
28 /*
29  * EHCI Host Controller Driver (EHCI)
30  *
31  * The EHCI driver is a software driver which interfaces to the Universal
32  * Serial Bus layer (USBA) and the Host Controller (HC). The interface to
33  * the Host Controller is defined by the EHCI Host Controller Interface.
34  *
35  * This module contains the EHCI driver interrupt code, which handles all
36  * Checking of status of USB transfers, error recovery and callbacks.
37  */
38 
39 #include <sys/usb/hcd/ehci/ehcid.h>
40 #include <sys/usb/hcd/ehci/ehci_xfer.h>
41 #include <sys/usb/hcd/ehci/ehci_util.h>
42 
43 /*
44  * EHCI Interrupt Handling functions.
45  */
46 void		ehci_handle_ue(ehci_state_t		*ehcip);
47 void		ehci_handle_frame_list_rollover(
48 				ehci_state_t		*ehcip);
49 void		ehci_handle_endpoint_reclaimation(
50 				ehci_state_t		*ehcip);
51 void		ehci_traverse_active_qtd_list(
52 				ehci_state_t		*ehcip);
53 static ehci_qtd_t *ehci_create_done_qtd_list(
54 				ehci_state_t		*ehcip);
55 static usb_cr_t ehci_parse_error(
56 				ehci_state_t		*ehcip,
57 				ehci_qtd_t		*qtd);
58 usb_cr_t	ehci_check_for_error(
59 				ehci_state_t		*ehcip,
60 				ehci_pipe_private_t	*pp,
61 				ehci_trans_wrapper_t	*tw,
62 				ehci_qtd_t		*qtd,
63 				uint_t			ctrl);
64 static usb_cr_t	ehci_check_for_short_xfer(
65 				ehci_state_t		*ehcip,
66 				ehci_pipe_private_t	*pp,
67 				ehci_trans_wrapper_t	*tw,
68 				ehci_qtd_t		*qtd);
69 void		ehci_handle_error(
70 				ehci_state_t		*ehcip,
71 				ehci_qtd_t		*qtd,
72 				usb_cr_t		error);
73 static void	ehci_cleanup_data_underrun(
74 				ehci_state_t		*ehcip,
75 				ehci_trans_wrapper_t	*tw,
76 				ehci_qtd_t		*qtd);
77 static void	ehci_handle_normal_qtd(
78 				ehci_state_t		*ehcip,
79 				ehci_qtd_t		*qtd,
80 				ehci_trans_wrapper_t	*tw);
81 void		ehci_handle_ctrl_qtd(
82 				ehci_state_t		*ehcip,
83 				ehci_pipe_private_t	*pp,
84 				ehci_trans_wrapper_t	*tw,
85 				ehci_qtd_t		*qtd,
86 				void			*);
87 void		ehci_handle_bulk_qtd(
88 				ehci_state_t		*ehcip,
89 				ehci_pipe_private_t	*pp,
90 				ehci_trans_wrapper_t	*tw,
91 				ehci_qtd_t		*qtd,
92 				void			*);
93 void		ehci_handle_intr_qtd(
94 				ehci_state_t		*ehcip,
95 				ehci_pipe_private_t	*pp,
96 				ehci_trans_wrapper_t	*tw,
97 				ehci_qtd_t		*qtd,
98 				void			*);
99 static void	ehci_handle_one_xfer_completion(
100 				ehci_state_t		*ehcip,
101 				ehci_trans_wrapper_t	*tw);
102 static void	ehci_sendup_qtd_message(
103 				ehci_state_t		*ehcip,
104 				ehci_pipe_private_t	*pp,
105 				ehci_trans_wrapper_t	*tw,
106 				ehci_qtd_t		*qtd,
107 				usb_cr_t		error);
108 
109 
110 /*
111  * Interrupt Handling functions
112  */
113 
114 /*
115  * ehci_handle_ue:
116  *
117  * Handling of Unrecoverable Error interrupt (UE).
118  */
119 void
120 ehci_handle_ue(ehci_state_t	*ehcip)
121 {
122 	usb_frame_number_t	before_frame_number, after_frame_number;
123 
124 	ASSERT(mutex_owned(&ehcip->ehci_int_mutex));
125 
126 	USB_DPRINTF_L3(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
127 	    "ehci_handle_ue: Handling of UE interrupt");
128 
129 	/*
130 	 * First check whether current UE error occurred due to USB or
131 	 * due to some other subsystem. This can be verified by reading
132 	 * usb frame numbers before & after a delay of few milliseconds.
133 	 * If usb frame number read after delay is greater than the one
134 	 * read before delay, then, USB subsystem is fine. In this case,
135 	 * disable UE error interrupt and return without shutdowning the
136 	 * USB subsystem.
137 	 *
138 	 * Otherwise, if usb frame number read after delay is less than
139 	 * or equal to one read before the delay, then, current UE error
140 	 * occurred from USB subsystem. In this case,go ahead with actual
141 	 * UE error recovery procedure.
142 	 *
143 	 * Get the current usb frame number before waiting for few
144 	 * milliseconds.
145 	 */
146 	before_frame_number = ehci_get_current_frame_number(ehcip);
147 
148 	/* Wait for few milliseconds */
149 	drv_usecwait(EHCI_TIMEWAIT);
150 
151 	/*
152 	 * Get the current usb frame number after waiting for
153 	 * milliseconds.
154 	 */
155 	after_frame_number = ehci_get_current_frame_number(ehcip);
156 
157 	USB_DPRINTF_L3(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
158 	    "ehci_handle_ue: Before Frame Number 0x%llx "
159 	    "After Frame Number 0x%llx", before_frame_number,
160 	    after_frame_number);
161 
162 	if (after_frame_number > before_frame_number) {
163 
164 		/* Disable UE interrupt */
165 		Set_OpReg(ehci_interrupt, (Get_OpReg(ehci_interrupt) &
166 		    ~EHCI_INTR_HOST_SYSTEM_ERROR));
167 
168 		return;
169 	}
170 
171 	/*
172 	 * This UE is due to USB hardware error. Reset ehci controller
173 	 * and reprogram to bring it back to functional state.
174 	 */
175 	if ((ehci_do_soft_reset(ehcip)) != USB_SUCCESS) {
176 
177 		USB_DPRINTF_L0(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
178 		    "Unrecoverable USB Hardware Error");
179 
180 		/* Disable UE interrupt */
181 		Set_OpReg(ehci_interrupt, (Get_OpReg(ehci_interrupt) &
182 		    ~EHCI_INTR_HOST_SYSTEM_ERROR));
183 
184 		/* Route all Root hub ports to Classic host controller */
185 		Set_OpReg(ehci_config_flag, EHCI_CONFIG_FLAG_CLASSIC);
186 
187 		/* Set host controller soft state to error */
188 		ehcip->ehci_hc_soft_state = EHCI_CTLR_ERROR_STATE;
189 	}
190 }
191 
192 
193 /*
194  * ehci_handle_frame_list_rollover:
195  *
196  * Update software based usb frame number part on every frame number
197  * overflow interrupt.
198  *
199  * Refer ehci spec 1.0, section 2.3.2, page 21 for more details.
200  *
201  * NOTE: This function is also called from POLLED MODE.
202  */
203 void
204 ehci_handle_frame_list_rollover(ehci_state_t *ehcip)
205 {
206 	ehcip->ehci_fno += (0x4000 -
207 	    (((Get_OpReg(ehci_frame_index) & 0x3FFF) ^
208 	    ehcip->ehci_fno) & 0x2000));
209 
210 	USB_DPRINTF_L4(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
211 	    "ehci_handle_frame_list_rollover:"
212 	    "Frame Number Higher Part 0x%llx\n", ehcip->ehci_fno);
213 }
214 
215 
216 /*
217  * ehci_handle_endpoint_reclamation:
218  *
219  * Reclamation of Host Controller (HC) Endpoint Descriptors (QH).
220  */
221 void
222 ehci_handle_endpoint_reclaimation(ehci_state_t	*ehcip)
223 {
224 	usb_frame_number_t	current_frame_number;
225 	usb_frame_number_t	endpoint_frame_number;
226 	ehci_qh_t		*reclaim_qh;
227 
228 	USB_DPRINTF_L4(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
229 	    "ehci_handle_endpoint_reclamation:");
230 
231 	ASSERT(mutex_owned(&ehcip->ehci_int_mutex));
232 
233 	current_frame_number = ehci_get_current_frame_number(ehcip);
234 
235 	/*
236 	 * Deallocate all Endpoint Descriptors (QH) which are on the
237 	 * reclamation list. These QH's are already removed from the
238 	 * interrupt lattice tree.
239 	 */
240 	while (ehcip->ehci_reclaim_list) {
241 
242 		reclaim_qh = ehcip->ehci_reclaim_list;
243 
244 		endpoint_frame_number = (usb_frame_number_t)(uintptr_t)
245 		    (EHCI_LOOKUP_ID(Get_QH(reclaim_qh->qh_reclaim_frame)));
246 
247 		USB_DPRINTF_L4(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
248 		    "ehci_handle_endpoint_reclamation:"
249 		    "current frame number 0x%llx endpoint frame number 0x%llx",
250 		    current_frame_number, endpoint_frame_number);
251 
252 		/*
253 		 * Deallocate current endpoint only if endpoint's usb frame
254 		 * number is less than or equal to current usb frame number.
255 		 *
256 		 * If endpoint's usb frame number is greater than the current
257 		 * usb frame number, ignore rest of the endpoints in the list
258 		 * since rest of the endpoints are inserted into the reclaim
259 		 * list later than the current reclaim endpoint.
260 		 */
261 		if (endpoint_frame_number > current_frame_number) {
262 			break;
263 		}
264 
265 		/* Get the next endpoint from the rec. list */
266 		ehcip->ehci_reclaim_list = ehci_qh_iommu_to_cpu(
267 		    ehcip, Get_QH(reclaim_qh->qh_reclaim_next));
268 
269 		/* Free 32bit ID */
270 		EHCI_FREE_ID((uint32_t)Get_QH(reclaim_qh->qh_reclaim_frame));
271 
272 		/* Deallocate the endpoint */
273 		ehci_deallocate_qh(ehcip, reclaim_qh);
274 	}
275 }
276 
277 
278 /*
279  * ehci_traverse_active_qtd_list:
280  */
281 void
282 ehci_traverse_active_qtd_list(
283 	ehci_state_t		*ehcip)
284 {
285 	uint_t			state;		/* QTD state */
286 	ehci_qtd_t		*curr_qtd = NULL; /* QTD pointers */
287 	ehci_qtd_t		*next_qtd = NULL; /* QTD pointers */
288 	usb_cr_t		error;		/* Error from QTD */
289 	ehci_trans_wrapper_t	*tw = NULL;	/* Transfer wrapper */
290 	ehci_pipe_private_t	*pp = NULL;	/* Pipe private field */
291 
292 	USB_DPRINTF_L4(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
293 	    "ehci_traverse_active_qtd_list:");
294 
295 	ASSERT(mutex_owned(&ehcip->ehci_int_mutex));
296 
297 	/* Sync QH and QTD pool */
298 	Sync_QH_QTD_Pool(ehcip);
299 
300 	/* Create done qtd list */
301 	curr_qtd = ehci_create_done_qtd_list(ehcip);
302 
303 	/* Traverse the list of transfer descriptors */
304 	while (curr_qtd) {
305 		/* Get next qtd from the active qtd list */
306 		next_qtd = ehci_qtd_iommu_to_cpu(ehcip,
307 		    Get_QTD(curr_qtd->qtd_active_qtd_next));
308 
309 		/* Check for QTD state */
310 		state = Get_QTD(curr_qtd->qtd_state);
311 
312 		USB_DPRINTF_L4(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
313 		    "ehci_traverse_active_qtd_list:\n\t"
314 		    "curr_qtd = 0x%p state = 0x%x", (void *)curr_qtd, state);
315 
316 		/* Obtain the  transfer wrapper  for this QTD */
317 		tw = (ehci_trans_wrapper_t *)EHCI_LOOKUP_ID(
318 		    (uint32_t)Get_QTD(curr_qtd->qtd_trans_wrapper));
319 
320 		ASSERT(tw != NULL);
321 
322 		pp = tw->tw_pipe_private;
323 
324 		USB_DPRINTF_L3(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
325 		    "ehci_traverse_active_qtd_list: "
326 		    "PP = 0x%p TW = 0x%p", pp, tw);
327 
328 		/*
329 		 * A QTD that is marked as RECLAIM has already been
330 		 * processed by QTD timeout handler & client driver
331 		 * has been informed through exception callback.
332 		 */
333 		if (state != EHCI_QTD_RECLAIM) {
334 			/* Look at the error status */
335 			error = ehci_parse_error(ehcip, curr_qtd);
336 
337 			if (error == USB_CR_OK) {
338 				ehci_handle_normal_qtd(ehcip, curr_qtd, tw);
339 			} else {
340 				/* handle the error condition */
341 				ehci_handle_error(ehcip, curr_qtd, error);
342 			}
343 		} else {
344 			USB_DPRINTF_L3(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
345 			    "ehci_traverse_active_qtd_list: "
346 			    "QTD State = %d", state);
347 		}
348 
349 		/* Deallocate this transfer descriptor */
350 		ehci_deallocate_qtd(ehcip, curr_qtd);
351 
352 		/*
353 		 * Deallocate the transfer wrapper if there are no more
354 		 * QTD's for the transfer wrapper.  ehci_deallocate_tw()
355 		 * will  not deallocate the tw for a periodic  endpoint
356 		 * since it will always have a QTD attached to it.
357 		 */
358 		ehci_deallocate_tw(ehcip, pp, tw);
359 
360 		curr_qtd = next_qtd;
361 	}
362 }
363 
364 
365 /*
366  * ehci_create_done_qtd_list:
367  *
368  * Create done qtd list from active qtd list.
369  */
370 ehci_qtd_t *
371 ehci_create_done_qtd_list(
372 	ehci_state_t		*ehcip)
373 {
374 	ehci_qtd_t		*curr_qtd = NULL, *next_qtd = NULL;
375 	ehci_qtd_t		*done_qtd_list = NULL, *last_done_qtd = NULL;
376 
377 	USB_DPRINTF_L4(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
378 	    "ehci_create_done_qtd_list:");
379 
380 	ASSERT(mutex_owned(&ehcip->ehci_int_mutex));
381 
382 	curr_qtd = ehcip->ehci_active_qtd_list;
383 
384 	while (curr_qtd) {
385 
386 		/* Get next qtd from the active qtd list */
387 		next_qtd = ehci_qtd_iommu_to_cpu(ehcip,
388 		    Get_QTD(curr_qtd->qtd_active_qtd_next));
389 
390 		/* Check this QTD has been processed by Host Controller */
391 		if (!(Get_QTD(curr_qtd->qtd_ctrl) &
392 		    EHCI_QTD_CTRL_ACTIVE_XACT)) {
393 
394 			/* Remove this QTD from active QTD list */
395 			ehci_remove_qtd_from_active_qtd_list(ehcip, curr_qtd);
396 
397 			Set_QTD(curr_qtd->qtd_active_qtd_next, NULL);
398 
399 			if (done_qtd_list) {
400 				Set_QTD(last_done_qtd->qtd_active_qtd_next,
401 				    ehci_qtd_cpu_to_iommu(ehcip, curr_qtd));
402 
403 				last_done_qtd = curr_qtd;
404 			} else {
405 				done_qtd_list = curr_qtd;
406 				last_done_qtd = curr_qtd;
407 			}
408 		}
409 
410 		curr_qtd = next_qtd;
411 	}
412 
413 	return (done_qtd_list);
414 }
415 
416 
417 /*
418  * ehci_parse_error:
419  *
420  * Parse the result for any errors.
421  */
422 static	usb_cr_t
423 ehci_parse_error(
424 	ehci_state_t		*ehcip,
425 	ehci_qtd_t		*qtd)
426 {
427 	uint_t			ctrl;
428 	ehci_trans_wrapper_t	*tw;
429 	ehci_pipe_private_t	*pp;
430 	uint_t			flag;
431 	usb_cr_t		error;
432 
433 	USB_DPRINTF_L4(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
434 	    "ehci_parse_error:");
435 
436 	ASSERT(mutex_owned(&ehcip->ehci_int_mutex));
437 
438 	ASSERT(qtd != NULL);
439 
440 	/* Obtain the transfer wrapper from the QTD */
441 	tw = (ehci_trans_wrapper_t *)
442 	    EHCI_LOOKUP_ID((uint32_t)Get_QTD(qtd->qtd_trans_wrapper));
443 
444 	ASSERT(tw != NULL);
445 
446 	/* Obtain the pipe private structure */
447 	pp = tw->tw_pipe_private;
448 
449 	USB_DPRINTF_L3(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
450 	    "ehci_parse_error: PP 0x%p TW 0x%p", pp, tw);
451 
452 	ctrl = (uint_t)Get_QTD(qtd->qtd_ctrl);
453 
454 	/*
455 	 * Check the condition code of completed QTD and report errors
456 	 * if any. This checking will be done both for the general and
457 	 * the isochronous QTDs.
458 	 */
459 	if ((error = ehci_check_for_error(ehcip, pp, tw, qtd, ctrl)) !=
460 	    USB_CR_OK) {
461 		flag = EHCI_REMOVE_XFER_ALWAYS;
462 	} else {
463 		flag  = EHCI_REMOVE_XFER_IFLAST;
464 	}
465 
466 	/* Stop the transfer timer */
467 	ehci_stop_xfer_timer(ehcip, tw, flag);
468 
469 	return (error);
470 }
471 
472 
473 /*
474  * ehci_check_for_error:
475  *
476  * Check for any errors.
477  *
478  * NOTE: This function is also called from POLLED MODE.
479  */
480 usb_cr_t
481 ehci_check_for_error(
482 	ehci_state_t		*ehcip,
483 	ehci_pipe_private_t	*pp,
484 	ehci_trans_wrapper_t	*tw,
485 	ehci_qtd_t		*qtd,
486 	uint_t			ctrl)
487 {
488 	usb_cr_t		error = USB_CR_OK;
489 	uint_t			status, speed, mask;
490 
491 	USB_DPRINTF_L4(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
492 	    "ehci_check_for_error: qtd = 0x%p ctrl = 0x%x",
493 	    qtd, ctrl);
494 
495 	/*
496 	 * Find the usb device speed and get the corresponding
497 	 * error status mask.
498 	 */
499 	speed = Get_QH(pp->pp_qh->qh_ctrl) & EHCI_QH_CTRL_ED_SPEED;
500 	mask = (speed == EHCI_QH_CTRL_ED_HIGH_SPEED)?
501 	    EHCI_QTD_CTRL_HS_XACT_STATUS : EHCI_QTD_CTRL_NON_HS_XACT_STATUS;
502 
503 	/* Exclude halted transaction error condition */
504 	status = ctrl & EHCI_QTD_CTRL_XACT_STATUS & ~EHCI_QTD_CTRL_HALTED_XACT;
505 
506 	switch (status & mask) {
507 	case EHCI_QTD_CTRL_NO_ERROR:
508 		USB_DPRINTF_L4(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
509 		    "ehci_check_for_error: No Error");
510 		error = USB_CR_OK;
511 		break;
512 	case EHCI_QTD_CTRL_ACTIVE_XACT:
513 		USB_DPRINTF_L4(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
514 		    "ehci_check_for_error: Not accessed");
515 		error = USB_CR_NOT_ACCESSED;
516 		break;
517 	case EHCI_QTD_CTRL_HALTED_XACT:
518 		USB_DPRINTF_L2(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
519 		    "ehci_check_for_error: Halted");
520 		error = USB_CR_STALL;
521 		break;
522 	case EHCI_QTD_CTRL_DATA_BUFFER_ERROR:
523 		if (tw->tw_direction == EHCI_QTD_CTRL_IN_PID) {
524 			USB_DPRINTF_L2(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
525 			    "ehci_check_for_error: Buffer Overrun");
526 			error = USB_CR_BUFFER_OVERRUN;
527 		} else	{
528 			USB_DPRINTF_L2(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
529 			    "ehci_check_for_error: Buffer Underrun");
530 			error = USB_CR_BUFFER_UNDERRUN;
531 		}
532 		break;
533 	case EHCI_QTD_CTRL_BABBLE_DETECTED:
534 		USB_DPRINTF_L2(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
535 		    "ehci_check_for_error: Babble Detected");
536 		error = USB_CR_DATA_OVERRUN;
537 		break;
538 	case EHCI_QTD_CTRL_XACT_ERROR:
539 		/*
540 		 * An xacterr bit of one is not necessarily an error,
541 		 * the transaction might have completed successfully
542 		 * after some retries.
543 		 *
544 		 * Try to detect the case when the queue is halted,
545 		 * because the error counter was decremented from one
546 		 * down to zero after a transaction error.
547 		 */
548 		if (ctrl & EHCI_QTD_CTRL_HALTED_XACT && (ctrl &
549 		    EHCI_QTD_CTRL_ERR_COUNT_MASK) == 0) {
550 			USB_DPRINTF_L2(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
551 			    "ehci_check_for_error: Transaction Error");
552 			error = USB_CR_DEV_NOT_RESP;
553 		} else {
554 			USB_DPRINTF_L4(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
555 			    "ehci_check_for_error: No Error");
556 			error = USB_CR_OK;
557 		}
558 		break;
559 	case EHCI_QTD_CTRL_MISSED_uFRAME:
560 		USB_DPRINTF_L2(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
561 		    "ehci_check_for_error: Missed uFrame");
562 		error = USB_CR_NOT_ACCESSED;
563 		break;
564 	case EHCI_QTD_CTRL_PRD_SPLIT_XACT_ERR:
565 		USB_DPRINTF_L2(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
566 		    "ehci_check_for_error: Periodic split-transaction "
567 		    "receives an error handshake");
568 		error = USB_CR_UNSPECIFIED_ERR;
569 		break;
570 	default:
571 		USB_DPRINTF_L2(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
572 		    "ehci_check_for_error: Unspecified Error");
573 		error = USB_CR_UNSPECIFIED_ERR;
574 		break;
575 	}
576 
577 	/*
578 	 * Check for halted transaction error condition.
579 	 * Under short xfer conditions, EHCI HC will not return an error
580 	 * or halt the QH.  This is done manually later in
581 	 * ehci_check_for_short_xfer.
582 	 */
583 	if ((ctrl & EHCI_QTD_CTRL_HALTED_XACT) && (error == USB_CR_OK)) {
584 		USB_DPRINTF_L2(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
585 		    "ehci_check_for_error: Halted");
586 		error = USB_CR_STALL;
587 	}
588 
589 	if (error == USB_CR_OK) {
590 		error = ehci_check_for_short_xfer(ehcip, pp, tw, qtd);
591 	}
592 
593 	if (error) {
594 		uint_t qh_ctrl =  Get_QH(pp->pp_qh->qh_ctrl);
595 
596 		USB_DPRINTF_L2(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
597 		    "ehci_check_for_error: Error %d Device address %d "
598 		    "Endpoint number %d", error,
599 		    (qh_ctrl & EHCI_QH_CTRL_DEVICE_ADDRESS),
600 		    ((qh_ctrl & EHCI_QH_CTRL_ED_NUMBER) >>
601 		    EHCI_QH_CTRL_ED_NUMBER_SHIFT));
602 	}
603 
604 	return (error);
605 }
606 
607 /*
608  * ehci_check_for_short_xfer:
609  *
610  * Check to see if there was a short xfer condition.
611  *
612  * NOTE: This function is also called from POLLED MODE.
613  *	 But it doesn't do anything.
614  */
615 static usb_cr_t
616 ehci_check_for_short_xfer(
617 	ehci_state_t		*ehcip,
618 	ehci_pipe_private_t	*pp,
619 	ehci_trans_wrapper_t	*tw,
620 	ehci_qtd_t		*qtd)
621 {
622 	usb_cr_t		error = USB_CR_OK;
623 	usb_ep_descr_t		*eptd;
624 	uchar_t			attributes;
625 	uint32_t		residue = 0;
626 	usb_req_attrs_t		xfer_attrs;
627 	size_t			length;
628 	mblk_t			*mp = NULL;
629 	usb_opaque_t		xfer_reqp;
630 
631 	USB_DPRINTF_L4(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
632 	    "ehci_check_for_short_xfer:");
633 
634 	if (pp->pp_flag == EHCI_POLLED_MODE_FLAG) {
635 
636 		return (error);
637 	}
638 
639 	/*
640 	 * Check for short xfer error.	If this is a control pipe, only check
641 	 * if it is in the data phase.
642 	 */
643 	eptd = &pp->pp_pipe_handle->p_ep;
644 	attributes = eptd->bmAttributes & USB_EP_ATTR_MASK;
645 
646 	switch (attributes) {
647 	case USB_EP_ATTR_CONTROL:
648 		if (Get_QTD(qtd->qtd_ctrl_phase) !=
649 		    EHCI_CTRL_DATA_PHASE) {
650 
651 			break;
652 		}
653 		/* FALLTHROUGH */
654 	case USB_EP_ATTR_BULK:
655 	case USB_EP_ATTR_INTR:
656 		/*
657 		 * If "Total bytes of xfer" in control field of
658 		 * Transfer Descriptor (QTD) is not equal to zero,
659 		 * then, we sent/received less data from the usb
660 		 * device than requested. In that case, get the
661 		 * actual received data size.
662 		 */
663 		residue = (Get_QTD(qtd->qtd_ctrl) &
664 		    EHCI_QTD_CTRL_BYTES_TO_XFER) >>
665 		    EHCI_QTD_CTRL_BYTES_TO_XFER_SHIFT;
666 
667 		break;
668 	case USB_EP_ATTR_ISOCH:
669 
670 		break;
671 	}
672 
673 	if (residue) {
674 		USB_DPRINTF_L2(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
675 		    "ehci_check_for_short_xfer: residue=%d direction=0x%x",
676 		    residue, tw->tw_direction);
677 
678 		length = Get_QTD(qtd->qtd_xfer_offs) +
679 		    Get_QTD(qtd->qtd_xfer_len) - residue;
680 
681 		if (tw->tw_direction == EHCI_QTD_CTRL_IN_PID) {
682 			xfer_attrs = ehci_get_xfer_attrs(ehcip, pp, tw);
683 
684 			if (xfer_attrs & USB_ATTRS_SHORT_XFER_OK) {
685 				ehci_cleanup_data_underrun(ehcip, tw, qtd);
686 			} else {
687 				/* Halt the pipe to mirror OHCI behavior */
688 				Set_QH(pp->pp_qh->qh_status,
689 				    ((Get_QH(pp->pp_qh->qh_status) &
690 				    ~EHCI_QH_STS_ACTIVE) |
691 				    EHCI_QH_STS_HALTED));
692 				error = USB_CR_DATA_UNDERRUN;
693 			}
694 
695 			USB_DPRINTF_L2(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
696 			    "ehci_check_for_short_xfer: requested data=%lu "
697 			    "received data=%lu", tw->tw_length, length);
698 
699 			switch (attributes) {
700 			case USB_EP_ATTR_CONTROL:
701 			case USB_EP_ATTR_BULK:
702 			case USB_EP_ATTR_INTR:
703 				/* Save the actual received length */
704 				tw->tw_length = length;
705 
706 				break;
707 			case USB_EP_ATTR_ISOCH:
708 			default:
709 
710 				break;
711 			}
712 		} else {
713 			USB_DPRINTF_L2(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
714 			    "ehci_check_for_short_xfer: requested data=%lu "
715 			    "sent data=%lu", tw->tw_length, length);
716 
717 			xfer_reqp = tw->tw_curr_xfer_reqp;
718 
719 			switch (attributes) {
720 			case USB_EP_ATTR_CONTROL:
721 
722 				break;
723 			case USB_EP_ATTR_BULK:
724 				mp = (mblk_t *)((usb_bulk_req_t *)
725 				    (xfer_reqp))->bulk_data;
726 
727 				/* Increment the read pointer */
728 				mp->b_rptr = mp->b_rptr + length;
729 
730 				break;
731 			case USB_EP_ATTR_INTR:
732 				mp = (mblk_t *)((usb_intr_req_t *)
733 				    (xfer_reqp))->intr_data;
734 
735 				/* Increment the read pointer */
736 				mp->b_rptr = mp->b_rptr + length;
737 
738 				break;
739 			case USB_EP_ATTR_ISOCH:
740 			default:
741 
742 				break;
743 			}
744 		}
745 	}
746 
747 	return (error);
748 }
749 
750 /*
751  * ehci_handle_error:
752  *
753  * Inform USBA about occurred transaction errors by calling the USBA callback
754  * routine.
755  */
756 void
757 ehci_handle_error(
758 	ehci_state_t		*ehcip,
759 	ehci_qtd_t		*qtd,
760 	usb_cr_t		error)
761 {
762 	ehci_trans_wrapper_t	*tw;
763 	usba_pipe_handle_data_t	*ph;
764 	ehci_pipe_private_t	*pp;
765 	ehci_qtd_t		*tw_qtd = qtd;
766 	uchar_t			attributes;
767 	usb_intr_req_t		*curr_intr_reqp;
768 
769 	USB_DPRINTF_L4(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
770 	    "ehci_handle_error: error = 0x%x", error);
771 
772 	ASSERT(mutex_owned(&ehcip->ehci_int_mutex));
773 
774 	ASSERT(qtd != NULL);
775 
776 	/* Print the values in the qtd */
777 	ehci_print_qtd(ehcip, qtd);
778 
779 	/* Obtain the transfer wrapper from the QTD */
780 	tw = (ehci_trans_wrapper_t *)
781 	    EHCI_LOOKUP_ID((uint32_t)Get_QTD(qtd->qtd_trans_wrapper));
782 
783 	ASSERT(tw != NULL);
784 
785 	/* Obtain the pipe private structure */
786 	pp = tw->tw_pipe_private;
787 
788 	ph = tw->tw_pipe_private->pp_pipe_handle;
789 	attributes = ph->p_ep.bmAttributes & USB_EP_ATTR_MASK;
790 
791 	/*
792 	 * Mark all QTDs belongs to this TW as RECLAIM
793 	 * so that we don't process them by mistake.
794 	 */
795 	while (tw_qtd) {
796 		/* Set QTD state to RECLAIM */
797 		Set_QTD(tw_qtd->qtd_state, EHCI_QTD_RECLAIM);
798 
799 		/* Get the next QTD from the wrapper */
800 		tw_qtd = ehci_qtd_iommu_to_cpu(ehcip,
801 		    Get_QTD(tw_qtd->qtd_tw_next_qtd));
802 	}
803 
804 	/*
805 	 * Special error handling
806 	 */
807 	if (tw->tw_direction == EHCI_QTD_CTRL_IN_PID) {
808 
809 		switch (attributes) {
810 		case USB_EP_ATTR_CONTROL:
811 			if (((ph->p_ep.bmAttributes &
812 			    USB_EP_ATTR_MASK) ==
813 			    USB_EP_ATTR_CONTROL) &&
814 			    (Get_QTD(qtd->qtd_ctrl_phase) ==
815 			    EHCI_CTRL_SETUP_PHASE)) {
816 
817 				break;
818 			}
819 			/* FALLTHROUGH */
820 		case USB_EP_ATTR_BULK:
821 			/*
822 			 * Call ehci_sendup_qtd_message
823 			 * to send message to upstream.
824 			 */
825 			ehci_sendup_qtd_message(ehcip, pp, tw, qtd, error);
826 
827 			return;
828 		case USB_EP_ATTR_INTR:
829 			curr_intr_reqp =
830 			    (usb_intr_req_t *)tw->tw_curr_xfer_reqp;
831 
832 			if (curr_intr_reqp->intr_attributes &
833 			    USB_ATTRS_ONE_XFER) {
834 
835 				ehci_handle_one_xfer_completion(ehcip, tw);
836 			}
837 
838 			/* Decrement periodic in request count */
839 			pp->pp_cur_periodic_req_cnt--;
840 			break;
841 		case USB_EP_ATTR_ISOCH:
842 			break;
843 		}
844 	}
845 
846 	ehci_hcdi_callback(ph, tw, error);
847 
848 	/* Check anybody is waiting for transfers completion event */
849 	ehci_check_for_transfers_completion(ehcip, pp);
850 }
851 
852 /*
853  * ehci_cleanup_data_underrun:
854  *
855  * Cleans up resources when a short xfer occurs.  Will only do cleanup if
856  * this pipe supports alternate_qtds.
857  *
858  * NOTE: This function is also called from POLLED MODE.
859  */
860 static void
861 ehci_cleanup_data_underrun(
862 	ehci_state_t		*ehcip,
863 	ehci_trans_wrapper_t	*tw,
864 	ehci_qtd_t		*qtd)
865 {
866 	ehci_qtd_t		*next_qtd, *temp_qtd;
867 
868 	USB_DPRINTF_L4(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
869 	    "ehci_cleanup_data_underrun: qtd=0x%p, tw=0x%p", qtd, tw);
870 
871 	/*
872 	 * Check if this transfer doesn't supports short_xfer or
873 	 * if this QTD is the last qtd in the tw.  If so there is
874 	 * no need for cleanup.
875 	 */
876 	if ((tw->tw_alt_qtd == NULL) || (qtd == tw->tw_qtd_tail)) {
877 		/* There is no need for cleanup */
878 		return;
879 	}
880 
881 	/* Start removing all the unused QTDs from the TW */
882 	next_qtd = (ehci_qtd_t *)ehci_qtd_iommu_to_cpu(ehcip,
883 	    Get_QTD(qtd->qtd_tw_next_qtd));
884 
885 	while (next_qtd) {
886 		tw->tw_num_qtds--;
887 
888 		ehci_remove_qtd_from_active_qtd_list(ehcip, next_qtd);
889 
890 		temp_qtd = next_qtd;
891 
892 		next_qtd = ehci_qtd_iommu_to_cpu(ehcip,
893 		    Get_QTD(next_qtd->qtd_tw_next_qtd));
894 
895 		ehci_deallocate_qtd(ehcip, temp_qtd);
896 	}
897 
898 	ASSERT(tw->tw_num_qtds == 1);
899 }
900 
901 /*
902  * ehci_handle_normal_qtd:
903  */
904 static void
905 ehci_handle_normal_qtd(
906 	ehci_state_t		*ehcip,
907 	ehci_qtd_t		*qtd,
908 	ehci_trans_wrapper_t	*tw)
909 {
910 	ehci_pipe_private_t	*pp;	/* Pipe private field */
911 
912 	USB_DPRINTF_L4(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
913 	    "ehci_handle_normal_qtd:");
914 
915 	ASSERT(mutex_owned(&ehcip->ehci_int_mutex));
916 	ASSERT(tw != NULL);
917 
918 	/* Obtain the pipe private structure */
919 	pp = tw->tw_pipe_private;
920 
921 	(*tw->tw_handle_qtd)(ehcip, pp, tw,
922 	    qtd, tw->tw_handle_callback_value);
923 
924 	/* Check anybody is waiting for transfers completion event */
925 	ehci_check_for_transfers_completion(ehcip, pp);
926 }
927 
928 
929 /*
930  * ehci_handle_ctrl_qtd:
931  *
932  * Handle a control Transfer Descriptor (QTD).
933  */
934 /* ARGSUSED */
935 void
936 ehci_handle_ctrl_qtd(
937 	ehci_state_t		*ehcip,
938 	ehci_pipe_private_t	*pp,
939 	ehci_trans_wrapper_t	*tw,
940 	ehci_qtd_t		*qtd,
941 	void			*tw_handle_callback_value)
942 {
943 	usba_pipe_handle_data_t	*ph = pp->pp_pipe_handle;
944 
945 	USB_DPRINTF_L4(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
946 	    "ehci_handle_ctrl_qtd: pp = 0x%p tw = 0x%p qtd = 0x%p state = 0x%x",
947 	    (void *)pp, (void *)tw, (void *)qtd, Get_QTD(qtd->qtd_ctrl_phase));
948 
949 	ASSERT(mutex_owned(&ehcip->ehci_int_mutex));
950 
951 	/*
952 	 * A control transfer consists of three phases:
953 	 *
954 	 * Setup
955 	 * Data (optional)
956 	 * Status
957 	 *
958 	 * There is a QTD per phase. A QTD for a given phase isn't
959 	 * enqueued until the previous phase is finished. EHCI
960 	 * spec allows more than one  control transfer on a pipe
961 	 * within a frame. However, we've found that some devices
962 	 * can't handle this.
963 	 */
964 	tw->tw_num_qtds--;
965 	switch (Get_QTD(qtd->qtd_ctrl_phase)) {
966 	case EHCI_CTRL_SETUP_PHASE:
967 		USB_DPRINTF_L3(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
968 		    "Setup complete: pp 0x%p qtd 0x%p",
969 		    (void *)pp, (void *)qtd);
970 
971 		break;
972 	case EHCI_CTRL_DATA_PHASE:
973 		USB_DPRINTF_L3(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
974 		    "Data complete: pp 0x%p qtd 0x%p",
975 		    (void *)pp, (void *)qtd);
976 
977 		break;
978 	case EHCI_CTRL_STATUS_PHASE:
979 		/*
980 		 * On some particular hardware, status phase is seen to
981 		 * finish before data phase gets timeouted. Don't handle
982 		 * the transfer result here if not all qtds are finished.
983 		 * Let the timeout handler handle it.
984 		 */
985 		if (tw->tw_num_qtds != 0) {
986 			USB_DPRINTF_L2(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
987 			    "Status complete, but the transfer is not done: "
988 			    "tw 0x%p, qtd 0x%p, tw_num_qtd 0x%d",
989 			    (void *)tw, (void *)qtd, tw->tw_num_qtds);
990 
991 			ehci_print_qh(ehcip, pp->pp_qh);
992 			ehci_print_qtd(ehcip, qtd);
993 
994 			break;
995 		}
996 
997 		if ((tw->tw_length) &&
998 		    (tw->tw_direction == EHCI_QTD_CTRL_IN_PID)) {
999 			/*
1000 			 * Call ehci_sendup_qtd_message
1001 			 * to send message to upstream.
1002 			 */
1003 			ehci_sendup_qtd_message(ehcip,
1004 			    pp, tw, qtd, USB_CR_OK);
1005 		} else {
1006 			ehci_hcdi_callback(ph, tw, USB_CR_OK);
1007 		}
1008 
1009 		USB_DPRINTF_L3(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
1010 		    "Status complete: pp 0x%p qtd 0x%p",
1011 		    (void *)pp, (void *)qtd);
1012 
1013 		break;
1014 	}
1015 }
1016 
1017 
1018 /*
1019  * ehci_handle_bulk_qtd:
1020  *
1021  * Handle a bulk Transfer Descriptor (QTD).
1022  */
1023 /* ARGSUSED */
1024 void
1025 ehci_handle_bulk_qtd(
1026 	ehci_state_t		*ehcip,
1027 	ehci_pipe_private_t	*pp,
1028 	ehci_trans_wrapper_t	*tw,
1029 	ehci_qtd_t		*qtd,
1030 	void			*tw_handle_callback_value)
1031 {
1032 	usba_pipe_handle_data_t	*ph = pp->pp_pipe_handle;
1033 	usb_ep_descr_t		*eptd = &ph->p_ep;
1034 
1035 	USB_DPRINTF_L4(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
1036 	    "ehci_handle_bulk_qtd:");
1037 
1038 	ASSERT(mutex_owned(&ehcip->ehci_int_mutex));
1039 
1040 	/*
1041 	 * Decrement the QTDs counter and check whether all the bulk
1042 	 * data has been send or received. If QTDs counter reaches
1043 	 * zero then inform client driver about completion current
1044 	 * bulk request. Other wise wait for completion of other bulk
1045 	 * QTDs or transactions on this pipe.
1046 	 */
1047 	if (--tw->tw_num_qtds != 0) {
1048 
1049 		USB_DPRINTF_L3(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
1050 		    "ehci_handle_bulk_qtd: Number of QTDs %d", tw->tw_num_qtds);
1051 
1052 		return;
1053 	}
1054 
1055 	/*
1056 	 * If this is a bulk in pipe, return the data to the client.
1057 	 * For a bulk out pipe, there is no need to do anything.
1058 	 */
1059 	if ((eptd->bEndpointAddress &
1060 	    USB_EP_DIR_MASK) == USB_EP_DIR_OUT) {
1061 
1062 		USB_DPRINTF_L3(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
1063 		    "ehci_handle_bulk_qtd: Bulk out pipe");
1064 
1065 		/* Do the callback */
1066 		ehci_hcdi_callback(ph, tw, USB_CR_OK);
1067 
1068 		return;
1069 	}
1070 
1071 	/* Call ehci_sendup_qtd_message to send message to upstream */
1072 	ehci_sendup_qtd_message(ehcip, pp, tw, qtd, USB_CR_OK);
1073 }
1074 
1075 
1076 /*
1077  * ehci_handle_intr_qtd:
1078  *
1079  * Handle a interrupt Transfer Descriptor (QTD).
1080  */
1081 /* ARGSUSED */
1082 void
1083 ehci_handle_intr_qtd(
1084 	ehci_state_t		*ehcip,
1085 	ehci_pipe_private_t	*pp,
1086 	ehci_trans_wrapper_t	*tw,
1087 	ehci_qtd_t		*qtd,
1088 	void			*tw_handle_callback_value)
1089 {
1090 	usb_intr_req_t		*curr_intr_reqp =
1091 	    (usb_intr_req_t *)tw->tw_curr_xfer_reqp;
1092 	usba_pipe_handle_data_t	*ph = pp->pp_pipe_handle;
1093 	usb_ep_descr_t		*eptd = &ph->p_ep;
1094 	usb_req_attrs_t		attrs;
1095 	int			error = USB_SUCCESS;
1096 
1097 	USB_DPRINTF_L4(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
1098 	    "ehci_handle_intr_qtd:");
1099 
1100 	ASSERT(mutex_owned(&ehcip->ehci_int_mutex));
1101 
1102 	/* Get the interrupt xfer attributes */
1103 	attrs = curr_intr_reqp->intr_attributes;
1104 
1105 	/*
1106 	 * For a Interrupt OUT pipe, we just callback and we are done
1107 	 */
1108 	if ((eptd->bEndpointAddress & USB_EP_DIR_MASK) == USB_EP_DIR_OUT) {
1109 
1110 		USB_DPRINTF_L3(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
1111 		    "ehci_handle_intr_qtd: Intr out pipe, intr_reqp=0x%p,"
1112 		    "data=0x%p", curr_intr_reqp, curr_intr_reqp->intr_data);
1113 
1114 		/* Do the callback */
1115 		ehci_hcdi_callback(ph, tw, USB_CR_OK);
1116 
1117 		return;
1118 	}
1119 
1120 	/* Decrement number of interrupt request count */
1121 	pp->pp_cur_periodic_req_cnt--;
1122 
1123 	/*
1124 	 * Check usb flag whether USB_FLAGS_ONE_XFER flag is set
1125 	 * and if so, free duplicate request.
1126 	 */
1127 	if (attrs & USB_ATTRS_ONE_XFER) {
1128 		ehci_handle_one_xfer_completion(ehcip, tw);
1129 	}
1130 
1131 	/* Call ehci_sendup_qtd_message to callback into client */
1132 	ehci_sendup_qtd_message(ehcip, pp, tw, qtd, USB_CR_OK);
1133 
1134 	/*
1135 	 * If interrupt pipe state is still active, insert next Interrupt
1136 	 * request into the Host Controller's Interrupt list.  Otherwise
1137 	 * you are done.
1138 	 */
1139 	if ((pp->pp_state != EHCI_PIPE_STATE_ACTIVE) ||
1140 	    (ehci_state_is_operational(ehcip) != USB_SUCCESS)) {
1141 
1142 		return;
1143 	}
1144 
1145 	if ((error = ehci_allocate_intr_in_resource(ehcip, pp, tw, 0)) ==
1146 	    USB_SUCCESS) {
1147 		curr_intr_reqp = (usb_intr_req_t *)tw->tw_curr_xfer_reqp;
1148 
1149 		ASSERT(curr_intr_reqp != NULL);
1150 
1151 		tw->tw_num_qtds = 1;
1152 
1153 		if (ehci_allocate_tds_for_tw(ehcip, pp, tw, tw->tw_num_qtds) !=
1154 		    USB_SUCCESS) {
1155 			ehci_deallocate_intr_in_resource(ehcip, pp, tw);
1156 			error = USB_FAILURE;
1157 		}
1158 	}
1159 
1160 	if (error != USB_SUCCESS) {
1161 		/*
1162 		 * Set pipe state to stop polling and error to no
1163 		 * resource. Don't insert any more interrupt polling
1164 		 * requests.
1165 		 */
1166 		pp->pp_state = EHCI_PIPE_STATE_STOP_POLLING;
1167 		pp->pp_error = USB_CR_NO_RESOURCES;
1168 	} else {
1169 		ehci_insert_intr_req(ehcip, pp, tw, 0);
1170 
1171 		/* Increment number of interrupt request count */
1172 		pp->pp_cur_periodic_req_cnt++;
1173 
1174 		ASSERT(pp->pp_cur_periodic_req_cnt ==
1175 		    pp->pp_max_periodic_req_cnt);
1176 	}
1177 }
1178 
1179 
1180 /*
1181  * ehci_handle_one_xfer_completion:
1182  */
1183 static void
1184 ehci_handle_one_xfer_completion(
1185 	ehci_state_t		*ehcip,
1186 	ehci_trans_wrapper_t	*tw)
1187 {
1188 	usba_pipe_handle_data_t	*ph = tw->tw_pipe_private->pp_pipe_handle;
1189 	ehci_pipe_private_t	*pp = tw->tw_pipe_private;
1190 	usb_intr_req_t		*curr_intr_reqp =
1191 	    (usb_intr_req_t *)tw->tw_curr_xfer_reqp;
1192 
1193 	USB_DPRINTF_L4(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
1194 	    "ehci_handle_one_xfer_completion: tw = 0x%p", tw);
1195 
1196 	ASSERT(mutex_owned(&ehcip->ehci_int_mutex));
1197 	ASSERT(curr_intr_reqp->intr_attributes & USB_ATTRS_ONE_XFER);
1198 
1199 	pp->pp_state = EHCI_PIPE_STATE_IDLE;
1200 
1201 	/*
1202 	 * For one xfer, we need to copy back data ptr
1203 	 * and free current request
1204 	 */
1205 	((usb_intr_req_t *)(pp->pp_client_periodic_in_reqp))->
1206 	    intr_data = ((usb_intr_req_t *)
1207 	    (tw->tw_curr_xfer_reqp))->intr_data;
1208 
1209 	((usb_intr_req_t *)tw->tw_curr_xfer_reqp)->intr_data = NULL;
1210 
1211 	/* Now free duplicate current request */
1212 	usb_free_intr_req((usb_intr_req_t *)tw-> tw_curr_xfer_reqp);
1213 
1214 	mutex_enter(&ph->p_mutex);
1215 	ph->p_req_count--;
1216 	mutex_exit(&ph->p_mutex);
1217 
1218 	/* Make client's request the current request */
1219 	tw->tw_curr_xfer_reqp = pp->pp_client_periodic_in_reqp;
1220 	pp->pp_client_periodic_in_reqp = NULL;
1221 }
1222 
1223 
1224 /*
1225  * ehci_sendup_qtd_message:
1226  *	copy data, if necessary and do callback
1227  */
1228 /* ARGSUSED */
1229 static void
1230 ehci_sendup_qtd_message(
1231 	ehci_state_t		*ehcip,
1232 	ehci_pipe_private_t	*pp,
1233 	ehci_trans_wrapper_t	*tw,
1234 	ehci_qtd_t		*qtd,
1235 	usb_cr_t		error)
1236 {
1237 	usb_ep_descr_t		*eptd = &pp->pp_pipe_handle->p_ep;
1238 	usba_pipe_handle_data_t	*ph = pp->pp_pipe_handle;
1239 	usb_opaque_t		curr_xfer_reqp = tw->tw_curr_xfer_reqp;
1240 	size_t			skip_len = 0;
1241 	size_t			length;
1242 	uchar_t			*buf;
1243 	mblk_t			*mp;
1244 
1245 	ASSERT(mutex_owned(&ehcip->ehci_int_mutex));
1246 
1247 	USB_DPRINTF_L4(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
1248 	    "ehci_sendup_qtd_message:");
1249 
1250 	ASSERT(tw != NULL);
1251 
1252 	length = tw->tw_length;
1253 
1254 	if ((eptd->bmAttributes & USB_EP_ATTR_MASK) == USB_EP_ATTR_CONTROL) {
1255 		/* Get the correct length */
1256 		if (((usb_ctrl_req_t *)curr_xfer_reqp)->ctrl_wLength)
1257 			length = length - EHCI_MAX_QTD_BUF_SIZE;
1258 		else
1259 			length = length - SETUP_SIZE;
1260 
1261 		/* Set the length of the buffer to skip */
1262 		skip_len = EHCI_MAX_QTD_BUF_SIZE;
1263 	}
1264 
1265 	/* Copy the data into the mblk_t */
1266 	buf = (uchar_t *)tw->tw_buf + skip_len;
1267 
1268 	USB_DPRINTF_L4(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
1269 	    "ehci_sendup_qtd_message: length %ld error %d", length, error);
1270 
1271 	/* Get the message block */
1272 	switch (eptd->bmAttributes & USB_EP_ATTR_MASK) {
1273 	case USB_EP_ATTR_CONTROL:
1274 		mp = ((usb_ctrl_req_t *)curr_xfer_reqp)->ctrl_data;
1275 		break;
1276 	case USB_EP_ATTR_BULK:
1277 		mp = ((usb_bulk_req_t *)curr_xfer_reqp)->bulk_data;
1278 		break;
1279 	case USB_EP_ATTR_INTR:
1280 		mp = ((usb_intr_req_t *)curr_xfer_reqp)->intr_data;
1281 		break;
1282 	case USB_EP_ATTR_ISOCH:
1283 		/* Isoc messages must not go through this path */
1284 		mp = NULL;
1285 		break;
1286 	}
1287 
1288 	ASSERT(mp != NULL);
1289 
1290 	if (length) {
1291 		/*
1292 		 * Update kstat byte counts
1293 		 * The control endpoints don't have direction bits so in
1294 		 * order for control stats to be counted correctly an in
1295 		 * bit must be faked on a control read.
1296 		 */
1297 		if ((eptd->bmAttributes & USB_EP_ATTR_MASK) ==
1298 		    USB_EP_ATTR_CONTROL) {
1299 			ehci_do_byte_stats(ehcip, length,
1300 			    eptd->bmAttributes, USB_EP_DIR_IN);
1301 		} else {
1302 			ehci_do_byte_stats(ehcip, length,
1303 			    eptd->bmAttributes, eptd->bEndpointAddress);
1304 		}
1305 
1306 		/* Sync IO buffer */
1307 		Sync_IO_Buffer(tw->tw_dmahandle, (skip_len + length));
1308 
1309 		/* since we specified NEVERSWAP, we can just use bcopy */
1310 		bcopy(buf, mp->b_rptr, length);
1311 
1312 		/* Increment the write pointer */
1313 		mp->b_wptr = mp->b_wptr + length;
1314 	} else {
1315 		USB_DPRINTF_L3(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
1316 		    "ehci_sendup_qtd_message: Zero length packet");
1317 	}
1318 
1319 	ehci_hcdi_callback(ph, tw, error);
1320 }
1321