130e7468fSPeter Dunlap /*
230e7468fSPeter Dunlap  * CDDL HEADER START
330e7468fSPeter Dunlap  *
430e7468fSPeter Dunlap  * The contents of this file are subject to the terms of the
530e7468fSPeter Dunlap  * Common Development and Distribution License (the "License").
630e7468fSPeter Dunlap  * You may not use this file except in compliance with the License.
730e7468fSPeter Dunlap  *
830e7468fSPeter Dunlap  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
930e7468fSPeter Dunlap  * or http://www.opensolaris.org/os/licensing.
1030e7468fSPeter Dunlap  * See the License for the specific language governing permissions
1130e7468fSPeter Dunlap  * and limitations under the License.
1230e7468fSPeter Dunlap  *
1330e7468fSPeter Dunlap  * When distributing Covered Code, include this CDDL HEADER in each
1430e7468fSPeter Dunlap  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
1530e7468fSPeter Dunlap  * If applicable, add the following below this CDDL HEADER, with the
1630e7468fSPeter Dunlap  * fields enclosed by brackets "[]" replaced with your own identifying
1730e7468fSPeter Dunlap  * information: Portions Copyright [yyyy] [name of copyright owner]
1830e7468fSPeter Dunlap  *
1930e7468fSPeter Dunlap  * CDDL HEADER END
2030e7468fSPeter Dunlap  */
2130e7468fSPeter Dunlap /*
2230e7468fSPeter Dunlap  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
2330e7468fSPeter Dunlap  * Use is subject to license terms.
24*48bbca81SDaniel Hoffman  * Copyright (c) 2016 by Delphix. All rights reserved.
2530e7468fSPeter Dunlap  */
2630e7468fSPeter Dunlap 
2730e7468fSPeter Dunlap #include <sys/types.h>
2830e7468fSPeter Dunlap #include <sys/ddi.h>
2930e7468fSPeter Dunlap #include <sys/types.h>
3030e7468fSPeter Dunlap #include <sys/socket.h>
3130e7468fSPeter Dunlap #include <netinet/in.h>
3230e7468fSPeter Dunlap #include <sys/sunddi.h>
3330e7468fSPeter Dunlap #include <sys/sdt.h>
3430e7468fSPeter Dunlap #include <sys/ib/ibtl/ibti.h>
3530e7468fSPeter Dunlap #include <sys/ib/ibtl/ibtl_types.h>
3630e7468fSPeter Dunlap 
3730e7468fSPeter Dunlap #include <sys/ib/clients/iser/iser.h>
3830e7468fSPeter Dunlap 
3930e7468fSPeter Dunlap /*
4030e7468fSPeter Dunlap  * iser_cq.c
4130e7468fSPeter Dunlap  *    Routines for completion queue handlers for iSER.
4230e7468fSPeter Dunlap  */
4330e7468fSPeter Dunlap static void iser_msg_handle(iser_chan_t *chan, iser_msg_t *msg);
4430e7468fSPeter Dunlap int iser_iscsihdr_handle(iser_chan_t *chan, iser_msg_t *msg);
4530e7468fSPeter Dunlap static int iser_ib_poll_send_completions(ibt_cq_hdl_t cq_hdl,
4630e7468fSPeter Dunlap     iser_chan_t *iser_chan);
4730e7468fSPeter Dunlap static int iser_ib_poll_recv_completions(ibt_cq_hdl_t cq_hdl,
4830e7468fSPeter Dunlap     iser_chan_t *iser_chan);
4930e7468fSPeter Dunlap 
5030e7468fSPeter Dunlap void
iser_ib_sendcq_handler(ibt_cq_hdl_t cq_hdl,void * arg)5130e7468fSPeter Dunlap iser_ib_sendcq_handler(ibt_cq_hdl_t cq_hdl, void *arg)
5230e7468fSPeter Dunlap {
5330e7468fSPeter Dunlap 	iser_chan_t	*iser_chan;
5430e7468fSPeter Dunlap 	ibt_status_t	status;
5530e7468fSPeter Dunlap 
5630e7468fSPeter Dunlap 	iser_chan = (iser_chan_t *)arg;
5730e7468fSPeter Dunlap 
58a82ec3cfSPriya Krishnan 	/*
59a82ec3cfSPriya Krishnan 	 * Poll for work request completion while successful. If the
60a82ec3cfSPriya Krishnan 	 * queue empties or otherwise becomes invalid, stop polling.
61a82ec3cfSPriya Krishnan 	 */
6230e7468fSPeter Dunlap 	do {
6330e7468fSPeter Dunlap 		status = iser_ib_poll_send_completions(cq_hdl, iser_chan);
64a82ec3cfSPriya Krishnan 	} while (status == IBT_SUCCESS);
6530e7468fSPeter Dunlap 
66a82ec3cfSPriya Krishnan 	if (status == IBT_CQ_EMPTY) {
67a82ec3cfSPriya Krishnan 		/* We've emptied the CQ, rearm it before we're done here */
68a82ec3cfSPriya Krishnan 		status = ibt_enable_cq_notify(cq_hdl, IBT_NEXT_COMPLETION);
69a82ec3cfSPriya Krishnan 		if (status != IBT_SUCCESS) {
70a82ec3cfSPriya Krishnan 			/* Unexpected error */
71a82ec3cfSPriya Krishnan 			ISER_LOG(CE_NOTE, "iser_ib_sendcq_handler: "
72a82ec3cfSPriya Krishnan 			    "ibt_enable_cq_notify error (%d)", status);
73a82ec3cfSPriya Krishnan 			return;
74a82ec3cfSPriya Krishnan 		}
7530e7468fSPeter Dunlap 
76a82ec3cfSPriya Krishnan 		/* Now, check for more completions after the rearm */
77a82ec3cfSPriya Krishnan 		do {
78a82ec3cfSPriya Krishnan 			status = iser_ib_poll_send_completions(
79a82ec3cfSPriya Krishnan 			    cq_hdl, iser_chan);
80a82ec3cfSPriya Krishnan 		} while (status == IBT_SUCCESS);
81a82ec3cfSPriya Krishnan 	}
8230e7468fSPeter Dunlap }
8330e7468fSPeter Dunlap 
8430e7468fSPeter Dunlap static int
iser_ib_poll_send_completions(ibt_cq_hdl_t cq_hdl,iser_chan_t * iser_chan)8530e7468fSPeter Dunlap iser_ib_poll_send_completions(ibt_cq_hdl_t cq_hdl, iser_chan_t *iser_chan)
8630e7468fSPeter Dunlap {
8730e7468fSPeter Dunlap 	ibt_wc_t	wc[ISER_IB_SCQ_POLL_MAX];
8830e7468fSPeter Dunlap 	ibt_wrid_t	wrid;
8930e7468fSPeter Dunlap 	idm_buf_t	*idb = NULL;
9030e7468fSPeter Dunlap 	idm_task_t	*idt = NULL;
9130e7468fSPeter Dunlap 	iser_wr_t	*wr = NULL;
9230e7468fSPeter Dunlap 	int		i;
9330e7468fSPeter Dunlap 	uint_t		npoll = 0;
9430e7468fSPeter Dunlap 	ibt_status_t	status;
9530e7468fSPeter Dunlap 	iser_conn_t	*iser_conn;
9630e7468fSPeter Dunlap 	idm_status_t	idm_status;
97a668b114SPriya Krishnan 	iser_mr_t	*mr;
9830e7468fSPeter Dunlap 
9930e7468fSPeter Dunlap 	iser_conn = iser_chan->ic_conn;
10030e7468fSPeter Dunlap 
101a82ec3cfSPriya Krishnan 	/* Poll ISER_IB_SCQ_POLL_MAX completions from the CQ */
10230e7468fSPeter Dunlap 	status = ibt_poll_cq(cq_hdl, wc, ISER_IB_SCQ_POLL_MAX, &npoll);
10330e7468fSPeter Dunlap 
10430e7468fSPeter Dunlap 	if (status != IBT_SUCCESS) {
10530e7468fSPeter Dunlap 		if (status != IBT_CQ_EMPTY) {
10630e7468fSPeter Dunlap 			/* Unexpected error */
10730e7468fSPeter Dunlap 			ISER_LOG(CE_NOTE, "iser_ib_sendcq_handler: ibt_poll_cq "
108a82ec3cfSPriya Krishnan 			    "unexpected error (%d)", status);
10930e7468fSPeter Dunlap 		}
11030e7468fSPeter Dunlap 		/* CQ is empty. Either way, move along... */
11130e7468fSPeter Dunlap 		return (status);
11230e7468fSPeter Dunlap 	}
11330e7468fSPeter Dunlap 
11430e7468fSPeter Dunlap 	/*
11530e7468fSPeter Dunlap 	 * Handle each of the completions we've polled
11630e7468fSPeter Dunlap 	 */
11730e7468fSPeter Dunlap 	for (i = 0; i < npoll; i++) {
11830e7468fSPeter Dunlap 
11930e7468fSPeter Dunlap 		DTRACE_PROBE3(iser__send__cqe, iser_chan_t *, iser_chan,
12030e7468fSPeter Dunlap 		    ibt_wc_t *, &wc[i], ibt_wc_status_t, wc[i].wc_status);
12130e7468fSPeter Dunlap 
12230e7468fSPeter Dunlap 		/* Grab the wrid of the completion */
12330e7468fSPeter Dunlap 		wrid = wc[i].wc_id;
12430e7468fSPeter Dunlap 
12530e7468fSPeter Dunlap 		/* Decrement this channel's SQ posted count */
12630e7468fSPeter Dunlap 		mutex_enter(&iser_chan->ic_sq_post_lock);
12730e7468fSPeter Dunlap 		iser_chan->ic_sq_post_count--;
12830e7468fSPeter Dunlap 		mutex_exit(&iser_chan->ic_sq_post_lock);
12930e7468fSPeter Dunlap 
13030e7468fSPeter Dunlap 		/* Pull in the wr handle */
13130e7468fSPeter Dunlap 		wr = (iser_wr_t *)(uintptr_t)wrid;
13230e7468fSPeter Dunlap 		ASSERT(wr != NULL);
13330e7468fSPeter Dunlap 
13430e7468fSPeter Dunlap 		/* Set an idm_status for return to IDM */
13530e7468fSPeter Dunlap 		idm_status = (wc[i].wc_status == IBT_WC_SUCCESS) ?
13630e7468fSPeter Dunlap 		    IDM_STATUS_SUCCESS : IDM_STATUS_FAIL;
13730e7468fSPeter Dunlap 
13830e7468fSPeter Dunlap 		/*
13930e7468fSPeter Dunlap 		 * A non-success status here indicates the QP went
14030e7468fSPeter Dunlap 		 * into an error state while this WR was being
14130e7468fSPeter Dunlap 		 * processed. This can also happen when the
14230e7468fSPeter Dunlap 		 * channel is closed on the remote end. Clean up
14330e7468fSPeter Dunlap 		 * the resources, then push CE_TRANSPORT_FAIL
14430e7468fSPeter Dunlap 		 * into IDM.
14530e7468fSPeter Dunlap 		 */
14630e7468fSPeter Dunlap 		if (wc[i].wc_status != IBT_WC_SUCCESS) {
14730e7468fSPeter Dunlap 			/*
14830e7468fSPeter Dunlap 			 * Free the resources attached to this
14930e7468fSPeter Dunlap 			 * completion.
15030e7468fSPeter Dunlap 			 */
15130e7468fSPeter Dunlap 			if (wr->iw_msg != NULL) {
15230e7468fSPeter Dunlap 				/* Free iser_msg handle */
15330e7468fSPeter Dunlap 				iser_msg_free(wr->iw_msg);
15430e7468fSPeter Dunlap 			}
15530e7468fSPeter Dunlap 
15630e7468fSPeter Dunlap 			if (wr->iw_pdu != NULL) {
15730e7468fSPeter Dunlap 				/* Complete the PDU */
15830e7468fSPeter Dunlap 				idm_pdu_complete(wr->iw_pdu, idm_status);
15930e7468fSPeter Dunlap 			}
16030e7468fSPeter Dunlap 
16130e7468fSPeter Dunlap 			if (wr->iw_buf != NULL) {
16230e7468fSPeter Dunlap 				/* Invoke buffer callback */
16330e7468fSPeter Dunlap 				idb = wr->iw_buf;
164a668b114SPriya Krishnan 				mr = ((iser_buf_t *)
165a668b114SPriya Krishnan 				    idb->idb_buf_private)->iser_mr;
16630e7468fSPeter Dunlap #ifdef DEBUG
16730e7468fSPeter Dunlap 				bcopy(&wc[i],
16830e7468fSPeter Dunlap 				    &((iser_buf_t *)idb->idb_buf_private)->
16930e7468fSPeter Dunlap 				    buf_wc, sizeof (ibt_wc_t));
17030e7468fSPeter Dunlap #endif
17130e7468fSPeter Dunlap 				idt = idb->idb_task_binding;
17230e7468fSPeter Dunlap 				mutex_enter(&idt->idt_mutex);
17330e7468fSPeter Dunlap 				if (wr->iw_type == ISER_WR_RDMAW) {
174a668b114SPriya Krishnan 					DTRACE_ISCSI_8(xfer__done,
175a668b114SPriya Krishnan 					    idm_conn_t *, idt->idt_ic,
176a668b114SPriya Krishnan 					    uintptr_t, idb->idb_buf,
177a668b114SPriya Krishnan 					    uint32_t, idb->idb_bufoffset,
178a668b114SPriya Krishnan 					    uint64_t, mr->is_mrva, uint32_t, 0,
179a668b114SPriya Krishnan 					    uint32_t, mr->is_mrrkey,
180a668b114SPriya Krishnan 					    uint32_t, idb->idb_xfer_len,
181a668b114SPriya Krishnan 					    int, XFER_BUF_TX_TO_INI);
18230e7468fSPeter Dunlap 					idm_buf_tx_to_ini_done(idt, idb,
18330e7468fSPeter Dunlap 					    IDM_STATUS_FAIL);
18430e7468fSPeter Dunlap 				} else { /* ISER_WR_RDMAR */
185a668b114SPriya Krishnan 					DTRACE_ISCSI_8(xfer__done,
186a668b114SPriya Krishnan 					    idm_conn_t *, idt->idt_ic,
187a668b114SPriya Krishnan 					    uintptr_t, idb->idb_buf,
188a668b114SPriya Krishnan 					    uint32_t, idb->idb_bufoffset,
189a668b114SPriya Krishnan 					    uint64_t, mr->is_mrva, uint32_t, 0,
190a668b114SPriya Krishnan 					    uint32_t, mr->is_mrrkey,
191a668b114SPriya Krishnan 					    uint32_t, idb->idb_xfer_len,
192a668b114SPriya Krishnan 					    int, XFER_BUF_RX_FROM_INI);
19330e7468fSPeter Dunlap 					idm_buf_rx_from_ini_done(idt, idb,
19430e7468fSPeter Dunlap 					    IDM_STATUS_FAIL);
19530e7468fSPeter Dunlap 				}
19630e7468fSPeter Dunlap 			}
19730e7468fSPeter Dunlap 
19830e7468fSPeter Dunlap 			/* Free the iser wr handle */
19930e7468fSPeter Dunlap 			iser_wr_free(wr);
20030e7468fSPeter Dunlap 
20130e7468fSPeter Dunlap 			/*
20230e7468fSPeter Dunlap 			 * Tell IDM that the channel has gone down,
203*48bbca81SDaniel Hoffman 			 * unless it already knows.
20430e7468fSPeter Dunlap 			 */
20530e7468fSPeter Dunlap 			mutex_enter(&iser_conn->ic_lock);
20630e7468fSPeter Dunlap 			switch (iser_conn->ic_stage) {
20730e7468fSPeter Dunlap 			case ISER_CONN_STAGE_IC_DISCONNECTED:
20830e7468fSPeter Dunlap 			case ISER_CONN_STAGE_IC_FREED:
20930e7468fSPeter Dunlap 			case ISER_CONN_STAGE_CLOSING:
21030e7468fSPeter Dunlap 			case ISER_CONN_STAGE_CLOSED:
21130e7468fSPeter Dunlap 				break;
21230e7468fSPeter Dunlap 
21330e7468fSPeter Dunlap 			default:
21430e7468fSPeter Dunlap 				idm_conn_event(iser_conn->ic_idmc,
21530e7468fSPeter Dunlap 				    CE_TRANSPORT_FAIL, idm_status);
21630e7468fSPeter Dunlap 				iser_conn->ic_stage = ISER_CONN_STAGE_CLOSING;
21730e7468fSPeter Dunlap 			}
21830e7468fSPeter Dunlap 			mutex_exit(&iser_conn->ic_lock);
21930e7468fSPeter Dunlap 
22030e7468fSPeter Dunlap 			/* Move onto the next completion */
22130e7468fSPeter Dunlap 			continue;
22230e7468fSPeter Dunlap 		}
22330e7468fSPeter Dunlap 
22430e7468fSPeter Dunlap 		/*
22530e7468fSPeter Dunlap 		 * For a success status, just invoke the PDU or
22630e7468fSPeter Dunlap 		 * buffer completion. We use our WR handle's
22730e7468fSPeter Dunlap 		 * "iw_type" here so that we can properly process
22830e7468fSPeter Dunlap 		 * because the CQE's opcode is invalid if the status
22930e7468fSPeter Dunlap 		 * is failed.
23030e7468fSPeter Dunlap 		 */
23130e7468fSPeter Dunlap 		switch (wr->iw_type) {
23230e7468fSPeter Dunlap 		case ISER_WR_SEND:
23330e7468fSPeter Dunlap 			/* Free the msg handle */
23430e7468fSPeter Dunlap 			ASSERT(wr->iw_msg != NULL);
23530e7468fSPeter Dunlap 			iser_msg_free(wr->iw_msg);
23630e7468fSPeter Dunlap 
23730e7468fSPeter Dunlap 			if (wr->iw_pdu == NULL) {
23830e7468fSPeter Dunlap 				/* This is a hello exchange message */
23930e7468fSPeter Dunlap 				mutex_enter(&iser_conn->ic_lock);
24030e7468fSPeter Dunlap 				if (iser_conn->ic_stage ==
24130e7468fSPeter Dunlap 				    ISER_CONN_STAGE_HELLOREPLY_SENT) {
24230e7468fSPeter Dunlap 					/*
24330e7468fSPeter Dunlap 					 * We're on the target side,
24430e7468fSPeter Dunlap 					 * and have just successfully
24530e7468fSPeter Dunlap 					 * sent the HelloReply msg.
24630e7468fSPeter Dunlap 					 */
24730e7468fSPeter Dunlap 					iser_conn->ic_stage =
24830e7468fSPeter Dunlap 					    ISER_CONN_STAGE_LOGGED_IN;
24930e7468fSPeter Dunlap 				}
25030e7468fSPeter Dunlap 				mutex_exit(&iser_conn->ic_lock);
25130e7468fSPeter Dunlap 			} else {
25230e7468fSPeter Dunlap 				/* This is a normal control message */
25330e7468fSPeter Dunlap 				idm_pdu_complete(wr->iw_pdu, idm_status);
25430e7468fSPeter Dunlap 			}
25530e7468fSPeter Dunlap 
25630e7468fSPeter Dunlap 			/* Free the wr handle */
25730e7468fSPeter Dunlap 			iser_wr_free(wr);
25830e7468fSPeter Dunlap 
25930e7468fSPeter Dunlap 			break;
26030e7468fSPeter Dunlap 
26130e7468fSPeter Dunlap 		case ISER_WR_RDMAW:
26230e7468fSPeter Dunlap 		case ISER_WR_RDMAR:
26330e7468fSPeter Dunlap 			/*
26430e7468fSPeter Dunlap 			 * Invoke the appropriate callback;
26530e7468fSPeter Dunlap 			 * the buffer will be freed there.
26630e7468fSPeter Dunlap 			 */
26730e7468fSPeter Dunlap 			idb = wr->iw_buf;
268a668b114SPriya Krishnan 			mr = ((iser_buf_t *)idb->idb_buf_private)->iser_mr;
26930e7468fSPeter Dunlap #ifdef DEBUG
27030e7468fSPeter Dunlap 			bcopy(&wc[i],
27130e7468fSPeter Dunlap 			    &((iser_buf_t *)idb->idb_buf_private)->buf_wc,
27230e7468fSPeter Dunlap 			    sizeof (ibt_wc_t));
27330e7468fSPeter Dunlap #endif
27430e7468fSPeter Dunlap 			idt = idb->idb_task_binding;
27530e7468fSPeter Dunlap 
27630e7468fSPeter Dunlap 			mutex_enter(&idt->idt_mutex);
27730e7468fSPeter Dunlap 			if (wr->iw_type == ISER_WR_RDMAW) {
278a668b114SPriya Krishnan 				DTRACE_ISCSI_8(xfer__done,
279a668b114SPriya Krishnan 				    idm_conn_t *, idt->idt_ic,
280a668b114SPriya Krishnan 				    uintptr_t, idb->idb_buf,
281a668b114SPriya Krishnan 				    uint32_t, idb->idb_bufoffset,
282a668b114SPriya Krishnan 				    uint64_t, mr->is_mrva, uint32_t, 0,
283a668b114SPriya Krishnan 				    uint32_t, mr->is_mrrkey,
284a668b114SPriya Krishnan 				    uint32_t, idb->idb_xfer_len,
285a668b114SPriya Krishnan 				    int, XFER_BUF_TX_TO_INI);
28630e7468fSPeter Dunlap 				idm_buf_tx_to_ini_done(idt, idb, idm_status);
28730e7468fSPeter Dunlap 			} else {
288a668b114SPriya Krishnan 				DTRACE_ISCSI_8(xfer__done,
289a668b114SPriya Krishnan 				    idm_conn_t *, idt->idt_ic,
290a668b114SPriya Krishnan 				    uintptr_t, idb->idb_buf,
291a668b114SPriya Krishnan 				    uint32_t, idb->idb_bufoffset,
292a668b114SPriya Krishnan 				    uint64_t, mr->is_mrva, uint32_t, 0,
293a668b114SPriya Krishnan 				    uint32_t, mr->is_mrrkey,
294a668b114SPriya Krishnan 				    uint32_t, idb->idb_xfer_len,
295a668b114SPriya Krishnan 				    int, XFER_BUF_RX_FROM_INI);
29630e7468fSPeter Dunlap 				idm_buf_rx_from_ini_done(idt, idb, idm_status);
29730e7468fSPeter Dunlap 			}
29830e7468fSPeter Dunlap 
29930e7468fSPeter Dunlap 			/* Free the wr handle */
30030e7468fSPeter Dunlap 			iser_wr_free(wr);
30130e7468fSPeter Dunlap 
30230e7468fSPeter Dunlap 			break;
30330e7468fSPeter Dunlap 
30430e7468fSPeter Dunlap 		default:
30530e7468fSPeter Dunlap 			ASSERT(0);
30630e7468fSPeter Dunlap 			break;
30730e7468fSPeter Dunlap 		}
30830e7468fSPeter Dunlap 	}
30930e7468fSPeter Dunlap 
31030e7468fSPeter Dunlap 	return (status);
31130e7468fSPeter Dunlap }
31230e7468fSPeter Dunlap 
31330e7468fSPeter Dunlap void
iser_ib_recvcq_handler(ibt_cq_hdl_t cq_hdl,void * arg)31430e7468fSPeter Dunlap iser_ib_recvcq_handler(ibt_cq_hdl_t cq_hdl, void *arg)
31530e7468fSPeter Dunlap {
31630e7468fSPeter Dunlap 	iser_chan_t	*iser_chan;
31730e7468fSPeter Dunlap 	ibt_status_t	status;
31830e7468fSPeter Dunlap 
31930e7468fSPeter Dunlap 	iser_chan = (iser_chan_t *)arg;
32030e7468fSPeter Dunlap 
321a82ec3cfSPriya Krishnan 	/*
322a82ec3cfSPriya Krishnan 	 * Poll for work request completion while successful. If the
323a82ec3cfSPriya Krishnan 	 * queue empties or otherwise becomes invalid, stop polling.
324a82ec3cfSPriya Krishnan 	 */
32530e7468fSPeter Dunlap 	do {
32630e7468fSPeter Dunlap 		status = iser_ib_poll_recv_completions(cq_hdl, iser_chan);
327a82ec3cfSPriya Krishnan 	} while (status == IBT_SUCCESS);
32830e7468fSPeter Dunlap 
329a82ec3cfSPriya Krishnan 	if (status == IBT_CQ_EMPTY) {
330a82ec3cfSPriya Krishnan 		/* We've emptied the CQ, rearm it before we're done here */
331a82ec3cfSPriya Krishnan 		status = ibt_enable_cq_notify(cq_hdl, IBT_NEXT_COMPLETION);
332a82ec3cfSPriya Krishnan 		if (status != IBT_SUCCESS) {
333a82ec3cfSPriya Krishnan 			/* Unexpected error */
334a82ec3cfSPriya Krishnan 			ISER_LOG(CE_NOTE, "iser_ib_recvcq_handler: "
335a82ec3cfSPriya Krishnan 			    "ibt_enable_cq_notify error (%d)", status);
336a82ec3cfSPriya Krishnan 			return;
337a82ec3cfSPriya Krishnan 		}
33830e7468fSPeter Dunlap 
339a82ec3cfSPriya Krishnan 		/* Now, check for more completions after the rearm */
340a82ec3cfSPriya Krishnan 		do {
341a82ec3cfSPriya Krishnan 			status = iser_ib_poll_recv_completions(
342a82ec3cfSPriya Krishnan 			    cq_hdl, iser_chan);
343a82ec3cfSPriya Krishnan 		} while (status == IBT_SUCCESS);
344a82ec3cfSPriya Krishnan 	}
34530e7468fSPeter Dunlap }
34630e7468fSPeter Dunlap 
34730e7468fSPeter Dunlap static int
iser_ib_poll_recv_completions(ibt_cq_hdl_t cq_hdl,iser_chan_t * iser_chan)34830e7468fSPeter Dunlap iser_ib_poll_recv_completions(ibt_cq_hdl_t cq_hdl, iser_chan_t *iser_chan)
34930e7468fSPeter Dunlap {
35030e7468fSPeter Dunlap 	ibt_wc_t	wc;
35130e7468fSPeter Dunlap 	iser_msg_t	*msg;
35230e7468fSPeter Dunlap 	iser_qp_t	*iser_qp;
35330e7468fSPeter Dunlap 	int		status;
35430e7468fSPeter Dunlap 
35530e7468fSPeter Dunlap 	iser_qp = &(iser_chan->ic_qp);
35630e7468fSPeter Dunlap 
35730e7468fSPeter Dunlap 	bzero(&wc, sizeof (ibt_wc_t));
35830e7468fSPeter Dunlap 	status = ibt_poll_cq(cq_hdl, &wc, 1, NULL);
35930e7468fSPeter Dunlap 	if (status == IBT_CQ_EMPTY) {
36030e7468fSPeter Dunlap 		/* CQ is empty, return */
36130e7468fSPeter Dunlap 		return (status);
36230e7468fSPeter Dunlap 	}
36330e7468fSPeter Dunlap 
36430e7468fSPeter Dunlap 	if (status != IBT_SUCCESS) {
36530e7468fSPeter Dunlap 		/* Unexpected error */
36630e7468fSPeter Dunlap 		ISER_LOG(CE_NOTE, "iser_ib_poll_recv_completions: "
36730e7468fSPeter Dunlap 		    "ibt_poll_cq error (%d)", status);
36830e7468fSPeter Dunlap 		mutex_enter(&iser_qp->qp_lock);
36930e7468fSPeter Dunlap 		iser_qp->rq_level--;
37030e7468fSPeter Dunlap 		mutex_exit(&iser_qp->qp_lock);
37130e7468fSPeter Dunlap 		/* Free the msg handle (if we got it back) */
37230e7468fSPeter Dunlap 		if ((msg = (iser_msg_t *)(uintptr_t)wc.wc_id) != NULL) {
37330e7468fSPeter Dunlap 			iser_msg_free(msg);
37430e7468fSPeter Dunlap 		}
37530e7468fSPeter Dunlap 		return (status);
37630e7468fSPeter Dunlap 	}
37730e7468fSPeter Dunlap 
37830e7468fSPeter Dunlap 	/* Retrieve the iSER msg handle */
37930e7468fSPeter Dunlap 	msg = (iser_msg_t *)(uintptr_t)wc.wc_id;
38030e7468fSPeter Dunlap 	ASSERT(msg != NULL);
38130e7468fSPeter Dunlap 
38230e7468fSPeter Dunlap 	/*
38330e7468fSPeter Dunlap 	 * Decrement the posted level in the RQ, then check
38430e7468fSPeter Dunlap 	 * to see if we need to fill the RQ back up (or if
38530e7468fSPeter Dunlap 	 * we are already on the taskq).
38630e7468fSPeter Dunlap 	 */
387cf8c0ebaSPeter Dunlap 	mutex_enter(&iser_chan->ic_conn->ic_lock);
38830e7468fSPeter Dunlap 	mutex_enter(&iser_qp->qp_lock);
38930e7468fSPeter Dunlap 	iser_qp->rq_level--;
39030e7468fSPeter Dunlap 
39130e7468fSPeter Dunlap 	if ((iser_qp->rq_taskqpending == B_FALSE) &&
39272cf3143Speter dunlap 	    (iser_qp->rq_level <= iser_qp->rq_lwm) &&
39372cf3143Speter dunlap 	    (iser_chan->ic_conn->ic_stage >= ISER_CONN_STAGE_IC_CONNECTED) &&
39472cf3143Speter dunlap 	    (iser_chan->ic_conn->ic_stage <= ISER_CONN_STAGE_LOGGED_IN)) {
39530e7468fSPeter Dunlap 		/* Set the pending flag and fire off a post_recv */
39630e7468fSPeter Dunlap 		iser_qp->rq_taskqpending = B_TRUE;
39730e7468fSPeter Dunlap 		mutex_exit(&iser_qp->qp_lock);
39830e7468fSPeter Dunlap 
399cf8c0ebaSPeter Dunlap 		status = iser_ib_post_recv_async(iser_chan->ic_chanhdl);
40030e7468fSPeter Dunlap 
40130e7468fSPeter Dunlap 		if (status != DDI_SUCCESS) {
40230e7468fSPeter Dunlap 			ISER_LOG(CE_NOTE, "iser_ib_poll_recv_completions: "
40330e7468fSPeter Dunlap 			    "task dispatch failed");
40430e7468fSPeter Dunlap 			/* Failure to launch, unset the pending flag */
40530e7468fSPeter Dunlap 			mutex_enter(&iser_qp->qp_lock);
40630e7468fSPeter Dunlap 			iser_qp->rq_taskqpending = B_FALSE;
40730e7468fSPeter Dunlap 			mutex_exit(&iser_qp->qp_lock);
40830e7468fSPeter Dunlap 		}
40930e7468fSPeter Dunlap 	} else {
41030e7468fSPeter Dunlap 		mutex_exit(&iser_qp->qp_lock);
41130e7468fSPeter Dunlap 	}
41230e7468fSPeter Dunlap 
41330e7468fSPeter Dunlap 	DTRACE_PROBE3(iser__recv__cqe, iser_chan_t *, iser_chan,
41430e7468fSPeter Dunlap 	    ibt_wc_t *, &wc, ibt_wc_status_t, wc.wc_status);
41530e7468fSPeter Dunlap 	if (wc.wc_status != IBT_WC_SUCCESS) {
41630e7468fSPeter Dunlap 		/*
41730e7468fSPeter Dunlap 		 * Tell IDM that the channel has gone down,
418*48bbca81SDaniel Hoffman 		 * unless it already knows.
41930e7468fSPeter Dunlap 		 */
42030e7468fSPeter Dunlap 		switch (iser_chan->ic_conn->ic_stage) {
42130e7468fSPeter Dunlap 		case ISER_CONN_STAGE_IC_DISCONNECTED:
42230e7468fSPeter Dunlap 		case ISER_CONN_STAGE_IC_FREED:
42330e7468fSPeter Dunlap 		case ISER_CONN_STAGE_CLOSING:
42430e7468fSPeter Dunlap 		case ISER_CONN_STAGE_CLOSED:
42530e7468fSPeter Dunlap 			break;
42630e7468fSPeter Dunlap 
42730e7468fSPeter Dunlap 		default:
42830e7468fSPeter Dunlap 			idm_conn_event(iser_chan->ic_conn->ic_idmc,
42930e7468fSPeter Dunlap 			    CE_TRANSPORT_FAIL, IDM_STATUS_FAIL);
43030e7468fSPeter Dunlap 			iser_chan->ic_conn->ic_stage =
43130e7468fSPeter Dunlap 			    ISER_CONN_STAGE_CLOSING;
43230e7468fSPeter Dunlap 		}
43330e7468fSPeter Dunlap 		mutex_exit(&iser_chan->ic_conn->ic_lock);
43430e7468fSPeter Dunlap 
43530e7468fSPeter Dunlap 		iser_msg_free(msg);
43630e7468fSPeter Dunlap 		return (DDI_SUCCESS);
43730e7468fSPeter Dunlap 	} else {
43872cf3143Speter dunlap 		mutex_exit(&iser_chan->ic_conn->ic_lock);
43972cf3143Speter dunlap 
44030e7468fSPeter Dunlap 		/*
44130e7468fSPeter Dunlap 		 * We have an iSER message in, let's handle it.
44230e7468fSPeter Dunlap 		 * We will free the iser_msg_t later in this path,
44330e7468fSPeter Dunlap 		 * depending upon the action required.
44430e7468fSPeter Dunlap 		 */
44530e7468fSPeter Dunlap 		iser_msg_handle(iser_chan, msg);
44630e7468fSPeter Dunlap 		return (DDI_SUCCESS);
44730e7468fSPeter Dunlap 	}
44830e7468fSPeter Dunlap }
44930e7468fSPeter Dunlap 
45030e7468fSPeter Dunlap static void
iser_msg_handle(iser_chan_t * chan,iser_msg_t * msg)45130e7468fSPeter Dunlap iser_msg_handle(iser_chan_t *chan, iser_msg_t *msg)
45230e7468fSPeter Dunlap {
45330e7468fSPeter Dunlap 	int		opcode;
45430e7468fSPeter Dunlap 	iser_ctrl_hdr_t	*hdr = NULL;
45530e7468fSPeter Dunlap 	iser_conn_t	*iser_conn = chan->ic_conn;
45630e7468fSPeter Dunlap 	int		status;
45730e7468fSPeter Dunlap 
45830e7468fSPeter Dunlap 	hdr = (iser_ctrl_hdr_t *)(uintptr_t)msg->msg_ds.ds_va;
45930e7468fSPeter Dunlap 	ASSERT(hdr != NULL);
46030e7468fSPeter Dunlap 
46130e7468fSPeter Dunlap 	opcode = hdr->opcode;
46230e7468fSPeter Dunlap 	if (opcode == ISER_OPCODE_CTRL_TYPE_PDU) {
46330e7468fSPeter Dunlap 		/*
46430e7468fSPeter Dunlap 		 * Handle an iSCSI Control PDU iSER message.
46530e7468fSPeter Dunlap 		 * Note we'll free the msg handle in the PDU callback.
46630e7468fSPeter Dunlap 		 */
46730e7468fSPeter Dunlap 		status = iser_iscsihdr_handle(chan, msg);
46830e7468fSPeter Dunlap 		if (status != DDI_SUCCESS) {
46930e7468fSPeter Dunlap 			/*
47030e7468fSPeter Dunlap 			 * We are unable to handle this message, and
47130e7468fSPeter Dunlap 			 * have no way to recover from this.  Fail the
47230e7468fSPeter Dunlap 			 * transport.
47330e7468fSPeter Dunlap 			 */
47430e7468fSPeter Dunlap 			ISER_LOG(CE_NOTE, "iser_msg_handle: failed "
47530e7468fSPeter Dunlap 			    "iser_iscsihdr_handle");
47630e7468fSPeter Dunlap 			iser_msg_free(msg);
47730e7468fSPeter Dunlap 			idm_conn_event(iser_conn->ic_idmc,
47830e7468fSPeter Dunlap 			    CE_TRANSPORT_FAIL, IDM_STATUS_FAIL);
47930e7468fSPeter Dunlap 		}
48030e7468fSPeter Dunlap 	} else if (opcode == ISER_OPCODE_HELLO_MSG) { /* at the target */
48130e7468fSPeter Dunlap 		/*
48230e7468fSPeter Dunlap 		 * We are currently not supporting Hello Exchange,
48330e7468fSPeter Dunlap 		 * since OFED iSER does not. May be revisited.
48430e7468fSPeter Dunlap 		 */
48530e7468fSPeter Dunlap 		ASSERT(opcode != ISER_OPCODE_HELLO_MSG);
48630e7468fSPeter Dunlap 
48730e7468fSPeter Dunlap 		if (iser_conn->ic_type != ISER_CONN_TYPE_TGT) {
48830e7468fSPeter Dunlap 			idm_conn_event(iser_conn->ic_idmc,
48930e7468fSPeter Dunlap 			    CE_TRANSPORT_FAIL, IDM_STATUS_FAIL);
49030e7468fSPeter Dunlap 		}
49130e7468fSPeter Dunlap 
49230e7468fSPeter Dunlap 		iser_hello_hdr_t *hello_hdr = (iser_hello_hdr_t *)hdr;
49330e7468fSPeter Dunlap 
49430e7468fSPeter Dunlap 		ISER_LOG(CE_NOTE, "received Hello message: opcode[%d], "
49530e7468fSPeter Dunlap 		    "maxver[%d], minver[%d], iser_ird[%d], msg (0x%p)",
49630e7468fSPeter Dunlap 		    hello_hdr->opcode, hello_hdr->maxver, hello_hdr->minver,
49730e7468fSPeter Dunlap 		    ntohs(hello_hdr->iser_ird), (void *)msg);
49830e7468fSPeter Dunlap 
49930e7468fSPeter Dunlap 		mutex_enter(&iser_conn->ic_lock);
50030e7468fSPeter Dunlap 
50130e7468fSPeter Dunlap 		if (iser_conn->ic_stage != ISER_CONN_STAGE_HELLO_WAIT) {
50230e7468fSPeter Dunlap 			/* target is not expected to receive a Hello */
50330e7468fSPeter Dunlap 			idm_conn_event(iser_conn->ic_idmc,
50430e7468fSPeter Dunlap 			    CE_TRANSPORT_FAIL, IDM_STATUS_FAIL);
50530e7468fSPeter Dunlap 		}
50630e7468fSPeter Dunlap 
50730e7468fSPeter Dunlap 		iser_conn->ic_stage = ISER_CONN_STAGE_HELLOREPLY_SENT;
50830e7468fSPeter Dunlap 		mutex_exit(&iser_conn->ic_lock);
50930e7468fSPeter Dunlap 
51030e7468fSPeter Dunlap 		/* Prepare and send a HelloReply message */
51130e7468fSPeter Dunlap 		status = iser_xfer_helloreply_msg(chan);
51230e7468fSPeter Dunlap 		if (status != ISER_STATUS_SUCCESS) {
51330e7468fSPeter Dunlap 
51430e7468fSPeter Dunlap 			mutex_enter(&iser_conn->ic_lock);
51530e7468fSPeter Dunlap 			iser_conn->ic_stage =
51630e7468fSPeter Dunlap 			    ISER_CONN_STAGE_HELLOREPLY_SENT_FAIL;
51730e7468fSPeter Dunlap 			mutex_exit(&iser_conn->ic_lock);
51830e7468fSPeter Dunlap 
51930e7468fSPeter Dunlap 			idm_conn_event(iser_conn->ic_idmc,
52030e7468fSPeter Dunlap 			    CE_TRANSPORT_FAIL, status);
52130e7468fSPeter Dunlap 		}
52230e7468fSPeter Dunlap 
52330e7468fSPeter Dunlap 		/* Free this msg handle */
52430e7468fSPeter Dunlap 		iser_msg_free(msg);
52530e7468fSPeter Dunlap 
52630e7468fSPeter Dunlap 	} else if (opcode == ISER_OPCODE_HELLOREPLY_MSG) { /* at initiator */
52730e7468fSPeter Dunlap 
52830e7468fSPeter Dunlap 		/*
52930e7468fSPeter Dunlap 		 * We are currently not supporting Hello Exchange,
53030e7468fSPeter Dunlap 		 * since OFED iSER does not. May be revisited.
53130e7468fSPeter Dunlap 		 */
53230e7468fSPeter Dunlap 		ASSERT(opcode != ISER_OPCODE_HELLOREPLY_MSG);
53330e7468fSPeter Dunlap 
53430e7468fSPeter Dunlap 		if (iser_conn->ic_type != ISER_CONN_TYPE_INI) {
53530e7468fSPeter Dunlap 			idm_conn_event(iser_conn->ic_idmc,
53630e7468fSPeter Dunlap 			    CE_TRANSPORT_FAIL, status);
53730e7468fSPeter Dunlap 		}
53830e7468fSPeter Dunlap 
53930e7468fSPeter Dunlap 		iser_helloreply_hdr_t *hello_hdr = (iser_helloreply_hdr_t *)hdr;
54030e7468fSPeter Dunlap 
54130e7468fSPeter Dunlap 		ISER_LOG(CE_NOTE, "received Hello Reply message: opcode[%d], "
54230e7468fSPeter Dunlap 		    "maxver[%d], curver[%d], iser_ord[%d], msg (0x%p)",
54330e7468fSPeter Dunlap 		    hello_hdr->opcode, hello_hdr->maxver, hello_hdr->curver,
54430e7468fSPeter Dunlap 		    ntohs(hello_hdr->iser_ord), (void *)msg);
54530e7468fSPeter Dunlap 
54630e7468fSPeter Dunlap 		/* Free this msg handle */
54730e7468fSPeter Dunlap 		iser_msg_free(msg);
54830e7468fSPeter Dunlap 
54930e7468fSPeter Dunlap 		/*
55030e7468fSPeter Dunlap 		 * Signal the receipt of HelloReply to the waiting thread
55130e7468fSPeter Dunlap 		 * so that the initiator can proceed to the Full Feature
55230e7468fSPeter Dunlap 		 * Phase.
55330e7468fSPeter Dunlap 		 */
55430e7468fSPeter Dunlap 		mutex_enter(&iser_conn->ic_lock);
55530e7468fSPeter Dunlap 		iser_conn->ic_stage = ISER_CONN_STAGE_HELLOREPLY_RCV;
55630e7468fSPeter Dunlap 		cv_signal(&iser_conn->ic_stage_cv);
55730e7468fSPeter Dunlap 		mutex_exit(&iser_conn->ic_lock);
55830e7468fSPeter Dunlap 	} else {
55930e7468fSPeter Dunlap 		/* Protocol error: free the msg handle and fail the session */
56030e7468fSPeter Dunlap 		ISER_LOG(CE_NOTE, "iser_msg_handle: unsupported opcode (0x%x): "
56130e7468fSPeter Dunlap 		    "terminating session on IDM handle (0x%p)", opcode,
56230e7468fSPeter Dunlap 		    (void *) iser_conn->ic_idmc);
56330e7468fSPeter Dunlap 
56430e7468fSPeter Dunlap 		iser_msg_free(msg);
56530e7468fSPeter Dunlap 		idm_conn_event(iser_conn->ic_idmc, CE_TRANSPORT_FAIL,
56630e7468fSPeter Dunlap 		    IDM_STATUS_FAIL);
56730e7468fSPeter Dunlap 	}
56830e7468fSPeter Dunlap }
56930e7468fSPeter Dunlap 
57030e7468fSPeter Dunlap #define	IDM_PDU_OPCODE(PDU) \
57130e7468fSPeter Dunlap 	((PDU)->isp_hdr->opcode & ISCSI_OPCODE_MASK)
57230e7468fSPeter Dunlap 
57330e7468fSPeter Dunlap /* network to host translation for 24b integers */
57430e7468fSPeter Dunlap static uint32_t
n2h24(uchar_t * ptr)57530e7468fSPeter Dunlap n2h24(uchar_t *ptr)
57630e7468fSPeter Dunlap {
57730e7468fSPeter Dunlap 	return ((ptr[0] << 16) | (ptr[1] << 8) | ptr[2]);
57830e7468fSPeter Dunlap }
57930e7468fSPeter Dunlap 
58030e7468fSPeter Dunlap /* ARGSUSED */
58130e7468fSPeter Dunlap static void
iser_rx_pdu_cb(idm_pdu_t * pdu,idm_status_t status)58230e7468fSPeter Dunlap iser_rx_pdu_cb(idm_pdu_t *pdu, idm_status_t status)
58330e7468fSPeter Dunlap {
58430e7468fSPeter Dunlap 	/* Free the iser msg handle and the PDU handle */
58530e7468fSPeter Dunlap 	iser_msg_free((iser_msg_t *)pdu->isp_transport_private);
58630e7468fSPeter Dunlap 	idm_pdu_free(pdu);
58730e7468fSPeter Dunlap }
58830e7468fSPeter Dunlap 
58930e7468fSPeter Dunlap int
iser_iscsihdr_handle(iser_chan_t * chan,iser_msg_t * msg)59030e7468fSPeter Dunlap iser_iscsihdr_handle(iser_chan_t *chan, iser_msg_t *msg)
59130e7468fSPeter Dunlap {
59230e7468fSPeter Dunlap 	idm_pdu_t	*pdu;
59330e7468fSPeter Dunlap 	uint8_t		*iser_hdrp;
59430e7468fSPeter Dunlap 	uint8_t		*iscsi_hdrp;
59530e7468fSPeter Dunlap 	iscsi_hdr_t	*bhs;
59630e7468fSPeter Dunlap 
59730e7468fSPeter Dunlap 	pdu = idm_pdu_alloc_nosleep(sizeof (iscsi_hdr_t), 0);
59830e7468fSPeter Dunlap 	pdu->isp_ic = chan->ic_conn->ic_idmc;
59930e7468fSPeter Dunlap 	ASSERT(pdu->isp_ic != NULL);
60030e7468fSPeter Dunlap 
60130e7468fSPeter Dunlap 	/* Set the iser_msg handle into the transport-private field */
60230e7468fSPeter Dunlap 	pdu->isp_transport_private = (void *)msg;
60330e7468fSPeter Dunlap 
60430e7468fSPeter Dunlap 	/* Set up a pointer in the pdu handle to the iSER header */
60530e7468fSPeter Dunlap 	iser_hdrp = (uint8_t *)(uintptr_t)msg->msg_ds.ds_va;
60630e7468fSPeter Dunlap 	if (iser_hdrp == NULL) {
60730e7468fSPeter Dunlap 		ISER_LOG(CE_NOTE, "iser_iscsihdr_handle: iser_hdrp is NULL");
60830e7468fSPeter Dunlap 		idm_pdu_free(pdu);
60930e7468fSPeter Dunlap 		return (ISER_STATUS_FAIL);
61030e7468fSPeter Dunlap 	}
61130e7468fSPeter Dunlap 	pdu->isp_transport_hdr = (void *)iser_hdrp;
61230e7468fSPeter Dunlap 	pdu->isp_transport_hdrlen = ISER_HEADER_LENGTH;
61330e7468fSPeter Dunlap 
61430e7468fSPeter Dunlap 	/*
61530e7468fSPeter Dunlap 	 * Set up a pointer to the iSCSI header, which is directly
61630e7468fSPeter Dunlap 	 * after the iSER header in the message.
61730e7468fSPeter Dunlap 	 */
61830e7468fSPeter Dunlap 	iscsi_hdrp = ((uint8_t *)(uintptr_t)msg->msg_ds.ds_va) +
61930e7468fSPeter Dunlap 	    ISER_HEADER_LENGTH;
62030e7468fSPeter Dunlap 	if (iscsi_hdrp == NULL) {
62130e7468fSPeter Dunlap 		ISER_LOG(CE_NOTE, "iser_iscsihdr_handle: iscsi_hdrp is NULL");
62230e7468fSPeter Dunlap 		idm_pdu_free(pdu);
62330e7468fSPeter Dunlap 		return (ISER_STATUS_FAIL);
62430e7468fSPeter Dunlap 	}
62530e7468fSPeter Dunlap 	pdu->isp_hdr = (iscsi_hdr_t *)(uintptr_t)iscsi_hdrp;
62630e7468fSPeter Dunlap 
62730e7468fSPeter Dunlap 	/* Fill in the BHS */
62830e7468fSPeter Dunlap 	bhs = pdu->isp_hdr;
62930e7468fSPeter Dunlap 	pdu->isp_hdrlen	= sizeof (iscsi_hdr_t) +
63030e7468fSPeter Dunlap 	    (bhs->hlength * sizeof (uint32_t));
63130e7468fSPeter Dunlap 	pdu->isp_datalen = n2h24(bhs->dlength);
63230e7468fSPeter Dunlap 	pdu->isp_callback = iser_rx_pdu_cb;
63330e7468fSPeter Dunlap 
63430e7468fSPeter Dunlap 	/*
63530e7468fSPeter Dunlap 	 * If datalen > 0, then non-scsi data may be present. Allocate
63630e7468fSPeter Dunlap 	 * space in the PDU handle and set a pointer to the data.
63730e7468fSPeter Dunlap 	 */
63830e7468fSPeter Dunlap 	if (pdu->isp_datalen) {
63930e7468fSPeter Dunlap 		pdu->isp_data = ((uint8_t *)(uintptr_t)pdu->isp_hdr) +
64030e7468fSPeter Dunlap 		    pdu->isp_hdrlen;
64130e7468fSPeter Dunlap 	}
64230e7468fSPeter Dunlap 
64330e7468fSPeter Dunlap 	/* Process RX PDU */
64430e7468fSPeter Dunlap 	idm_pdu_rx(pdu->isp_ic, pdu);
64530e7468fSPeter Dunlap 
64630e7468fSPeter Dunlap 	return (DDI_SUCCESS);
64730e7468fSPeter Dunlap }
648